Uws11 update

README.changes

@@ -1,3 +1,12 @@
+2016/03/22
+==========
+
+UWS update (branch uws11_update): new field 'creationTime' needed in Uws_Jobs table
+on the web server:
+
+ALTER TABLE `Uws_Jobs` ADD COLUMN `creationTime` VARCHAR(256) AFTER quote;
+
+
 2015/09/08
 ==========
 
@@ -198,4 +207,4 @@ From now on, add the following config entry to your instance:
 
 set this to 1 if you want the user account to be validated by a moderator, the default value is 0.
 
-WARNING! not setting this might be a potential security problem!
+WARNING! not setting this might be a potential security problem!

modules/query/models/Query.php

@@ -111,15 +111,16 @@ public function validate($sql, $plan = false, $table, array &$errors, array &$so
     }
 
     /**
-     * Querys the database with a plain text query.
+     * Queries the database with a plain text query.
      * @param string $sql the sql string
      * @param bool $plan flag for plan creation
      * @param string $table result table
      * @param array $options for further options that are handeled by the queue
+     * @param string $creationTime time of creation for uws-jobs (from pending phase) (for new UWS 1.1)
      * @param string $jobType job type of the new job
      * @return array $response
      */
-    public function query($sql, $plan = false, $table, $sources, $options = array(), $jobType = 'web') {
+    public function query($sql, $plan = false, $table, $sources, $options = array(), $creationTime = NULL, $jobType = 'web') {
         // init error array
         $errors = array();
 
@@ -183,6 +184,9 @@ public function query($sql, $plan = false, $table, $sources, $options = array(),
         }
         $job['sources'] = implode(',',$tmp);
 
+        // also provide the creationTime as option to submitJob
+        $options['creationTime'] = $creationTime;
+
         // submit job
         $this->_queue->submitJob($job, $errors, $options);
         if (!empty($errors)) {

modules/query/models/Resource/DirectQuery.php

@@ -71,7 +71,7 @@ public function submitJob(&$job, array &$errors, $options = false) {
 
         // get jobId from the options, or not
         if (!empty($options) && array_key_exists('jobId', $options)) {
-            $job['id'] = "{$options['jobId']}";
+            $job['id'] = "{$options['jobId']}"; // will be overwritten later on by "true" jobId => remove?
         }
 
         // create the actual sql statement
@@ -156,10 +156,19 @@ public function submitJob(&$job, array &$errors, $options = false) {
         }
 
         // set other fields in job object
-        $job['time'] = date("Y-m-d\TH:i:s");
+
+        // if no time is set yet (or it is Null/0's), then create it now, otherwise keep the stored (creation) time
+        if (!empty($options) && array_key_exists('creationTime', $options)) {
+            $job['time'] = $options['creationTime'];
+        }
+        // check if time is 0, even if it was already set (because if creationTime was NULL, we do not want to use this)
+        if ($job['time'] == false || $job['time'] == "0000-00-00 00:00:00" || $job['time'] == NULL) {
+            $job['time'] = date("Y-m-d H:i:s");
+        }
+
         $job['complete'] = true;
 
-        // insert job into jobs table
+        // insert job into jobs table // is this the one on the server??
         $job['id'] = $this->getJobResource()->insertRow($job);
     }
 

modules/query/models/Resource/QQueueQuery.php

@@ -153,11 +153,20 @@ public function submitJob(&$job, array &$errors, $options = false) {
             return Query_Model_Resource_QQueueQuery::$_status['error'];
         }
 
-        // fetch the new jobs row for the timestamp and the status
+        // fetch the new jobs row for the status
         $row = $this->fetchRow($job['id']);
-        $job['time'] = $row['timeSubmit'];
         $job['status_id'] = $row['status_id'];
 
+        // if no time is set yet, then create it now, otherwise keep the stored (creation) time
+        // set the job's 'time' to the creationTime provided in $options;
+        // if no creationTime given (or Null/000), use "timeSubmit" from the server instead
+        if (!empty($options) && array_key_exists('creationTime', $options)) {
+            $job['time'] = $options['creationTime'];
+        }
+        if ($job['time'] == false || $job['time'] == "0000-00-00 00:00:00" || $job['time'] == NULL) {
+            $job['time'] = $row['timeSubmit'];
+        }
+
         // insert job into jobs table
         $this->getJobResource()->insertRow($job);
     }

modules/query/models/Uws.php

@@ -103,6 +103,7 @@ public function getJobList($params) {
             $wherestatus = NULL;
         } else {
             // statuslist is empty and no phase-filter was given
+            // => shall return everything except removed jobs (ARCHIVED phase)
             $wherestatus = array('status_id != ?' => $this->getResource()->getStatusId('removed'));
         }
 
@@ -112,9 +113,9 @@ public function getJobList($params) {
         // check LAST keyword and set $limit accordingly
         if (array_key_exists('LAST', $params)) {
             $last = $params['LAST'];
-            // check if string contains only digits (positive integer!),
+            // check if string contains only digits (-> positive integers) and not 0
             // if so, convert to integer
-            if (isset($last) && ctype_digit($last)) {
+            if (isset($last) && ctype_digit($last) && $last > 0) {
                 $last = intval($last);
                 // set limit (maybe restrict to max. limit here?)
                 $limit = $last;
@@ -155,7 +156,16 @@ public function getJobList($params) {
             foreach ($rows as $job) {
                 $href = Daiquiri_Config::getInstance()->getSiteUrl() . "/uws/" . urlencode($params['moduleName']) . "/" . urlencode($job['id']);
                 $status = Query_Model_Uws::$status[Query_Model_Uws::$statusQueue[$job['status']]];
-                $joblist[] = array('id' => $job['table'], 'href' => $href, 'status' => $status, 'time' => $job['time']);
+                $jobId = $job['id'];
+
+                $joblist[] = array('id' => $jobId,
+                                   'href' => $href,
+                                   'status' => $status,
+                                   'time' => $job['time'],
+                                   'creationTime' => $job['time'],
+                                   'runId' => $job['table'],
+                                   'ownerId' => Daiquiri_Auth::getInstance()->getCurrentUsername()
+                                   );
             }
         }
 
@@ -169,56 +179,81 @@ public function getJobList($params) {
             // add AFTER condition and/or ignore startTime=NULL in case of AFTER/LAST
             $whereafter = array();
             if (isset($after)) {
-                $whereafter = array("CONVERT_TZ(startTime, 'SYSTEM', '+0:00') > ?" => date('Y-m-d H:i:s', $after));
+                $whereafter = array("CONVERT_TZ(creationTime, 'SYSTEM', '+0:00') > ?" => date('Y-m-d H:i:s', $after));
             }
             else if (isset($last)) {
-                $whereafter = array('startTime IS NOT NULL');
+                $whereafter = array('creationTime IS NOT NULL'); // should never happen actually ... (maybe older jobs with no creation time yet ...)
             }
 
             $pendingRows = $resUWSJobs->fetchRows(array(
                 'where' => $whereafter,
-                'order' => array('startTime DESC'),
-                'order' => array('jobId DESC'), // add this here for useful ordering if startTime=NULL (PENDING jobs)
+                'order' => array('creationTime DESC'),
+                'order' => array('jobId DESC'), // add this here for useful ordering if creationTime=NULL (PENDING jobs)
                 'limit' => $limit,
             )); // the check for the userId is inside UWSJobs
 
             foreach ($pendingRows as $job) {
                 $href = Daiquiri_Config::getInstance()->getSiteUrl() . "/uws/" . urlencode($params['moduleName']) . "/" . urlencode($job['jobId']);
                 $status = $job['phase'];
+                $jobId = $job['jobId'];
 
                 if ( empty($phase) || in_array($status, $phases) ) {
-                    $pendingJoblist[] = array('id' => $job['jobId'], 'href' => $href, 'status' => $status, 'time' => $job['startTime']);
+                    $pendingJoblist[] = array('id' => $jobId,
+                                              'href' => $href,
+                                              'status' => $status,
+                                              'time' => $job['startTime'],
+                                              'creationTime' => $job['creationTime'],
+                                              'runId' => $job['runId'],
+                                              'ownerId' => $job['ownerId']
+                                             );
                 }
             }
         }
 
         // reverse job sort order, since standard requires for LAST/AFTER keywords
         // *ascending* startTimes (but needed initial DESC order in queries for most recent limit)
-        $joblist = array_reverse($joblist);
-        $pendingJoblist = array_reverse($pendingJoblist);
+        // -- reverse ordering not needed anymore! Yeah! (Feb 2016)
 
         // merge both lists ...
         $joblist = array_merge($joblist, $pendingJoblist);
 
-        // and sort by startTime (< 10 ms for 10,000 jobs, so it's fast enough to do it always)
+        // ... convert time formats and update list ...
+        // (Before sorting, so that we have a consistent
+        // format, no matter what may have been different in the two lists.)
+        // (This would be a good opportunity to convert to UTC time zone as well,
+        //  but currently this is not required by the standard.)
+
+        foreach ($joblist as $key => $job) {
+            $creationdatetime = new DateTime($job['creationTime']);
+            $creationdatetime = $creationdatetime->format('c');
+            $joblist[$key]['creationTime'] = $creationdatetime;
+        }
+
+        // ... and sort by creationTime
+        // (< 10 ms for 10,000 jobs, so it's fast enough to do it always)
         $sortcolumn = array();
         foreach ($joblist as $job) {
-            $sortcolumn[] = $job['time'];
+            $sortcolumn[] = $job['creationTime'];
         }
-        array_multisort($sortcolumn, SORT_ASC, $joblist);
+        array_multisort($sortcolumn, SORT_DESC, $joblist);
 
         // apply final cut, if required
         if (array_key_exists('LAST', $params)) {
             // cut the list, only keep number of jobs required by LAST or given by limit
             if (!is_null($limit)) {
-                $joblist = array_slice($joblist, -$limit, $limit);
+                $joblist = array_slice($joblist, 0, $limit);
             }
         }
 
         // copy jobs into jobs-object
         $jobs = new Uws_Model_Resource_Jobs();
         foreach ($joblist as $job) {
-            $jobs->addJob($job['id'], $job['href'], array($job['status']));
+           $jobs->addJob($job['id'],
+                          $job['href'],
+                          array($job['status']),
+                          $job['creationTime'],
+                          $job['runId'],
+                          $job['ownerId']);
         }
 
         return $jobs;
@@ -245,9 +280,11 @@ public function getJob($requestParams) {
         $jobUWS->jobId = $row['id'];
         $jobUWS->ownerId = Daiquiri_Auth::getInstance()->getCurrentUsername();
         $jobUWS->phase = Query_Model_Uws::$status[Query_Model_Uws::$statusQueue[$row['status']]];
+        $jobUWS->runId = $row['table']; // our interpretation of runId: use it as the job name = table name
 
         // convert timestamps to ISO 8601
         if (get_class($this->getResource()) == 'Query_Model_Resource_QQueueQuery') {
+
             if ($row['timeExecute'] !== "0000-00-00 00:00:00") {
                 $datetimeStart = new DateTime($row['timeExecute']);
                 $jobUWS->startTime = $datetimeStart->format('c');
@@ -257,11 +294,18 @@ public function getJob($requestParams) {
                 $datetimeEnd = new DateTime($row['timeFinish']);
                 $jobUWS->endTime = $datetimeEnd->format('c');
             }
+
+            if ($row['time'] != "0000-00-00 00:00:00") {
+                $datetimeCreation = new DateTime($row['time']);
+                $jobUWS->creationTime = $datetimeCreation->format('c');
+            }
+
         } else {
             // for simple queue
             $datetime = new DateTime($row['time']);
             $jobUWS->startTime = $datetime->format('c');
             $jobUWS->endTime = $datetime->format('c');
+            $jobUWS->creationTime = $datetime->format('c');
         }
 
         if ($this->getResource()->hasQueues()) {
@@ -408,7 +452,7 @@ public function runJob(Uws_Model_Resource_JobSummaryType &$job) {
             $resource->updateRow($job->jobId, array("phase" => "ERROR", "errorSummary" => Zend_Json::encode($job->errorSummary)));
 
             // job is in final state now, add startTime and endTime
-            $now = date('Y-m-d\TH:i:s');
+            $now = date('Y-m-d H:i:s');
             $resource->updateRow($job->jobId, array("startTime" => $now, "endTime" => $now));
 
             return;
@@ -432,8 +476,8 @@ public function runJob(Uws_Model_Resource_JobSummaryType &$job) {
         // validate query
         // We will set a startTime here only if the validation results in an
         // error; otherwise the startTime will be set once execution of the
-        // job starts. This is the behaviour expected by a client as discussed
-        // on the IVOA mailing list.
+        // job starts. This is the expected behaviour as discussed
+        // on the IVOA mailing list (fall 2015).
         $job->resetErrors();
         $model = new Query_Model_Query();
         try {
@@ -446,8 +490,8 @@ public function runJob(Uws_Model_Resource_JobSummaryType &$job) {
                 $resource = new Uws_Model_Resource_UWSJobs();
                 $resource->updateRow($job->jobId, array("phase" => "ERROR", "errorSummary" => Zend_Json::encode($job->errorSummary)));
 
-                // job is in final state now, add startTime and endTime
-                $now = date('Y-m-d\TH:i:s');
+                // job is in final state now (ERROR), add startTime and endTime
+                $now = date('Y-m-d H:i:s');
                 $resource->updateRow($job->jobId, array("startTime" => $now, "endTime" => $now));
 
                 return;
@@ -458,18 +502,26 @@ public function runJob(Uws_Model_Resource_JobSummaryType &$job) {
             $resource = new Uws_Model_Resource_UWSJobs();
             $resource->updateRow($job->jobId, array("phase" => "ERROR", "errorSummary" => Zend_Json::encode($job->errorSummary)));
 
-            // job is in final state now, add startTime and endTime
-            $now = date('Y-m-d\TH:i:s');
+            // job is in final state now (ERROR), add startTime and endTime
+            $now = date('Y-m-d H:i:s');
             $resource->updateRow($job->jobId, array("startTime" => $now, "endTime" => $now));
 
             return;
         }
 
         // submit query
+
+        // first get the creationTime from UWSJobs (pending job list),
+        // because we want to store it in the main table
+        $resource = new Uws_Model_Resource_UWSJobs();
+        $jobUws = $resource->fetchRow($job->jobId);
+        $creationTime = $jobUws['creationTime'];
+
+
         if ($this->getResource()->hasQueues()) {
-            $response = $model->query($sql, false, $tablename, $sources, array("queue" => $queue, "jobId" => $job->jobId),'uws');
+            $response = $model->query($sql, false, $tablename, $sources, array("queue" => $queue, "jobId" => $job->jobId), $creationTime, 'uws');
         } else {
-            $response = $model->query($sql, false, $tablename, $sources, array("jobId" => $job->jobId),'uws');
+            $response = $model->query($sql, false, $tablename, $sources, array("jobId" => $job->jobId), $creationTime, 'uws');
         }
 
         if ($response['status'] !== 'ok') {
@@ -480,14 +532,14 @@ public function runJob(Uws_Model_Resource_JobSummaryType &$job) {
             $resource = new Uws_Model_Resource_UWSJobs();
             $resource->updateRow($job->jobId, array("phase" => "ERROR", "errorSummary" => Zend_Json::encode($job->errorSummary)));
 
-            // job is in final state now, add startTime and endTime
-            $now = date('Y-m-d\TH:i:s');
+            // job is in final state now (ERROR), add startTime and endTime
+            $now = date('Y-m-d H:i:s');
             $resource->updateRow($job->jobId, array("startTime" => $now, "endTime" => $now));
 
             return;
         }
 
-        // clean up stuff (basically just remove the job in the temorary UWS job store - if we are here
+        // clean up stuff (basically just remove the job in the temporary UWS job store - if we are here
         // everything has been handled by the queue)
         $resource = new Uws_Model_Resource_UWSJobs();
         $resource->deleteRow($job->jobId);

modules/uws/db/schema.sql

@@ -4,6 +4,7 @@ CREATE TABLE IF NOT EXISTS Uws_Jobs (
   `ownerId` VARCHAR(256),
   `phase` VARCHAR(64) NOT NULL,
   `quote` DATETIME,
+  `creationTime` DATETIME,
   `startTime` DATETIME,
   `endTime` DATETIME,
   `executionDuration` BIGINT,

modules/uws/models/Resource/JobSummaryType.php

@@ -36,6 +36,7 @@ public function __construct($name) {
         $this->ownerId = false;
         $this->phase = "PENDING";
         $this->quote = false;
+        $this->creationTime = false;
         $this->startTime = false;
         $this->endTime = false;
         $this->executionDuration = 0;
@@ -99,10 +100,11 @@ public function toXML(&$xmlDoc, &$node = false) {
         $job->appendChild($version);
 
         $this->_writeXMLElement($xmlDoc, $job, "jobId");
-        $this->_writeXMLElement($xmlDoc, $job, "runId", true);
+        $this->_writeXMLElement($xmlDoc, $job, "runId"); //, true, do not add "true: in order to set the element even if null
         $this->_writeXMLElement($xmlDoc, $job, "ownerId");
         $this->_writeXMLElement($xmlDoc, $job, "phase");
         $this->_writeXMLElement($xmlDoc, $job, "quote");
+        $this->_writeXMLElement($xmlDoc, $job, "creationTime");
         $this->_writeXMLElement($xmlDoc, $job, "startTime");
         $this->_writeXMLElement($xmlDoc, $job, "endTime");
         $this->_writeXMLElement($xmlDoc, $job, "executionDuration");

modules/uws/models/Resource/Jobs.php

@@ -32,13 +32,18 @@ public function __construct() {
         $this->jobref = array();
     }
 
-    public function addJob($id, $href, array $phases) {
+    public function addJob($id, $href, array $phases, $creationTime, $runId, $ownerId) {
         $newJob = new Uws_Model_Resource_ShortJobDescriptionType("jobref");
 
         $newJob->id = $id;
         $newJob->reference->href = $href;
         $newJob->phase = $phases;
 
+        // optional new keywords for updated UWS 1.1 version
+        $newJob->creationTime = $creationTime;
+        $newJob->runId = $runId;
+        $newJob->ownerId = $ownerId;
+
         $this->jobref[] = $newJob;
     }