Merge pull request #242 from lbr38/devel

4.17.0

www/controllers/App/Config/Settings.php

@@ -101,6 +101,22 @@ public static function get()
             }
         }
 
+        if (!defined('TASK_QUEUING')) {
+            if (!empty($settings['TASK_QUEUING'])) {
+                define('TASK_QUEUING', $settings['TASK_QUEUING']);
+            } else {
+                define('TASK_QUEUING', 'false');
+            }
+        }
+
+        if (!defined('TASK_QUEUING_MAX_SIMULTANEOUS')) {
+            if (!empty($settings['TASK_QUEUING_MAX_SIMULTANEOUS'])) {
+                define('TASK_QUEUING_MAX_SIMULTANEOUS', $settings['TASK_QUEUING_MAX_SIMULTANEOUS']);
+            } else {
+                define('TASK_QUEUING_MAX_SIMULTANEOUS', 2);
+            }
+        }
+
         if (!defined('TASK_EXECUTION_MEMORY_LIMIT')) {
             if (!empty($settings['TASK_EXECUTION_MEMORY_LIMIT'])) {
                 define('TASK_EXECUTION_MEMORY_LIMIT', $settings['TASK_EXECUTION_MEMORY_LIMIT']);

www/controllers/Layout/Container/vars/tasks/log.vars.inc.php

@@ -8,6 +8,10 @@
  *  Get the log file of the task
  */
 try {
+    /**
+     *  If a task Id is provided in the URL, use it
+     *  Otherwise, get the latest task Id
+     */
     if (!empty(__ACTUAL_URI__[2]) and (is_numeric(__ACTUAL_URI__[2]))) {
         $taskId = __ACTUAL_URI__[2];
     } else {
@@ -29,7 +33,12 @@
         $taskLogController = new \Controllers\Task\Log\Log($taskId);
 
         // Get raw params from the task
-        $rawParams = json_decode($taskInfo['Raw_params'], true);
+        try {
+            $rawParams = json_decode($taskInfo['Raw_params'], true, 512, JSON_THROW_ON_ERROR);
+        } catch (JsonException $e) {
+            throw new Exception('Error while reading log: could not get task params from task #' . $taskId . ': ' . $e->getMessage());
+        }
+
         $repoId = null;
         $snapId = null;
         $envId = null;

www/controllers/Layout/Table/vars/tasks/list-queued.vars.inc.php

@@ -0,0 +1,30 @@
+<?php
+$myTask = new \Controllers\Task\Task();
+$reloadableTableOffset = 0;
+
+/**
+ *  Retrieve offset from cookie if exists
+ */
+if (!empty($_COOKIE['tables/tasks/list-queued/offset']) and is_numeric($_COOKIE['tables/tasks/list-queued/offset'])) {
+    $reloadableTableOffset = $_COOKIE['tables/tasks/list-queued/offset'];
+}
+
+/**
+ *  Get list of queued tasks, with offset
+ */
+$reloadableTableContent = $myTask->listQueued('', true, $reloadableTableOffset);
+
+/**
+ *  Get list of ALL queued tasks, without offset, for the total count
+ */
+$reloadableTableTotalItems = count($myTask->listQueued());
+
+/**
+ *  Count total pages for the pagination
+ */
+$reloadableTableTotalPages = ceil($reloadableTableTotalItems / 10);
+
+/**
+ *  Calculate current page number
+ */
+$reloadableTableCurrentPage = ceil($reloadableTableOffset / 10) + 1;

www/controllers/Repo/Package.php

@@ -288,35 +288,37 @@ public function delete(int $snapId, array $packages)
              *  Check that the file path starts with REPOS_DIR
              *  Prevents a malicious person from providing a path that has nothing to do with the repo directory (e.g. /etc/...)
              */
-            if (preg_match("#^" . REPOS_DIR . "#", $path)) {
-                /**
-                 *  Check that the file ends with .deb or .rpm otherwise we move on to the next one
-                 */
-                if (!preg_match("#.deb$#", $path) and !preg_match("#.rpm$#", $path)) {
-                    continue;
-                }
-
-                /**
-                 *  If the file does not exist, we ignore it and move on to the next one
-                 */
-                if (!file_exists($path)) {
-                    continue;
-                }
+            if (!preg_match("#^" . REPOS_DIR . "#", realpath($path))) {
+                throw new Exception('Invalid package path ' . $path);
+            }
 
-                /**
-                 *  Delete package
-                 */
-                if (!unlink($path)) {
-                    throw new Exception('Unable to delete package ' . $path);
-                }
+            /**
+             *  Check that the file ends with .deb or .rpm otherwise we move on to the next one
+             */
+            if (!preg_match("#.deb$#", $path) and !preg_match("#.rpm$#", $path)) {
+                continue;
+            }
 
-                $deletedPackages[] = str_replace($repoPath . '/', '', $path);
+            /**
+             *  If the file does not exist, we ignore it and move on to the next one
+             */
+            if (!file_exists($path)) {
+                continue;
+            }
 
-                /**
-                 *  Set repo rebuild status to 'needed'
-                 */
-                $myrepo->snapSetRebuild($snapId, 'needed');
+            /**
+             *  Delete package
+             */
+            if (!unlink($path)) {
+                throw new Exception('Unable to delete package ' . $path);
             }
+
+            $deletedPackages[] = str_replace($repoPath . '/', '', $path);
+
+            /**
+             *  Set repo rebuild status to 'needed'
+             */
+            $myrepo->snapSetRebuild($snapId, 'needed');
         }
 
         return $deletedPackages;

www/controllers/Service/File.php

@@ -23,15 +23,15 @@ public function cleanUp()
 
         try {
             /**
-             *  Clean temp files and directories older than 7 days
+             *  Clean temp files and directories older than 3 days
              */
             if (is_dir(DATA_DIR . '/.temp')) {
                 $files = \Controllers\Common::findRecursive(DATA_DIR . '/.temp');
                 $dirs = \Controllers\Common::findDirRecursive(DATA_DIR . '/.temp');
 
                 if (!empty($files)) {
                     foreach ($files as $file) {
-                        if (filemtime($file) < strtotime('-7 days')) {
+                        if (filemtime($file) < strtotime('-3 days')) {
                             if (!unlink($file)) {
                                 throw new Exception('Could not clean temporary file <b>' . $file . '</b>');
                             }

www/controllers/Service/ScheduledTask.php

@@ -146,10 +146,15 @@ public function execute()
                 echo $this->getDate() . ' Launching scheduled task #' . $taskId . '...' . PHP_EOL;
 
                 try {
+                    // Add the scheduled task to the queue and execute it
+                    $this->taskController->updateStatus($taskId, 'queued');
                     $this->taskController->executeId($taskId);
                 } catch (Exception $e) {
                     $this->logController->log('error', 'Service', 'Error while launching scheduled task: ' . $e->getMessage());
                 }
+
+                // Let some time between each task, to make sure the queue system works properly
+                sleep(1);
             }
         }
     }

www/controllers/Settings.php

@@ -74,6 +74,18 @@ public function apply(array $sendSettings)
             $settingsToApply['PROXY'] = '';
         }
 
+        if (!empty($sendSettings['task-queuing'])) {
+            if ($sendSettings['task-queuing'] == 'true') {
+                $settingsToApply['TASK_QUEUING'] = 'true';
+            } else {
+                $settingsToApply['TASK_QUEUING'] = 'false';
+            }
+        }
+
+        if (!empty($sendSettings['task-queuing-max-simultaneous']) and is_numeric($sendSettings['task-queuing-max-simultaneous']) and $sendSettings['task-queuing-max-simultaneous'] > 0) {
+            $settingsToApply['TASK_QUEUING_MAX_SIMULTANEOUS'] = \Controllers\Common::validateData($sendSettings['task-queuing-max-simultaneous']);
+        }
+
         if (!empty($sendSettings['task-execution-memory-limit']) and is_numeric($sendSettings['task-execution-memory-limit']) and $sendSettings['task-execution-memory-limit'] > 2) {
             $settingsToApply['TASK_EXECUTION_MEMORY_LIMIT'] = \Controllers\Common::validateData($sendSettings['task-execution-memory-limit']);
         }

www/controllers/Task/Task.php

@@ -156,6 +156,16 @@ public function updateDuration(int $id, string $duration) : void
         $this->model->updateDuration($id, $duration);
     }
 
+    /**
+     *  List all queued tasks
+     *  It is possible to filter the type of task ('immediate' or 'scheduled')
+     *  It is possible to add an offset to the request
+     */
+    public function listQueued(string $type = '', bool $withOffset = false, int $offset = 0)
+    {
+        return $this->model->listQueued($type, $withOffset, $offset);
+    }
+
     /**
      *  List all running tasks
      *  It is possible to filter the type of task ('immediate' or 'scheduled')
@@ -281,17 +291,18 @@ private function new(array $params) : int
     {
         /**
          *  Default values
-         *  By default the task is new and immediate
+         *  By default the task is immediate and is queued
          */
-        $status = 'new';
-        $type   = 'immediate';
+        $type = 'immediate';
+        $status = 'queued';
 
         /**
-         *  If task is scheduled
+         *  If task is scheduled then overwrite the type and status
+         *  Task is not queued immediately, it will be queued at the scheduled time (when the service will launch the task)
          */
         if ($params['schedule']['scheduled'] == 'true') {
+            $type = 'scheduled';
             $status = 'scheduled';
-            $type   = 'scheduled';
         }
 
         /**
@@ -325,10 +336,16 @@ private function new(array $params) : int
             }
         }
 
+        try {
+            $paramsJson = json_encode($params, JSON_THROW_ON_ERROR);
+        } catch (JsonException $e) {
+            throw new Exception('Could not encode task parameters: ' . $e->getMessage());
+        }
+
         /**
          *  Add the task in database
          */
-        $taskId = $this->model->new($type, json_encode($params), $status);
+        $taskId = $this->model->new($type, $paramsJson, $status);
 
         return $taskId;
     }
@@ -548,6 +565,10 @@ public function end()
      */
     public function relaunch(int $id)
     {
+        if (!IS_ADMIN) {
+            throw new Exception('You are not allowed to relaunch a task');
+        }
+
         /**
          *  First, duplicate task in database
          */
@@ -574,6 +595,10 @@ private function duplicate(int $id) : int
      */
     public function kill(string $taskId)
     {
+        if (!IS_ADMIN) {
+            throw new Exception('You are not allowed to relaunch a task');
+        }
+
         if (!file_exists(PID_DIR . '/' . $taskId . '.pid')) {
             throw new Exception('Specified task PID does not exist');
         }
@@ -738,15 +763,15 @@ public function getChildrenPid(int $pid)
     }
 
     /**
-     *  Enable a task
+     *  Enable a recurrent task
      */
     public function enable(int $id)
     {
         $this->model->enable($id);
     }
 
     /**
-     *  Disable a task
+     *  Disable a recurrent task
      */
     public function disable(int $id)
     {

www/controllers/ajax/settings.php

@@ -77,13 +77,20 @@
  *  Get websocker server log content
  */
 if ($action == 'get-wss-log' and !empty([$_POST['logfile']])) {
+    $logfile = \Controllers\Common::validateData($_POST['logfile']);
+
+    // Check if the log file is allowed and is not outside the logs directory. Verify that the user is not trying to do something malicious.
+    if (!preg_match('#^' . WS_LOGS_DIR . '#', realpath(WS_LOGS_DIR . '/' . $logfile))) {
+        response(HTTP_BAD_REQUEST, 'Invalid log file');
+    }
+
     // Check if the log file exists
-    if (!file_exists(WS_LOGS_DIR . '/' . $_POST['logfile'])) {
+    if (!file_exists(WS_LOGS_DIR . '/' . $logfile)) {
         response(HTTP_BAD_REQUEST, 'Log file not found');
     }
 
     // Get the log content
-    $content = file_get_contents(WS_LOGS_DIR . '/' . $_POST['logfile']);
+    $content = file_get_contents(WS_LOGS_DIR . '/' . $logfile);
 
     // Check if the log content was read successfully
     if ($content === false) {

www/models/Connection.php

@@ -531,6 +531,8 @@ private function generateMainTables()
         EMAIL_RECIPIENT VARCHAR(255),
         PROXY VARCHAR(255),
         TASK_EXECUTION_MEMORY_LIMIT INTEGER,
+        TASK_QUEUING CHAR(5),
+        TASK_QUEUING_MAX_SIMULTANEOUS INTEGER,
         /* Repo settings */
         RETENTION INTEGER,
         REPO_CONF_FILES_PREFIX VARCHAR(255),
@@ -587,6 +589,8 @@ private function generateMainTables()
                 REPO_CONF_FILES_PREFIX,
                 TIMEZONE,
                 TASK_EXECUTION_MEMORY_LIMIT,
+                TASK_QUEUING,
+                TASK_QUEUING_MAX_SIMULTANEOUS,
                 MIRRORING_PACKAGE_DOWNLOAD_TIMEOUT,
                 RPM_REPO,
                 RPM_SIGN_PACKAGES,
@@ -614,6 +618,8 @@ private function generateMainTables()
                 'repomanager-',
                 'Europe/Paris',
                 '1024',
+                'false',
+                '2',
                 '300',
                 'true',
                 'true',