first commit

lib/pkp/classes/queue/JobRunner.php

@@ -0,0 +1,349 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @file classes/queue/JobRunner.php
+ *
+ * Copyright (c) 2014-2022 Simon Fraser University
+ * Copyright (c) 2000-2022 John Willinsky
+ * Distributed under the GNU GPL v3. For full terms see the file docs/COPYING.
+ *
+ * @class JobRunner
+ *
+ * @brief   Synchronous job runner that will attempt to execute jobs at the end of web request
+ *          life cycle. It class is designed to execute jobs when not running via a worker
+ *          or a cron job.
+ */
+
+namespace PKP\queue;
+
+use Illuminate\Database\Eloquent\Builder as EloquentBuilder;
+use PKP\config\Config;
+use PKP\core\PKPQueueProvider;
+
+class JobRunner
+{
+    /**
+     * The core queue job running service provider
+     */
+    protected PKPQueueProvider $jobQueue;
+
+    /**
+     * Should have max job number running constrain
+     */
+    protected bool $hasMaxJobsConstrain = false;
+
+    /**
+     * Max jobs to process in a single run
+     */
+    protected ?int $maxJobsToProcess = null;
+
+    /**
+     * Should have max run time constrain when processing jobs
+     */
+    protected bool $hasMaxExecutionTimeConstrain = false;
+
+    /**
+     * Max job processing run time in SECOND
+     */
+    protected ?int $maxTimeToProcessJobs = null;
+
+    /**
+     * Should have max consumed memory constrain when processing jobs
+     */
+    protected bool $hasMaxMemoryConstrain = false;
+
+    /**
+     * Max memory to consume when processing jobs in BYTE
+     */
+    protected ?int $maxMemoryToConsumed = null;
+
+    /**
+     * Should estimate next job possible processing time to allow next job to be processed/run
+     */
+    protected bool $hasEstimatedTimeToProcessNextJobConstrain = false;
+
+    /**
+     * Create a new instance
+     *
+     */
+    public function __construct(PKPQueueProvider $jobQueue = null)
+    {
+        $this->jobQueue = $jobQueue ?? app('pkpJobQueue');
+    }
+
+    /**
+     * Set the max job number running constrain
+     *
+     */
+    public function withMaxJobsConstrain(): self
+    {
+        $this->hasMaxJobsConstrain = true;
+
+        if (! $this->getMaxJobsToProcess()) {
+            $this->setMaxJobsToProcess(Config::getVar('queues', 'job_runner_max_jobs', 30));
+        }
+
+        return $this;
+    }
+
+    /**
+     * Set the max allowed jobs number to process
+     *
+     */
+    public function setMaxJobsToProcess(int $maxJobsToProcess): self
+    {
+        $this->maxJobsToProcess = $maxJobsToProcess;
+
+        return $this;
+    }
+
+    /**
+     * Get the max allowed jobs number to process
+     *
+     */
+    public function getMaxJobsToProcess(): ?int
+    {
+        return $this->maxJobsToProcess;
+    }
+
+    /**
+     * Set the max run time constrain when processing jobs
+     *
+     */
+    public function withMaxExecutionTimeConstrain(): self
+    {
+        $this->hasMaxExecutionTimeConstrain = true;
+
+        if (! $this->getMaxTimeToProcessJobs()) {
+            $this->setMaxTimeToProcessJobs($this->deduceSafeMaxExecutionTime());
+        }
+
+        return $this;
+    }
+
+    /**
+     * Set the max allowed run time to process jobs in SECONDS
+     *
+     */
+    public function setMaxTimeToProcessJobs(int $maxTimeToProcessJobs): self
+    {
+        $this->maxTimeToProcessJobs = $maxTimeToProcessJobs;
+
+        return $this;
+    }
+
+    /**
+     * Get the max allowed run time to process jobs in SECONDS
+     *
+     */
+    public function getMaxTimeToProcessJobs(): ?int
+    {
+        return $this->maxTimeToProcessJobs;
+    }
+
+    /**
+     * Set the max consumable memory constrain when processing jobs
+     *
+     */
+    public function withMaxMemoryConstrain(): self
+    {
+        $this->hasMaxMemoryConstrain = true;
+
+        if (! $this->getMaxMemoryToConsumed()) {
+            $this->setMaxMemoryToConsumed($this->deduceSafeMaxAllowedMemory());
+        }
+
+        return $this;
+    }
+
+    /**
+     * Set the max allowed consumable memory to process jobs in BYTES
+     *
+     */
+    public function setMaxMemoryToConsumed(int $maxMemoryToConsumed): self
+    {
+        $this->maxMemoryToConsumed = $maxMemoryToConsumed;
+
+        return $this;
+    }
+
+    /**
+     * get the max allowed consumable memory to process jobs in BYTES
+     *
+     */
+    public function getMaxMemoryToConsumed(): ?int
+    {
+        return $this->maxMemoryToConsumed;
+    }
+
+    /**
+     * Set constrain to estimate next job possible processing time to allow next job to be processed/run
+     *
+     */
+    public function withEstimatedTimeToProcessNextJobConstrain(): self
+    {
+        $this->hasEstimatedTimeToProcessNextJobConstrain = true;
+
+        return $this;
+    }
+
+    /**
+     * Process/Run/Execute jobs off CLI
+     *
+     */
+    public function processJobs(EloquentBuilder $jobBuilder = null): bool
+    {
+        $jobBuilder ??= $this->jobQueue->getJobModelBuilder();
+
+        $jobProcessedCount = 0;
+        $jobProcessingStartTime = time();
+
+        while ($jobBuilder->count()) {
+            if ($this->exceededJobLimit($jobProcessedCount)) {
+                return true;
+            }
+
+            if ($this->exceededTimeLimit($jobProcessingStartTime)) {
+                return true;
+            }
+
+            if ($this->exceededMemoryLimit()) {
+                return true;
+            }
+
+            if ($this->mayExceedMemoryLimitAtNextJob($jobProcessedCount, $jobProcessingStartTime)) {
+                return true;
+            }
+
+            $this->jobQueue->runJobInQueue();
+
+            $jobProcessedCount = $jobProcessedCount + 1;
+        }
+
+        return true;
+    }
+
+    /**
+     * Check if max job has processed or not
+     *
+     * @param int $jobProcessedCount    The number of jobs that has processed so far
+     */
+    protected function exceededJobLimit(int $jobProcessedCount): bool
+    {
+        if (! $this->hasMaxJobsConstrain) {
+            return false;
+        }
+
+        return $jobProcessedCount >= $this->getMaxJobsToProcess();
+    }
+
+    /**
+     * Check if max run time to process jobs has exceed or not
+     *
+     * @param int $jobProcessingStartTime   The start time since job processing has started in seconds
+     */
+    protected function exceededTimeLimit(int $jobProcessingStartTime): bool
+    {
+        if (! $this->hasMaxExecutionTimeConstrain) {
+            return false;
+        }
+
+        return (time() - $jobProcessingStartTime) >= $this->getMaxTimeToProcessJobs();
+    }
+
+    /**
+     * Check if memory consumed since job processing started has exceed defined max memory
+     *
+     */
+    protected function exceededMemoryLimit(): bool
+    {
+        if (! $this->hasMaxMemoryConstrain) {
+            return false;
+        }
+
+        return memory_get_usage(true) >= $this->getMaxMemoryToConsumed();
+    }
+
+    /**
+     * Estimate if next job processing time will likely exceed defined max processing run time
+     *
+     * @param int $jobProcessedCount        The number of jobs that has processed so far
+     * @param int $jobProcessingStartTime   The start time since job processing has started in seconds
+     *
+     */
+    protected function mayExceedMemoryLimitAtNextJob(int $jobProcessedCount, int $jobProcessingStartTime): bool
+    {
+        if (! $this->hasEstimatedTimeToProcessNextJobConstrain) {
+            return false;
+        }
+
+        // if no max run time constrain set, no point in estimating the likely time exceed
+        if (!$this->hasMaxExecutionTimeConstrain) {
+            return false;
+        }
+
+        // if no job has processed yet, no way to calculate next job possible processing time
+        if ($jobProcessedCount <= 0) {
+            return false;
+        }
+
+        $currentTotalExecutionTime = time() - $jobProcessingStartTime;
+        $timePerJob = (int)($currentTotalExecutionTime / $jobProcessedCount);
+        $totalTimeByNextJobComplete = $currentTotalExecutionTime + ($timePerJob * 3);
+
+        return $totalTimeByNextJobComplete > $this->getMaxTimeToProcessJobs();
+    }
+
+    /**
+     * Deduce possible safe max job processing run time in SECONDS
+     *
+     * It will consider both what defined in the ini file and application config file
+     * and will take a minimum one based on those two values
+     *
+     */
+    protected function deduceSafeMaxExecutionTime(): int
+    {
+        $maxExecutionTimeSetToINI = (int)ini_get('max_execution_time');
+        $maxExecutionTimeSetToConfig = (int)Config::getVar('queues', 'job_runner_max_execution_time', 20);
+
+        return $maxExecutionTimeSetToINI <= 0
+            ? $maxExecutionTimeSetToConfig
+            : min($maxExecutionTimeSetToINI, $maxExecutionTimeSetToConfig);
+    }
+
+    /**
+     * Deduce possible safe max consumable memory for job processing in BYTES
+     *
+     * It will consider both of what defined in the ini file and the application config file
+     * and will take a minimum one based on those two values
+     *
+     * In the application config file, max memory can be defined as INT or STRING value in following manner
+     *
+     *      If defined as INT (e.g 90), it will be calculated as percentage value of
+     *      what currently defined in the ini file.
+     *
+     *      If defined as STRING (e.g 128M), it will try to calculate it as memory defined in megabytes
+     *      but if failed, will try to cast to INT to apply percentage rule
+     *
+     */
+    protected function deduceSafeMaxAllowedMemory(): int
+    {
+        $maxMemoryLimitSetToINI = function_exists('ini_get') ? ini_get('memory_limit') : '128M';
+
+        if (!$maxMemoryLimitSetToINI || -1 === $maxMemoryLimitSetToINI || '-1' === $maxMemoryLimitSetToINI) {
+            $maxMemoryLimitSetToINI = '128M'; // Unlimited, set to 128M.
+        }
+
+        $maxMemoryLimitSetToINIInBytes = convertHrToBytes($maxMemoryLimitSetToINI);
+
+        $maxMemoryLimitSetToConfig = Config::getVar('queues', 'job_runner_max_memory', 80);
+
+        $maxMemoryLimitSetToConfigInBytes = in_array(strtolower(substr((string)$maxMemoryLimitSetToConfig, -1)), ['k', 'm', 'g'], true)
+            ? convertHrToBytes($maxMemoryLimitSetToConfig)
+            : (int)($maxMemoryLimitSetToINIInBytes * ((float)((int)$maxMemoryLimitSetToConfig / 100)));
+
+        return min($maxMemoryLimitSetToConfigInBytes, $maxMemoryLimitSetToINIInBytes);
+    }
+}