From 2f439e3f91a3276dde010a694c2808ef9016ff73 Mon Sep 17 00:00:00 2001 From: Vincent Vanwaelscappel Date: Thu, 18 Jan 2024 18:44:54 +0100 Subject: [PATCH] wait #6642 @3 --- .docker/config/supervisor/default-worker.conf | 2 +- .../config/supervisor/download-worker.conf | 2 +- .../supervisor/fluidbook-process-worker.conf | 2 +- .../supervisor/medialibrary-worker.conf | 2 +- .docker/config/supervisor/theme-worker.conf | 2 +- .docker/config/supervisor/ws2-worker.conf | 2 +- app/Fluidbook/Farm.php | 4 +- app/Jobs/Base.php | 3 +- app/Jobs/FluidbookDocumentFileProcess.php | 4 +- app/Jobs/FluidbookDocumentUpload.php | 2 +- app/Models/FluidbookDocument.php | 59 +++++++++++-------- config/queue.php | 7 ++- 12 files changed, 52 insertions(+), 39 deletions(-) diff --git a/.docker/config/supervisor/default-worker.conf b/.docker/config/supervisor/default-worker.conf index 4cd21561c..5cfe635a0 100644 --- a/.docker/config/supervisor/default-worker.conf +++ b/.docker/config/supervisor/default-worker.conf @@ -1,6 +1,6 @@ [program:default-worker] process_name=%(program_name)s_%(process_num)02d -command=php /application/artisan queue:work --queue=default --timeout=120 --max-time=3600 --memory=12000 +command=/usr/bin/php /application/artisan queue:work --queue=default --timeout=3600000 --memory=12000 autostart=true autorestart=true user=toolbox diff --git a/.docker/config/supervisor/download-worker.conf b/.docker/config/supervisor/download-worker.conf index 4a5f24515..39c2f547f 100644 --- a/.docker/config/supervisor/download-worker.conf +++ b/.docker/config/supervisor/download-worker.conf @@ -1,6 +1,6 @@ [program:download-worker] process_name=%(program_name)s_%(process_num)02d -command=php /application/artisan queue:work --queue=download --timeout=7200 --max-time=3600 --memory=12000 +command=/usr/bin/php /application/artisan queue:work --queue=download --timeout=3600000 --memory=12000 autostart=true autorestart=true user=toolbox diff --git a/.docker/config/supervisor/fluidbook-process-worker.conf b/.docker/config/supervisor/fluidbook-process-worker.conf index 2d5633c56..0cbb72e99 100644 --- a/.docker/config/supervisor/fluidbook-process-worker.conf +++ b/.docker/config/supervisor/fluidbook-process-worker.conf @@ -1,6 +1,6 @@ [program:fluidbook-process-worker] process_name=%(program_name)s_%(process_num)02d -command=php /application/artisan queue:work --queue=fluidbookprocess --timeout=7200 --max-time=3600 --memory=12000 +command=/usr/bin/php /application/artisan queue:work --queue=fluidbookprocess --timeout=3600000 --memory=12000 autostart=true autorestart=true user=toolbox diff --git a/.docker/config/supervisor/medialibrary-worker.conf b/.docker/config/supervisor/medialibrary-worker.conf index 492bff15e..2f947f093 100644 --- a/.docker/config/supervisor/medialibrary-worker.conf +++ b/.docker/config/supervisor/medialibrary-worker.conf @@ -1,6 +1,6 @@ [program:medialibrary-worker] process_name=%(program_name)s_%(process_num)02d -command=php /application/artisan queue:work --queue=medialibrary --timeout=0 +command=/usr/bin/php /application/artisan queue:work --queue=medialibrary --timeout=3600000 autostart=true autorestart=true user=toolbox diff --git a/.docker/config/supervisor/theme-worker.conf b/.docker/config/supervisor/theme-worker.conf index 793bc083d..b2aa670ad 100644 --- a/.docker/config/supervisor/theme-worker.conf +++ b/.docker/config/supervisor/theme-worker.conf @@ -1,6 +1,6 @@ [program:theme-worker] process_name=%(program_name)s_%(process_num)02d -command=php /application/artisan queue:work --queue=theme --timeout=360 --max-time=3600 --memory=12000 +command=php /application/artisan queue:work --queue=theme --timeout=3600000 --memory=12000 autostart=true autorestart=true user=toolbox diff --git a/.docker/config/supervisor/ws2-worker.conf b/.docker/config/supervisor/ws2-worker.conf index 3ca86be92..0beb44436 100644 --- a/.docker/config/supervisor/ws2-worker.conf +++ b/.docker/config/supervisor/ws2-worker.conf @@ -1,6 +1,6 @@ [program:ws2-worker] process_name=%(program_name)s_%(process_num)02d -command=php /application/artisan queue:work --queue=ws2 --timeout=360 --max-time=3600 --memory=12000 +command=/usr/bin/php /application/artisan queue:work --queue=ws2 --timeout=3600000 --memory=12000 autostart=true autorestart=true user=toolbox diff --git a/app/Fluidbook/Farm.php b/app/Fluidbook/Farm.php index 6b69c0db5..354508844 100644 --- a/app/Fluidbook/Farm.php +++ b/app/Fluidbook/Farm.php @@ -14,8 +14,8 @@ use Mockery\Exception; class Farm { protected static $_outPDF = []; - //protected static $_forceServer = false; - protected static $_forceServer = 'alphaville'; + protected static $_forceServer = false; + //protected static $_forceServer = 'alphaville'; protected static $_region = Region::EUROPE; diff --git a/app/Jobs/Base.php b/app/Jobs/Base.php index ce42b8a58..763688bab 100644 --- a/app/Jobs/Base.php +++ b/app/Jobs/Base.php @@ -18,7 +18,8 @@ class Base extends \Cubist\Backpack\Jobs\Base protected string $_cacheKey; public $failOnTimeout = false; - public $timeout = 120000; + public $timeout = 3600000; + public $retryAfter = 3600000; public function __construct() { diff --git a/app/Jobs/FluidbookDocumentFileProcess.php b/app/Jobs/FluidbookDocumentFileProcess.php index 32ee499e9..c74ee8f8c 100644 --- a/app/Jobs/FluidbookDocumentFileProcess.php +++ b/app/Jobs/FluidbookDocumentFileProcess.php @@ -57,9 +57,9 @@ class FluidbookDocumentFileProcess extends Base /** * @return boolean */ - public function isDone($forceCheck = false) + public function isDone($forceCheck = false, $checkFile = true) { - return $this->finish || $this->isOK($forceCheck); + return $this->finish || ($checkFile && $this->isOK($forceCheck)); } public function isOK($forceCheck = false) diff --git a/app/Jobs/FluidbookDocumentUpload.php b/app/Jobs/FluidbookDocumentUpload.php index f79ee278d..2cfe5c235 100644 --- a/app/Jobs/FluidbookDocumentUpload.php +++ b/app/Jobs/FluidbookDocumentUpload.php @@ -50,7 +50,7 @@ class FluidbookDocumentUpload extends Base public static function updateProgression($id, $document_id, $message, $progress) { - Cache::put('FluidbookDocumentUpload_' . $id, ['message' => $message, 'document' => $document_id, 'progress' => $progress]); + Cache::put('FluidbookDocumentUpload_' . $id, ['message' => $message, 'document' => $document_id, 'progress' => $progress, 'update' => time()]); } public static function getProgression($id) diff --git a/app/Models/FluidbookDocument.php b/app/Models/FluidbookDocument.php index e77b97661..83fe3a0bb 100644 --- a/app/Models/FluidbookDocument.php +++ b/app/Models/FluidbookDocument.php @@ -18,6 +18,7 @@ use Cubist\Util\Files\Files; use Cubist\Util\Gzip; use Cubist\Util\Math; use Illuminate\Support\Facades\Cache; +use Illuminate\Support\Facades\Log; // __('!!Paramètres des fluidbooks') class FluidbookDocument extends ToolboxModel @@ -103,6 +104,8 @@ class FluidbookDocument extends ToolboxModel ['svg', 150, true, false, 'html'], ]; + Log::debug('Sync ? ' . $sync); + $nbfiles = count($files); for ($i = 1; $i <= $this->pages; $i++) { foreach ($files as $file) { @@ -112,13 +115,21 @@ class FluidbookDocument extends ToolboxModel } else { dispatch($job); } + $jobs[] = $job; - $this->_checkJobs($uploadID, $jobs, $nbfiles); + if ($sync) { + $this->_checkJobs($uploadID, $jobs, $nbfiles, true); + } } + + if (!$sync && $i % 4 === 0) { + $this->_checkJobs($uploadID, $jobs, $nbfiles, false); + } + } while (true) { - if ($this->_checkJobs($uploadID, $jobs, $nbfiles) === 1) { + if ($this->_checkJobs($uploadID, $jobs, $nbfiles, true) === 1) { return; } usleep(0.25 * 1000000); @@ -145,14 +156,14 @@ class FluidbookDocument extends ToolboxModel Farm::extractLinks($this->getPDFSource(), $this->path()); } - protected function _checkJobs($uploadID, $jobs, $nbfiles) + protected function _checkJobs($uploadID, $jobs, $nbfiles, $checkFile = true) { $this->checkInfos(); $nbjobs = $nbfiles * $this->pages; $done = 0; foreach ($jobs as $job) { /** @var $job FluidbookDocumentFileProcess */ - if ($job->isDone()) { + if ($job->isDone(false, $checkFile)) { $done++; } } @@ -169,6 +180,7 @@ class FluidbookDocument extends ToolboxModel public function checkInfos($force = false) { if ($force || null === $this->pdf_data) { + Log::debug('check document infos'); $infos = $this->_getInfos(); $this->pages = $infos->pages; @@ -305,31 +317,30 @@ class FluidbookDocument extends ToolboxModel public function _getPath($page, $format = 'jpg', $resolution = 150, $quality = 85, $withText = true, $withGraphics = true, $version = 'html') { $this->_normalize($format, $resolution, $quality, $withText, $withGraphics, $version); - $cacheKey = $this->fileCacheKey($page, $format, $resolution, $quality, $withText, $withGraphics, $version); - if (Cache::has($cacheKey)) { - return Cache::get($cacheKey); - } + $cacheKey = $this->fileCacheKey($page, $format, $resolution, $quality, $withText, $withGraphics, $version) . '_path'; - $dir = $this->path($version) . '/'; + return Cache::rememberForever($cacheKey, function () use ($page, $format, $resolution, $quality, $withText, $withGraphics, $version) { + $dir = $this->path($version) . '/'; - if ($format === 'svg') { - $prefix = $withGraphics ? 'f' : 't'; - $file = $dir . $prefix . 'o' . $page; - if ($withGraphics) { - $file .= '-' . $resolution; - } - $file .= '.svg'; - } else { - $q = ($format === 'jpg' && $quality != 85) ? '-' . $quality : ''; - $prefix = $withText ? 't' : 'h'; - if ($resolution === 'thumb') { - $file = $dir . 'p' . $page . $q . '.' . $format; + if ($format === 'svg') { + $prefix = $withGraphics ? 'f' : 't'; + $file = $dir . $prefix . 'o' . $page; + if ($withGraphics) { + $file .= '-' . $resolution; + } + $file .= '.svg'; } else { - $file = $dir . $prefix . $page . '-' . $resolution . $q . '.' . $format; + $q = ($format === 'jpg' && $quality != 85) ? '-' . $quality : ''; + $prefix = $withText ? 't' : 'h'; + if ($resolution === 'thumb') { + $file = $dir . 'p' . $page . $q . '.' . $format; + } else { + $file = $dir . $prefix . $page . '-' . $resolution . $q . '.' . $format; + } } - } - return $file; + return $file; + }); } protected function fileCacheKey($page, $format = 'jpg', $resolution = 150, $quality = 85, $withText = true, $withGraphics = true, $version = 'html') diff --git a/config/queue.php b/config/queue.php index 5a0356205..fad4222a1 100644 --- a/config/queue.php +++ b/config/queue.php @@ -39,14 +39,15 @@ return [ 'table' => 'jobs', 'queue' => 'default', 'after_commit' => true, - 'retry_after' => 90, + 'retry_after' => 3600000, + 'timeout' => 3600000, ], 'beanstalkd' => [ 'driver' => 'beanstalkd', 'host' => 'localhost', 'queue' => 'default', - 'retry_after' => 90, + 'retry_after' => 3600000, 'block_for' => 0, ], @@ -63,7 +64,7 @@ return [ 'driver' => 'redis', 'connection' => 'default', 'queue' => env('REDIS_QUEUE', 'default'), - 'retry_after' => 90, + 'retry_after' => 3600000, 'block_for' => null, ], -- 2.39.5