From: Vincent Vanwaelscappel Date: Thu, 24 Nov 2022 08:45:01 +0000 (+0100) Subject: wait #5605 @2 X-Git-Url: http://git.cubedesigners.com/?a=commitdiff_plain;h=1090f755a0c3334ad2659e3f2892000ff06d2ee7;p=fluidbook-toolbox.git wait #5605 @2 --- diff --git a/.docker/config/sshd/sshd_config b/.docker/config/sshd/sshd_config new file mode 100644 index 000000000..e69de29bb diff --git a/.docker/config/supervisor/ws2-worker.conf b/.docker/config/supervisor/ws2-worker.conf new file mode 100644 index 000000000..d5d556ebf --- /dev/null +++ b/.docker/config/supervisor/ws2-worker.conf @@ -0,0 +1,11 @@ +[program:default-worker] +process_name=%(program_name)s_%(process_num)02d +command=php /application/artisan queue:work --queue=ws2 --timeout=0 +autostart=true +autorestart=true +user=toolbox +group=www-data +numprocs=4 +redirect_stderr=true +stdout_logfile=/proc/self/fd/2 +stopwaitsecs=3600 diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml index 041cd6014..055e8f397 100644 --- a/.docker/docker-compose.yml +++ b/.docker/docker-compose.yml @@ -5,10 +5,18 @@ services: build: /home/toolbox/www/.docker/images/httpd working_dir: /application volumes: - - '/home/toolbox/www/:/application/' - - '/mnt/sshfs/godzilla/data/fluidbook/docs/:/data/extranet/www/fluidbook/docs/' - - '/home/toolbox/www/storage/app/public/:/usr/local/apache2/htdocs/storage/' + # Config - '/home/toolbox/www/.docker/config/httpd/httpd.conf:/usr/local/apache2/conf/httpd.conf' + # Files + - '/home/extranet/share:/application/share' + - '/mnt:/mnt' + - '/home/toolbox/www:/application' + - '/home/toolbox/www/storage/app/public/:/application/public/storage/' + - '/home/toolbox/www/.docker/config/php.ini:/etc/php/8.1/fpm/conf.d/99-overrides.ini' + - '/home/toolbox/www/.docker/config/cron/crontab:/etc/crontab' + - '/mnt/sshfs/godzilla/data/fluidbook/docs/:/application/protected/fluidbook/docs/' + - '/home/extranet:/home/extranet' + - '/data/extranet:/data/extranet' ports: - '37126:80' environment: @@ -40,13 +48,14 @@ services: - '/home/toolbox/www/.docker/config/monit/:/etc/monit/' - '/home/toolbox/www/.docker/config/sudoers:/etc/sudoers.d/toolbox' - '/home/toolbox/www/.docker/config/monit/id:/var/lib/monit/id' + # Files - '/home/extranet/share:/application/share' - '/mnt:/mnt' - '/home/toolbox/www:/application' - '/home/toolbox/www/storage/app/public/:/application/public/storage/' - '/home/toolbox/www/.docker/config/php.ini:/etc/php/8.1/fpm/conf.d/99-overrides.ini' - '/home/toolbox/www/.docker/config/cron/crontab:/etc/crontab' - - '/mnt/sshfs/godzilla/data/fluidbook/docs/:/data/extranet/www/fluidbook/docs/' + - '/mnt/sshfs/godzilla/data/fluidbook/docs/:/application/protected/fluidbook/docs/' - '/home/extranet:/home/extranet' - '/data/extranet:/data/extranet' tmpfs: diff --git a/.docker/update b/.docker/update index 65004891b..4e68f34f5 100644 --- a/.docker/update +++ b/.docker/update @@ -5,3 +5,4 @@ docker network create fluidbook-toolbox docker compose down docker compose up -d docker exec -it fluidbook-toolbox /application/scripts/update +/home/toolbox/www/scripts/fixrights diff --git a/app/Fields/FluidbookExportVersion.php b/app/Fields/FluidbookExportVersion.php index 508695f0d..cd187468f 100644 --- a/app/Fields/FluidbookExportVersion.php +++ b/app/Fields/FluidbookExportVersion.php @@ -7,20 +7,36 @@ use Cubist\Backpack\Magic\Fields\SelectFromArray; class FluidbookExportVersion extends SelectFromArray { - public function getOptions() + /** + * @return array[] + */ + public static function getVersions() { - return [ - 'online' => __('Version online - Version par défaut'), - 'sharepoint' => __('Version Sharepoint - Version par défaut'), - 'scorm' => __('Version SCORM - Version par défaut'), - 'win_inss_html' => __('Version offline - Executable Windows'), - 'win_ins_html' => __('Version offline - Installeur Auto-executable Windows'), - 'win_exe_html' => __('Version offline - ZIP Windows'), - 'mac_exe_html' => __('Version offline - Exécutable Mac OS X'), - 'win_cd_html' => __('Version offline - CD-ROM / Clé USB'), - 'win_html' => __('Version offline - HTML (Non adaptée à l\'installation sur un serveur web)'), - 'precompiled' => __('Version precompilée'), + 'online' => ['label' => __('Version online - Version par défaut'), 'short' => __('Online')], + 'sharepoint' => ['label' => __('Version Sharepoint - Version par défaut'), 'short' => 'Sharepoint'], + 'scorm' => ['label' => __('Version SCORM - Version par défaut'), 'short' => 'SCORM'], + 'win_inss_html' => ['label' => __('Version offline - Executable Windows'), 'short' => __('Exécutable Windows')], + 'win_ins_html' => ['label' => __('Version offline - Installeur Auto-executable Windows'), 'short' => __('Installeur windows')], + 'win_exe_html' => ['label' => __('Version offline - ZIP Windows'), 'short' => __('Zip Windows')], + 'mac_exe_html' => ['label' => __('Version offline - Exécutable Mac OS X'), 'short' => __('Exécutable Mac')], + 'win_cd_html' => ['label' => __('Version offline - CD-ROM / Clé USB'), 'short' => __('CD-Rom / Clé USB')], + 'win_html' => ['label' => __('Version offline - HTML (Non adaptée à l\'installation sur un serveur web)'), 'short' => __('Exécutable windows HTML')], + 'precompiled' => ['label' => __('Version precompilée'), 'short' => __('précompilé')], ]; } + + /** + * @return array + */ + public function getOptions() + { + + $versions = self::getVersions(); + $res = []; + foreach ($versions as $key => $version) { + $res[$key] = $version['label']; + } + return $res; + } } diff --git a/app/Http/Controllers/Admin/Operations/FluidbookCollection/DownloadOperation.php b/app/Http/Controllers/Admin/Operations/FluidbookCollection/DownloadOperation.php index 849bc86e2..e7c7fea27 100644 --- a/app/Http/Controllers/Admin/Operations/FluidbookCollection/DownloadOperation.php +++ b/app/Http/Controllers/Admin/Operations/FluidbookCollection/DownloadOperation.php @@ -2,8 +2,10 @@ namespace App\Http\Controllers\Admin\Operations\FluidbookCollection; +use App\Http\Middleware\CheckIfAdmin; use App\Jobs\FluidbookCollectionDownload; use App\Models\FluidbookCollection; +use Cubist\Backpack\Http\Controllers\Base\XSendFileController; use Illuminate\Support\Facades\Route; use Prologue\Alerts\Facades\Alert; @@ -12,6 +14,7 @@ trait DownloadOperation protected function setupDownloadRoutes($segment, $routeName, $controller) { Route::match(['get'], $segment . '/{id}/download/{action}', $controller . '@download'); + Route::match(['get'], $segment . '/{id}/downloadfile/{rand}/{path}', $controller . '@downloadFile')->withoutMiddleware([CheckIfAdmin::class]); } protected function setupDownloadDefaults() @@ -25,4 +28,10 @@ trait DownloadOperation Alert::add('success', __('La compilation a été placée en file d\'attente. Vous recevrez un email lorsqu\'elle sera terminée.'))->flash(); return redirect(backpack_url('fluidbook-collection')); } + + protected function downloadFile($id, $rand, $path) + { + $file = protected_path('fluidbookcollection/final/' . $id . '/' . $rand . '/' . $path); + return XSendFileController::sendfile($file); + } } diff --git a/app/Jobs/Base.php b/app/Jobs/Base.php index 81ddec496..cd15bc800 100644 --- a/app/Jobs/Base.php +++ b/app/Jobs/Base.php @@ -3,6 +3,7 @@ namespace App\Jobs; use App\Models\User; +use Illuminate\Support\Facades\Cache; class Base extends \Cubist\Backpack\Jobs\Base { @@ -11,6 +12,12 @@ class Base extends \Cubist\Backpack\Jobs\Base * @var User */ protected $user; + protected string $_cacheKey; + + public function __construct() + { + $this->generateCacheKey(); + } /** * @param User $user @@ -27,4 +34,21 @@ class Base extends \Cubist\Backpack\Jobs\Base { return $this->user; } + + public function generateCacheKey() + { + $this->_cacheKey = 'job_state_' . uniqid(); + } + + protected function _getState($key, $default = null) + { + return Cache::get($this->_cacheKey . '_' . $key, $default); + } + + protected function _setState($key, $value) + { + return Cache::put($this->_cacheKey . '_' . $key, $value); + } + + } diff --git a/app/Jobs/DownloadBase.php b/app/Jobs/DownloadBase.php index 030bea9e7..17b7cc8d9 100644 --- a/app/Jobs/DownloadBase.php +++ b/app/Jobs/DownloadBase.php @@ -120,7 +120,6 @@ class DownloadBase extends Base $subject = __($this->_subject, ['title' => $this->_title(), 'nb' => $this->_id()]); $text = ''; $actions = ['Télécharger' => $url]; - try { if ($this->action === 'scormcloud') { $scormURL = ScormCloud::send($url, 'toolbox_' . $this->type . '_' . $this->_id()); diff --git a/app/Jobs/FluidbookCollectionDownload.php b/app/Jobs/FluidbookCollectionDownload.php index fbf02c28c..f4671d8cc 100644 --- a/app/Jobs/FluidbookCollectionDownload.php +++ b/app/Jobs/FluidbookCollectionDownload.php @@ -2,12 +2,11 @@ namespace App\Jobs; -use App\Services\ScormCloud; +use App\Models\User; use App\Services\WorkshopV2; use Cubist\Util\Files\Files; use Cubist\Util\PHP; use Cubist\Util\Str; -use Cubist\Util\Zip; class FluidbookCollectionDownload extends DownloadBase { @@ -22,13 +21,16 @@ class FluidbookCollectionDownload extends DownloadBase { if ($this->action === 'install_hosting') { $url = $this->installHosting($this->entry->getPageData()); - $this->sendNotification(__('Collection ":title" (#:nb) installée sur le serveur hosting', ['title' => $this->_title(), 'nb' => $this->_id()]), '', $url); + } else if ($this->action === 'export' && in_array($this->entry->version, ['win_inss_html', 'win_ins_html'])) { + $url = $this->downloadList($this->entry->getPageData()); + $this->sendNotification(__('Collection ":title" (#:nb) prête au téléchargement', ['title' => $this->_title(), 'nb' => $this->_id()]), '', $url); } else { parent::handle(); } } + protected function _compile() { $compilepath = protected_path('collection/final/' . $this->entry->id); @@ -55,7 +57,7 @@ class FluidbookCollectionDownload extends DownloadBase protected function _getws2() { - $ws = new WorkshopV2($this->user); + $ws = new WorkshopV2(); $ws->login($this->user->email, $this->user->api_token); return $ws; } @@ -204,29 +206,112 @@ window.location='./' + locale + '/index.html'; } } + protected function installHosting($data) { $ws = $this->_getws2(); $options = $this->getCollectionGlobalSettings(); - $res = []; $updatedPublications = []; + $jobs = []; foreach ($data->publications as $publication) { $fbid = $publication['fluidbook']; if (isset($publication['dir']) && $publication['dir']) { $options['dir'] = $publication['dir']; } else { $metadata = $ws->getMetadata($fbid); - $publication['dir'] = $options['dir'] = $metadata->export->install_hosting->{$data->version}->dir ?? Str::slug($metadata->title); + $publication['dir'] = $options['dir'] = $metadata->export->install_hosting->online->dir ?? Str::slug($metadata->title); } - $res['Fluidbook #' . $fbid] = $ws->installBookOnHosting($fbid, $options, $data->version); + + $job = new FluidbookWS2Download(); + $job->setBookId($fbid); + $job->setVersion('online'); + $job->setOptions($options); + $job->setAction('install_hosting'); + $job->setJobName('install_hosting_' . $fbid); + $u = backpack_user() ?? User::withoutGlobalScopes()->find(5); + $job->setCredentials([$u->email, $u->api_token]); + //$job->handle(); + dispatch($job)->onQueue('ws2'); + $jobs['Fluidbook #' . $fbid] = $job; $updatedPublications[] = $publication; } $this->entry->publications = $updatedPublications; $this->entry->saveQuietly(); + + while (!$this->_checkJobs($jobs)) { + usleep(1000000 * 0.25); + } + + $res = []; + foreach ($jobs as $label => $job) { + $res[$label] = $job->getResult(); + } return $res; } + protected function downloadList($data) + { + $ws = $this->_getws2(); + $options = $this->getCollectionGlobalSettings(); + + $jobs = []; + $res = []; + $rand = sha1(uniqid()); + $compilepath = Files::mkdir(protected_path('fluidbookcollection/final/' . $this->entry->id . '/' . $rand)); + foreach ($data->publications as $publication) { + $fbid = $publication['fluidbook']; + if ($publication['export']) { + $name = $publication['export']; + } else { + $metadata = $ws->getMetadata($fbid); + $name = Str::slug($metadata->title); + } + $name .= '.exe'; + + $job = new FluidbookWS2Download(); + $job->setBookId($fbid); + $job->setVersion($this->entry->version); + $job->setOptions($options); + $job->setAction('export'); + $job->setJobName('export_' . $fbid); + $job->setDestination($compilepath . '/' . $name); + $u = backpack_user() ?? User::withoutGlobalScopes()->find(5); + $job->setCredentials([$u->email, $u->api_token]); + dispatch($job)->onQueue('ws2'); + //$job->handle(); + + $jobs['Fluidbook #' . $fbid] = $job; + $res['Fluidbook #' . $fbid] = url('fluidbook-collection/' . $this->entry->id . '/downloadfile/' . $rand . '/' . $name); + } + while (!$this->_checkJobs($jobs)) { + usleep(1000000 * 0.25); + } + + return $res; + } + + + /** + * @param $jobs FluidbookWS2Download[] + * @return bool + */ + protected function _checkJobs($jobs) + { + $nbjobs = count($jobs); + $done = 0; + foreach ($jobs as $label => $job) { + if ($job->isDone()) { + $done++; + } + } + if (rand(1, 10) == 5) { + echo $done . '/' . $nbjobs . "\n"; + + } + return $done === $nbjobs; + } + protected function compileExport($data, $path) { $ws = $this->_getws2(); diff --git a/app/Jobs/FluidbookImagesPreprocess.php b/app/Jobs/FluidbookImagesPreprocess.php index ae2c7ba7e..f598a7732 100644 --- a/app/Jobs/FluidbookImagesPreprocess.php +++ b/app/Jobs/FluidbookImagesPreprocess.php @@ -85,7 +85,6 @@ class FluidbookImagesPreprocess extends Base $done++; } } - echo $done . '/' . $nbjobs . "\n"; return $done === $nbjobs; } diff --git a/app/Jobs/FluidbookWS2Download.php b/app/Jobs/FluidbookWS2Download.php new file mode 100644 index 000000000..9891ef238 --- /dev/null +++ b/app/Jobs/FluidbookWS2Download.php @@ -0,0 +1,203 @@ +_bookId = $bookId; + } + + /** + * @return int|string + */ + public function getBookId(): int|string + { + return $this->_bookId; + } + + /** + * @return string + */ + public function getDestination(): string + { + return $this->_destination; + } + + /** + * @param string $destination + */ + public function setDestination(string $destination): void + { + $this->_destination = $destination; + } + + /** + * @return string + */ + public function getVersion(): string + { + return $this->_version; + } + + /** + * @param string $version + */ + public function setVersion(string $version): void + { + $this->_version = $version; + } + + /** + * @return int + */ + public function getTries(): int + { + return $this->_tries; + } + + /** + * @param int $tries + */ + public function setTries(int $tries): void + { + $this->_tries = $tries; + } + + /** + * @return string + */ + public function getAction(): string + { + return $this->_action; + } + + /** + * @param string $action + */ + public function setAction(string $action): void + { + $this->_action = $action; + } + + /** + * @param array $options + */ + public function setOptions(array $options): void + { + $this->_options = $options; + } + + /** + * @return array + */ + public function getOptions(): array + { + return $this->_options; + } + + /** + * @return mixed + */ + public function getResult(): mixed + { + return $this->_getState('result', null); + } + + /** + * @param mixed $result + */ + public function setResult(mixed $result): void + { + $this->_setState('result', $result); + } + + /** + * @return bool + */ + public function isDone(): bool + { + return $this->_getState('done', false); + } + + /** + * @param bool $done + */ + public function setDone(bool $done): void + { + $this->_setState('done', $done); + } + + /** + * @return \Exception|null + */ + public function getException(): \Exception|null + { + return $this->_getState('exception', null); + } + + /** + * @param \Exception|null $exception + */ + public function setException(\Exception|null $exception): void + { + $this->_setState('exception', $exception); + } + + /** + * @return array + */ + public function getCredentials(): array + { + return $this->_credentials; + } + + /** + * @param array $credentials + */ + public function setCredentials(array $credentials): void + { + $this->_credentials = $credentials; + } + + + public function handle() + { + try { + $ws = new WorkshopV2(); + $ws->login($this->getCredentials()[0], $this->getCredentials()[1]); + if ($this->getAction() === 'install_hosting') { + $res = $ws->installBookOnHosting($this->getBookId(), $this->getOptions(), 'online', $this->getTries()); + } else if ($this->getAction() === 'export') { + if (in_array($this->getVersion(), ['win_ins_html', 'win_inss_html'])) { + $res = $ws->downloadBookExport($this->getBookId(), $this->getDestination(), $this->getOptions(), $this->getVersion(), $this->getTries()); + } + } + $this->setResult($res); + } catch (\Exception $e) { + $this->setException($e); + } + $this->setDone(true); + } + +} diff --git a/app/Jobs/UpdateWS2ThemeTable.php b/app/Jobs/UpdateWS2ThemeTable.php index aa848f1b7..912e8a0d3 100644 --- a/app/Jobs/UpdateWS2ThemeTable.php +++ b/app/Jobs/UpdateWS2ThemeTable.php @@ -82,7 +82,7 @@ class UpdateWS2ThemeTable extends Base $dest = $this->_t3dir . '/' . $theme->id . '.jpg'; $preview = storage_path('themes/' . $theme->id . '.jpg'); - \Cubist\Util\Files\Files::copyFile($preview, $dest); + \Cubist\Util\Files\Files::copyFile($preview, $dest, false, true, false); return $res; } diff --git a/app/Services/WorkshopV2.php b/app/Services/WorkshopV2.php index 98004c48a..5fe8ea55c 100644 --- a/app/Services/WorkshopV2.php +++ b/app/Services/WorkshopV2.php @@ -23,20 +23,9 @@ class WorkshopV2 /** @var string */ protected $_domain = 'https://workshop.fluidbook.com/'; - /** - * @var User - */ - protected $user; - - public function __construct($user) + public function __construct() { - $this->user = $user; - if (null !== $this->user) { - $cookie_id = $this->user->id; - } else { - $cookie_id = Str::random(5); - } - $this->_cookies = new FileCookieJar(Files::mkdir(protected_path('ws2cookies/')) . $cookie_id, true); + $this->_cookies = new FileCookieJar(Files::mkdir(protected_path('ws2cookies/')) . Str::random(5), true); $this->_http = new Client(['base_uri' => $this->_domain, 'timeout' => 60000, 'read_timeout' => 60000, 'cookies' => $this->_cookies]); } @@ -47,7 +36,6 @@ class WorkshopV2 } $res = $this->_request('/', 'POST', ['user_email' => $username, 'api_token' => $api_key]); if (!$this->isLoggedIn()) { - dd($res); throw new Exception('Login failed'); } return $res; @@ -67,9 +55,9 @@ class WorkshopV2 } - public function installBookOnHosting($id, $options = [], $version = 'online') + public function installBookOnHosting($id, $options = [], $version = 'online', $tries = 3) { - return $this->installBook($id, null, $options + ['action' => 'install_hosting'], $version, 3, null, '_nothing'); + return $this->installBook($id, null, $options + ['action' => 'install_hosting'], $version, $tries, null, '_nothing'); } public function installBookIfNeeded($id, $dir, $options = [], $timestamp = 'auto', $version = 'online') @@ -146,7 +134,7 @@ class WorkshopV2 $this->$function((string)$xml->redirection, $dir, $beforeInstallCallback); return true; } else { - if ($tries == 0) { + if ($tries <= 0) { throw new Exception('Unable to download book'); } $this->validDownloadBook($id); diff --git a/resources/views/vendor/backpack/crud/buttons/fluidbook_collection/download.blade.php b/resources/views/vendor/backpack/crud/buttons/fluidbook_collection/download.blade.php index 2295349e4..832eb0b53 100644 --- a/resources/views/vendor/backpack/crud/buttons/fluidbook_collection/download.blade.php +++ b/resources/views/vendor/backpack/crud/buttons/fluidbook_collection/download.blade.php @@ -1,5 +1,7 @@ @php - $actions=['download'=>__('Télécharger'),'install_hosting'=>__('Installer sur hosting')]; + $v=$entry->version?:'online'; + $vname=\App\Fields\FluidbookExportVersion::getVersions()[$v]['short']; + $actions=['download'=>__('Télécharger la version :version',['version'=>$vname]),'install_hosting'=>__('Installer la version online sur hosting')]; if($entry->type==='scorm_multilang'){ $actions['scormcloud']=__('Tester sur Scorm Cloud'); } @@ -11,5 +13,5 @@ data-context-route="{{$crud->route}}/$id/download/$action" data-context-id="{{$entry->getKey()}}" > - {{__('Exporter')}} + {{__('Export')}} diff --git a/scripts/fixrights b/scripts/fixrights new file mode 100644 index 000000000..6612dd217 --- /dev/null +++ b/scripts/fixrights @@ -0,0 +1,4 @@ +#!/bin/sh +chmod -R 755 /home/toolbox/www/scripts +chmod -R 775 /home/toolbox/www/protected +chmod -R 775 /home/toolbox/www/public diff --git a/scripts/restartworkers.bat b/scripts/restartworkers.bat new file mode 100644 index 000000000..9f7710080 --- /dev/null +++ b/scripts/restartworkers.bat @@ -0,0 +1,5 @@ +@echo off +cls +C:\tools\cygwin\bin\ssh.exe -t root@toolbox.fluidbook.com 'docker exec -it -u toolbox fluidbook-toolbox /application/scripts/restartworkers' +exit +exit diff --git a/scripts/updatenpm b/scripts/updatenpm index ceeb091fc..61bc08bc6 100644 --- a/scripts/updatenpm +++ b/scripts/updatenpm @@ -1,6 +1,6 @@ #!/bin/sh -npm install -npm update +npm install --prefer-offline --no-audit --progress=false +#npm update npm run elearningmedia-prod npm run elearningpackage-prod npm run quiz-prod