From d08a6bbaffd8138fe86c254287adf86f65d867b9 Mon Sep 17 00:00:00 2001 From: Vincent Vanwaelscappel Date: Wed, 13 Jul 2022 18:12:52 +0200 Subject: [PATCH] wip #4209 @4 --- app/Console/Commands/FluidbookFarmPing.php | 19 +++ app/Console/Commands/FluidbookFarmUpdate.php | 19 +++ app/Console/Kernel.php | 1 + .../CompositionOperation.php | 5 +- app/Models/FluidbookDocument.php | 140 +++++++++++++++++- app/Models/FluidbookPublication.php | 28 ++++ app/Util/FluidbookFarm.php | 137 +++++++++++++++++ 7 files changed, 343 insertions(+), 6 deletions(-) create mode 100644 app/Console/Commands/FluidbookFarmPing.php create mode 100644 app/Console/Commands/FluidbookFarmUpdate.php create mode 100644 app/Util/FluidbookFarm.php diff --git a/app/Console/Commands/FluidbookFarmPing.php b/app/Console/Commands/FluidbookFarmPing.php new file mode 100644 index 000000000..bf2fdb090 --- /dev/null +++ b/app/Console/Commands/FluidbookFarmPing.php @@ -0,0 +1,19 @@ +command('cubist:magic:precache')->everyFiveMinutes(); $schedule->command('job:dispatch ProcessTotals')->everyTwoHours(); + $schedule->command('fluidbook:farm:ping')->everyMinute(); } diff --git a/app/Http/Controllers/Admin/Operations/FluidbookPublication/CompositionOperation.php b/app/Http/Controllers/Admin/Operations/FluidbookPublication/CompositionOperation.php index 595849b9a..2ba979823 100644 --- a/app/Http/Controllers/Admin/Operations/FluidbookPublication/CompositionOperation.php +++ b/app/Http/Controllers/Admin/Operations/FluidbookPublication/CompositionOperation.php @@ -2,6 +2,7 @@ namespace App\Http\Controllers\Admin\Operations\FluidbookPublication; +use App\Models\FluidbookDocument; use Cubist\Util\Files\Files; use Illuminate\Support\Facades\Route; @@ -19,7 +20,9 @@ trait CompositionOperation protected function getThumb($doc_id, $doc_page) { - $path = '/mnt/sshfs/godzilla/data/fluidbook/docs/' . $doc_id . '/p' . $doc_page . '.jpg'; + /** @var FluidbookDocument $doc */ + $doc = FluidbookDocument::find($doc_id); + $path = $doc->getFile($doc_page, 'jpg', 'thumb', true, true, ''); return response(null)->header('Content-Type', Files::_getMimeType($path))->header('X-Sendfile', $path); } diff --git a/app/Models/FluidbookDocument.php b/app/Models/FluidbookDocument.php index a1bd83a06..70b1d0a6e 100644 --- a/app/Models/FluidbookDocument.php +++ b/app/Models/FluidbookDocument.php @@ -3,19 +3,22 @@ namespace App\Models; use App\Models\Base\ToolboxModel; -use Cubist\Backpack\Magic\Fields\Datetime; +use App\Util\FluidbookFarm; use Cubist\Backpack\Magic\Fields\Integer; use Cubist\Backpack\Magic\Fields\Text; use Cubist\Backpack\Magic\Fields\Textarea; -use Cubist\Backpack\Magic\Models\CubistMagicAbstractModel; +use Cubist\PDF\PDFTools; class FluidbookDocument extends ToolboxModel { + public const WS_DOCS = '/data1/extranet/www/fluidbook/docs/'; protected $table = 'fluidbook_document'; protected $_options = ['name' => 'fluidbook-document', 'singular' => 'document', 'plural' => 'documents']; protected static $_permissionBase = 'fluidbook-document'; + protected $_filesData = null; + protected $casts = ['bookmarks' => 'array', 'pdf_data' => 'array', 'file_data' => 'array']; public function setFields() { @@ -24,8 +27,135 @@ class FluidbookDocument extends ToolboxModel $this->addField('file', Text::class); $this->addField('pages', Integer::class); $this->addOwnerField(); - $this->addField('bookmarks', Textarea::class); - $this->addField('pdf_data', Textarea::class); - $this->addField('file_data', Textarea::class); + $this->addField('bookmarks', Textarea::class, ['cast' => 'array']); + $this->addField('pdf_data', Textarea::class, ['cast' => 'array']); + $this->addField('file_data', Textarea::class, ['cast' => 'array']); + $this->addField('pagesnumbers', Text::class); + } + + public function getResolutionRatio() + { + $this->checkInfos(); + $a4surface = 500990; // en pt² + $docSurface = $this->pdf_data['size'][0] * $this->pdf_data['size'][1]; // en pt² + // to have the same surface resulting in px, we have to sqrt the ratio between the two surfaces defined above + return sqrt($a4surface / $docSurface); + } + + public function getMobileFirstRatio() + { + $this->checkInfos(); + return 620 / $this->pdf_data['size'][0]; + } + + public function checkInfos() + { + if (null === $this->pdf_data) { + $infos = PDFTools::infos(); + $this->pdf_data = $infos['infos']; + $this->bookmarks = $infos['bookmarks']; + $this->pagenumbers = $infos['numberSections']; + $this->saveWithoutFlushingCache(); + } + } + + /** + * @return string + */ + public function path($path = ''): string + { + return rtrim(self::WS_DOCS . $this->id . ($path ? DIRECTORY_SEPARATOR . ltrim($path, DIRECTORY_SEPARATOR) : $path), DIRECTORY_SEPARATOR); + } + + public function getFilesData() + { + if ($this->_filesData === null) { + $limit = time() - 604800; + $f = $this->path('filesdata.json'); + if (file_exists($f) && filemtime($f) > $limit) { + $this->_filesData = json_decode(file_get_contents($f), true); + } else { + $this->_filesData = []; + } + } + return $this->_filesData; + } + + public function getFile($page, $format = 'jpg', $resolution = 150, $withText = true, $withGraphics = true, $version = 'html', $force = false) + { + $this->getFilesData(); + $cacheKey = md5($page . '||' . $format . '//' . $resolution . '""' . ($withText ? '1' : '0') . '---' . ($withGraphics ? '1' : '0') . '%%' . $version); + if (!isset($this->_filesData[$cacheKey]) || $force) { + $this->_filesData[$cacheKey] = $this->_getFile($page, $format, $resolution, $withText, $withGraphics, $version, $force); + } + return $this->_filesData[$cacheKey]; + } + + + public function _getFile($page, $format = 'jpg', $resolution = 150, $withText = true, $withGraphics = true, $version = 'html', $force = false) + { + if ($format === 'jpeg') { + $format = 'jpg'; + } + if ($format === 'svg') { + $version = 'html'; + } else if ($format === 'swf') { + $version = ''; + } + + if ($resolution === 'thumb') { + $withGraphics = true; + $withText = true; + $version = ''; + } + + $dir = $this->path($version) . '/'; + $minsize = 1; + if ($format === 'svg') { + $prefix = $withGraphics ? 'f' : 't'; + $file = $dir . $prefix . 'o' . $page; + if ($withGraphics) { + $file .= '-' . $resolution; + } + $file .= '.svg'; + + $reffile = $this->path('/html/fp' . $page . '.svg'); + $minsize = 100; + } else if ($format === 'png' || $format === 'jpg') { + $prefix = $withText ? 't' : 'h'; + if ($resolution === 'thumb') { + $file = $dir . 'p' . $page . '.' . $format; + } else { + $file = $dir . $prefix . $page . '-' . $resolution . '.' . $format; + } + } else if ($format === 'swf') { + $file = $dir . 'p' . $page . '.' . $format; + } + + $do = false; + if (!file_exists($file)) { + $do = true; + } else if (filesize($file) < $minsize) { + $do = true; + } else if (isset($reffile) && (!file_exists($reffile) || filemtime($file) < filemtime($reffile))) { + $do = true; + } else if ($format === 'svg') { + $t = filemtime($file); + $do = $t < 1603181003 && $t > 1602662603; + } + + if ($do || $force) { + return FluidbookFarm::getFile($page, $format, $resolution, $withText, $withGraphics, $version, $this->getResolutionRatio(), $this->getMobileFirstRatio(), $this->path(), $force); + } + + touch($file); + return $file; + } + + public function __destruct() + { + if (null !== $this->_filesData) { + file_put_contents($this->path('filesdata.json'), json_encode($this->_filesData)); + } } } diff --git a/app/Models/FluidbookPublication.php b/app/Models/FluidbookPublication.php index fc272d7f0..fe18c7507 100644 --- a/app/Models/FluidbookPublication.php +++ b/app/Models/FluidbookPublication.php @@ -31,6 +31,8 @@ class FluidbookPublication extends ToolboxModel protected $_enableTrackNonDefaultValues = true; protected static $_permissionBase = 'fluidbook-publication'; + protected static $_docs = []; + protected $_operations = [CompositionOperation::class]; use PublicationSettings; @@ -150,4 +152,30 @@ class FluidbookPublication extends ToolboxModel return '/data1/extranet/www/fluidbook/books/working/' . $this->id; } + /** + * @param $documentID + * @return FluidbookDocument + */ + protected static function _getDocument($documentID) + { + if (!isset(self::$_docs[$documentID])) { + self::$_docs[$documentID] = FluidbookDocument::find($documentID); + } + return self::$_docs[$documentID]; + } + + + public function getFile($page, $format = 'jpg', $resolution = 150, $withText = true, $withGraphics = true, $version = 'html', $force = false) + { + if ($format === 'jpg') { + $q = $this->JPEGQuality ?? 85; + } else { + $q = 85; + } + if ($q != 85) { + $resolution .= '-' . $q; + } + $compo = $this->composition[$page]; + return self::_getDocument($compo[0])->getFile($compo[1], $format, $resolution, $withText, $withGraphics, $version, $force); + } } diff --git a/app/Util/FluidbookFarm.php b/app/Util/FluidbookFarm.php new file mode 100644 index 000000000..8cf491923 --- /dev/null +++ b/app/Util/FluidbookFarm.php @@ -0,0 +1,137 @@ + 'elephantman', 'host' => 'elephantman.cubedesigners.com', 'port' => 22822, 'weight' => 1], + ['name' => 'fastandfurious', 'host' => 'fastandfurious.cubedesigners.com', 'port' => 22822, 'weight' => 1], + ['name' => 'vmparis', 'host' => 'paris.cubedesigners.com', 'port' => 22922, 'weight' => 2], + ['name' => 'vincent', 'host' => 'paris.cubedesigners.com', 'port' => 22923, 'weight' => 1], + ['name' => 'godzilla', 'host' => 'godzilla.cubedesigners.com', 'weight' => 3], + ['name' => 'dracula', 'host' => 'dracula.cubedesigners.com', 'weight' => 3], + ['name' => 'her', 'host' => 'her2.cubedesigners.com', 'weight' => 4], + ['name' => 'brazil', 'host' => 'brazil.cubedesigners.com', 'weight' => 6], + ]; + + protected static function _pingCache() + { + return Files::mkdir(storage_path('fluidbookfarm')) . '/pings'; + } + + public static function getServers() + { + return self::$_farmServers; + } + + public static function pickOneServer() + { + $hat = []; + $pingCache = self::_pingCache(); + if (!file_exists($pingCache)) { + self::ping(false); + } + $pings = json_decode(file_get_contents(self::_pingCache())); + + foreach (self::$_farmServers as $k => $farmServer) { + if (!isset($pings[$k]) || !$pings[$k]) { + continue; + } + for ($i = 0; $i < $farmServer['weight']; $i++) { + $hat[] = $k; + } + } + shuffle($hat); + $i = array_pop($hat); + return self::$_farmServers[$i]; + } + + public static function getFile($page, $format, $resolution, $withText, $withGraphics, $version, $resolutionRatio, $mobileFirstRatio, $path, $force) + { + $start = microtime(true); + $farmer = self::pickOneServer(); + + $params = ['page' => $page, 'format' => $format, 'resolution' => $resolution, 'withText' => $withText, 'withGraphics' => $withGraphics, 'version' => $version, 'force' => $force, 'out' => $path, 'resolutionRatio' => $resolutionRatio, 'mobileRatio' => $mobileFirstRatio]; + $cl = new CommandLine('/usr/local/fluidbook_processfarm/bin/process', null, true); + $cl->setSSH($farmer['host'], 'fluidbookfarmer', '', $farmer['port'] ?? 22, '/home/extranet/.ssh/id_rsa'); + $cl->setManualArg(base64_encode(json_encode($params))); + $cl->execute(); + + $o = trim($cl->getOutput()); + if (preg_match('|/data1/extranet/www/[^\s]+|', $o, $matches)) { + $o = $matches[0]; + } + + if (file_exists($o)) { + $res = $o; + } else { + echo $cl->getOutput(); + $res = false; + } + $time = round(microtime(true) - $start, 4); + $log = '[' . $farmer['name'] . ']' . "\t" . date('Y-m-d H:i:s') . "\t\t\t\t" . $time . "\t\t\t\t$page|$format|$resolution|$withText|$withGraphics|$version\t\t\t\t$res\t\t\t\t" + $cl->getOutput() + "\n"; + + $fp = fopen($path . '/farm.log', 'a+'); + fwrite($fp, $log); + fclose($fp); + + return $res; + } + + public static function ping($echo = true) + { + $originalConnectionTimeout = ini_get('default_socket_timeout'); + ini_set('default_socket_timeout', 5); + $cache = self::_pingCache(); + $servers = self::getServers(); + $pings = []; + if (file_exists($cache)) { + $cached = json_decode(file_get_contents($cache)); + if (count($cached) === count($servers)) { + $pings = $cached; + } + } + + foreach ($servers as $id => $farmer) { + if ($echo) { + echo $farmer['name'] . ' (' . $id . ') || '; + } + if (isset($pings[$id]) && !$pings[$id]) { + // If ping failed recently, we wait a bit before trying again. + if (rand(0, 9) != 5) { + if ($echo) { + echo 'Skipped, will try again soon' . "\n"; + } + continue; + } + } + + $cl = new CommandLine('/usr/local/fluidbook_processfarm/bin/ping'); + $cl->setSSH($farmer['host'], 'fluidbookfarmer', '', $farmer['port'] ?? 22, '/home/extranet/.ssh/id_rsa'); + $cl->execute(); + $ok = trim($cl->output) == '1'; + if ($echo) { + echo ($ok ? 'OK' : 'KO') . ' : ' . trim($cl->getOutput()) . "\n"; + } + + $pings[$id] = $ok; + } + file_put_contents($cache, json_encode($pings)); + ini_set('default_socket_timeout', $originalConnectionTimeout); + } + + public static function update() + { + foreach (self::getServers() as $id => $farmer) { + echo $farmer['host'] . ' (' . $id . ')' . "\n"; + $cl = new CommandLine('sudo /usr/local/fluidbook_processfarm/bin/update'); + $cl->setSSH($farmer['host'], 'fluidbookfarmer', '', $farmer['port'] ?? 22, '/home/extranet/.ssh/id_rsa'); + $cl->execute(); + echo $cl->getCommand() . "\n\n--\n\n" . $cl->getOutput() . "\n\n"; + } + } +} -- 2.39.5