--- /dev/null
+<?php
+
+
+namespace App\Console\Commands;
+
+use App\Util\FluidbookFarm;
+use Cubist\Backpack\Console\Commands\CubistCommand;
+
+
+class FluidbookFarmPing extends CubistCommand
+{
+ protected $signature = 'fluidbook:farm:ping';
+ protected $description = 'Manage fluidbook farm';
+
+ public function handle()
+ {
+ FluidbookFarm::ping();
+ }
+}
--- /dev/null
+<?php
+
+
+namespace App\Console\Commands;
+
+use App\Util\FluidbookFarm;
+use Cubist\Backpack\Console\Commands\CubistCommand;
+
+
+class FluidbookFarmUpdate extends CubistCommand
+{
+ protected $signature = 'fluidbook:farm:update';
+ protected $description = 'Manage fluidbook farm';
+
+ public function handle()
+ {
+ FluidbookFarm::update();
+ }
+}
parent::schedule($schedule);
$schedule->command('cubist:magic:precache')->everyFiveMinutes();
$schedule->command('job:dispatch ProcessTotals')->everyTwoHours();
+ $schedule->command('fluidbook:farm:ping')->everyMinute();
}
namespace App\Http\Controllers\Admin\Operations\FluidbookPublication;
+use App\Models\FluidbookDocument;
use Cubist\Util\Files\Files;
use Illuminate\Support\Facades\Route;
protected function getThumb($doc_id, $doc_page)
{
- $path = '/mnt/sshfs/godzilla/data/fluidbook/docs/' . $doc_id . '/p' . $doc_page . '.jpg';
+ /** @var FluidbookDocument $doc */
+ $doc = FluidbookDocument::find($doc_id);
+ $path = $doc->getFile($doc_page, 'jpg', 'thumb', true, true, '');
return response(null)->header('Content-Type', Files::_getMimeType($path))->header('X-Sendfile', $path);
}
namespace App\Models;
use App\Models\Base\ToolboxModel;
-use Cubist\Backpack\Magic\Fields\Datetime;
+use App\Util\FluidbookFarm;
use Cubist\Backpack\Magic\Fields\Integer;
use Cubist\Backpack\Magic\Fields\Text;
use Cubist\Backpack\Magic\Fields\Textarea;
-use Cubist\Backpack\Magic\Models\CubistMagicAbstractModel;
+use Cubist\PDF\PDFTools;
class FluidbookDocument extends ToolboxModel
{
+ public const WS_DOCS = '/data1/extranet/www/fluidbook/docs/';
protected $table = 'fluidbook_document';
protected $_options = ['name' => 'fluidbook-document',
'singular' => 'document',
'plural' => 'documents'];
protected static $_permissionBase = 'fluidbook-document';
+ protected $_filesData = null;
+ protected $casts = ['bookmarks' => 'array', 'pdf_data' => 'array', 'file_data' => 'array'];
public function setFields()
{
$this->addField('file', Text::class);
$this->addField('pages', Integer::class);
$this->addOwnerField();
- $this->addField('bookmarks', Textarea::class);
- $this->addField('pdf_data', Textarea::class);
- $this->addField('file_data', Textarea::class);
+ $this->addField('bookmarks', Textarea::class, ['cast' => 'array']);
+ $this->addField('pdf_data', Textarea::class, ['cast' => 'array']);
+ $this->addField('file_data', Textarea::class, ['cast' => 'array']);
+ $this->addField('pagesnumbers', Text::class);
+ }
+
+ public function getResolutionRatio()
+ {
+ $this->checkInfos();
+ $a4surface = 500990; // en pt²
+ $docSurface = $this->pdf_data['size'][0] * $this->pdf_data['size'][1]; // en pt²
+ // to have the same surface resulting in px, we have to sqrt the ratio between the two surfaces defined above
+ return sqrt($a4surface / $docSurface);
+ }
+
+ public function getMobileFirstRatio()
+ {
+ $this->checkInfos();
+ return 620 / $this->pdf_data['size'][0];
+ }
+
+ public function checkInfos()
+ {
+ if (null === $this->pdf_data) {
+ $infos = PDFTools::infos();
+ $this->pdf_data = $infos['infos'];
+ $this->bookmarks = $infos['bookmarks'];
+ $this->pagenumbers = $infos['numberSections'];
+ $this->saveWithoutFlushingCache();
+ }
+ }
+
+ /**
+ * @return string
+ */
+ public function path($path = ''): string
+ {
+ return rtrim(self::WS_DOCS . $this->id . ($path ? DIRECTORY_SEPARATOR . ltrim($path, DIRECTORY_SEPARATOR) : $path), DIRECTORY_SEPARATOR);
+ }
+
+ public function getFilesData()
+ {
+ if ($this->_filesData === null) {
+ $limit = time() - 604800;
+ $f = $this->path('filesdata.json');
+ if (file_exists($f) && filemtime($f) > $limit) {
+ $this->_filesData = json_decode(file_get_contents($f), true);
+ } else {
+ $this->_filesData = [];
+ }
+ }
+ return $this->_filesData;
+ }
+
+ public function getFile($page, $format = 'jpg', $resolution = 150, $withText = true, $withGraphics = true, $version = 'html', $force = false)
+ {
+ $this->getFilesData();
+ $cacheKey = md5($page . '||' . $format . '//' . $resolution . '""' . ($withText ? '1' : '0') . '---' . ($withGraphics ? '1' : '0') . '%%' . $version);
+ if (!isset($this->_filesData[$cacheKey]) || $force) {
+ $this->_filesData[$cacheKey] = $this->_getFile($page, $format, $resolution, $withText, $withGraphics, $version, $force);
+ }
+ return $this->_filesData[$cacheKey];
+ }
+
+
+ public function _getFile($page, $format = 'jpg', $resolution = 150, $withText = true, $withGraphics = true, $version = 'html', $force = false)
+ {
+ if ($format === 'jpeg') {
+ $format = 'jpg';
+ }
+ if ($format === 'svg') {
+ $version = 'html';
+ } else if ($format === 'swf') {
+ $version = '';
+ }
+
+ if ($resolution === 'thumb') {
+ $withGraphics = true;
+ $withText = true;
+ $version = '';
+ }
+
+ $dir = $this->path($version) . '/';
+ $minsize = 1;
+ if ($format === 'svg') {
+ $prefix = $withGraphics ? 'f' : 't';
+ $file = $dir . $prefix . 'o' . $page;
+ if ($withGraphics) {
+ $file .= '-' . $resolution;
+ }
+ $file .= '.svg';
+
+ $reffile = $this->path('/html/fp' . $page . '.svg');
+ $minsize = 100;
+ } else if ($format === 'png' || $format === 'jpg') {
+ $prefix = $withText ? 't' : 'h';
+ if ($resolution === 'thumb') {
+ $file = $dir . 'p' . $page . '.' . $format;
+ } else {
+ $file = $dir . $prefix . $page . '-' . $resolution . '.' . $format;
+ }
+ } else if ($format === 'swf') {
+ $file = $dir . 'p' . $page . '.' . $format;
+ }
+
+ $do = false;
+ if (!file_exists($file)) {
+ $do = true;
+ } else if (filesize($file) < $minsize) {
+ $do = true;
+ } else if (isset($reffile) && (!file_exists($reffile) || filemtime($file) < filemtime($reffile))) {
+ $do = true;
+ } else if ($format === 'svg') {
+ $t = filemtime($file);
+ $do = $t < 1603181003 && $t > 1602662603;
+ }
+
+ if ($do || $force) {
+ return FluidbookFarm::getFile($page, $format, $resolution, $withText, $withGraphics, $version, $this->getResolutionRatio(), $this->getMobileFirstRatio(), $this->path(), $force);
+ }
+
+ touch($file);
+ return $file;
+ }
+
+ public function __destruct()
+ {
+ if (null !== $this->_filesData) {
+ file_put_contents($this->path('filesdata.json'), json_encode($this->_filesData));
+ }
}
}
protected $_enableTrackNonDefaultValues = true;
protected static $_permissionBase = 'fluidbook-publication';
+ protected static $_docs = [];
+
protected $_operations = [CompositionOperation::class];
use PublicationSettings;
return '/data1/extranet/www/fluidbook/books/working/' . $this->id;
}
+ /**
+ * @param $documentID
+ * @return FluidbookDocument
+ */
+ protected static function _getDocument($documentID)
+ {
+ if (!isset(self::$_docs[$documentID])) {
+ self::$_docs[$documentID] = FluidbookDocument::find($documentID);
+ }
+ return self::$_docs[$documentID];
+ }
+
+
+ public function getFile($page, $format = 'jpg', $resolution = 150, $withText = true, $withGraphics = true, $version = 'html', $force = false)
+ {
+ if ($format === 'jpg') {
+ $q = $this->JPEGQuality ?? 85;
+ } else {
+ $q = 85;
+ }
+ if ($q != 85) {
+ $resolution .= '-' . $q;
+ }
+ $compo = $this->composition[$page];
+ return self::_getDocument($compo[0])->getFile($compo[1], $format, $resolution, $withText, $withGraphics, $version, $force);
+ }
}
--- /dev/null
+<?php
+
+namespace App\Util;
+
+use Cubist\Util\CommandLine;
+use Cubist\Util\Files\Files;
+
+class FluidbookFarm
+{
+ protected static $_farmServers = [
+ ['name' => 'elephantman', 'host' => 'elephantman.cubedesigners.com', 'port' => 22822, 'weight' => 1],
+ ['name' => 'fastandfurious', 'host' => 'fastandfurious.cubedesigners.com', 'port' => 22822, 'weight' => 1],
+ ['name' => 'vmparis', 'host' => 'paris.cubedesigners.com', 'port' => 22922, 'weight' => 2],
+ ['name' => 'vincent', 'host' => 'paris.cubedesigners.com', 'port' => 22923, 'weight' => 1],
+ ['name' => 'godzilla', 'host' => 'godzilla.cubedesigners.com', 'weight' => 3],
+ ['name' => 'dracula', 'host' => 'dracula.cubedesigners.com', 'weight' => 3],
+ ['name' => 'her', 'host' => 'her2.cubedesigners.com', 'weight' => 4],
+ ['name' => 'brazil', 'host' => 'brazil.cubedesigners.com', 'weight' => 6],
+ ];
+
+ protected static function _pingCache()
+ {
+ return Files::mkdir(storage_path('fluidbookfarm')) . '/pings';
+ }
+
+ public static function getServers()
+ {
+ return self::$_farmServers;
+ }
+
+ public static function pickOneServer()
+ {
+ $hat = [];
+ $pingCache = self::_pingCache();
+ if (!file_exists($pingCache)) {
+ self::ping(false);
+ }
+ $pings = json_decode(file_get_contents(self::_pingCache()));
+
+ foreach (self::$_farmServers as $k => $farmServer) {
+ if (!isset($pings[$k]) || !$pings[$k]) {
+ continue;
+ }
+ for ($i = 0; $i < $farmServer['weight']; $i++) {
+ $hat[] = $k;
+ }
+ }
+ shuffle($hat);
+ $i = array_pop($hat);
+ return self::$_farmServers[$i];
+ }
+
+ public static function getFile($page, $format, $resolution, $withText, $withGraphics, $version, $resolutionRatio, $mobileFirstRatio, $path, $force)
+ {
+ $start = microtime(true);
+ $farmer = self::pickOneServer();
+
+ $params = ['page' => $page, 'format' => $format, 'resolution' => $resolution, 'withText' => $withText, 'withGraphics' => $withGraphics, 'version' => $version, 'force' => $force, 'out' => $path, 'resolutionRatio' => $resolutionRatio, 'mobileRatio' => $mobileFirstRatio];
+ $cl = new CommandLine('/usr/local/fluidbook_processfarm/bin/process', null, true);
+ $cl->setSSH($farmer['host'], 'fluidbookfarmer', '', $farmer['port'] ?? 22, '/home/extranet/.ssh/id_rsa');
+ $cl->setManualArg(base64_encode(json_encode($params)));
+ $cl->execute();
+
+ $o = trim($cl->getOutput());
+ if (preg_match('|/data1/extranet/www/[^\s]+|', $o, $matches)) {
+ $o = $matches[0];
+ }
+
+ if (file_exists($o)) {
+ $res = $o;
+ } else {
+ echo $cl->getOutput();
+ $res = false;
+ }
+ $time = round(microtime(true) - $start, 4);
+ $log = '[' . $farmer['name'] . ']' . "\t" . date('Y-m-d H:i:s') . "\t\t\t\t" . $time . "\t\t\t\t$page|$format|$resolution|$withText|$withGraphics|$version\t\t\t\t$res\t\t\t\t" + $cl->getOutput() + "\n";
+
+ $fp = fopen($path . '/farm.log', 'a+');
+ fwrite($fp, $log);
+ fclose($fp);
+
+ return $res;
+ }
+
+ public static function ping($echo = true)
+ {
+ $originalConnectionTimeout = ini_get('default_socket_timeout');
+ ini_set('default_socket_timeout', 5);
+ $cache = self::_pingCache();
+ $servers = self::getServers();
+ $pings = [];
+ if (file_exists($cache)) {
+ $cached = json_decode(file_get_contents($cache));
+ if (count($cached) === count($servers)) {
+ $pings = $cached;
+ }
+ }
+
+ foreach ($servers as $id => $farmer) {
+ if ($echo) {
+ echo $farmer['name'] . ' (' . $id . ') || ';
+ }
+ if (isset($pings[$id]) && !$pings[$id]) {
+ // If ping failed recently, we wait a bit before trying again.
+ if (rand(0, 9) != 5) {
+ if ($echo) {
+ echo 'Skipped, will try again soon' . "\n";
+ }
+ continue;
+ }
+ }
+
+ $cl = new CommandLine('/usr/local/fluidbook_processfarm/bin/ping');
+ $cl->setSSH($farmer['host'], 'fluidbookfarmer', '', $farmer['port'] ?? 22, '/home/extranet/.ssh/id_rsa');
+ $cl->execute();
+ $ok = trim($cl->output) == '1';
+ if ($echo) {
+ echo ($ok ? 'OK' : 'KO') . ' : ' . trim($cl->getOutput()) . "\n";
+ }
+
+ $pings[$id] = $ok;
+ }
+ file_put_contents($cache, json_encode($pings));
+ ini_set('default_socket_timeout', $originalConnectionTimeout);
+ }
+
+ public static function update()
+ {
+ foreach (self::getServers() as $id => $farmer) {
+ echo $farmer['host'] . ' (' . $id . ')' . "\n";
+ $cl = new CommandLine('sudo /usr/local/fluidbook_processfarm/bin/update');
+ $cl->setSSH($farmer['host'], 'fluidbookfarmer', '', $farmer['port'] ?? 22, '/home/extranet/.ssh/id_rsa');
+ $cl->execute();
+ echo $cl->getCommand() . "\n\n--\n\n" . $cl->getOutput() . "\n\n";
+ }
+ }
+}
projects / fluidbook-toolbox.git / commitdiff