From: Vincent Vanwaelscappel <vincent@cubedesigners.com>
Date: Wed, 27 Dec 2023 11:09:14 +0000 (+0100)
Subject: wip #6598 @3
X-Git-Url: http://git.cubedesigners.com/?a=commitdiff_plain;h=42220765d22059374b95ced4a691cf01ee475fbe;p=fluidbook-toolbox.git

wip #6598 @3
---

diff --git a/.docker/dev/docker-compose.yml b/.docker/dev/docker-compose.yml
index f2fc02e5e..fea626a9e 100644
--- a/.docker/dev/docker-compose.yml
+++ b/.docker/dev/docker-compose.yml
@@ -178,6 +178,9 @@ volumes:
     driver_opts:
       sshcmd: "fluidbook@kingkong.cubedesigners.com:/home/fluidbook/data"
       allow_other: ""
+      cache: "no"
+      uid: 1001
+      gid: 33
       password: "Qu2E9uq48bBzsqGiZA"
 
   docs:
@@ -185,4 +188,7 @@ volumes:
     driver_opts:
       sshcmd: "fluidbook@dracula.cubedesigners.com:/data/fluidbook/docs"
       allow_other: ""
+      cache: "no"
+      uid: 1001
+      gid: 33
       password: "Qu2E9uq48bBzsqGiZA"
diff --git a/.docker/syncprodtodev.exclude b/.docker/syncprodtodev.exclude
index 21e2359e3..dd6248a04 100644
--- a/.docker/syncprodtodev.exclude
+++ b/.docker/syncprodtodev.exclude
@@ -3,6 +3,7 @@ fluidbookpublication/final
 fluidbookpublication/seo
 fluidbookpublication/pdf
 fluidbookcollection/final
+fluidbookcollection/download
 collection/final
 elearningpackage/final
 elearningmedia/final
@@ -13,3 +14,4 @@ app/public/elearningpackage
 app/public/fluidbookcollection
 app/public/quiz
 protected/youtube
+
diff --git a/app/Fluidbook/Farm.php b/app/Fluidbook/Farm.php
index 66bbb881a..ee04f0b83 100644
--- a/app/Fluidbook/Farm.php
+++ b/app/Fluidbook/Farm.php
@@ -2,6 +2,7 @@
 
 namespace App\Fluidbook;
 
+use Cubist\PDF\PDFTools;
 use Cubist\Util\Files\Files;
 use Fluidbook\Tools\Jobs\ProcessFile;
 use hollodotme\FastCGI\Client;
@@ -14,9 +15,9 @@ use Mockery\Exception;
 class Farm
 {
     protected static $_outPDF = [];
-    protected static $_forceServer = false;
+    //protected static $_forceServer = false;
 
-    //protected static $_forceServer = 'alphaville';
+    protected static $_forceServer = 'alphaville';
 
     protected static $_region = Region::EUROPE;
 
@@ -114,7 +115,8 @@ class Farm
 
     public static function getFile($page, $format, $resolution, $quality, $withText, $withGraphics, $version, $resolutionRatio, $mobileFirstRatio, $path, $force = false)
     {
-        $params = ['toolbox' => 1,
+        $params = [
+            'operation' => 'documentfile',
             'page' => $page,
             'format' => $format,
             'resolution' => $resolution,
@@ -144,7 +146,9 @@ class Farm
             $out = self::_getOutDirFromPDF($pdf);
         }
 
-        $attrs = ['pdf' => $pdf,
+        $attrs = [
+            'operation' => 'pdffile',
+            'pdf' => $pdf,
             'page' => $page,
             'format' => $format,
             'resolution' => $resolution,
@@ -156,7 +160,7 @@ class Farm
 
         $cachekey = 'filefrompdf_' . hash('sha256', print_r($attrs, true));
 
-        $params = array_merge($attrs, ['toolbox' => 1,
+        $params = array_merge($attrs, [
             'force' => $force,
         ]);
 
@@ -184,30 +188,95 @@ class Farm
         return $res;
     }
 
+    public static function fixPDF($pdf, $out)
+    {
+        return self::_getFile(['operation' => 'fixpdf',
+            'pdf' => $pdf,
+            'out' => $out], 0,false);
+    }
+
+    public static function cutPDF($pdf, $mode, $out)
+    {
+        return self::_getFile(['operation' => 'cutpdf',
+            'pdf' => $pdf,
+            'out' => $out,
+            'mode' => $mode,
+        ], 0,false);
+    }
+
+    public static function trimPDF($pdf, $out)
+    {
+        return self::_getFile(['operation' => 'trimpdf',
+            'pdf' => $pdf,
+            'out' => $out,
+        ], 0,false);
+    }
+
+    public static function splitPDF($pdf, $out)
+    {
+        return self::_getFile(['operation' => 'splitpdf',
+            'pdf' => $pdf,
+            'out' => $out], 0,false);
+    }
+
+    public static function extractLinks($pdf, $out)
+    {
+        return self::_getFile(['operation' => 'extractlinks',
+            'pdf' => $pdf,
+            'out' => $out], 0,false);
+    }
+
+    public static function extractTexts($pdf, $out, $mode, $extractionMethod, $ignoreSeparators)
+    {
+        return self::_getFile(
+            ['operation' => 'extracttexts',
+                'pdf' => $pdf,
+                'out' => $out,
+                'mode' => $mode,
+                'method' => $extractionMethod,
+                'ignoreseparators' => $ignoreSeparators], 0,false);
+    }
+
+    public static function extractHighlightsData($pdf, $out, $mode, $ignoreSeparators)
+    {
+        return self::_getFile(
+            ['operation' => 'extracthightlightsdata',
+                'pdf' => $pdf,
+                'out' => $out,
+                'mode' => $mode,
+                'ignoreseparators' => $ignoreSeparators], 0,false);
+    }
+
     /**
      * @throws \Exception
      */
-    protected static function _getFile($params, $attempts = 3)
+    protected static function _getFile($params, $attempts = 3, $checkOutput = true)
     {
         $start = microtime(true);
         $farmer = self::pickOneServer();
 
+        $params['toolbox'] = '1';
+
         try {
             $output = self::sendRequest($farmer, 'process.php', $params);
         } catch (\Exception $e) {
             Log::error('Farm server error ' . $farmer['name'] . ' : ' . json_encode($params) . ' : ' . $e->getMessage());
             if ($attempts > 0) {
-                return static::_getFile($params, $attempts - 1);
+                return static::_getFile($params, $attempts - 1, $checkOutput);
             }
             return false;
         }
 
-        $output = trim($output);
+        if ($checkOutput) {
+            $output = trim($output);
 
-        if (file_exists($output)) {
-            $res = $output;
+            if (file_exists($output)) {
+                $res = $output;
+            } else {
+                $res = false;
+            }
         } else {
-            $res = false;
+            $res = true;
         }
 
         $time = round(microtime(true) - $start, 4);
@@ -216,7 +285,7 @@ class Farm
         if (!$res) {
             if ($attempts > 0) {
                 Log::warning('Farm failure (' . $farmer['name'] . ') : ' . $output . ' // ' . $log);
-                return static::_getFile($params, $attempts - 1);
+                return static::_getFile($params, $attempts - 1, $checkOutput);
             }
             throw new \Exception('Farm failure  (' . $farmer['name'] . ')  : ' . $output . ' // ' . $log);
         }
diff --git a/app/Models/FluidbookDocument.php b/app/Models/FluidbookDocument.php
index c95e2bbdc..56751891f 100644
--- a/app/Models/FluidbookDocument.php
+++ b/app/Models/FluidbookDocument.php
@@ -137,13 +137,13 @@ class FluidbookDocument extends ToolboxModel
 
     public function extractTexts()
     {
-        PDFTools::extractTexts($this->getPDFSource(), $this->path(), 'standard', 'fluidbook', '');
-        PDFTools::extractHighlightsData($this->getPDFSource(), $this->path(), 'standard');
+        Farm::extractTexts($this->getPDFSource(), $this->path(), 'standard', 'fluidbook', '');
+        Farm::extractHighlightsData($this->getPDFSource(), $this->path(), 'standard', 'standard');
     }
 
     public function extractLinks()
     {
-        PDFTools::extractLinks($this->getPDFSource(), $this->path());
+        Farm::extractLinks($this->getPDFSource(), $this->path());
     }
 
     protected function _checkJobs($uploadID, $jobs, $nbfiles)
@@ -191,17 +191,17 @@ class FluidbookDocument extends ToolboxModel
 
     public function fixPDF()
     {
-        $fixed = $this->getPDFSource('fixed',false);
+        $fixed = $this->getPDFSource('fixed');
         $original = $this->getPDFSource('original');
         if (!file_exists($fixed) || filesize($fixed) === 0 || filemtime($fixed) < filemtime($original)) {
-            PDFTools::fixPDF($original, $fixed);
+            Farm::fixPDF($original, $fixed);
         }
         return $fixed;
     }
 
     public function splitPDF()
     {
-        PDFTools::split($this->getPDFSource(), $this->path('pdf'));
+        Farm::splitPDF($this->getPDFSource(), $this->path('pdf'));
     }
 
     /**
@@ -395,7 +395,10 @@ class FluidbookDocument extends ToolboxModel
     {
         $path = $this->_getTextPath($page, $type, $mode, $extractionMethod, $ignoreSeparators);
         if (!$this->_checkTextFile($path)) {
-            PDFTools::extractTexts($this->getPDFSource(), $this->path(), $mode, $extractionMethod, $ignoreSeparators);
+            Farm::extractTexts($this->getPDFSource(), $this->path(), $mode, $extractionMethod, $ignoreSeparators);
+            if (!Gzip::file_exists($path)) {
+                throw new \Exception('An error occured while producing file ' . $path);
+            }
         }
         return Gzip::path($path);
     }
@@ -409,7 +412,7 @@ class FluidbookDocument extends ToolboxModel
     {
         $path = $this->_getHightlightFilePath($page, $mode, $ignoreSeparators);
         if (!$this->_checkTextFile($path)) {
-            PDFTools::extractHighlightsData($this->getPDFSource(), $this->path(), $mode, $ignoreSeparators);
+            Farm::extractHighlightsData($this->getPDFSource(), $this->path(), $mode, $ignoreSeparators);
             if (!Gzip::file_exists($path)) {
                 throw new \Exception('An error occured while producing file ' . $path);
             }
@@ -424,68 +427,48 @@ class FluidbookDocument extends ToolboxModel
     }
 
     public function _getHightlightFilePath($page, $mode = 'standard', $ignoreSeparators = '')
+    {
+        return $this->_getTextBasePath($mode, $ignoreSeparators) . 'p' . $page . '.fby';
+    }
+
+    public function _getTextBasePath($mode = 'standard', $ignoreSeparators = '')
     {
         $sepFolder = '';
         if ($ignoreSeparators) {
             $sepFolder = 'sep_' . md5($ignoreSeparators) . '/';
         }
         $dir = $mode !== 'standard' ? $mode . '/' : '';
-        return $this->path('texts/' . $dir . $sepFolder) . 'p' . $page . '.fby';
+        return $this->path('texts/' . $dir . $sepFolder);
     }
 
     public function _getTextPath($page, $type = self::TEXT_PLAIN, $mode = 'standard', $extractionMethod = 'fluidbook', $ignoreSeparators = '')
     {
-        $sepFolder = '';
-        if ($ignoreSeparators) {
-            $sepFolder = 'sep_' . md5($ignoreSeparators) . '/';
-        }
-
         if ($mode === 'robust') {
             $extractionMethod = 'fluidbook';
         }
 
         $prefix = self::_getTextFilePrefix($extractionMethod);
-        $dir = $mode !== 'standard' ? $mode . '/' : '';
-        return $this->path('texts/' . $dir . $sepFolder) . $prefix . $type . $page . '.txt';
+        return $this->_getTextBasePath($mode, $ignoreSeparators) . $prefix . $type . $page . '.txt';
     }
 
-    public function getPDFSource($type = 'crop', $checkIfExists = true): string
+    public function getPDFSource($type = 'crop'): string
     {
-        $res = $this->path() . $type . '.pdf';
-        if ($checkIfExists && !file_exists($res)) {
-            if ($type === 'original') {
-                throw new \Exception('Missing original PDF file ' . $res);
-            } else if ($type === 'fixed') {
-                $this->fixPDF();
-            } else if ($type === 'crop') {
-                $this->cropAndCut();
-            }
-        }
-        return $res;
+        return $this->path() . $type . '.pdf';
     }
 
     public function cutDocument($mode)
     {
-        $fwstk = new FWSTK();
-        $fwstk->setArg('--input ' . $this->getPDFSource('fixed'));
-        $fwstk->setArg('--cut ' . $mode);
-        $fwstk->setArg('--output ' . $this->getPDFSource('crop', false));
-        $fwstk->execute();
+        Farm::cutPDF($this->getPDFSource('fixed'), $mode, $this->getPDFSource());
     }
 
     public function trimDocument()
     {
-
-        $fwstk = new FWSTK();
-        $fwstk->setArg('--input ' . $this->getPDFSource('fixed'));
-        $fwstk->setArg('--trim');
-        $fwstk->setArg('--output ' . $this->getPDFSource('crop', false));
-        $fwstk->execute();
+        Farm::trimPDF($this->getPDFSource('fixed'), $this->getPDFSource());
     }
 
     public function lnCrop()
     {
-        link($this->getPDFSource('fixed'), $this->getPDFSource('crop', false));
+        link($this->getPDFSource('fixed'), $this->getPDFSource());
     }
 
     protected function isCropped()
@@ -498,7 +481,7 @@ class FluidbookDocument extends ToolboxModel
         $infos = $this->_getInfos('fixed');
 
         $this->detectSpreads($infos);
-        $this->detectPageDifferences($infos);
+        //$this->detectPageDifferences($infos);
 
         if (!$this->isCropped()) {
             $this->lnCrop();
diff --git a/app/Models/FluidbookPublication.php b/app/Models/FluidbookPublication.php
index 18ecc9f5d..52b41b7c9 100644
--- a/app/Models/FluidbookPublication.php
+++ b/app/Models/FluidbookPublication.php
@@ -38,7 +38,6 @@ use App\Jobs\FluidbookImagesPreprocess;
 use App\Jobs\FluidbookSocialImage;
 use App\Jobs\GenerateDeliveryThumbnailsPreview;
 use App\Models\Base\ToolboxDownloadable;
-use App\Models\Base\ToolboxSettingsModel;
 use App\Models\Base\ToolboxStatusModel;
 use App\Models\Traits\CheckHash;
 use App\Models\Traits\PublicationSettings;
@@ -604,11 +603,11 @@ class FluidbookPublication extends ToolboxStatusModel
         return $this->composition_updated_at->getTimestamp();
     }
 
-    public function getPDFSource($page, $type = 'crop', $checkIfExist = true): string
+    public function getPDFSource($page, $type = 'crop'): string
     {
         $compo = $this->getComposition()[$page];
         if ($type == 'crop') {
-            return self::_getDocument($compo[0])->getPDFSource($type, $checkIfExist);
+            return self::_getDocument($compo[0])->getPDFSource($type);
         }
     }
 
diff --git a/bin/startup.sh b/bin/startup.sh
index 0c28109a9..9dddf8bfd 100644
--- a/bin/startup.sh
+++ b/bin/startup.sh
@@ -66,7 +66,7 @@ if mountpoint -q "/application/protected/fluidbookpublication/docs"; then
     :
 else
     #sshfs -o reconnect,ServerAliveInterval=15,ServerAliveCountMax=3 -o cache=no -o sshfs_sync -o uid=1002 -o gid=33 -o allow_other root@godzilla.cubedesigners.com:/data/fluidbook/docs /application/protected/fluidbookpublication/docs
-    sshfs -o reconnect,ServerAliveInterval=15,ServerAliveCountMax=3 -o cache=no -o sshfs_sync -o uid=1002 -o gid=33 -o allow_other root@dracula.cubedesigners.com:/data/fluidbook/docs /application/protected/fluidbookpublication/docs
+    sshfs -o reconnect,ServerAliveInterval=15,ServerAliveCountMax=3 -o cache=no -o uid=1002 -o gid=33 -o allow_other root@dracula.cubedesigners.com:/data/fluidbook/docs /application/protected/fluidbookpublication/docs
 fi
 
 # Launch PHP
diff --git a/scripts/update.sh b/scripts/update.sh
index 980b824b5..a57eb9b8b 100644
--- a/scripts/update.sh
+++ b/scripts/update.sh
@@ -17,7 +17,7 @@ sudo /application/scripts/restartworkers.sh
 /application/scripts/updatenpm.sh
 sudo chown -R 1001:33 /application/storage/framework
 sudo chown -R 1001:33 /application/storage/framework
-php artisan optimize:clear
+#php artisan optimize:clear
 php artisan cubist:magic:precache
 php artisan toolbox:precache
 sudo chown -R 1001:33 /application/storage/framework