From 1bdd1a9b3ee3f293bde9ad4ce572a9d216208e75 Mon Sep 17 00:00:00 2001 From: soufiane Date: Tue, 18 Nov 2025 15:32:08 +0100 Subject: [PATCH] wait #7825 10:00 --- .docker/docker-compose.yml | 20 ++-- .../AuditLinksOperation.php | 105 ++++++++++++++++++ app/Jobs/AuditLink.php | 14 ++- app/Jobs/registerLinksForAudit.php | 5 +- app/Models/FluidbookAuditLink.php | 73 +----------- app/Models/FluidbookCollection.php | 1 + ..._17_184637_create_fluidbook_audit_link.php | 39 +++++++ routes/web.php | 23 ++++ 8 files changed, 192 insertions(+), 88 deletions(-) create mode 100644 database/migrations/2025_11_17_184637_create_fluidbook_audit_link.php diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml index 76760271e..077336997 100644 --- a/.docker/docker-compose.yml +++ b/.docker/docker-compose.yml @@ -174,16 +174,18 @@ services: networks: - fluidbook-toolbox - typesense: - container_name: presquot-typesense - image: typesense/typesense:29.0 - restart: unless-stopped - volumes: - - ./typesense:/data - command: '--data-dir /data --api-key=K4fae5KYZTVj6Wucp5q9 --enable-cors' + flaresolverr: + # DockerHub mirror flaresolverr/flaresolverr:latest + image: ghcr.io/flaresolverr/flaresolverr:latest + container_name: flaresolverr environment: - VIRTUAL_HOST: typesense.dev.prescription-quotidien.com - LETSENCRYPT_HOST: typesense.dev.prescription-quotidien.com + - LOG_LEVEL=${LOG_LEVEL:-info} + - LOG_HTML=${LOG_HTML:-false} + - CAPTCHA_SOLVER=${CAPTCHA_SOLVER:-none} + - TZ=Europe/London + ports: + - "${PORT:-8191}:8191" + restart: unless-stopped networks: fluidbook-toolbox: diff --git a/app/Http/Controllers/Admin/Operations/FluidbookCollection/AuditLinksOperation.php b/app/Http/Controllers/Admin/Operations/FluidbookCollection/AuditLinksOperation.php index 0e4e95d7c..48a9bdd25 100644 --- a/app/Http/Controllers/Admin/Operations/FluidbookCollection/AuditLinksOperation.php +++ b/app/Http/Controllers/Admin/Operations/FluidbookCollection/AuditLinksOperation.php @@ -15,6 +15,9 @@ use PhpOffice\PhpSpreadsheet\Writer\Xlsx; trait AuditLinksOperation { + protected static $curlOpt = []; + protected static $timeout = 30; + protected function setupAuditLinksRoutes($segment, $routeName, $controller) { Route::match(['get','post'],$segment . '/{id}/export_excel', $controller . '@exportExcel')->name("download_audit_links"); @@ -115,4 +118,106 @@ trait AuditLinksOperation return response()->download($tmpfile, $filename)->deleteFileAfterSend(); } + + public static function getHttpCode($url) { + $ch = curl_init($url); + self::setCurlOpt([CURLOPT_FOLLOWLOCATION => false]); + curl_setopt_array($ch, self::$curlOpt); + curl_exec($ch); + + $httpcode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + curl_close($ch); + + $ch = curl_init($url); + self::$curlOpt[CURLOPT_FOLLOWLOCATION] = true; + curl_setopt_array($ch, self::$curlOpt); + curl_exec($ch); + + $finalHttpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + $finalUrl = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL); + + curl_close($ch); + + return ['httpcode' => $httpcode, 'finalurl' => $finalUrl, 'finalHttpCode' => $finalHttpCode]; + } + + public static function getHttpCodeCloudflare($url) { + $apiUrl = 'http://flaresolverr:8191/v1'; + + $payload = json_encode([ + 'cmd' => 'request.get', + 'url' => $url, + 'maxTimeout' => 60000 + ]); + + /*$flaresolverrOptions = [ + CURLOPT_POST => true, + CURLOPT_HTTPHEADER => ['Content-Type: application/json'], + CURLOPT_POSTFIELDS => $payload, + ];*/ + + $ch = curl_init($apiUrl); + self::$curlOpt = [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_HEADER => true, + CURLOPT_NOBODY => true, + CURLOPT_TIMEOUT => self::$timeout, + CURLOPT_POST => true, + CURLOPT_HTTPHEADER => ['Content-Type: application/json'], + CURLOPT_POSTFIELDS => $payload, + ]; + curl_setopt_array($ch, self::$curlOpt); + $response = curl_exec($ch); + + if(preg_match('/(error|code) \b(301|302|308|404|401|403|405|500|502|503)\b/', $response, $matches)) { + $httpcode = $matches[2] ?? ''; + $finalUrl = ''; + $finalHttpCode = ''; + } + + curl_close($ch); + + return ['httpcode' => $httpcode, 'finalurl' => $finalUrl, 'finalHttpCode' => $finalHttpCode]; + } + + protected static function setCurlOpt($moreOptions = []) { + self::$curlOpt = [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_HEADER => true, + CURLOPT_NOBODY => true, + CURLOPT_TIMEOUT => self::$timeout, + ...$moreOptions + ]; + } + + public static function getUrlInfo($url) { + $headers = get_headers($url, 1); + if($headers['Server'] === "cloudflare") { + $response = self::getHttpCodeCloudflare($url); + }else { + $response = self::getHttpCode($url); + } + + return $response; + } + + public static function getHttpCodeComment($httpcode) + { + switch ($httpcode) { + case 301: + return " - Moved Permanently: the resource has a new permanent home — update your bookmarks or links."; + case 302: + return " - Found: tells the client to look at (browse to) another URL."; + //case 307: + //case 308: + case 400: + return " - Bad request: this and all future requests should be directed to the given URI."; + } + } + + public static function youtubeVideoExist($videoID) + { + $headers = get_headers('https://www.youtube.com/oembed?format=json&url=http://www.youtube.com/watch?v=' . $videoID); + return (is_array($headers) && preg_match('/^HTTP\\/\\d+\\.\\d+\\s+2\\d\\d\\s+.*$/', $headers[0])); + } } diff --git a/app/Jobs/AuditLink.php b/app/Jobs/AuditLink.php index d1f301e7f..e82699d88 100644 --- a/app/Jobs/AuditLink.php +++ b/app/Jobs/AuditLink.php @@ -33,7 +33,7 @@ class AuditLink extends Base // Error code start with 4 or 5 // Redirection code start with 3 try { - $curlResponse = FluidbookAuditLink::getHttpCode($link['url']); + $curlResponse = FluidbookCollection::getUrlInfo($link['url']); }catch (\Error $e) { echo "Error when trying to get http code: " . $e->getMessage().PHP_EOL; continue; @@ -51,7 +51,7 @@ class AuditLink extends Base $webvideo = WebVideo::parse($link['url'], true); if($webvideo !== false) { if($webvideo['service'] === 'youtube') { - if (!FluidbookAuditLink::youtubeVideoExist($webvideo['id'])) { + if (!FluidbookCollection::youtubeVideoExist($webvideo['id'])) { $httpCode = "404"; } } @@ -60,7 +60,7 @@ class AuditLink extends Base if (str_starts_with($httpCode, 3)) { try { $finalurl = $curlResponse['finalurl']; - $finalcodeurl = FluidbookAuditLink::getHttpCode($curlResponse['finalurl'])['httpcode']; + $finalcodeurl = $curlResponse['finalHttpCode']; }catch (\Error $e) { echo "Error when trying to get final url: " . $e->getMessage().PHP_EOL; continue; @@ -70,7 +70,7 @@ class AuditLink extends Base } } - $comment = FluidbookAuditLink::getHttpCodeComment($httpCode); + $comment = FluidbookCollection::getHttpCodeComment($httpCode); $firstTimeError = ''; if(str_starts_with($httpCode, 3) || str_starts_with($httpCode, 4) || str_starts_with($httpCode, 5)) { @@ -79,6 +79,10 @@ class AuditLink extends Base $externalLinks[] = [ 'id' => $link['id'], + 'fluidbook_id' => $link['fluidbook_id'], + 'page' => $link['page'], + 'link_id' => $link['link_id'], + 'url' => $link['url'], 'error_code' => str_starts_with($httpCode, 4) || str_starts_with($httpCode, 5) ? $httpCode.$comment : "", 'first_time_error' => $firstTimeError, // Datetime of the first time we saw this error 'last_date_test' => date('Y-m-d H:i:s'), @@ -92,7 +96,7 @@ class AuditLink extends Base if($externalLinks) { $keys = array_slice(array_keys($externalLinks[0]), 1); - DB::table('fluidbook_audit_link')->upsert($externalLinks, ['id'], $keys); + FluidbookAuditLink::upsert($externalLinks, ['id'], $keys); } Log::info('Job exécuté avec succès'); diff --git a/app/Jobs/registerLinksForAudit.php b/app/Jobs/registerLinksForAudit.php index 625eb55b8..08e14798e 100644 --- a/app/Jobs/registerLinksForAudit.php +++ b/app/Jobs/registerLinksForAudit.php @@ -63,12 +63,9 @@ class registerLinksForAudit extends Base 'updated_at'=> date('Y-m-d H:i:s') ]; } - - //print_r($publication.' => OK'. PHP_EOL); } - dd('exit'); - FluidbookAuditLink::updateOrCreate($externalLinks); + FluidbookAuditLink::upsert($externalLinks, ['link_id','fluidbook_id'], ['url','updated_at','page']); Log::info('Job exécuté avec succès'); } diff --git a/app/Models/FluidbookAuditLink.php b/app/Models/FluidbookAuditLink.php index d7c8d34f1..650343466 100644 --- a/app/Models/FluidbookAuditLink.php +++ b/app/Models/FluidbookAuditLink.php @@ -9,8 +9,10 @@ use Cubist\Backpack\Magic\Fields\Integer; use Cubist\Backpack\Magic\Fields\Text; use App\Http\Controllers\Admin\Operations\ChangeownerOperation; use App\Http\Controllers\Admin\Operations\ChangestatusOperation; +use Illuminate\Database\Eloquent\Model; +use Illuminate\Routing\Route; -class FluidbookAuditLink extends ToolboxModel +class FluidbookAuditLink extends Model { use CheckHash; use ToolboxDownloadable; @@ -21,73 +23,4 @@ class FluidbookAuditLink extends ToolboxModel 'plural' => 'auditlinks']; protected static $_permissionBase = 'fluidbook-collection'; - - public function setFields() - { - parent::setFields(); - - $this->addField('fluidbook_id', Integer::class, '', []); - $this->addField('page', Integer::class, '', []); - $this->addField('link_id', Text::class, '', []); - $this->addField('error_code', Text::class, '', []); - $this->addField('first_time_error', Text::class, '', ['default' => '']); - $this->addField('last_date_test', Text::class, '', ['default' => '']); - $this->addField('url', Text::class, '', []); - $this->addField('new_url', Text::class, '', ['default' => '']); - $this->addField('redirection_code', Text::class, '', []); - $this->addField('final_code_url', Text::class, '', ['default' => '']); - $this->addField('final_target', Text::class, '', ['default' => '']); - } - - public static function youtubeVideoExist($videoID) - { - $headers = get_headers('https://www.youtube.com/oembed?format=json&url=http://www.youtube.com/watch?v=' . $videoID); - return (is_array($headers) && preg_match('/^HTTP\\/\\d+\\.\\d+\\s+2\\d\\d\\s+.*$/', $headers[0])); - } - - public static function getHttpCode($url, $timeout = 30) - { - $flaresolverrUrl = 'http://flaresolverr:8191/v1'; - - $payload = json_encode([ - 'cmd' => 'request.get', - 'url' => $url, - 'maxTimeout' => 60000, - ]); - - $ch = curl_init($flaresolverrUrl); - - curl_setopt_array($ch, [ - CURLOPT_RETURNTRANSFER => true, - CURLOPT_HEADER => true, - CURLOPT_NOBODY => true, - CURLOPT_FOLLOWLOCATION => true, - CURLOPT_TIMEOUT => $timeout, - CURLOPT_POST => true, - CURLOPT_HTTPHEADER => ['Content-Type: application/json'], - CURLOPT_POSTFIELDS => $payload, - ]); - - curl_exec($ch); - - $httpcode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - $finalUrl = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL); - - curl_close($ch); - return ['httpcode' => $httpcode, 'finalurl' => $finalUrl]; - } - - public static function getHttpCodeComment($httpcode) - { - switch ($httpcode) { - case 301: - return " - Moved Permanently: the resource has a new permanent home — update your bookmarks or links."; - case 302: - return " - Found: tells the client to look at (browse to) another URL."; - //case 307: - //case 308: - case 400: - return " - Bad request: this and all future requests should be directed to the given URI."; - } - } } diff --git a/app/Models/FluidbookCollection.php b/app/Models/FluidbookCollection.php index 5da41def1..4317c0c7c 100644 --- a/app/Models/FluidbookCollection.php +++ b/app/Models/FluidbookCollection.php @@ -60,6 +60,7 @@ class FluidbookCollection extends ToolboxStatusModel { use CheckHash; use ToolboxDownloadable; + use AuditLinksOperation; protected $table = 'fluidbook_collection'; protected $_options = ['name' => 'fluidbook-collection', diff --git a/database/migrations/2025_11_17_184637_create_fluidbook_audit_link.php b/database/migrations/2025_11_17_184637_create_fluidbook_audit_link.php new file mode 100644 index 000000000..8480d26de --- /dev/null +++ b/database/migrations/2025_11_17_184637_create_fluidbook_audit_link.php @@ -0,0 +1,39 @@ +id(); + $table->integer('fluidbook_id'); + $table->integer('page'); + $table->string('link_id'); + $table->string('error_code')->nullable(); + $table->string('first_time_error')->nullable(); + $table->string('last_date_test')->nullable(); + $table->string('url'); + $table->string('new_url')->nullable(); + $table->string('redirection_code')->nullable(); + $table->string('final_code_url')->nullable(); + $table->string('final_target')->nullable(); + $table->unique(['link_id', 'fluidbook_id']); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('fluidbook_audit_link'); + } +}; diff --git a/routes/web.php b/routes/web.php index 68baf6ee5..73668f8aa 100644 --- a/routes/web.php +++ b/routes/web.php @@ -36,4 +36,27 @@ Route::group([ Route::any('slack/endpoint', [\App\Http\Controllers\SlackController::class, 'endpoint']); }); +Route::get('/httpcode/{error}/{redirection?}', function($error, $redirection = null){ + if($error == 200) { + return "ok"; + }elseif($error == 401){ + abort(401); + }elseif($error == 403){ + abort(403); + }elseif($error == 404){ + abort(404); + }elseif($error == 500){ + abort(500); + }elseif($error == 502){ + abort(502); + }elseif($error == 503){ + abort(503); + }elseif($error == 308 && !$redirection){ + return redirect('/httpcode/308/urlderedirection', 308); + }elseif($error == 302 && !$redirection){ + return redirect('/httpcode/302/urlderedirection'); + }elseif($error == 301 && !$redirection){ + return redirect('/httpcode/301/urlderedirection', 301); + } +}); -- 2.39.5