networks:
- fluidbook-toolbox
- typesense:
- container_name: presquot-typesense
- image: typesense/typesense:29.0
- restart: unless-stopped
- volumes:
- - ./typesense:/data
- command: '--data-dir /data --api-key=K4fae5KYZTVj6Wucp5q9 --enable-cors'
+ flaresolverr:
+ # DockerHub mirror flaresolverr/flaresolverr:latest
+ image: ghcr.io/flaresolverr/flaresolverr:latest
+ container_name: flaresolverr
environment:
- VIRTUAL_HOST: typesense.dev.prescription-quotidien.com
- LETSENCRYPT_HOST: typesense.dev.prescription-quotidien.com
+ - LOG_LEVEL=${LOG_LEVEL:-info}
+ - LOG_HTML=${LOG_HTML:-false}
+ - CAPTCHA_SOLVER=${CAPTCHA_SOLVER:-none}
+ - TZ=Europe/London
+ ports:
+ - "${PORT:-8191}:8191"
+ restart: unless-stopped
networks:
fluidbook-toolbox:
trait AuditLinksOperation
{
+ protected static $curlOpt = [];
+ protected static $timeout = 30;
+
protected function setupAuditLinksRoutes($segment, $routeName, $controller)
{
Route::match(['get','post'],$segment . '/{id}/export_excel', $controller . '@exportExcel')->name("download_audit_links");
return response()->download($tmpfile, $filename)->deleteFileAfterSend();
}
+
+ public static function getHttpCode($url) {
+ $ch = curl_init($url);
+ self::setCurlOpt([CURLOPT_FOLLOWLOCATION => false]);
+ curl_setopt_array($ch, self::$curlOpt);
+ curl_exec($ch);
+
+ $httpcode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+ curl_close($ch);
+
+ $ch = curl_init($url);
+ self::$curlOpt[CURLOPT_FOLLOWLOCATION] = true;
+ curl_setopt_array($ch, self::$curlOpt);
+ curl_exec($ch);
+
+ $finalHttpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+ $finalUrl = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
+
+ curl_close($ch);
+
+ return ['httpcode' => $httpcode, 'finalurl' => $finalUrl, 'finalHttpCode' => $finalHttpCode];
+ }
+
+ public static function getHttpCodeCloudflare($url) {
+ $apiUrl = 'http://flaresolverr:8191/v1';
+
+ $payload = json_encode([
+ 'cmd' => 'request.get',
+ 'url' => $url,
+ 'maxTimeout' => 60000
+ ]);
+
+ /*$flaresolverrOptions = [
+ CURLOPT_POST => true,
+ CURLOPT_HTTPHEADER => ['Content-Type: application/json'],
+ CURLOPT_POSTFIELDS => $payload,
+ ];*/
+
+ $ch = curl_init($apiUrl);
+ self::$curlOpt = [
+ CURLOPT_RETURNTRANSFER => true,
+ CURLOPT_HEADER => true,
+ CURLOPT_NOBODY => true,
+ CURLOPT_TIMEOUT => self::$timeout,
+ CURLOPT_POST => true,
+ CURLOPT_HTTPHEADER => ['Content-Type: application/json'],
+ CURLOPT_POSTFIELDS => $payload,
+ ];
+ curl_setopt_array($ch, self::$curlOpt);
+ $response = curl_exec($ch);
+
+ if(preg_match('/(error|code) \b(301|302|308|404|401|403|405|500|502|503)\b/', $response, $matches)) {
+ $httpcode = $matches[2] ?? '';
+ $finalUrl = '';
+ $finalHttpCode = '';
+ }
+
+ curl_close($ch);
+
+ return ['httpcode' => $httpcode, 'finalurl' => $finalUrl, 'finalHttpCode' => $finalHttpCode];
+ }
+
+ protected static function setCurlOpt($moreOptions = []) {
+ self::$curlOpt = [
+ CURLOPT_RETURNTRANSFER => true,
+ CURLOPT_HEADER => true,
+ CURLOPT_NOBODY => true,
+ CURLOPT_TIMEOUT => self::$timeout,
+ ...$moreOptions
+ ];
+ }
+
+ public static function getUrlInfo($url) {
+ $headers = get_headers($url, 1);
+ if($headers['Server'] === "cloudflare") {
+ $response = self::getHttpCodeCloudflare($url);
+ }else {
+ $response = self::getHttpCode($url);
+ }
+
+ return $response;
+ }
+
+ public static function getHttpCodeComment($httpcode)
+ {
+ switch ($httpcode) {
+ case 301:
+ return " - Moved Permanently: the resource has a new permanent home — update your bookmarks or links.";
+ case 302:
+ return " - Found: tells the client to look at (browse to) another URL.";
+ //case 307:
+ //case 308:
+ case 400:
+ return " - Bad request: this and all future requests should be directed to the given URI.";
+ }
+ }
+
+ public static function youtubeVideoExist($videoID)
+ {
+ $headers = get_headers('https://www.youtube.com/oembed?format=json&url=http://www.youtube.com/watch?v=' . $videoID);
+ return (is_array($headers) && preg_match('/^HTTP\\/\\d+\\.\\d+\\s+2\\d\\d\\s+.*$/', $headers[0]));
+ }
}
// Error code start with 4 or 5
// Redirection code start with 3
try {
- $curlResponse = FluidbookAuditLink::getHttpCode($link['url']);
+ $curlResponse = FluidbookCollection::getUrlInfo($link['url']);
}catch (\Error $e) {
echo "Error when trying to get http code: " . $e->getMessage().PHP_EOL;
continue;
$webvideo = WebVideo::parse($link['url'], true);
if($webvideo !== false) {
if($webvideo['service'] === 'youtube') {
- if (!FluidbookAuditLink::youtubeVideoExist($webvideo['id'])) {
+ if (!FluidbookCollection::youtubeVideoExist($webvideo['id'])) {
$httpCode = "404";
}
}
if (str_starts_with($httpCode, 3)) {
try {
$finalurl = $curlResponse['finalurl'];
- $finalcodeurl = FluidbookAuditLink::getHttpCode($curlResponse['finalurl'])['httpcode'];
+ $finalcodeurl = $curlResponse['finalHttpCode'];
}catch (\Error $e) {
echo "Error when trying to get final url: " . $e->getMessage().PHP_EOL;
continue;
}
}
- $comment = FluidbookAuditLink::getHttpCodeComment($httpCode);
+ $comment = FluidbookCollection::getHttpCodeComment($httpCode);
$firstTimeError = '';
if(str_starts_with($httpCode, 3) || str_starts_with($httpCode, 4) || str_starts_with($httpCode, 5)) {
$externalLinks[] = [
'id' => $link['id'],
+ 'fluidbook_id' => $link['fluidbook_id'],
+ 'page' => $link['page'],
+ 'link_id' => $link['link_id'],
+ 'url' => $link['url'],
'error_code' => str_starts_with($httpCode, 4) || str_starts_with($httpCode, 5) ? $httpCode.$comment : "",
'first_time_error' => $firstTimeError, // Datetime of the first time we saw this error
'last_date_test' => date('Y-m-d H:i:s'),
if($externalLinks) {
$keys = array_slice(array_keys($externalLinks[0]), 1);
- DB::table('fluidbook_audit_link')->upsert($externalLinks, ['id'], $keys);
+ FluidbookAuditLink::upsert($externalLinks, ['id'], $keys);
}
Log::info('Job exécuté avec succès');
'updated_at'=> date('Y-m-d H:i:s')
];
}
-
- //print_r($publication.' => OK'. PHP_EOL);
}
- dd('exit');
- FluidbookAuditLink::updateOrCreate($externalLinks);
+ FluidbookAuditLink::upsert($externalLinks, ['link_id','fluidbook_id'], ['url','updated_at','page']);
Log::info('Job exécuté avec succès');
}
use Cubist\Backpack\Magic\Fields\Text;
use App\Http\Controllers\Admin\Operations\ChangeownerOperation;
use App\Http\Controllers\Admin\Operations\ChangestatusOperation;
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Routing\Route;
-class FluidbookAuditLink extends ToolboxModel
+class FluidbookAuditLink extends Model
{
use CheckHash;
use ToolboxDownloadable;
'plural' => 'auditlinks'];
protected static $_permissionBase = 'fluidbook-collection';
-
- public function setFields()
- {
- parent::setFields();
-
- $this->addField('fluidbook_id', Integer::class, '', []);
- $this->addField('page', Integer::class, '', []);
- $this->addField('link_id', Text::class, '', []);
- $this->addField('error_code', Text::class, '', []);
- $this->addField('first_time_error', Text::class, '', ['default' => '']);
- $this->addField('last_date_test', Text::class, '', ['default' => '']);
- $this->addField('url', Text::class, '', []);
- $this->addField('new_url', Text::class, '', ['default' => '']);
- $this->addField('redirection_code', Text::class, '', []);
- $this->addField('final_code_url', Text::class, '', ['default' => '']);
- $this->addField('final_target', Text::class, '', ['default' => '']);
- }
-
- public static function youtubeVideoExist($videoID)
- {
- $headers = get_headers('https://www.youtube.com/oembed?format=json&url=http://www.youtube.com/watch?v=' . $videoID);
- return (is_array($headers) && preg_match('/^HTTP\\/\\d+\\.\\d+\\s+2\\d\\d\\s+.*$/', $headers[0]));
- }
-
- public static function getHttpCode($url, $timeout = 30)
- {
- $flaresolverrUrl = 'http://flaresolverr:8191/v1';
-
- $payload = json_encode([
- 'cmd' => 'request.get',
- 'url' => $url,
- 'maxTimeout' => 60000,
- ]);
-
- $ch = curl_init($flaresolverrUrl);
-
- curl_setopt_array($ch, [
- CURLOPT_RETURNTRANSFER => true,
- CURLOPT_HEADER => true,
- CURLOPT_NOBODY => true,
- CURLOPT_FOLLOWLOCATION => true,
- CURLOPT_TIMEOUT => $timeout,
- CURLOPT_POST => true,
- CURLOPT_HTTPHEADER => ['Content-Type: application/json'],
- CURLOPT_POSTFIELDS => $payload,
- ]);
-
- curl_exec($ch);
-
- $httpcode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
- $finalUrl = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
-
- curl_close($ch);
- return ['httpcode' => $httpcode, 'finalurl' => $finalUrl];
- }
-
- public static function getHttpCodeComment($httpcode)
- {
- switch ($httpcode) {
- case 301:
- return " - Moved Permanently: the resource has a new permanent home — update your bookmarks or links.";
- case 302:
- return " - Found: tells the client to look at (browse to) another URL.";
- //case 307:
- //case 308:
- case 400:
- return " - Bad request: this and all future requests should be directed to the given URI.";
- }
- }
}
{
use CheckHash;
use ToolboxDownloadable;
+ use AuditLinksOperation;
protected $table = 'fluidbook_collection';
protected $_options = ['name' => 'fluidbook-collection',
--- /dev/null
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+ /**
+ * Run the migrations.
+ */
+ public function up(): void
+ {
+ Schema::create('fluidbook_audit_link', function (Blueprint $table) {
+ $table->id();
+ $table->integer('fluidbook_id');
+ $table->integer('page');
+ $table->string('link_id');
+ $table->string('error_code')->nullable();
+ $table->string('first_time_error')->nullable();
+ $table->string('last_date_test')->nullable();
+ $table->string('url');
+ $table->string('new_url')->nullable();
+ $table->string('redirection_code')->nullable();
+ $table->string('final_code_url')->nullable();
+ $table->string('final_target')->nullable();
+ $table->unique(['link_id', 'fluidbook_id']);
+ $table->timestamps();
+ });
+ }
+
+ /**
+ * Reverse the migrations.
+ */
+ public function down(): void
+ {
+ Schema::dropIfExists('fluidbook_audit_link');
+ }
+};
Route::any('slack/endpoint', [\App\Http\Controllers\SlackController::class, 'endpoint']);
});
+Route::get('/httpcode/{error}/{redirection?}', function($error, $redirection = null){
+ if($error == 200) {
+ return "ok";
+ }elseif($error == 401){
+ abort(401);
+ }elseif($error == 403){
+ abort(403);
+ }elseif($error == 404){
+ abort(404);
+ }elseif($error == 500){
+ abort(500);
+ }elseif($error == 502){
+ abort(502);
+ }elseif($error == 503){
+ abort(503);
+ }elseif($error == 308 && !$redirection){
+ return redirect('/httpcode/308/urlderedirection', 308);
+ }elseif($error == 302 && !$redirection){
+ return redirect('/httpcode/302/urlderedirection');
+ }elseif($error == 301 && !$redirection){
+ return redirect('/httpcode/301/urlderedirection', 301);
+ }
+});
projects / fluidbook-toolbox.git / commitdiff