From 955232677ff7b5fd11808123fc39c54478093c0d Mon Sep 17 00:00:00 2001 From: Vincent Vanwaelscappel Date: Thu, 27 Jun 2024 15:35:36 +0200 Subject: [PATCH] wip #6964 --- .../Operations/Tools/WebflowOperation.php | 2 +- app/Jobs/WebflowPublish.php | 14 +- app/Models/ToolWebflow.php | 126 ++++++++++++++++-- app/Services/Webflow.php | 112 ++++++++++++++-- .../views/fields/webflow/texts.blade.php | 12 +- 5 files changed, 231 insertions(+), 35 deletions(-) diff --git a/app/Http/Controllers/Admin/Operations/Tools/WebflowOperation.php b/app/Http/Controllers/Admin/Operations/Tools/WebflowOperation.php index dce574f41..09a9515fa 100644 --- a/app/Http/Controllers/Admin/Operations/Tools/WebflowOperation.php +++ b/app/Http/Controllers/Admin/Operations/Tools/WebflowOperation.php @@ -30,7 +30,7 @@ trait WebflowOperation } $wf = ToolWebflow::withoutGlobalScopes()->find($id); Webflow::setToken($wf->webflow_api_token); - return response()->json(Webflow::getEditableData($wf->webflow)); + return response()->json($wf->getEditableData()); }); } diff --git a/app/Jobs/WebflowPublish.php b/app/Jobs/WebflowPublish.php index 5117b6831..19bc02f0f 100644 --- a/app/Jobs/WebflowPublish.php +++ b/app/Jobs/WebflowPublish.php @@ -30,17 +30,27 @@ class WebflowPublish extends Base start_measure('Webflow Publish ' . $this->id . ' ' . $this->mode); /** @var ToolWebflow $wf */ $wf = ToolWebflow::withoutGlobalScopes()->find($this->id); + $subject = __('Site :name publié', ['name' => $wf->name]); $notify = true; - if ($this->mode === 'webflow' && $wf->webflow_refresh_on_publish) { + if ($this->mode === 'api') { + $wf->getEditableData(); + $notify = false; + } else if ($this->mode === 'force') { $wf->refreshFormDataFromAPI(); $wf->mirror(false); + $notify = false; + } else if ($this->mode === 'mirror') { + $wf->mirror(false); + } else if ($this->mode === 'webflow' && $wf->webflow_refresh_on_publish) { + $wf->mirror(false, true); + $wf->refreshFormDataFromAPI(); $text = __('Le site vient d\'être républié suite à une mise à jour de webflow'); } else if ($this->mode === 'auto') { $text = __('Le site vient d\'être républié suite à une mise à jour des contenus'); } else { $text = __('Le site vient d\'être républié suite à une déclenchement manuel'); - $notify=false; + $notify = false; } $wf->compile(); diff --git a/app/Models/ToolWebflow.php b/app/Models/ToolWebflow.php index 281c8a769..02d814a9d 100644 --- a/app/Models/ToolWebflow.php +++ b/app/Models/ToolWebflow.php @@ -54,6 +54,7 @@ class ToolWebflow extends ToolboxTranslatableModel $this->addField('webflow_refresh_on_publish', Checkbox::class, __('Recharger tous les contenus lors de la publication sur Webflow'), ['default' => true, 'tab' => __('Paramètres')]); $this->addField('publish_on_save', Checkbox::class, __('Publier le site lors de la modification de contenus'), ['default' => true, 'tab' => __('Paramètres')]); $this->addField('domains', Textarea::class, __('Domaines à télécharger'), ['translatable' => false, 'tab' => __('Paramètres')]); + $this->addField('exclude_domains', Textarea::class, __('Domaines à exclure'), ['translatable' => false, 'tab' => __('Paramètres')]); $this->addField('locales_domains', Table::class, __('Langues'), ['translatable' => false, 'columns' => ['locale' => __('Code langue'), 'url' => __('URL')], 'tab' => __('Paramètres')]); $this->addField('slack', Text::class, __('Notification slack'), ['translatable' => false, 'tab' => __('Paramètres')]); $s = StaticSiteUploader::getSites(); @@ -67,6 +68,7 @@ class ToolWebflow extends ToolboxTranslatableModel $this->addField('texts', WebflowTexts::class, '', ['tab' => __('Textes'), 'translatable' => true, 'hint' => __('Modifier un texte ici ne produira aucun changement sur webflow')]); $this->addField('images', WebflowImages::class, '', ['tab' => __('Images'), 'translatable' => true]); $this->addField('seo', BunchOfFieldsMultiple::class, '', ['translatable' => true, 'edit_label' => '%url | %seo_title', 'allows_add' => false, 'allows_delete' => false, 'allows_clone' => false, 'allows_reorder' => false, 'bunch' => SEOPage::class, 'tab' => __('SEO')]); + $this->addField('former_sitemap', Code::class, __('Ancienne sitemap'), ['language' => 'xml', 'tab' => __('Redirections')]); $this->addField('redirections', BunchOfFieldsMultiple::class, '', ['translatable' => false, 'edit_label' => '%from → %to', 'bunch' => Redirection::class, 'tab' => __('Redirections')]); $this->addField('api', Hidden::class); } @@ -105,6 +107,27 @@ class ToolWebflow extends ToolboxTranslatableModel public function onSaving(): bool { $this->saveDataInWebflow(); + + $change = false; + $froms = []; + $r = $this->redirections; + foreach ($r as $redirection) { + $froms[] = trim($redirection['from']); + } + + $sitemap = simplexml_load_string($this->former_sitemap); + $sitemap->registerXPathNamespace('s', "http://www.sitemaps.org/schemas/sitemap/0.9"); + foreach ($sitemap->xpath('//s:loc') as $url) { + $u = parse_url($url); + $p = trim($u['path'], '/'); + if (!in_array($p, $froms)) { + $change = true; + $r[] = ['from' => $p, 'to' => '']; + } + } + if ($change) { + $this->redirections = $r; + } return parent::onSaving(); } @@ -142,7 +165,7 @@ class ToolWebflow extends ToolboxTranslatableModel $page['og_description'] = $page['seo_description']; } - if (!$this->_compareArrays($api['seo'][$page['id']], $page)) { + if (isset($api['seo'][$page['id']]) && $api['seo'][$page['id']]['type'] !== 'page_html' && !$this->_compareArrays($api['seo'][$page['id']], $page)) { $api['seo'][$page['id']] = $page; $hasChanged = true; Webflow::savePageMeta($page); @@ -150,11 +173,19 @@ class ToolWebflow extends ToolboxTranslatableModel } foreach ($images[$mainLocale] as $id => $alt) { - $apiAlt = $api['images'][$id]['alt'] ?? str_starts_with($api['images'][$id]['alt'], '__wf_') ? '' : $api['images'][$id]['alt']; + $apiAlt = $api['images'][$id]['alt']; + if (str_starts_with($apiAlt, '__wf_')) { + $apiAlt = ''; + } + if (null === $alt) { + $alt = ''; + } if ($apiAlt != $alt) { + start_measure('saveImageAlt ' . $apiAlt . '!=' . $alt); $api['images'][$id]['alt'] = $alt; Webflow::saveImageAlt($id, $alt); $hasChanged = true; + stop_measure('saveImageAlt ' . $apiAlt . '!=' . $alt); } if ($alt) { @@ -207,9 +238,10 @@ class ToolWebflow extends ToolboxTranslatableModel { start_measure("Webflow refresh data from api"); $lock = $this->getLock(); + //dd(Webflow::getAllData($this->webflow)['cms']); try { Webflow::clearCache(); - $this->api = Webflow::getEditableData($this->webflow); + $this->api = $this->getEditableData(); $mainLocale = $this->getMainLocale(); $locales = $this->getLocalesCodes(); @@ -332,10 +364,13 @@ class ToolWebflow extends ToolboxTranslatableModel 'uploads-ssl.webflow.com', 'uploads.webflow.com', 'cdn.prod.website-files.com', - 'cdn.embedly.com', ] + \Cubist\Util\Text::splitLines($this->domains)); + $exclude = array_unique([ + ] + \Cubist\Util\Text::splitLines($this->exclude_domains)); + $wget->setArg("domains", implode(',', $domains)); + $wget->setArg("exclude-domains", implode(',', $exclude)); $wget->setArg("compression", 'auto'); if (!$force) { $wget->setArg('N'); @@ -350,7 +385,7 @@ class ToolWebflow extends ToolboxTranslatableModel public function listPages() { - $seo = Webflow::getEditableData($this->webflow)['seo']; + $seo = $this->getEditableData()['seo']; $res = []; foreach ($seo as $s) { $res[$s['id']] = $s['url']; @@ -394,19 +429,24 @@ class ToolWebflow extends ToolboxTranslatableModel protected function compileLocale($locale) { $mirror = $this->getMirrorPath(); - $path = Files::mkdir(protected_path('webflow/final/' . $this->id . '/' . $locale)); + $path = Files::emptyDir(protected_path('webflow/final/' . $this->id . '/' . $locale)); $rsync = new CommandLine\Rsync($mirror, $path, true); $rsync->execute(); file_put_contents(Files::mkdir($path . '/css') . 'custom.css', $this->getCustomCSS()); file_put_contents(Files::mkdir($path . '/js') . 'custom.js', $this->getCustomJS()); + $isMainLocale = $this->getMainLocale() === $locale; + $texts = $this->getTextTranslationsForCompilation($locale); + $images = $this->getTranslation('images', $locale); + $seo = $this->getTranslation('seo', $locale); + foreach (Files::getRecursiveDirectoryIterator($path) as $f) { /** @var $f \SplFileInfo */ if ($f->isDir() || $f->getExtension() !== 'html') { continue; } - $this->compileHTMLFile($f, $locale); + $this->compileHTMLFile($f, $isMainLocale, $locale, $texts, $images, $seo); } } @@ -415,25 +455,55 @@ class ToolWebflow extends ToolboxTranslatableModel * @param $locale string * @return void */ - protected function compileHTMLFile($f, $locale) + protected function compileHTMLFile($f, $isMainLocale, $locale, $texts, $images, $seo) { + $html = file_get_contents($f->getPathname()); + $regex = '/("https:\/\/' . $this->webflow . '.webflow.io\/\\\\")(.*)(\\\\"")/'; + $html = preg_replace($regex, '\"$2\"', $html); + + if (!preg_match('/data-wf-page="([^\"]+)"/', $html, $m)) { + return; + } + $pageId = $m[1]; + + $html = str_replace('', '' . "\n" . '', $html); $html = str_replace('', '' . "\n" . '', $html); $html = preg_replace('/lang=\"[a-zA-Z\-_]{2,6}\"/', 'lang="' . $locale . '"', $html); + // Texts - $texts = $this->getTextTranslationsForCompilation($locale); foreach ($texts as $text => $translation) { - $html = str_replace($text, $translation, $html); + $html = str_replace('>' . $text . '<', '>' . $translation . '<', $html); } - // Images - $images = $this->getTranslation('images', $locale); + // SEO - $html = preg_replace('/[^<]*<\/title>/', '<title>Nimp', $html); + if (!$isMainLocale) { + foreach ($seo as $s) { + if ($s['id'] === $pageId) { + $currentPage = $s; + break; + } + } + + if (isset($currentPage)) { + $og_title = e($currentPage['og_title_copied'] ? $currentPage['seo_title'] : $currentPage['og_title']); + $og_desc = e($currentPage['og_description_copied'] ? $currentPage['seo_description'] : $currentPage['og_description']); + + $html = preg_replace('/\s*\s*/', '', $html); + $meta = ' + + + + + '; + $html = preg_replace('/[^<]*<\/title>/', '<title>' . e($currentPage['seo_title']) . '' . $meta, $html); + } + } file_put_contents($f->getPathname(), $html); } @@ -485,19 +555,47 @@ class ToolWebflow extends ToolboxTranslatableModel Cache::put('webflow_' . $this->id . '_locales', $locales); + } catch (LockTimeoutException $e) { } finally { $lock?->forceRelease(); } + $seoTranslations = $this->getTranslations('seo'); + foreach ($seoTranslations as $locale => $seo) { + $translation = []; + foreach ($seo as $item) { + if (!is_array($item) || !isset($item['id']) || !$item['id']) { + continue; + } + if (!isset($pages[$item['id']])) { + continue; + } + $translation[] = $item; + } + + + uasort($translation, function ($a, $b) { + if ($a['url'] === '/index.html') { + return -1000; + } + if ($b['url'] === '/index.html') { + return 1000; + } + return strcmp($a['url'], $b['url']); + }); + + $this->setTranslation('seo', $locale, $translation); + } + return parent::onRetrieved(); } public function getEditableData() { Webflow::setToken($this->webflow_api_token); - return Webflow::getEditableData($this->webflow); + return Webflow::getEditableData($this->webflow, $this->getMirrorPath()); } public function getAvailableLocales() diff --git a/app/Services/Webflow.php b/app/Services/Webflow.php index 66d0f5b99..90176f4da 100644 --- a/app/Services/Webflow.php +++ b/app/Services/Webflow.php @@ -3,6 +3,7 @@ namespace App\Services; use App\Services\Webflow\Excel; +use Cubist\Util\Files\Files; use Cubist\Util\Html; use Illuminate\Http\Client\Response; use Illuminate\Support\Facades\Cache; @@ -79,7 +80,7 @@ class Webflow public static function request($url, $data = [], $method = 'get', $ttl = 86400, $force = false) { - start_measure('Webflow API : ' . $url); + start_measure('Webflow API (' . $method . '): ' . $url); $cacheKey = self::getCacheKey($url, $data, $method); if ($force) { Cache::forget($cacheKey); @@ -90,16 +91,20 @@ class Webflow } /** @var Response $response */ $response = Http::withToken(self::getToken())->$method(self::BASE_URL . $url, $data); - if ((int)$response->header('X-RateLimit-Remaining') <= 1) { + $xRateLimitRemaining = $response->header('X-RateLimit-Remaining') === '' ? 10 : (int)$response->header('X-RateLimit-Remaining'); + + if ($xRateLimitRemaining <= 1) { + start_measure('wait for API : ' . $response->header('X-RateLimit-Remaining')); Cache::set('webflow_' . static::getToken() . '_wait', true, 60); sleep(60); + stop_measure('wait for API : ' . $response->header('X-RateLimit-Remaining')); } return $response->json(); }); if (null === $res && !$force) { return self::request($url, $data, $method, $ttl, true); } - stop_measure('Webflow API : ' . $url); + stop_measure('Webflow API (' . $method . '): ' . $url); return $res; } @@ -163,6 +168,8 @@ class Webflow ]; } + //file_put_contents(Files::mkdir(protected_path('webflow/' . $shortname . '/')) . 'cms.json', json_encode($res)); + return $res; } @@ -217,7 +224,7 @@ class Webflow public static function listCMSCollections($shortname) { - return self::paginatedRequest('sites/' . self::getSiteId($shortname) . '/collections', 'collections'); + return self::request('sites/' . self::getSiteId($shortname) . '/collections')['collections']; } public static function getPageContents($pageID) @@ -246,7 +253,7 @@ class Webflow return self::request('assets/' . $id, ['altText' => $alt], 'post', 0, true); } - public static function getEditableData($shortname) + public static function getEditableData($shortname, $mirrorPath) { start_measure('Webflow : get editable data'); @@ -273,7 +280,11 @@ class Webflow $type = isset($details['publishedPath']) ? 'folder' : 'page'; if ($type === 'page') { $url .= '.html'; + if (!file_exists($mirrorPath . $url)) { + continue; + } } + $url = '/' . $url; @@ -322,15 +333,45 @@ class Webflow } } - uasort($res['seo'], function ($a, $b) { - if ($a['url'] === '/index.html') { - return -1000; + $ignoreNames = ['slug', 'json']; + $allowTypes = ['RichText', 'PlainText']; + + foreach ($data['cms'] as $collection) { + $fields = []; + + foreach ($collection['details']['fields'] as $field) { + if (!in_array($field['slug'], $ignoreNames) && in_array($field['type'], $allowTypes)) { + $fields[$field['slug']] = $field['type']; + } } - if ($b['url'] === '/index.html') { - return 1000; + + foreach ($collection['contents'] as $item) { + foreach ($fields as $f => $type) { + if (!isset($item['fieldData'][$f])) { + continue; + } + $t = $item['fieldData'][$f]; + if (!$t) { + continue; + } + if ($type === 'PlainText') { + $texts = [$t]; + } else if ($type === 'RichText') { + $texts = Html::getTextNodes($t); + } + foreach ($texts as $t) { + if (!isset($res['texts'][$t])) { + $res['texts'][$t] = ['key' => base64_encode($t), 'occurences' => 0]; + } + $res['texts'][$t]['occurences']++; + } + } } - return strcmp($a['url'], $b['url']); - }); + } + + $res['seo'] = static::getPagesFromHTMLFiles($res['seo'], $mirrorPath); + + self::$_editableData[$key] = $res; @@ -339,4 +380,51 @@ class Webflow stop_measure('Webflow : get editable data'); return self::$_editableData[$key]; } + + public static function getPagesFromHTMLFiles($seo, $mirrorPath) + { + foreach (Files::getRecursiveDirectoryIterator($mirrorPath) as $f) { + /** @var $f \SplFileInfo */ + if ($f->isDir() || $f->getExtension() !== 'html') { + continue; + } + $relative = '/' . ltrim(str_replace($mirrorPath, '', $f->getPathname()), "/"); + + $found = false; + foreach ($seo as $s) { + if ($s['url'] === $relative) { + $found = true; + break; + } + } + if ($found) { + continue; + } + + $slug = $f->getBasename('.html'); + $html = file_get_contents($f->getPathname()); + $title = ''; + $description = ''; + if (preg_match('/([^>]*)<\/title>/', $html, $matches)) { + $title = $matches[1]; + } + + + $seo[$relative] = [ + 'id' => $relative, + 'url' => $relative, + 'type' => 'page_html', + 'slug' => $slug, + 'draft' => false, + 'seo_title' => $title, + 'seo_description' => $description, + 'og_title' => $title, + 'og_description' => $description, + 'og_title_copied' => true, + 'og_description_copied' => true, + ];; + } + + return $seo; + } } diff --git a/resources/views/fields/webflow/texts.blade.php b/resources/views/fields/webflow/texts.blade.php index a2aa08932..5f096aabc 100644 --- a/resources/views/fields/webflow/texts.blade.php +++ b/resources/views/fields/webflow/texts.blade.php @@ -2,12 +2,12 @@ $value=old(square_brackets_to_dots($field['name'])) ?? $field['value'] ?? $field['default'] ?? '[]'; $translations=\Cubist\Util\Json::decode($value,\Cubist\Util\Json::TYPE_ARRAY)??[]; $texts=\Cubist\Util\Json::decode($crud->entry->api,\Cubist\Util\Json::TYPE_ARRAY)['texts']; - foreach ($translations as $key=>$trans) { - $t=base64_decode($key); - if(!isset($texts[$t])){ - $texts[$t]=['occurences'=>0,'key'=>$key]; - } - } +// foreach ($translations as $key=>$trans) { +// $t=base64_decode($key); +// if(!isset($texts[$t])){ +// $texts[$t]=['occurences'=>0,'key'=>$key]; +// } +// } @endphp @include('crud::fields.inc.wrapper_start') -- 2.39.5