From 795cbeccc358095e31dc98b9efc03472afa0a0a3 Mon Sep 17 00:00:00 2001 From: "ext.jeremy.guillot@maxicoffee.domains" Date: Mon, 16 Mar 2026 00:08:50 +0100 Subject: [PATCH 1/3] =?UTF-8?q?feat(setting):=20=C3=A9tendre=20ContentSour?= =?UTF-8?q?ce=20avec=20champs=20de=20test=20et=20domain=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Ajouter testSlug, testChapterNumber, baseUrl sur ContentSource (entité, domain model, migration) - Exposer ces champs dans les Resources, Processors, Providers et Mapper - Mettre à jour store Pinia, repository API et composants Vue (form, card, liste) --- .../application/store/contentSourceStore.js | 37 ++++++- .../domain/model/ScraperHealthStatus.js | 6 ++ .../api/apiContentSourceRepository.js | 11 ++ .../components/ContentSourceCard.vue | 31 ++++++ .../components/ContentSourceForm.vue | 48 +++++---- .../pages/ScrapperConfigurations.vue | 42 +++++++- config/services.yaml | 7 ++ migrations/Version20260315221706.php | 41 +++++++ .../Command/UpsertContentSourceCommand.php | 2 + .../UpsertContentSourceCommandHandler.php | 4 + .../Response/ContentSourceResponse.php | 10 ++ .../Setting/Domain/Model/ContentSource.php | 51 +++++++++ .../Resource/GetContentSourceResource.php | 5 + .../Resource/UpsertContentSourceResource.php | 2 + .../UpsertContentSourceStateProcessor.php | 2 + .../GetContentSourceStateProvider.php | 5 + .../ListContentSourceStateProvider.php | 5 + .../Mapper/ContentSourceMapper.php | 19 +++- ...eContentSourceForHealthCheckRepository.php | 102 ++++++++++++++++++ src/Entity/ContentSource.php | 75 +++++++++++++ 20 files changed, 475 insertions(+), 30 deletions(-) create mode 100644 assets/vue/app/domain/setting/domain/model/ScraperHealthStatus.js create mode 100644 migrations/Version20260315221706.php create mode 100644 src/Domain/Setting/Infrastructure/Persistence/Repository/DoctrineContentSourceForHealthCheckRepository.php diff --git a/assets/vue/app/domain/setting/application/store/contentSourceStore.js b/assets/vue/app/domain/setting/application/store/contentSourceStore.js index 4cdabdd..17b710d 100644 --- a/assets/vue/app/domain/setting/application/store/contentSourceStore.js +++ b/assets/vue/app/domain/setting/application/store/contentSourceStore.js @@ -23,7 +23,11 @@ export const useContentSourceStore = defineStore('contentSource', { importing: false, exporting: false, importError: null, - exportError: null + exportError: null, + + // Health check state + checkingHealth: false, + checkHealthError: null, }), getters: { @@ -174,6 +178,36 @@ export const useContentSourceStore = defineStore('contentSource', { this.currentSourceError = null; }, + // Check all scrapers health + async checkAllHealth() { + if (this.checkingHealth) return; + + this.checkingHealth = true; + this.checkHealthError = null; + + try { + await contentSourceRepository.checkAllHealth(); + } catch (error) { + this.checkHealthError = error.message; + console.error('Erreur lors du health check:', error); + throw error; + } finally { + this.checkingHealth = false; + } + }, + + // Update health status of a single source (called from Mercure) + updateSourceHealth(sourceId, status, error = null) { + const index = this.sources.findIndex(s => s.id === sourceId); + if (index !== -1) { + this.sources[index] = { + ...this.sources[index], + healthStatus: status, + healthLastError: error, + }; + } + }, + // Clear errors clearErrors() { this.sourcesError = null; @@ -181,6 +215,7 @@ export const useContentSourceStore = defineStore('contentSource', { this.saveError = null; this.importError = null; this.exportError = null; + this.checkHealthError = null; } } }); diff --git a/assets/vue/app/domain/setting/domain/model/ScraperHealthStatus.js b/assets/vue/app/domain/setting/domain/model/ScraperHealthStatus.js new file mode 100644 index 0000000..7c9183e --- /dev/null +++ b/assets/vue/app/domain/setting/domain/model/ScraperHealthStatus.js @@ -0,0 +1,6 @@ +export const ScraperHealthStatus = { + UNKNOWN: 'unknown', + OK: 'ok', + KO: 'ko', + TESTING: 'testing', +}; diff --git a/assets/vue/app/domain/setting/infrastructure/api/apiContentSourceRepository.js b/assets/vue/app/domain/setting/infrastructure/api/apiContentSourceRepository.js index f068008..0e2c78a 100644 --- a/assets/vue/app/domain/setting/infrastructure/api/apiContentSourceRepository.js +++ b/assets/vue/app/domain/setting/infrastructure/api/apiContentSourceRepository.js @@ -82,6 +82,17 @@ export class ApiContentSourceRepository { } } + /** + * Déclenche le test de santé de tous les scrapers + */ + async checkAllHealth() { + try { + await this.apiClient.post('/scraping/check-all-health', {}); + } catch (error) { + throw new Error(error.response?.data?.message || 'Erreur lors du lancement du health check'); + } + } + /** * Teste une configuration de scraper */ diff --git a/assets/vue/app/domain/setting/presentation/components/ContentSourceCard.vue b/assets/vue/app/domain/setting/presentation/components/ContentSourceCard.vue index b8a2140..8713ae6 100644 --- a/assets/vue/app/domain/setting/presentation/components/ContentSourceCard.vue +++ b/assets/vue/app/domain/setting/presentation/components/ContentSourceCard.vue @@ -30,6 +30,14 @@ class="px-2 py-1 text-xs font-medium"> {{ getOrientation(source) }} + + + + {{ getHealthLabel(source.healthStatus) }} + @@ -39,6 +47,7 @@ diff --git a/assets/vue/app/domain/setting/presentation/components/ContentSourceForm.vue b/assets/vue/app/domain/setting/presentation/components/ContentSourceForm.vue index 2017a1a..0761ad0 100644 --- a/assets/vue/app/domain/setting/presentation/components/ContentSourceForm.vue +++ b/assets/vue/app/domain/setting/presentation/components/ContentSourceForm.vue @@ -108,17 +108,17 @@
-

Test de la configuration

+

Configuration de test (health check)

-
@@ -191,12 +191,9 @@ const form = ref({ nextPageSelector: '', chapterSelector: '', scrapingType: 'html', - token: '' -}); - -const testData = ref({ - mangaSlug: '', - chapterNumber: '' + token: '', + testSlug: '', + testChapterNumber: '', }); const testing = ref(false); @@ -204,17 +201,17 @@ const testing = ref(false); const canTest = computed(() => { return form.value.baseUrl && form.value.chapterUrlFormat && - testData.value.mangaSlug && - testData.value.chapterNumber; + form.value.testSlug && + form.value.testChapterNumber; }); const generatedTestUrl = computed(() => { - if (!form.value.chapterUrlFormat || !testData.value.mangaSlug || !testData.value.chapterNumber) { + if (!form.value.chapterUrlFormat || !form.value.testSlug || !form.value.testChapterNumber) { return ''; } return form.value.chapterUrlFormat - .replace('{slug}', testData.value.mangaSlug) - .replace('{chapterNumber}', testData.value.chapterNumber); + .replace('{slug}', form.value.testSlug) + .replace('{chapterNumber}', form.value.testChapterNumber); }); watch(() => props.source, (newSource) => { @@ -226,7 +223,9 @@ watch(() => props.source, (newSource) => { nextPageSelector: newSource.nextPageSelector || '', chapterSelector: newSource.chapterSelector || '', scrapingType: (newSource.scrapingType || 'html').toLowerCase(), - token: newSource.token || '' + token: newSource.token || '', + testSlug: newSource.testSlug || '', + testChapterNumber: newSource.testChapterNumber ?? '', }; } else { form.value = { @@ -236,7 +235,9 @@ watch(() => props.source, (newSource) => { nextPageSelector: '', chapterSelector: '', scrapingType: 'html', - token: '' + token: '', + testSlug: '', + testChapterNumber: '', }; } }, { immediate: true }); @@ -253,8 +254,9 @@ const testConfiguration = async () => { await emit('test', { configuration: { ...form.value }, testData: { - ...testData.value, - testUrl: generatedTestUrl.value + mangaSlug: form.value.testSlug, + chapterNumber: form.value.testChapterNumber, + testUrl: generatedTestUrl.value, } }); } finally { diff --git a/assets/vue/app/domain/setting/presentation/pages/ScrapperConfigurations.vue b/assets/vue/app/domain/setting/presentation/pages/ScrapperConfigurations.vue index f482d89..4d991e3 100644 --- a/assets/vue/app/domain/setting/presentation/pages/ScrapperConfigurations.vue +++ b/assets/vue/app/domain/setting/presentation/pages/ScrapperConfigurations.vue @@ -91,10 +91,11 @@ import { ArrowPathIcon, ArrowUpTrayIcon, ExclamationTriangleIcon, + HeartIcon, PlusIcon } from '@heroicons/vue/24/outline'; import { storeToRefs } from 'pinia'; -import { computed, onMounted, ref } from 'vue'; +import { computed, onMounted, onUnmounted, ref } from 'vue'; import { useRouter } from 'vue-router'; import Toolbar from '../../../../shared/components/ui/Toolbar.vue'; import { useContentSourceStore } from '../../application/store/contentSourceStore'; @@ -108,9 +109,13 @@ const { loadingSources, sourcesError, importing, - exporting + exporting, + checkingHealth, } = storeToRefs(contentSourceStore); +// Mercure — écoute des mises à jour health +let mercureEventSource = null; + // Local state const showImportModal = ref(false); const showExportSuccess = ref(false); @@ -120,12 +125,32 @@ const importData = ref(''); // Load sources on mount and clear current source onMounted(async () => { try { - contentSourceStore.clearCurrentSource(); // Clear any previously loaded source - contentSourceStore.clearErrors(); // Clear any previous errors + contentSourceStore.clearCurrentSource(); + contentSourceStore.clearErrors(); await contentSourceStore.loadSources(); } catch (error) { console.error('Erreur lors du chargement des sources:', error); } + + // Écoute Mercure pour les mises à jour de health status + const url = new URL('/.well-known/mercure', window.location.href); + sources.value.forEach(source => { + url.searchParams.append('topic', `scrapers/health/${source.id}`); + }); + + mercureEventSource = new EventSource(url.toString()); + mercureEventSource.onmessage = (event) => { + try { + const data = JSON.parse(event.data); + contentSourceStore.updateSourceHealth(data.sourceId, data.status, data.error); + } catch (e) { + console.error('Erreur parsing Mercure event:', e); + } + }; +}); + +onUnmounted(() => { + mercureEventSource?.close(); }); // Toolbar configuration @@ -135,6 +160,7 @@ const toolbarConfig = computed(() => ({ ], rightSection: [ { type: 'button', icon: ArrowPathIcon, label: 'Actualiser', onClick: () => contentSourceStore.loadSources(), disabled: loadingSources.value }, + { type: 'button', icon: HeartIcon, label: 'Tester tous', onClick: handleCheckAllHealth, disabled: checkingHealth.value }, { type: 'button', icon: ArrowDownTrayIcon, label: 'Exporter', onClick: handleExport, disabled: exporting.value }, { type: 'button', icon: ArrowUpTrayIcon, label: 'Importer', onClick: () => showImportModal.value = true }, ], @@ -156,6 +182,14 @@ const openSourceLink = (url) => { window.open(url, '_blank'); }; +async function handleCheckAllHealth() { + try { + await contentSourceStore.checkAllHealth(); + } catch (error) { + console.error('Erreur lors du health check:', error); + } +} + async function handleExport() { try { const exportData = await contentSourceStore.exportSources(); diff --git a/config/services.yaml b/config/services.yaml index d3d6b41..0b10b2b 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -180,6 +180,13 @@ services: tags: - { name: messenger.message_handler, bus: command.bus } + # Scraper Health Check + App\Domain\Scraping\Domain\Contract\Repository\ContentSourceForHealthCheckInterface: + alias: App\Domain\Setting\Infrastructure\Persistence\Repository\DoctrineContentSourceForHealthCheckRepository + + App\Domain\Scraping\Domain\Contract\Repository\ContentSourceHealthRepositoryInterface: + alias: App\Domain\Setting\Infrastructure\Persistence\Repository\DoctrineContentSourceForHealthCheckRepository + # Import Domain Services App\Domain\Import\Infrastructure\Service\FilenameAnalyzer: ~ diff --git a/migrations/Version20260315221706.php b/migrations/Version20260315221706.php new file mode 100644 index 0000000..65c4f69 --- /dev/null +++ b/migrations/Version20260315221706.php @@ -0,0 +1,41 @@ +addSql('ALTER TABLE content_source ADD test_slug VARCHAR(255) DEFAULT NULL'); + $this->addSql('ALTER TABLE content_source ADD test_chapter_number DOUBLE PRECISION DEFAULT NULL'); + $this->addSql('ALTER TABLE content_source ADD health_status VARCHAR(20) DEFAULT \'unknown\' NOT NULL'); + $this->addSql('ALTER TABLE content_source ADD health_last_tested_at TIMESTAMP(0) WITHOUT TIME ZONE DEFAULT NULL'); + $this->addSql('ALTER TABLE content_source ADD health_last_error TEXT DEFAULT NULL'); + $this->addSql('COMMENT ON COLUMN content_source.health_last_tested_at IS \'(DC2Type:datetime_immutable)\''); + } + + public function down(Schema $schema): void + { + // this down() migration is auto-generated, please modify it to your needs + $this->addSql('CREATE SCHEMA public'); + $this->addSql('ALTER TABLE content_source DROP test_slug'); + $this->addSql('ALTER TABLE content_source DROP test_chapter_number'); + $this->addSql('ALTER TABLE content_source DROP health_status'); + $this->addSql('ALTER TABLE content_source DROP health_last_tested_at'); + $this->addSql('ALTER TABLE content_source DROP health_last_error'); + } +} diff --git a/src/Domain/Setting/Application/Command/UpsertContentSourceCommand.php b/src/Domain/Setting/Application/Command/UpsertContentSourceCommand.php index 3949457..bffca24 100644 --- a/src/Domain/Setting/Application/Command/UpsertContentSourceCommand.php +++ b/src/Domain/Setting/Application/Command/UpsertContentSourceCommand.php @@ -12,6 +12,8 @@ readonly class UpsertContentSourceCommand public ?string $imageSelector = null, public ?string $nextPageSelector = null, public ?string $chapterSelector = null, + public ?string $testSlug = null, + public ?float $testChapterNumber = null, ) { } } diff --git a/src/Domain/Setting/Application/CommandHandler/UpsertContentSourceCommandHandler.php b/src/Domain/Setting/Application/CommandHandler/UpsertContentSourceCommandHandler.php index 4fcfb36..cb9cac5 100644 --- a/src/Domain/Setting/Application/CommandHandler/UpsertContentSourceCommandHandler.php +++ b/src/Domain/Setting/Application/CommandHandler/UpsertContentSourceCommandHandler.php @@ -26,6 +26,8 @@ readonly class UpsertContentSourceCommandHandler imageSelector: $command->imageSelector, nextPageSelector: $command->nextPageSelector, chapterSelector: $command->chapterSelector, + testSlug: $command->testSlug, + testChapterNumber: $command->testChapterNumber, ); $this->contentSourceRepository->save($contentSource); } @@ -38,6 +40,8 @@ readonly class UpsertContentSourceCommandHandler imageSelector: $command->imageSelector, nextPageSelector: $command->nextPageSelector, chapterSelector: $command->chapterSelector, + testSlug: $command->testSlug, + testChapterNumber: $command->testChapterNumber, ); $this->contentSourceRepository->save($contentSource); } diff --git a/src/Domain/Setting/Application/Response/ContentSourceResponse.php b/src/Domain/Setting/Application/Response/ContentSourceResponse.php index b9841cd..fdc932e 100644 --- a/src/Domain/Setting/Application/Response/ContentSourceResponse.php +++ b/src/Domain/Setting/Application/Response/ContentSourceResponse.php @@ -15,6 +15,11 @@ readonly class ContentSourceResponse public ?string $nextPageSelector, public ?string $chapterSelector, public string $cleanBaseUrl, + public ?string $testSlug = null, + public ?float $testChapterNumber = null, + public string $healthStatus = 'unknown', + public ?\DateTimeImmutable $healthLastTestedAt = null, + public ?string $healthLastError = null, ) { } @@ -29,6 +34,11 @@ readonly class ContentSourceResponse nextPageSelector: $contentSource->getNextPageSelector(), chapterSelector: $contentSource->getChapterSelector(), cleanBaseUrl: $contentSource->getCleanBaseUrl(), + testSlug: $contentSource->getTestSlug(), + testChapterNumber: $contentSource->getTestChapterNumber(), + healthStatus: $contentSource->getHealthStatus(), + healthLastTestedAt: $contentSource->getHealthLastTestedAt(), + healthLastError: $contentSource->getHealthLastError(), ); } } diff --git a/src/Domain/Setting/Domain/Model/ContentSource.php b/src/Domain/Setting/Domain/Model/ContentSource.php index 2dbe928..c860d2b 100644 --- a/src/Domain/Setting/Domain/Model/ContentSource.php +++ b/src/Domain/Setting/Domain/Model/ContentSource.php @@ -12,6 +12,11 @@ final class ContentSource private ?string $imageSelector = null, private ?string $nextPageSelector = null, private ?string $chapterSelector = null, + private ?string $testSlug = null, + private ?float $testChapterNumber = null, + private string $healthStatus = 'unknown', + private ?\DateTimeImmutable $healthLastTestedAt = null, + private ?string $healthLastError = null, ) { } @@ -50,6 +55,44 @@ final class ContentSource return $this->chapterSelector; } + public function getTestSlug(): ?string + { + return $this->testSlug; + } + + public function getTestChapterNumber(): ?float + { + return $this->testChapterNumber; + } + + public function getHealthStatus(): string + { + return $this->healthStatus; + } + + public function getHealthLastTestedAt(): ?\DateTimeImmutable + { + return $this->healthLastTestedAt; + } + + public function getHealthLastError(): ?string + { + return $this->healthLastError; + } + + public function updateTestConfig(?string $testSlug, ?float $testChapterNumber): void + { + $this->testSlug = $testSlug; + $this->testChapterNumber = $testChapterNumber; + } + + public function updateHealthStatus(string $status, ?\DateTimeImmutable $testedAt = null, ?string $error = null): void + { + $this->healthStatus = $status; + $this->healthLastTestedAt = $testedAt; + $this->healthLastError = $error; + } + public function updateId(int $id): void { $this->id = $id; @@ -71,6 +114,8 @@ final class ContentSource ?string $imageSelector = null, ?string $nextPageSelector = null, ?string $chapterSelector = null, + ?string $testSlug = null, + ?float $testChapterNumber = null, ): self { return new self( id: null, @@ -80,6 +125,8 @@ final class ContentSource imageSelector: $imageSelector, nextPageSelector: $nextPageSelector, chapterSelector: $chapterSelector, + testSlug: $testSlug, + testChapterNumber: $testChapterNumber, ); } @@ -90,6 +137,8 @@ final class ContentSource ?string $imageSelector = null, ?string $nextPageSelector = null, ?string $chapterSelector = null, + ?string $testSlug = null, + ?float $testChapterNumber = null, ): void { $this->baseUrl = $baseUrl; $this->chapterUrlFormat = $chapterUrlFormat; @@ -97,5 +146,7 @@ final class ContentSource $this->imageSelector = $imageSelector; $this->nextPageSelector = $nextPageSelector; $this->chapterSelector = $chapterSelector; + $this->testSlug = $testSlug; + $this->testChapterNumber = $testChapterNumber; } } diff --git a/src/Domain/Setting/Infrastructure/ApiPlatform/Resource/GetContentSourceResource.php b/src/Domain/Setting/Infrastructure/ApiPlatform/Resource/GetContentSourceResource.php index 1fa0ffc..639a557 100644 --- a/src/Domain/Setting/Infrastructure/ApiPlatform/Resource/GetContentSourceResource.php +++ b/src/Domain/Setting/Infrastructure/ApiPlatform/Resource/GetContentSourceResource.php @@ -30,6 +30,11 @@ class GetContentSourceResource public readonly ?string $nextPageSelector, public readonly ?string $chapterSelector, public readonly string $cleanBaseUrl, + public readonly ?string $testSlug = null, + public readonly ?float $testChapterNumber = null, + public readonly string $healthStatus = 'unknown', + public readonly ?\DateTimeImmutable $healthLastTestedAt = null, + public readonly ?string $healthLastError = null, ) { } } diff --git a/src/Domain/Setting/Infrastructure/ApiPlatform/Resource/UpsertContentSourceResource.php b/src/Domain/Setting/Infrastructure/ApiPlatform/Resource/UpsertContentSourceResource.php index cf86496..a147f59 100644 --- a/src/Domain/Setting/Infrastructure/ApiPlatform/Resource/UpsertContentSourceResource.php +++ b/src/Domain/Setting/Infrastructure/ApiPlatform/Resource/UpsertContentSourceResource.php @@ -43,6 +43,8 @@ class UpsertContentSourceResource public readonly ?string $imageSelector = null, public readonly ?string $nextPageSelector = null, public readonly ?string $chapterSelector = null, + public readonly ?string $testSlug = null, + public readonly ?float $testChapterNumber = null, ) { } } diff --git a/src/Domain/Setting/Infrastructure/ApiPlatform/State/Processor/UpsertContentSourceStateProcessor.php b/src/Domain/Setting/Infrastructure/ApiPlatform/State/Processor/UpsertContentSourceStateProcessor.php index ef685a6..51b62d7 100644 --- a/src/Domain/Setting/Infrastructure/ApiPlatform/State/Processor/UpsertContentSourceStateProcessor.php +++ b/src/Domain/Setting/Infrastructure/ApiPlatform/State/Processor/UpsertContentSourceStateProcessor.php @@ -30,6 +30,8 @@ readonly class UpsertContentSourceStateProcessor implements ProcessorInterface imageSelector: $data->imageSelector, nextPageSelector: $data->nextPageSelector, chapterSelector: $data->chapterSelector, + testSlug: $data->testSlug, + testChapterNumber: $data->testChapterNumber, ); $this->handler->handle($command); diff --git a/src/Domain/Setting/Infrastructure/ApiPlatform/State/Provider/GetContentSourceStateProvider.php b/src/Domain/Setting/Infrastructure/ApiPlatform/State/Provider/GetContentSourceStateProvider.php index 7bb9ae5..43d3503 100644 --- a/src/Domain/Setting/Infrastructure/ApiPlatform/State/Provider/GetContentSourceStateProvider.php +++ b/src/Domain/Setting/Infrastructure/ApiPlatform/State/Provider/GetContentSourceStateProvider.php @@ -32,6 +32,11 @@ readonly class GetContentSourceStateProvider implements ProviderInterface nextPageSelector: $response->nextPageSelector, chapterSelector: $response->chapterSelector, cleanBaseUrl: $response->cleanBaseUrl, + testSlug: $response->testSlug, + testChapterNumber: $response->testChapterNumber, + healthStatus: $response->healthStatus, + healthLastTestedAt: $response->healthLastTestedAt, + healthLastError: $response->healthLastError, ); } catch (ContentSourceNotFoundException $e) { throw new NotFoundHttpException($e->getMessage()); diff --git a/src/Domain/Setting/Infrastructure/ApiPlatform/State/Provider/ListContentSourceStateProvider.php b/src/Domain/Setting/Infrastructure/ApiPlatform/State/Provider/ListContentSourceStateProvider.php index f28f4f2..a95253a 100644 --- a/src/Domain/Setting/Infrastructure/ApiPlatform/State/Provider/ListContentSourceStateProvider.php +++ b/src/Domain/Setting/Infrastructure/ApiPlatform/State/Provider/ListContentSourceStateProvider.php @@ -30,6 +30,11 @@ readonly class ListContentSourceStateProvider implements ProviderInterface nextPageSelector: $contentSourceResponse->nextPageSelector, chapterSelector: $contentSourceResponse->chapterSelector, cleanBaseUrl: $contentSourceResponse->cleanBaseUrl, + testSlug: $contentSourceResponse->testSlug, + testChapterNumber: $contentSourceResponse->testChapterNumber, + healthStatus: $contentSourceResponse->healthStatus, + healthLastTestedAt: $contentSourceResponse->healthLastTestedAt, + healthLastError: $contentSourceResponse->healthLastError, ), $response->contentSources ); diff --git a/src/Domain/Setting/Infrastructure/Persistence/Mapper/ContentSourceMapper.php b/src/Domain/Setting/Infrastructure/Persistence/Mapper/ContentSourceMapper.php index 93d59ca..4dfa6e0 100644 --- a/src/Domain/Setting/Infrastructure/Persistence/Mapper/ContentSourceMapper.php +++ b/src/Domain/Setting/Infrastructure/Persistence/Mapper/ContentSourceMapper.php @@ -17,6 +17,11 @@ readonly class ContentSourceMapper imageSelector: $entity->getImageSelector(), nextPageSelector: $entity->getNextPageSelector(), chapterSelector: $entity->getChapterSelector(), + testSlug: $entity->getTestSlug(), + testChapterNumber: $entity->getTestChapterNumber(), + healthStatus: $entity->getHealthStatus(), + healthLastTestedAt: $entity->getHealthLastTestedAt(), + healthLastError: $entity->getHealthLastError(), ); } @@ -29,7 +34,12 @@ readonly class ContentSourceMapper ->setScrapingType($contentSource->getScrapingType()) ->setImageSelector($contentSource->getImageSelector()) ->setNextPageSelector($contentSource->getNextPageSelector()) - ->setChapterSelector($contentSource->getChapterSelector()); + ->setChapterSelector($contentSource->getChapterSelector()) + ->setTestSlug($contentSource->getTestSlug()) + ->setTestChapterNumber($contentSource->getTestChapterNumber()) + ->setHealthStatus($contentSource->getHealthStatus()) + ->setHealthLastTestedAt($contentSource->getHealthLastTestedAt()) + ->setHealthLastError($contentSource->getHealthLastError()); return $entity; } @@ -41,7 +51,12 @@ readonly class ContentSourceMapper ->setScrapingType($contentSource->getScrapingType()) ->setImageSelector($contentSource->getImageSelector()) ->setNextPageSelector($contentSource->getNextPageSelector()) - ->setChapterSelector($contentSource->getChapterSelector()); + ->setChapterSelector($contentSource->getChapterSelector()) + ->setTestSlug($contentSource->getTestSlug()) + ->setTestChapterNumber($contentSource->getTestChapterNumber()) + ->setHealthStatus($contentSource->getHealthStatus()) + ->setHealthLastTestedAt($contentSource->getHealthLastTestedAt()) + ->setHealthLastError($contentSource->getHealthLastError()); return $entity; } diff --git a/src/Domain/Setting/Infrastructure/Persistence/Repository/DoctrineContentSourceForHealthCheckRepository.php b/src/Domain/Setting/Infrastructure/Persistence/Repository/DoctrineContentSourceForHealthCheckRepository.php new file mode 100644 index 0000000..4e3006e --- /dev/null +++ b/src/Domain/Setting/Infrastructure/Persistence/Repository/DoctrineContentSourceForHealthCheckRepository.php @@ -0,0 +1,102 @@ +entityManager->getRepository(ContentSourceEntity::class)->findAll(); + + return array_map( + fn (ContentSourceEntity $entity) => new ContentSourceHealthCheckData( + id: $entity->getId(), + baseUrl: $entity->getBaseUrl(), + chapterUrlFormat: $entity->getChapterUrlFormat(), + scrapingType: $entity->getScrapingType(), + imageSelector: $entity->getImageSelector(), + nextPageSelector: $entity->getNextPageSelector(), + chapterSelector: $entity->getChapterSelector(), + testSlug: $entity->getTestSlug(), + testChapterNumber: $entity->getTestChapterNumber(), + ), + $entities + ); + } + + public function markAsTesting(int $sourceId): void + { + $entity = $this->entityManager->find(ContentSourceEntity::class, $sourceId); + if (!$entity) { + return; + } + + $entity->setHealthStatus('testing') + ->setHealthLastError(null); + + $this->entityManager->flush(); + $this->publishUpdate($sourceId, 'testing', null); + } + + public function markAsHealthy(int $sourceId, \DateTimeImmutable $testedAt): void + { + $entity = $this->entityManager->find(ContentSourceEntity::class, $sourceId); + if (!$entity) { + return; + } + + $entity->setHealthStatus('ok') + ->setHealthLastTestedAt($testedAt) + ->setHealthLastError(null); + + $this->entityManager->flush(); + $this->publishUpdate($sourceId, 'ok', null); + } + + public function markAsUnhealthy(int $sourceId, \DateTimeImmutable $testedAt, string $error): void + { + $entity = $this->entityManager->find(ContentSourceEntity::class, $sourceId); + if (!$entity) { + return; + } + + $entity->setHealthStatus('ko') + ->setHealthLastTestedAt($testedAt) + ->setHealthLastError($error); + + $this->entityManager->flush(); + $this->publishUpdate($sourceId, 'ko', $error); + } + + private function publishUpdate(int $sourceId, string $status, ?string $error): void + { + try { + $this->hub->publish(new Update( + "scrapers/health/{$sourceId}", + json_encode(['sourceId' => $sourceId, 'status' => $status, 'error' => $error]) + )); + } catch (\Throwable $e) { + $this->logger->warning('Mercure publish failed for scraper health update', [ + 'sourceId' => $sourceId, + 'status' => $status, + 'error' => $e->getMessage(), + ]); + } + } +} diff --git a/src/Entity/ContentSource.php b/src/Entity/ContentSource.php index 73b91ce..0f8ccaa 100644 --- a/src/Entity/ContentSource.php +++ b/src/Entity/ContentSource.php @@ -36,6 +36,21 @@ class ContentSource #[ORM\Column(length: 255, nullable: true)] private ?string $ChapterSelector = null; + #[ORM\Column(length: 255, nullable: true)] + private ?string $testSlug = null; + + #[ORM\Column(nullable: true)] + private ?float $testChapterNumber = null; + + #[ORM\Column(length: 20, options: ['default' => 'unknown'])] + private string $healthStatus = 'unknown'; + + #[ORM\Column(nullable: true)] + private ?\DateTimeImmutable $healthLastTestedAt = null; + + #[ORM\Column(type: 'text', nullable: true)] + private ?string $healthLastError = null; + public function getId(): ?int { return $this->id; @@ -119,6 +134,66 @@ class ContentSource return $this; } + public function getTestSlug(): ?string + { + return $this->testSlug; + } + + public function setTestSlug(?string $testSlug): static + { + $this->testSlug = $testSlug; + + return $this; + } + + public function getTestChapterNumber(): ?float + { + return $this->testChapterNumber; + } + + public function setTestChapterNumber(?float $testChapterNumber): static + { + $this->testChapterNumber = $testChapterNumber; + + return $this; + } + + public function getHealthStatus(): string + { + return $this->healthStatus; + } + + public function setHealthStatus(string $healthStatus): static + { + $this->healthStatus = $healthStatus; + + return $this; + } + + public function getHealthLastTestedAt(): ?\DateTimeImmutable + { + return $this->healthLastTestedAt; + } + + public function setHealthLastTestedAt(?\DateTimeImmutable $healthLastTestedAt): static + { + $this->healthLastTestedAt = $healthLastTestedAt; + + return $this; + } + + public function getHealthLastError(): ?string + { + return $this->healthLastError; + } + + public function setHealthLastError(?string $healthLastError): static + { + $this->healthLastError = $healthLastError; + + return $this; + } + public function getCleanBaseUrl(): string { return preg_replace( From 01474c264b8c43b96b6865f5446c687534477afd Mon Sep 17 00:00:00 2001 From: "ext.jeremy.guillot@maxicoffee.domains" Date: Mon, 16 Mar 2026 00:08:57 +0100 Subject: [PATCH 2/3] =?UTF-8?q?feat(scraping):=20impl=C3=A9menter=20le=20h?= =?UTF-8?q?ealth=20check=20de=20tous=20les=20scrapers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Commande CheckAllScrapersHealth + handler avec ports dédiés - Value Object ContentSourceHealthCheckData - Resource API Platform et State Processor - Adapters InMemory et tests unitaires + fonctionnels --- .../Command/CheckAllScrapersHealth.php | 7 + .../CheckAllScrapersHealthHandler.php | 64 ++++++++ .../ContentSourceForHealthCheckInterface.php | 11 ++ ...ContentSourceHealthRepositoryInterface.php | 12 ++ .../ContentSourceHealthCheckData.php | 19 +++ .../CheckAllScrapersHealthResource.php | 23 +++ .../CheckAllScrapersHealthStateProcessor.php | 23 +++ ...yContentSourceForHealthCheckRepository.php | 27 +++ .../InMemoryContentSourceHealthRepository.php | 41 +++++ .../CheckAllScrapersHealthHandlerTest.php | 154 ++++++++++++++++++ .../Scraping/CheckAllScrapersHealthTest.php | 72 ++++++++ 11 files changed, 453 insertions(+) create mode 100644 src/Domain/Scraping/Application/Command/CheckAllScrapersHealth.php create mode 100644 src/Domain/Scraping/Application/CommandHandler/CheckAllScrapersHealthHandler.php create mode 100644 src/Domain/Scraping/Domain/Contract/Repository/ContentSourceForHealthCheckInterface.php create mode 100644 src/Domain/Scraping/Domain/Contract/Repository/ContentSourceHealthRepositoryInterface.php create mode 100644 src/Domain/Scraping/Domain/Model/ValueObject/ContentSourceHealthCheckData.php create mode 100644 src/Domain/Scraping/Infrastructure/ApiPlatform/Resource/CheckAllScrapersHealthResource.php create mode 100644 src/Domain/Scraping/Infrastructure/ApiPlatform/State/Processor/CheckAllScrapersHealthStateProcessor.php create mode 100644 tests/Domain/Scraping/Adapter/InMemoryContentSourceForHealthCheckRepository.php create mode 100644 tests/Domain/Scraping/Adapter/InMemoryContentSourceHealthRepository.php create mode 100644 tests/Domain/Scraping/Application/CommandHandler/CheckAllScrapersHealthHandlerTest.php create mode 100644 tests/Feature/Scraping/CheckAllScrapersHealthTest.php diff --git a/src/Domain/Scraping/Application/Command/CheckAllScrapersHealth.php b/src/Domain/Scraping/Application/Command/CheckAllScrapersHealth.php new file mode 100644 index 0000000..2f74bd5 --- /dev/null +++ b/src/Domain/Scraping/Application/Command/CheckAllScrapersHealth.php @@ -0,0 +1,7 @@ +contentSourceForHealthCheckRepo->getAll(); + + foreach ($sources as $source) { + if ($source->testSlug === null || $source->testChapterNumber === null) { + $this->logger->warning('ContentSource {id} has no test config, skipping health check.', ['id' => $source->id]); + continue; + } + + try { + $this->contentSourceHealthRepo->markAsTesting($source->id); + $testUrl = str_replace( + ['{slug}', '{chapterNumber}'], + [$source->testSlug, $source->testChapterNumber], + $source->chapterUrlFormat + ); + + $testCommand = new TestScraperConfiguration( + baseUrl: $source->baseUrl, + chapterUrlFormat: $source->chapterUrlFormat, + scrapingType: $source->scrapingType, + testUrl: $testUrl, + mangaSlug: $source->testSlug, + chapterNumber: $source->testChapterNumber, + imageSelector: $source->imageSelector, + nextPageSelector: $source->nextPageSelector, + chapterSelector: $source->chapterSelector, + ); + + $response = $this->testScraperConfigurationHandler->handle($testCommand); + + if ($response->success) { + $this->contentSourceHealthRepo->markAsHealthy($source->id, new \DateTimeImmutable()); + } else { + $firstError = $response->errors[0]['message'] ?? 'Erreur inconnue'; + $this->contentSourceHealthRepo->markAsUnhealthy($source->id, new \DateTimeImmutable(), $firstError); + } + } catch (\Exception $e) { + $this->contentSourceHealthRepo->markAsUnhealthy($source->id, new \DateTimeImmutable(), $e->getMessage()); + } + } + } +} diff --git a/src/Domain/Scraping/Domain/Contract/Repository/ContentSourceForHealthCheckInterface.php b/src/Domain/Scraping/Domain/Contract/Repository/ContentSourceForHealthCheckInterface.php new file mode 100644 index 0000000..6bf1d3a --- /dev/null +++ b/src/Domain/Scraping/Domain/Contract/Repository/ContentSourceForHealthCheckInterface.php @@ -0,0 +1,11 @@ +handler->handle(new CheckAllScrapersHealth()); + + return null; + } +} diff --git a/tests/Domain/Scraping/Adapter/InMemoryContentSourceForHealthCheckRepository.php b/tests/Domain/Scraping/Adapter/InMemoryContentSourceForHealthCheckRepository.php new file mode 100644 index 0000000..e3c6158 --- /dev/null +++ b/tests/Domain/Scraping/Adapter/InMemoryContentSourceForHealthCheckRepository.php @@ -0,0 +1,27 @@ +sources[] = $data; + } + + public function getAll(): array + { + return $this->sources; + } + + public function clear(): void + { + $this->sources = []; + } +} diff --git a/tests/Domain/Scraping/Adapter/InMemoryContentSourceHealthRepository.php b/tests/Domain/Scraping/Adapter/InMemoryContentSourceHealthRepository.php new file mode 100644 index 0000000..6bd827d --- /dev/null +++ b/tests/Domain/Scraping/Adapter/InMemoryContentSourceHealthRepository.php @@ -0,0 +1,41 @@ + */ + private array $statuses = []; + + public function markAsTesting(int $sourceId): void + { + $this->statuses[$sourceId] = ['status' => 'testing', 'testedAt' => null, 'error' => null]; + } + + public function markAsHealthy(int $sourceId, \DateTimeImmutable $testedAt): void + { + $this->statuses[$sourceId] = ['status' => 'ok', 'testedAt' => $testedAt, 'error' => null]; + } + + public function markAsUnhealthy(int $sourceId, \DateTimeImmutable $testedAt, string $error): void + { + $this->statuses[$sourceId] = ['status' => 'ko', 'testedAt' => $testedAt, 'error' => $error]; + } + + public function getStatus(int $sourceId): ?string + { + return $this->statuses[$sourceId]['status'] ?? null; + } + + public function getError(int $sourceId): ?string + { + return $this->statuses[$sourceId]['error'] ?? null; + } + + public function clear(): void + { + $this->statuses = []; + } +} diff --git a/tests/Domain/Scraping/Application/CommandHandler/CheckAllScrapersHealthHandlerTest.php b/tests/Domain/Scraping/Application/CommandHandler/CheckAllScrapersHealthHandlerTest.php new file mode 100644 index 0000000..d75ff39 --- /dev/null +++ b/tests/Domain/Scraping/Application/CommandHandler/CheckAllScrapersHealthHandlerTest.php @@ -0,0 +1,154 @@ +sourceRepo = new InMemoryContentSourceForHealthCheckRepository(); + $this->healthRepo = new InMemoryContentSourceHealthRepository(); + $this->scraperFactory = new InMemoryScraperFactory(); + $this->scraperFactory->addScraper('html', new InMemoryScraperAdapter()); + + $testScraperHandler = new TestScraperConfigurationHandler($this->scraperFactory); + + $this->handler = new CheckAllScrapersHealthHandler( + $this->sourceRepo, + $this->healthRepo, + $testScraperHandler, + new NullLogger(), + ); + } + + public function testSourceWithoutTestSlugIsSkipped(): void + { + $this->sourceRepo->add(new ContentSourceHealthCheckData( + id: 1, + baseUrl: 'https://example.com', + chapterUrlFormat: 'https://example.com/{slug}/{chapterNumber}', + scrapingType: 'html', + imageSelector: 'img', + nextPageSelector: null, + chapterSelector: null, + testSlug: null, + testChapterNumber: null, + )); + + $this->handler->handle(new CheckAllScrapersHealth()); + + $this->assertNull($this->healthRepo->getStatus(1)); + } + + public function testSourceWithTestSlugIsMarkedAsHealthyOnSuccess(): void + { + $this->sourceRepo->add(new ContentSourceHealthCheckData( + id: 2, + baseUrl: 'https://example.com', + chapterUrlFormat: 'https://example.com/{slug}/{chapterNumber}', + scrapingType: 'html', + imageSelector: 'img', + nextPageSelector: null, + chapterSelector: null, + testSlug: 'one-piece', + testChapterNumber: 1.0, + )); + + $this->handler->handle(new CheckAllScrapersHealth()); + + $this->assertSame('ok', $this->healthRepo->getStatus(2)); + $this->assertNull($this->healthRepo->getError(2)); + } + + public function testSourceIsMarkedAsUnhealthyWhenScraperThrows(): void + { + $failingScraper = new InMemoryScraperAdapter(); + $failingScraper->simulateError(new \RuntimeException('Connexion refusée')); + $this->scraperFactory->addScraper('html', $failingScraper); + + $this->sourceRepo->add(new ContentSourceHealthCheckData( + id: 3, + baseUrl: 'https://example.com', + chapterUrlFormat: 'https://example.com/{slug}/{chapterNumber}', + scrapingType: 'html', + imageSelector: 'img', + nextPageSelector: null, + chapterSelector: null, + testSlug: 'one-piece', + testChapterNumber: 1.0, + )); + + $this->handler->handle(new CheckAllScrapersHealth()); + + $this->assertSame('ko', $this->healthRepo->getStatus(3)); + $this->assertNotNull($this->healthRepo->getError(3)); + } + + public function testMultipleSourcesAreAllProcessed(): void + { + $this->sourceRepo->add(new ContentSourceHealthCheckData( + id: 10, + baseUrl: 'https://siteA.com', + chapterUrlFormat: 'https://siteA.com/{slug}/{chapterNumber}', + scrapingType: 'html', + imageSelector: 'img', + nextPageSelector: null, + chapterSelector: null, + testSlug: 'manga-a', + testChapterNumber: 1.0, + )); + + $this->sourceRepo->add(new ContentSourceHealthCheckData( + id: 11, + baseUrl: 'https://siteB.com', + chapterUrlFormat: 'https://siteB.com/{slug}/{chapterNumber}', + scrapingType: 'html', + imageSelector: 'img', + nextPageSelector: null, + chapterSelector: null, + testSlug: null, + testChapterNumber: null, + )); + + $this->sourceRepo->add(new ContentSourceHealthCheckData( + id: 12, + baseUrl: 'https://siteC.com', + chapterUrlFormat: 'https://siteC.com/{slug}/{chapterNumber}', + scrapingType: 'html', + imageSelector: 'img', + nextPageSelector: null, + chapterSelector: null, + testSlug: 'manga-c', + testChapterNumber: 3.0, + )); + + $this->handler->handle(new CheckAllScrapersHealth()); + + $this->assertSame('ok', $this->healthRepo->getStatus(10)); + $this->assertNull($this->healthRepo->getStatus(11)); // skippée + $this->assertSame('ok', $this->healthRepo->getStatus(12)); + } + + protected function tearDown(): void + { + $this->sourceRepo->clear(); + $this->healthRepo->clear(); + $this->scraperFactory->clear(); + } +} diff --git a/tests/Feature/Scraping/CheckAllScrapersHealthTest.php b/tests/Feature/Scraping/CheckAllScrapersHealthTest.php new file mode 100644 index 0000000..547d293 --- /dev/null +++ b/tests/Feature/Scraping/CheckAllScrapersHealthTest.php @@ -0,0 +1,72 @@ +request('POST', '/api/scraping/check-all-health', [ + 'json' => new \stdClass(), + ]); + } + + public function testItReturns202WithNoSources(): void + { + $this->post(); + + $this->assertResponseStatusCodeSame(Response::HTTP_ACCEPTED); + } + + public function testItReturns202WithSourcesHavingNoTestConfig(): void + { + $source = new ContentSource(); + $source->setBaseUrl('https://example.com') + ->setChapterUrlFormat('https://example.com/{slug}/{chapterNumber}') + ->setScrapingType('html'); + + $this->entityManager->persist($source); + $this->entityManager->flush(); + + $this->post(); + + $this->assertResponseStatusCodeSame(Response::HTTP_ACCEPTED); + + // La source sans testSlug ne doit pas avoir son statut modifié + $this->entityManager->clear(); + $reloaded = $this->entityManager->find(ContentSource::class, $source->getId()); + $this->assertSame('unknown', $reloaded->getHealthStatus()); + } + + public function testHealthStatusIsUpdatedForSourcesWithTestConfig(): void + { + $source = new ContentSource(); + $source->setBaseUrl('https://example.com') + ->setChapterUrlFormat('https://example.com/{slug}/{chapterNumber}') + ->setScrapingType('html') + ->setTestSlug('one-piece') + ->setTestChapterNumber(1.0); + + $this->entityManager->persist($source); + $this->entityManager->flush(); + + $this->post(); + + $this->assertResponseStatusCodeSame(Response::HTTP_ACCEPTED); + + // Le statut ne doit plus être 'unknown' après le test + $this->entityManager->clear(); + $reloaded = $this->entityManager->find(ContentSource::class, $source->getId()); + $this->assertNotSame('unknown', $reloaded->getHealthStatus()); + $this->assertNotSame('testing', $reloaded->getHealthStatus()); // doit être terminé + } +} From 874003eb35152538fef199f8310ebd9a8f58f9cb Mon Sep 17 00:00:00 2001 From: "ext.jeremy.guillot@maxicoffee.domains" Date: Mon, 16 Mar 2026 00:09:01 +0100 Subject: [PATCH 3/3] fix(scraping): corriger les 403 sur les images avec protection anti-hotlink MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Ajouter le header Referer (origin de l'image) dans ImageDownloader pour les téléchargements backend - Ajouter referrerpolicy="no-referrer" sur les de la modale de test pour les previews navigateur --- .../domain/setting/presentation/pages/ScrapperEdit.vue | 6 ++++++ .../Scraping/Infrastructure/Service/ImageDownloader.php | 9 ++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/assets/vue/app/domain/setting/presentation/pages/ScrapperEdit.vue b/assets/vue/app/domain/setting/presentation/pages/ScrapperEdit.vue index 7db1ece..39aecd4 100644 --- a/assets/vue/app/domain/setting/presentation/pages/ScrapperEdit.vue +++ b/assets/vue/app/domain/setting/presentation/pages/ScrapperEdit.vue @@ -86,6 +86,7 @@ :src="imageUrl" :alt="`Image ${index + 1}`" class="w-full h-32 object-cover border border-gray-200 dark:border-gray-600" + referrerpolicy="no-referrer" @error="handleImageError" @load="handleImageLoad" />
@@ -278,6 +279,11 @@ const handleTest = async ({ configuration, testData }) => { testResults.value = {}; try { + // Persister testSlug + testChapterNumber avant de lancer le test + if (isEditing.value) { + await contentSourceStore.updateSource(route.params.id, configuration); + } + // Préparer les données selon le format de l'API const testConfiguration = { baseUrl: configuration.baseUrl, diff --git a/src/Domain/Scraping/Infrastructure/Service/ImageDownloader.php b/src/Domain/Scraping/Infrastructure/Service/ImageDownloader.php index a886bcc..69ffdb6 100644 --- a/src/Domain/Scraping/Infrastructure/Service/ImageDownloader.php +++ b/src/Domain/Scraping/Infrastructure/Service/ImageDownloader.php @@ -20,7 +20,14 @@ readonly class ImageDownloader implements ImageDownloaderInterface public function download(string $url, string $destination): void { - $response = $this->httpClient->request('GET', $url); + $urlParts = parse_url($url); + $referer = ($urlParts['scheme'] ?? 'https') . '://' . ($urlParts['host'] ?? ''); + + $response = $this->httpClient->request('GET', $url, [ + 'headers' => [ + 'Referer' => $referer, + ], + ]); $contentType = $response->getHeaders()['content-type'][0] ?? ''; if (!str_starts_with($contentType, 'image/')) {