@@ -191,12 +191,9 @@ const form = ref({
nextPageSelector: '',
chapterSelector: '',
scrapingType: 'html',
- token: ''
-});
-
-const testData = ref({
- mangaSlug: '',
- chapterNumber: ''
+ token: '',
+ testSlug: '',
+ testChapterNumber: '',
});
const testing = ref(false);
@@ -204,17 +201,17 @@ const testing = ref(false);
const canTest = computed(() => {
return form.value.baseUrl &&
form.value.chapterUrlFormat &&
- testData.value.mangaSlug &&
- testData.value.chapterNumber;
+ form.value.testSlug &&
+ form.value.testChapterNumber;
});
const generatedTestUrl = computed(() => {
- if (!form.value.chapterUrlFormat || !testData.value.mangaSlug || !testData.value.chapterNumber) {
+ if (!form.value.chapterUrlFormat || !form.value.testSlug || !form.value.testChapterNumber) {
return '';
}
return form.value.chapterUrlFormat
- .replace('{slug}', testData.value.mangaSlug)
- .replace('{chapterNumber}', testData.value.chapterNumber);
+ .replace('{slug}', form.value.testSlug)
+ .replace('{chapterNumber}', form.value.testChapterNumber);
});
watch(() => props.source, (newSource) => {
@@ -226,7 +223,9 @@ watch(() => props.source, (newSource) => {
nextPageSelector: newSource.nextPageSelector || '',
chapterSelector: newSource.chapterSelector || '',
scrapingType: (newSource.scrapingType || 'html').toLowerCase(),
- token: newSource.token || ''
+ token: newSource.token || '',
+ testSlug: newSource.testSlug || '',
+ testChapterNumber: newSource.testChapterNumber ?? '',
};
} else {
form.value = {
@@ -236,7 +235,9 @@ watch(() => props.source, (newSource) => {
nextPageSelector: '',
chapterSelector: '',
scrapingType: 'html',
- token: ''
+ token: '',
+ testSlug: '',
+ testChapterNumber: '',
};
}
}, { immediate: true });
@@ -253,8 +254,9 @@ const testConfiguration = async () => {
await emit('test', {
configuration: { ...form.value },
testData: {
- ...testData.value,
- testUrl: generatedTestUrl.value
+ mangaSlug: form.value.testSlug,
+ chapterNumber: form.value.testChapterNumber,
+ testUrl: generatedTestUrl.value,
}
});
} finally {
diff --git a/assets/vue/app/domain/setting/presentation/pages/ScrapperConfigurations.vue b/assets/vue/app/domain/setting/presentation/pages/ScrapperConfigurations.vue
index f482d89..4d991e3 100644
--- a/assets/vue/app/domain/setting/presentation/pages/ScrapperConfigurations.vue
+++ b/assets/vue/app/domain/setting/presentation/pages/ScrapperConfigurations.vue
@@ -91,10 +91,11 @@ import {
ArrowPathIcon,
ArrowUpTrayIcon,
ExclamationTriangleIcon,
+ HeartIcon,
PlusIcon
} from '@heroicons/vue/24/outline';
import { storeToRefs } from 'pinia';
-import { computed, onMounted, ref } from 'vue';
+import { computed, onMounted, onUnmounted, ref } from 'vue';
import { useRouter } from 'vue-router';
import Toolbar from '../../../../shared/components/ui/Toolbar.vue';
import { useContentSourceStore } from '../../application/store/contentSourceStore';
@@ -108,9 +109,13 @@ const {
loadingSources,
sourcesError,
importing,
- exporting
+ exporting,
+ checkingHealth,
} = storeToRefs(contentSourceStore);
+// Mercure — écoute des mises à jour health
+let mercureEventSource = null;
+
// Local state
const showImportModal = ref(false);
const showExportSuccess = ref(false);
@@ -120,12 +125,32 @@ const importData = ref('');
// Load sources on mount and clear current source
onMounted(async () => {
try {
- contentSourceStore.clearCurrentSource(); // Clear any previously loaded source
- contentSourceStore.clearErrors(); // Clear any previous errors
+ contentSourceStore.clearCurrentSource();
+ contentSourceStore.clearErrors();
await contentSourceStore.loadSources();
} catch (error) {
console.error('Erreur lors du chargement des sources:', error);
}
+
+ // Écoute Mercure pour les mises à jour de health status
+ const url = new URL('/.well-known/mercure', window.location.href);
+ sources.value.forEach(source => {
+ url.searchParams.append('topic', `scrapers/health/${source.id}`);
+ });
+
+ mercureEventSource = new EventSource(url.toString());
+ mercureEventSource.onmessage = (event) => {
+ try {
+ const data = JSON.parse(event.data);
+ contentSourceStore.updateSourceHealth(data.sourceId, data.status, data.error);
+ } catch (e) {
+ console.error('Erreur parsing Mercure event:', e);
+ }
+ };
+});
+
+onUnmounted(() => {
+ mercureEventSource?.close();
});
// Toolbar configuration
@@ -135,6 +160,7 @@ const toolbarConfig = computed(() => ({
],
rightSection: [
{ type: 'button', icon: ArrowPathIcon, label: 'Actualiser', onClick: () => contentSourceStore.loadSources(), disabled: loadingSources.value },
+ { type: 'button', icon: HeartIcon, label: 'Tester tous', onClick: handleCheckAllHealth, disabled: checkingHealth.value },
{ type: 'button', icon: ArrowDownTrayIcon, label: 'Exporter', onClick: handleExport, disabled: exporting.value },
{ type: 'button', icon: ArrowUpTrayIcon, label: 'Importer', onClick: () => showImportModal.value = true },
],
@@ -156,6 +182,14 @@ const openSourceLink = (url) => {
window.open(url, '_blank');
};
+async function handleCheckAllHealth() {
+ try {
+ await contentSourceStore.checkAllHealth();
+ } catch (error) {
+ console.error('Erreur lors du health check:', error);
+ }
+}
+
async function handleExport() {
try {
const exportData = await contentSourceStore.exportSources();
diff --git a/assets/vue/app/domain/setting/presentation/pages/ScrapperEdit.vue b/assets/vue/app/domain/setting/presentation/pages/ScrapperEdit.vue
index 7db1ece..39aecd4 100644
--- a/assets/vue/app/domain/setting/presentation/pages/ScrapperEdit.vue
+++ b/assets/vue/app/domain/setting/presentation/pages/ScrapperEdit.vue
@@ -86,6 +86,7 @@
:src="imageUrl"
:alt="`Image ${index + 1}`"
class="w-full h-32 object-cover border border-gray-200 dark:border-gray-600"
+ referrerpolicy="no-referrer"
@error="handleImageError"
@load="handleImageLoad" />
@@ -278,6 +279,11 @@ const handleTest = async ({ configuration, testData }) => {
testResults.value = {};
try {
+ // Persister testSlug + testChapterNumber avant de lancer le test
+ if (isEditing.value) {
+ await contentSourceStore.updateSource(route.params.id, configuration);
+ }
+
// Préparer les données selon le format de l'API
const testConfiguration = {
baseUrl: configuration.baseUrl,
diff --git a/config/services.yaml b/config/services.yaml
index d3d6b41..0b10b2b 100644
--- a/config/services.yaml
+++ b/config/services.yaml
@@ -180,6 +180,13 @@ services:
tags:
- { name: messenger.message_handler, bus: command.bus }
+ # Scraper Health Check
+ App\Domain\Scraping\Domain\Contract\Repository\ContentSourceForHealthCheckInterface:
+ alias: App\Domain\Setting\Infrastructure\Persistence\Repository\DoctrineContentSourceForHealthCheckRepository
+
+ App\Domain\Scraping\Domain\Contract\Repository\ContentSourceHealthRepositoryInterface:
+ alias: App\Domain\Setting\Infrastructure\Persistence\Repository\DoctrineContentSourceForHealthCheckRepository
+
# Import Domain Services
App\Domain\Import\Infrastructure\Service\FilenameAnalyzer: ~
diff --git a/migrations/Version20260315221706.php b/migrations/Version20260315221706.php
new file mode 100644
index 0000000..65c4f69
--- /dev/null
+++ b/migrations/Version20260315221706.php
@@ -0,0 +1,41 @@
+addSql('ALTER TABLE content_source ADD test_slug VARCHAR(255) DEFAULT NULL');
+ $this->addSql('ALTER TABLE content_source ADD test_chapter_number DOUBLE PRECISION DEFAULT NULL');
+ $this->addSql('ALTER TABLE content_source ADD health_status VARCHAR(20) DEFAULT \'unknown\' NOT NULL');
+ $this->addSql('ALTER TABLE content_source ADD health_last_tested_at TIMESTAMP(0) WITHOUT TIME ZONE DEFAULT NULL');
+ $this->addSql('ALTER TABLE content_source ADD health_last_error TEXT DEFAULT NULL');
+ $this->addSql('COMMENT ON COLUMN content_source.health_last_tested_at IS \'(DC2Type:datetime_immutable)\'');
+ }
+
+ public function down(Schema $schema): void
+ {
+ // this down() migration is auto-generated, please modify it to your needs
+ $this->addSql('CREATE SCHEMA public');
+ $this->addSql('ALTER TABLE content_source DROP test_slug');
+ $this->addSql('ALTER TABLE content_source DROP test_chapter_number');
+ $this->addSql('ALTER TABLE content_source DROP health_status');
+ $this->addSql('ALTER TABLE content_source DROP health_last_tested_at');
+ $this->addSql('ALTER TABLE content_source DROP health_last_error');
+ }
+}
diff --git a/src/Domain/Scraping/Application/Command/CheckAllScrapersHealth.php b/src/Domain/Scraping/Application/Command/CheckAllScrapersHealth.php
new file mode 100644
index 0000000..2f74bd5
--- /dev/null
+++ b/src/Domain/Scraping/Application/Command/CheckAllScrapersHealth.php
@@ -0,0 +1,7 @@
+contentSourceForHealthCheckRepo->getAll();
+
+ foreach ($sources as $source) {
+ if ($source->testSlug === null || $source->testChapterNumber === null) {
+ $this->logger->warning('ContentSource {id} has no test config, skipping health check.', ['id' => $source->id]);
+ continue;
+ }
+
+ try {
+ $this->contentSourceHealthRepo->markAsTesting($source->id);
+ $testUrl = str_replace(
+ ['{slug}', '{chapterNumber}'],
+ [$source->testSlug, $source->testChapterNumber],
+ $source->chapterUrlFormat
+ );
+
+ $testCommand = new TestScraperConfiguration(
+ baseUrl: $source->baseUrl,
+ chapterUrlFormat: $source->chapterUrlFormat,
+ scrapingType: $source->scrapingType,
+ testUrl: $testUrl,
+ mangaSlug: $source->testSlug,
+ chapterNumber: $source->testChapterNumber,
+ imageSelector: $source->imageSelector,
+ nextPageSelector: $source->nextPageSelector,
+ chapterSelector: $source->chapterSelector,
+ );
+
+ $response = $this->testScraperConfigurationHandler->handle($testCommand);
+
+ if ($response->success) {
+ $this->contentSourceHealthRepo->markAsHealthy($source->id, new \DateTimeImmutable());
+ } else {
+ $firstError = $response->errors[0]['message'] ?? 'Erreur inconnue';
+ $this->contentSourceHealthRepo->markAsUnhealthy($source->id, new \DateTimeImmutable(), $firstError);
+ }
+ } catch (\Exception $e) {
+ $this->contentSourceHealthRepo->markAsUnhealthy($source->id, new \DateTimeImmutable(), $e->getMessage());
+ }
+ }
+ }
+}
diff --git a/src/Domain/Scraping/Domain/Contract/Repository/ContentSourceForHealthCheckInterface.php b/src/Domain/Scraping/Domain/Contract/Repository/ContentSourceForHealthCheckInterface.php
new file mode 100644
index 0000000..6bf1d3a
--- /dev/null
+++ b/src/Domain/Scraping/Domain/Contract/Repository/ContentSourceForHealthCheckInterface.php
@@ -0,0 +1,11 @@
+handler->handle(new CheckAllScrapersHealth());
+
+ return null;
+ }
+}
diff --git a/src/Domain/Scraping/Infrastructure/Service/ImageDownloader.php b/src/Domain/Scraping/Infrastructure/Service/ImageDownloader.php
index a886bcc..69ffdb6 100644
--- a/src/Domain/Scraping/Infrastructure/Service/ImageDownloader.php
+++ b/src/Domain/Scraping/Infrastructure/Service/ImageDownloader.php
@@ -20,7 +20,14 @@ readonly class ImageDownloader implements ImageDownloaderInterface
public function download(string $url, string $destination): void
{
- $response = $this->httpClient->request('GET', $url);
+ $urlParts = parse_url($url);
+ $referer = ($urlParts['scheme'] ?? 'https') . '://' . ($urlParts['host'] ?? '');
+
+ $response = $this->httpClient->request('GET', $url, [
+ 'headers' => [
+ 'Referer' => $referer,
+ ],
+ ]);
$contentType = $response->getHeaders()['content-type'][0] ?? '';
if (!str_starts_with($contentType, 'image/')) {
diff --git a/src/Domain/Setting/Application/Command/UpsertContentSourceCommand.php b/src/Domain/Setting/Application/Command/UpsertContentSourceCommand.php
index 3949457..bffca24 100644
--- a/src/Domain/Setting/Application/Command/UpsertContentSourceCommand.php
+++ b/src/Domain/Setting/Application/Command/UpsertContentSourceCommand.php
@@ -12,6 +12,8 @@ readonly class UpsertContentSourceCommand
public ?string $imageSelector = null,
public ?string $nextPageSelector = null,
public ?string $chapterSelector = null,
+ public ?string $testSlug = null,
+ public ?float $testChapterNumber = null,
) {
}
}
diff --git a/src/Domain/Setting/Application/CommandHandler/UpsertContentSourceCommandHandler.php b/src/Domain/Setting/Application/CommandHandler/UpsertContentSourceCommandHandler.php
index 4fcfb36..cb9cac5 100644
--- a/src/Domain/Setting/Application/CommandHandler/UpsertContentSourceCommandHandler.php
+++ b/src/Domain/Setting/Application/CommandHandler/UpsertContentSourceCommandHandler.php
@@ -26,6 +26,8 @@ readonly class UpsertContentSourceCommandHandler
imageSelector: $command->imageSelector,
nextPageSelector: $command->nextPageSelector,
chapterSelector: $command->chapterSelector,
+ testSlug: $command->testSlug,
+ testChapterNumber: $command->testChapterNumber,
);
$this->contentSourceRepository->save($contentSource);
}
@@ -38,6 +40,8 @@ readonly class UpsertContentSourceCommandHandler
imageSelector: $command->imageSelector,
nextPageSelector: $command->nextPageSelector,
chapterSelector: $command->chapterSelector,
+ testSlug: $command->testSlug,
+ testChapterNumber: $command->testChapterNumber,
);
$this->contentSourceRepository->save($contentSource);
}
diff --git a/src/Domain/Setting/Application/Response/ContentSourceResponse.php b/src/Domain/Setting/Application/Response/ContentSourceResponse.php
index b9841cd..fdc932e 100644
--- a/src/Domain/Setting/Application/Response/ContentSourceResponse.php
+++ b/src/Domain/Setting/Application/Response/ContentSourceResponse.php
@@ -15,6 +15,11 @@ readonly class ContentSourceResponse
public ?string $nextPageSelector,
public ?string $chapterSelector,
public string $cleanBaseUrl,
+ public ?string $testSlug = null,
+ public ?float $testChapterNumber = null,
+ public string $healthStatus = 'unknown',
+ public ?\DateTimeImmutable $healthLastTestedAt = null,
+ public ?string $healthLastError = null,
) {
}
@@ -29,6 +34,11 @@ readonly class ContentSourceResponse
nextPageSelector: $contentSource->getNextPageSelector(),
chapterSelector: $contentSource->getChapterSelector(),
cleanBaseUrl: $contentSource->getCleanBaseUrl(),
+ testSlug: $contentSource->getTestSlug(),
+ testChapterNumber: $contentSource->getTestChapterNumber(),
+ healthStatus: $contentSource->getHealthStatus(),
+ healthLastTestedAt: $contentSource->getHealthLastTestedAt(),
+ healthLastError: $contentSource->getHealthLastError(),
);
}
}
diff --git a/src/Domain/Setting/Domain/Model/ContentSource.php b/src/Domain/Setting/Domain/Model/ContentSource.php
index 2dbe928..c860d2b 100644
--- a/src/Domain/Setting/Domain/Model/ContentSource.php
+++ b/src/Domain/Setting/Domain/Model/ContentSource.php
@@ -12,6 +12,11 @@ final class ContentSource
private ?string $imageSelector = null,
private ?string $nextPageSelector = null,
private ?string $chapterSelector = null,
+ private ?string $testSlug = null,
+ private ?float $testChapterNumber = null,
+ private string $healthStatus = 'unknown',
+ private ?\DateTimeImmutable $healthLastTestedAt = null,
+ private ?string $healthLastError = null,
) {
}
@@ -50,6 +55,44 @@ final class ContentSource
return $this->chapterSelector;
}
+ public function getTestSlug(): ?string
+ {
+ return $this->testSlug;
+ }
+
+ public function getTestChapterNumber(): ?float
+ {
+ return $this->testChapterNumber;
+ }
+
+ public function getHealthStatus(): string
+ {
+ return $this->healthStatus;
+ }
+
+ public function getHealthLastTestedAt(): ?\DateTimeImmutable
+ {
+ return $this->healthLastTestedAt;
+ }
+
+ public function getHealthLastError(): ?string
+ {
+ return $this->healthLastError;
+ }
+
+ public function updateTestConfig(?string $testSlug, ?float $testChapterNumber): void
+ {
+ $this->testSlug = $testSlug;
+ $this->testChapterNumber = $testChapterNumber;
+ }
+
+ public function updateHealthStatus(string $status, ?\DateTimeImmutable $testedAt = null, ?string $error = null): void
+ {
+ $this->healthStatus = $status;
+ $this->healthLastTestedAt = $testedAt;
+ $this->healthLastError = $error;
+ }
+
public function updateId(int $id): void
{
$this->id = $id;
@@ -71,6 +114,8 @@ final class ContentSource
?string $imageSelector = null,
?string $nextPageSelector = null,
?string $chapterSelector = null,
+ ?string $testSlug = null,
+ ?float $testChapterNumber = null,
): self {
return new self(
id: null,
@@ -80,6 +125,8 @@ final class ContentSource
imageSelector: $imageSelector,
nextPageSelector: $nextPageSelector,
chapterSelector: $chapterSelector,
+ testSlug: $testSlug,
+ testChapterNumber: $testChapterNumber,
);
}
@@ -90,6 +137,8 @@ final class ContentSource
?string $imageSelector = null,
?string $nextPageSelector = null,
?string $chapterSelector = null,
+ ?string $testSlug = null,
+ ?float $testChapterNumber = null,
): void {
$this->baseUrl = $baseUrl;
$this->chapterUrlFormat = $chapterUrlFormat;
@@ -97,5 +146,7 @@ final class ContentSource
$this->imageSelector = $imageSelector;
$this->nextPageSelector = $nextPageSelector;
$this->chapterSelector = $chapterSelector;
+ $this->testSlug = $testSlug;
+ $this->testChapterNumber = $testChapterNumber;
}
}
diff --git a/src/Domain/Setting/Infrastructure/ApiPlatform/Resource/GetContentSourceResource.php b/src/Domain/Setting/Infrastructure/ApiPlatform/Resource/GetContentSourceResource.php
index 1fa0ffc..639a557 100644
--- a/src/Domain/Setting/Infrastructure/ApiPlatform/Resource/GetContentSourceResource.php
+++ b/src/Domain/Setting/Infrastructure/ApiPlatform/Resource/GetContentSourceResource.php
@@ -30,6 +30,11 @@ class GetContentSourceResource
public readonly ?string $nextPageSelector,
public readonly ?string $chapterSelector,
public readonly string $cleanBaseUrl,
+ public readonly ?string $testSlug = null,
+ public readonly ?float $testChapterNumber = null,
+ public readonly string $healthStatus = 'unknown',
+ public readonly ?\DateTimeImmutable $healthLastTestedAt = null,
+ public readonly ?string $healthLastError = null,
) {
}
}
diff --git a/src/Domain/Setting/Infrastructure/ApiPlatform/Resource/UpsertContentSourceResource.php b/src/Domain/Setting/Infrastructure/ApiPlatform/Resource/UpsertContentSourceResource.php
index cf86496..a147f59 100644
--- a/src/Domain/Setting/Infrastructure/ApiPlatform/Resource/UpsertContentSourceResource.php
+++ b/src/Domain/Setting/Infrastructure/ApiPlatform/Resource/UpsertContentSourceResource.php
@@ -43,6 +43,8 @@ class UpsertContentSourceResource
public readonly ?string $imageSelector = null,
public readonly ?string $nextPageSelector = null,
public readonly ?string $chapterSelector = null,
+ public readonly ?string $testSlug = null,
+ public readonly ?float $testChapterNumber = null,
) {
}
}
diff --git a/src/Domain/Setting/Infrastructure/ApiPlatform/State/Processor/UpsertContentSourceStateProcessor.php b/src/Domain/Setting/Infrastructure/ApiPlatform/State/Processor/UpsertContentSourceStateProcessor.php
index ef685a6..51b62d7 100644
--- a/src/Domain/Setting/Infrastructure/ApiPlatform/State/Processor/UpsertContentSourceStateProcessor.php
+++ b/src/Domain/Setting/Infrastructure/ApiPlatform/State/Processor/UpsertContentSourceStateProcessor.php
@@ -30,6 +30,8 @@ readonly class UpsertContentSourceStateProcessor implements ProcessorInterface
imageSelector: $data->imageSelector,
nextPageSelector: $data->nextPageSelector,
chapterSelector: $data->chapterSelector,
+ testSlug: $data->testSlug,
+ testChapterNumber: $data->testChapterNumber,
);
$this->handler->handle($command);
diff --git a/src/Domain/Setting/Infrastructure/ApiPlatform/State/Provider/GetContentSourceStateProvider.php b/src/Domain/Setting/Infrastructure/ApiPlatform/State/Provider/GetContentSourceStateProvider.php
index 7bb9ae5..43d3503 100644
--- a/src/Domain/Setting/Infrastructure/ApiPlatform/State/Provider/GetContentSourceStateProvider.php
+++ b/src/Domain/Setting/Infrastructure/ApiPlatform/State/Provider/GetContentSourceStateProvider.php
@@ -32,6 +32,11 @@ readonly class GetContentSourceStateProvider implements ProviderInterface
nextPageSelector: $response->nextPageSelector,
chapterSelector: $response->chapterSelector,
cleanBaseUrl: $response->cleanBaseUrl,
+ testSlug: $response->testSlug,
+ testChapterNumber: $response->testChapterNumber,
+ healthStatus: $response->healthStatus,
+ healthLastTestedAt: $response->healthLastTestedAt,
+ healthLastError: $response->healthLastError,
);
} catch (ContentSourceNotFoundException $e) {
throw new NotFoundHttpException($e->getMessage());
diff --git a/src/Domain/Setting/Infrastructure/ApiPlatform/State/Provider/ListContentSourceStateProvider.php b/src/Domain/Setting/Infrastructure/ApiPlatform/State/Provider/ListContentSourceStateProvider.php
index f28f4f2..a95253a 100644
--- a/src/Domain/Setting/Infrastructure/ApiPlatform/State/Provider/ListContentSourceStateProvider.php
+++ b/src/Domain/Setting/Infrastructure/ApiPlatform/State/Provider/ListContentSourceStateProvider.php
@@ -30,6 +30,11 @@ readonly class ListContentSourceStateProvider implements ProviderInterface
nextPageSelector: $contentSourceResponse->nextPageSelector,
chapterSelector: $contentSourceResponse->chapterSelector,
cleanBaseUrl: $contentSourceResponse->cleanBaseUrl,
+ testSlug: $contentSourceResponse->testSlug,
+ testChapterNumber: $contentSourceResponse->testChapterNumber,
+ healthStatus: $contentSourceResponse->healthStatus,
+ healthLastTestedAt: $contentSourceResponse->healthLastTestedAt,
+ healthLastError: $contentSourceResponse->healthLastError,
),
$response->contentSources
);
diff --git a/src/Domain/Setting/Infrastructure/Persistence/Mapper/ContentSourceMapper.php b/src/Domain/Setting/Infrastructure/Persistence/Mapper/ContentSourceMapper.php
index 93d59ca..4dfa6e0 100644
--- a/src/Domain/Setting/Infrastructure/Persistence/Mapper/ContentSourceMapper.php
+++ b/src/Domain/Setting/Infrastructure/Persistence/Mapper/ContentSourceMapper.php
@@ -17,6 +17,11 @@ readonly class ContentSourceMapper
imageSelector: $entity->getImageSelector(),
nextPageSelector: $entity->getNextPageSelector(),
chapterSelector: $entity->getChapterSelector(),
+ testSlug: $entity->getTestSlug(),
+ testChapterNumber: $entity->getTestChapterNumber(),
+ healthStatus: $entity->getHealthStatus(),
+ healthLastTestedAt: $entity->getHealthLastTestedAt(),
+ healthLastError: $entity->getHealthLastError(),
);
}
@@ -29,7 +34,12 @@ readonly class ContentSourceMapper
->setScrapingType($contentSource->getScrapingType())
->setImageSelector($contentSource->getImageSelector())
->setNextPageSelector($contentSource->getNextPageSelector())
- ->setChapterSelector($contentSource->getChapterSelector());
+ ->setChapterSelector($contentSource->getChapterSelector())
+ ->setTestSlug($contentSource->getTestSlug())
+ ->setTestChapterNumber($contentSource->getTestChapterNumber())
+ ->setHealthStatus($contentSource->getHealthStatus())
+ ->setHealthLastTestedAt($contentSource->getHealthLastTestedAt())
+ ->setHealthLastError($contentSource->getHealthLastError());
return $entity;
}
@@ -41,7 +51,12 @@ readonly class ContentSourceMapper
->setScrapingType($contentSource->getScrapingType())
->setImageSelector($contentSource->getImageSelector())
->setNextPageSelector($contentSource->getNextPageSelector())
- ->setChapterSelector($contentSource->getChapterSelector());
+ ->setChapterSelector($contentSource->getChapterSelector())
+ ->setTestSlug($contentSource->getTestSlug())
+ ->setTestChapterNumber($contentSource->getTestChapterNumber())
+ ->setHealthStatus($contentSource->getHealthStatus())
+ ->setHealthLastTestedAt($contentSource->getHealthLastTestedAt())
+ ->setHealthLastError($contentSource->getHealthLastError());
return $entity;
}
diff --git a/src/Domain/Setting/Infrastructure/Persistence/Repository/DoctrineContentSourceForHealthCheckRepository.php b/src/Domain/Setting/Infrastructure/Persistence/Repository/DoctrineContentSourceForHealthCheckRepository.php
new file mode 100644
index 0000000..4e3006e
--- /dev/null
+++ b/src/Domain/Setting/Infrastructure/Persistence/Repository/DoctrineContentSourceForHealthCheckRepository.php
@@ -0,0 +1,102 @@
+entityManager->getRepository(ContentSourceEntity::class)->findAll();
+
+ return array_map(
+ fn (ContentSourceEntity $entity) => new ContentSourceHealthCheckData(
+ id: $entity->getId(),
+ baseUrl: $entity->getBaseUrl(),
+ chapterUrlFormat: $entity->getChapterUrlFormat(),
+ scrapingType: $entity->getScrapingType(),
+ imageSelector: $entity->getImageSelector(),
+ nextPageSelector: $entity->getNextPageSelector(),
+ chapterSelector: $entity->getChapterSelector(),
+ testSlug: $entity->getTestSlug(),
+ testChapterNumber: $entity->getTestChapterNumber(),
+ ),
+ $entities
+ );
+ }
+
+ public function markAsTesting(int $sourceId): void
+ {
+ $entity = $this->entityManager->find(ContentSourceEntity::class, $sourceId);
+ if (!$entity) {
+ return;
+ }
+
+ $entity->setHealthStatus('testing')
+ ->setHealthLastError(null);
+
+ $this->entityManager->flush();
+ $this->publishUpdate($sourceId, 'testing', null);
+ }
+
+ public function markAsHealthy(int $sourceId, \DateTimeImmutable $testedAt): void
+ {
+ $entity = $this->entityManager->find(ContentSourceEntity::class, $sourceId);
+ if (!$entity) {
+ return;
+ }
+
+ $entity->setHealthStatus('ok')
+ ->setHealthLastTestedAt($testedAt)
+ ->setHealthLastError(null);
+
+ $this->entityManager->flush();
+ $this->publishUpdate($sourceId, 'ok', null);
+ }
+
+ public function markAsUnhealthy(int $sourceId, \DateTimeImmutable $testedAt, string $error): void
+ {
+ $entity = $this->entityManager->find(ContentSourceEntity::class, $sourceId);
+ if (!$entity) {
+ return;
+ }
+
+ $entity->setHealthStatus('ko')
+ ->setHealthLastTestedAt($testedAt)
+ ->setHealthLastError($error);
+
+ $this->entityManager->flush();
+ $this->publishUpdate($sourceId, 'ko', $error);
+ }
+
+ private function publishUpdate(int $sourceId, string $status, ?string $error): void
+ {
+ try {
+ $this->hub->publish(new Update(
+ "scrapers/health/{$sourceId}",
+ json_encode(['sourceId' => $sourceId, 'status' => $status, 'error' => $error])
+ ));
+ } catch (\Throwable $e) {
+ $this->logger->warning('Mercure publish failed for scraper health update', [
+ 'sourceId' => $sourceId,
+ 'status' => $status,
+ 'error' => $e->getMessage(),
+ ]);
+ }
+ }
+}
diff --git a/src/Entity/ContentSource.php b/src/Entity/ContentSource.php
index 73b91ce..0f8ccaa 100644
--- a/src/Entity/ContentSource.php
+++ b/src/Entity/ContentSource.php
@@ -36,6 +36,21 @@ class ContentSource
#[ORM\Column(length: 255, nullable: true)]
private ?string $ChapterSelector = null;
+ #[ORM\Column(length: 255, nullable: true)]
+ private ?string $testSlug = null;
+
+ #[ORM\Column(nullable: true)]
+ private ?float $testChapterNumber = null;
+
+ #[ORM\Column(length: 20, options: ['default' => 'unknown'])]
+ private string $healthStatus = 'unknown';
+
+ #[ORM\Column(nullable: true)]
+ private ?\DateTimeImmutable $healthLastTestedAt = null;
+
+ #[ORM\Column(type: 'text', nullable: true)]
+ private ?string $healthLastError = null;
+
public function getId(): ?int
{
return $this->id;
@@ -119,6 +134,66 @@ class ContentSource
return $this;
}
+ public function getTestSlug(): ?string
+ {
+ return $this->testSlug;
+ }
+
+ public function setTestSlug(?string $testSlug): static
+ {
+ $this->testSlug = $testSlug;
+
+ return $this;
+ }
+
+ public function getTestChapterNumber(): ?float
+ {
+ return $this->testChapterNumber;
+ }
+
+ public function setTestChapterNumber(?float $testChapterNumber): static
+ {
+ $this->testChapterNumber = $testChapterNumber;
+
+ return $this;
+ }
+
+ public function getHealthStatus(): string
+ {
+ return $this->healthStatus;
+ }
+
+ public function setHealthStatus(string $healthStatus): static
+ {
+ $this->healthStatus = $healthStatus;
+
+ return $this;
+ }
+
+ public function getHealthLastTestedAt(): ?\DateTimeImmutable
+ {
+ return $this->healthLastTestedAt;
+ }
+
+ public function setHealthLastTestedAt(?\DateTimeImmutable $healthLastTestedAt): static
+ {
+ $this->healthLastTestedAt = $healthLastTestedAt;
+
+ return $this;
+ }
+
+ public function getHealthLastError(): ?string
+ {
+ return $this->healthLastError;
+ }
+
+ public function setHealthLastError(?string $healthLastError): static
+ {
+ $this->healthLastError = $healthLastError;
+
+ return $this;
+ }
+
public function getCleanBaseUrl(): string
{
return preg_replace(
diff --git a/tests/Domain/Scraping/Adapter/InMemoryContentSourceForHealthCheckRepository.php b/tests/Domain/Scraping/Adapter/InMemoryContentSourceForHealthCheckRepository.php
new file mode 100644
index 0000000..e3c6158
--- /dev/null
+++ b/tests/Domain/Scraping/Adapter/InMemoryContentSourceForHealthCheckRepository.php
@@ -0,0 +1,27 @@
+sources[] = $data;
+ }
+
+ public function getAll(): array
+ {
+ return $this->sources;
+ }
+
+ public function clear(): void
+ {
+ $this->sources = [];
+ }
+}
diff --git a/tests/Domain/Scraping/Adapter/InMemoryContentSourceHealthRepository.php b/tests/Domain/Scraping/Adapter/InMemoryContentSourceHealthRepository.php
new file mode 100644
index 0000000..6bd827d
--- /dev/null
+++ b/tests/Domain/Scraping/Adapter/InMemoryContentSourceHealthRepository.php
@@ -0,0 +1,41 @@
+ */
+ private array $statuses = [];
+
+ public function markAsTesting(int $sourceId): void
+ {
+ $this->statuses[$sourceId] = ['status' => 'testing', 'testedAt' => null, 'error' => null];
+ }
+
+ public function markAsHealthy(int $sourceId, \DateTimeImmutable $testedAt): void
+ {
+ $this->statuses[$sourceId] = ['status' => 'ok', 'testedAt' => $testedAt, 'error' => null];
+ }
+
+ public function markAsUnhealthy(int $sourceId, \DateTimeImmutable $testedAt, string $error): void
+ {
+ $this->statuses[$sourceId] = ['status' => 'ko', 'testedAt' => $testedAt, 'error' => $error];
+ }
+
+ public function getStatus(int $sourceId): ?string
+ {
+ return $this->statuses[$sourceId]['status'] ?? null;
+ }
+
+ public function getError(int $sourceId): ?string
+ {
+ return $this->statuses[$sourceId]['error'] ?? null;
+ }
+
+ public function clear(): void
+ {
+ $this->statuses = [];
+ }
+}
diff --git a/tests/Domain/Scraping/Application/CommandHandler/CheckAllScrapersHealthHandlerTest.php b/tests/Domain/Scraping/Application/CommandHandler/CheckAllScrapersHealthHandlerTest.php
new file mode 100644
index 0000000..d75ff39
--- /dev/null
+++ b/tests/Domain/Scraping/Application/CommandHandler/CheckAllScrapersHealthHandlerTest.php
@@ -0,0 +1,154 @@
+sourceRepo = new InMemoryContentSourceForHealthCheckRepository();
+ $this->healthRepo = new InMemoryContentSourceHealthRepository();
+ $this->scraperFactory = new InMemoryScraperFactory();
+ $this->scraperFactory->addScraper('html', new InMemoryScraperAdapter());
+
+ $testScraperHandler = new TestScraperConfigurationHandler($this->scraperFactory);
+
+ $this->handler = new CheckAllScrapersHealthHandler(
+ $this->sourceRepo,
+ $this->healthRepo,
+ $testScraperHandler,
+ new NullLogger(),
+ );
+ }
+
+ public function testSourceWithoutTestSlugIsSkipped(): void
+ {
+ $this->sourceRepo->add(new ContentSourceHealthCheckData(
+ id: 1,
+ baseUrl: 'https://example.com',
+ chapterUrlFormat: 'https://example.com/{slug}/{chapterNumber}',
+ scrapingType: 'html',
+ imageSelector: 'img',
+ nextPageSelector: null,
+ chapterSelector: null,
+ testSlug: null,
+ testChapterNumber: null,
+ ));
+
+ $this->handler->handle(new CheckAllScrapersHealth());
+
+ $this->assertNull($this->healthRepo->getStatus(1));
+ }
+
+ public function testSourceWithTestSlugIsMarkedAsHealthyOnSuccess(): void
+ {
+ $this->sourceRepo->add(new ContentSourceHealthCheckData(
+ id: 2,
+ baseUrl: 'https://example.com',
+ chapterUrlFormat: 'https://example.com/{slug}/{chapterNumber}',
+ scrapingType: 'html',
+ imageSelector: 'img',
+ nextPageSelector: null,
+ chapterSelector: null,
+ testSlug: 'one-piece',
+ testChapterNumber: 1.0,
+ ));
+
+ $this->handler->handle(new CheckAllScrapersHealth());
+
+ $this->assertSame('ok', $this->healthRepo->getStatus(2));
+ $this->assertNull($this->healthRepo->getError(2));
+ }
+
+ public function testSourceIsMarkedAsUnhealthyWhenScraperThrows(): void
+ {
+ $failingScraper = new InMemoryScraperAdapter();
+ $failingScraper->simulateError(new \RuntimeException('Connexion refusée'));
+ $this->scraperFactory->addScraper('html', $failingScraper);
+
+ $this->sourceRepo->add(new ContentSourceHealthCheckData(
+ id: 3,
+ baseUrl: 'https://example.com',
+ chapterUrlFormat: 'https://example.com/{slug}/{chapterNumber}',
+ scrapingType: 'html',
+ imageSelector: 'img',
+ nextPageSelector: null,
+ chapterSelector: null,
+ testSlug: 'one-piece',
+ testChapterNumber: 1.0,
+ ));
+
+ $this->handler->handle(new CheckAllScrapersHealth());
+
+ $this->assertSame('ko', $this->healthRepo->getStatus(3));
+ $this->assertNotNull($this->healthRepo->getError(3));
+ }
+
+ public function testMultipleSourcesAreAllProcessed(): void
+ {
+ $this->sourceRepo->add(new ContentSourceHealthCheckData(
+ id: 10,
+ baseUrl: 'https://siteA.com',
+ chapterUrlFormat: 'https://siteA.com/{slug}/{chapterNumber}',
+ scrapingType: 'html',
+ imageSelector: 'img',
+ nextPageSelector: null,
+ chapterSelector: null,
+ testSlug: 'manga-a',
+ testChapterNumber: 1.0,
+ ));
+
+ $this->sourceRepo->add(new ContentSourceHealthCheckData(
+ id: 11,
+ baseUrl: 'https://siteB.com',
+ chapterUrlFormat: 'https://siteB.com/{slug}/{chapterNumber}',
+ scrapingType: 'html',
+ imageSelector: 'img',
+ nextPageSelector: null,
+ chapterSelector: null,
+ testSlug: null,
+ testChapterNumber: null,
+ ));
+
+ $this->sourceRepo->add(new ContentSourceHealthCheckData(
+ id: 12,
+ baseUrl: 'https://siteC.com',
+ chapterUrlFormat: 'https://siteC.com/{slug}/{chapterNumber}',
+ scrapingType: 'html',
+ imageSelector: 'img',
+ nextPageSelector: null,
+ chapterSelector: null,
+ testSlug: 'manga-c',
+ testChapterNumber: 3.0,
+ ));
+
+ $this->handler->handle(new CheckAllScrapersHealth());
+
+ $this->assertSame('ok', $this->healthRepo->getStatus(10));
+ $this->assertNull($this->healthRepo->getStatus(11)); // skippée
+ $this->assertSame('ok', $this->healthRepo->getStatus(12));
+ }
+
+ protected function tearDown(): void
+ {
+ $this->sourceRepo->clear();
+ $this->healthRepo->clear();
+ $this->scraperFactory->clear();
+ }
+}
diff --git a/tests/Feature/Scraping/CheckAllScrapersHealthTest.php b/tests/Feature/Scraping/CheckAllScrapersHealthTest.php
new file mode 100644
index 0000000..547d293
--- /dev/null
+++ b/tests/Feature/Scraping/CheckAllScrapersHealthTest.php
@@ -0,0 +1,72 @@
+request('POST', '/api/scraping/check-all-health', [
+ 'json' => new \stdClass(),
+ ]);
+ }
+
+ public function testItReturns202WithNoSources(): void
+ {
+ $this->post();
+
+ $this->assertResponseStatusCodeSame(Response::HTTP_ACCEPTED);
+ }
+
+ public function testItReturns202WithSourcesHavingNoTestConfig(): void
+ {
+ $source = new ContentSource();
+ $source->setBaseUrl('https://example.com')
+ ->setChapterUrlFormat('https://example.com/{slug}/{chapterNumber}')
+ ->setScrapingType('html');
+
+ $this->entityManager->persist($source);
+ $this->entityManager->flush();
+
+ $this->post();
+
+ $this->assertResponseStatusCodeSame(Response::HTTP_ACCEPTED);
+
+ // La source sans testSlug ne doit pas avoir son statut modifié
+ $this->entityManager->clear();
+ $reloaded = $this->entityManager->find(ContentSource::class, $source->getId());
+ $this->assertSame('unknown', $reloaded->getHealthStatus());
+ }
+
+ public function testHealthStatusIsUpdatedForSourcesWithTestConfig(): void
+ {
+ $source = new ContentSource();
+ $source->setBaseUrl('https://example.com')
+ ->setChapterUrlFormat('https://example.com/{slug}/{chapterNumber}')
+ ->setScrapingType('html')
+ ->setTestSlug('one-piece')
+ ->setTestChapterNumber(1.0);
+
+ $this->entityManager->persist($source);
+ $this->entityManager->flush();
+
+ $this->post();
+
+ $this->assertResponseStatusCodeSame(Response::HTTP_ACCEPTED);
+
+ // Le statut ne doit plus être 'unknown' après le test
+ $this->entityManager->clear();
+ $reloaded = $this->entityManager->find(ContentSource::class, $source->getId());
+ $this->assertNotSame('unknown', $reloaded->getHealthStatus());
+ $this->assertNotSame('testing', $reloaded->getHealthStatus()); // doit être terminé
+ }
+}