4 Commits

Author SHA1 Message Date
ext.jeremy.guillot@maxicoffee.domains
01b6628fa6 fix(scraping): corriger les 403 sur les images avec protection anti-hotlink
- Ajouter le header Referer (origin de l'image) dans ImageDownloader pour les téléchargements backend
- Ajouter referrerpolicy="no-referrer" sur les <img> de la modale de test pour les previews navigateur
2026-03-16 00:09:19 +01:00
ext.jeremy.guillot@maxicoffee.domains
ae7a485195 feat(scraping): implémenter le health check de tous les scrapers
- Commande CheckAllScrapersHealth + handler avec ports dédiés
- Value Object ContentSourceHealthCheckData
- Resource API Platform et State Processor
- Adapters InMemory et tests unitaires + fonctionnels
2026-03-16 00:09:19 +01:00
ext.jeremy.guillot@maxicoffee.domains
734dea569c feat(setting): étendre ContentSource avec champs de test et domain model
- Ajouter testSlug, testChapterNumber, baseUrl sur ContentSource (entité, domain model, migration)
- Exposer ces champs dans les Resources, Processors, Providers et Mapper
- Mettre à jour store Pinia, repository API et composants Vue (form, card, liste)
2026-03-16 00:09:19 +01:00
ext.jeremy.guillot@maxicoffee.domains
71d6bb5ee9 feat(ui): harmoniser les pages Scrapers sur le design system Mangarr
- Layout canonique px-6 py-8 + sections border-t (suppression container mx-auto)
- Toolbar : label titre + bouton retour (ScrapperEdit) + boutons actions (ScrapperConfigurations)
- Bouton submit déplacé dans la toolbar droite via defineExpose/ref
- ContentSourceForm aplati (suppression du wrapper carte et du header)
- Séparation des sections du formulaire par border-t
- Suppression de tous les rounded-* sur les 4 composants
- Suppression du bloc debug "aucune source" et du h1 volant
2026-03-16 00:09:19 +01:00
33 changed files with 1108 additions and 257 deletions

View File

@@ -23,7 +23,11 @@ export const useContentSourceStore = defineStore('contentSource', {
importing: false, importing: false,
exporting: false, exporting: false,
importError: null, importError: null,
exportError: null exportError: null,
// Health check state
checkingHealth: false,
checkHealthError: null,
}), }),
getters: { getters: {
@@ -174,6 +178,36 @@ export const useContentSourceStore = defineStore('contentSource', {
this.currentSourceError = null; this.currentSourceError = null;
}, },
// Check all scrapers health
async checkAllHealth() {
if (this.checkingHealth) return;
this.checkingHealth = true;
this.checkHealthError = null;
try {
await contentSourceRepository.checkAllHealth();
} catch (error) {
this.checkHealthError = error.message;
console.error('Erreur lors du health check:', error);
throw error;
} finally {
this.checkingHealth = false;
}
},
// Update health status of a single source (called from Mercure)
updateSourceHealth(sourceId, status, error = null) {
const index = this.sources.findIndex(s => s.id === sourceId);
if (index !== -1) {
this.sources[index] = {
...this.sources[index],
healthStatus: status,
healthLastError: error,
};
}
},
// Clear errors // Clear errors
clearErrors() { clearErrors() {
this.sourcesError = null; this.sourcesError = null;
@@ -181,6 +215,7 @@ export const useContentSourceStore = defineStore('contentSource', {
this.saveError = null; this.saveError = null;
this.importError = null; this.importError = null;
this.exportError = null; this.exportError = null;
this.checkHealthError = null;
} }
} }
}); });

View File

@@ -0,0 +1,6 @@
export const ScraperHealthStatus = {
UNKNOWN: 'unknown',
OK: 'ok',
KO: 'ko',
TESTING: 'testing',
};

View File

@@ -82,6 +82,17 @@ export class ApiContentSourceRepository {
} }
} }
/**
* Déclenche le test de santé de tous les scrapers
*/
async checkAllHealth() {
try {
await this.apiClient.post('/scraping/check-all-health', {});
} catch (error) {
throw new Error(error.response?.data?.message || 'Erreur lors du lancement du health check');
}
}
/** /**
* Teste une configuration de scraper * Teste une configuration de scraper
*/ */

View File

@@ -1,7 +1,7 @@
<template> <template>
<div <div
@click="$emit('edit', source)" @click="$emit('edit', source)"
class="bg-white dark:bg-gray-800 rounded-lg shadow-md border border-gray-200 dark:border-gray-700 p-6 hover:shadow-lg transition-shadow duration-200 cursor-pointer"> class="bg-white dark:bg-gray-800 shadow-md border border-gray-200 dark:border-gray-700 p-6 hover:shadow-lg transition-shadow duration-200 cursor-pointer">
<!-- Header avec URL et icône externe --> <!-- Header avec URL et icône externe -->
<div class="flex items-center justify-between mb-4"> <div class="flex items-center justify-between mb-4">
<h3 class="text-lg font-semibold text-gray-900 dark:text-white truncate" :title="source.cleanBaseUrl"> <h3 class="text-lg font-semibold text-gray-900 dark:text-white truncate" :title="source.cleanBaseUrl">
@@ -20,16 +20,24 @@
<!-- Badge type de scraping --> <!-- Badge type de scraping -->
<span <span
:class="getScrapingTypeBadgeClass(source.scrapingType)" :class="getScrapingTypeBadgeClass(source.scrapingType)"
class="px-2 py-1 text-xs font-medium rounded-md"> class="px-2 py-1 text-xs font-medium">
{{ source.scrapingType?.toLowerCase() || 'N/A' }} {{ source.scrapingType?.toLowerCase() || 'N/A' }}
</span> </span>
<!-- Badge orientation basé sur les sélecteurs --> <!-- Badge orientation basé sur les sélecteurs -->
<span <span
:class="getOrientationBadgeClass(source)" :class="getOrientationBadgeClass(source)"
class="px-2 py-1 text-xs font-medium rounded-md"> class="px-2 py-1 text-xs font-medium">
{{ getOrientation(source) }} {{ getOrientation(source) }}
</span> </span>
<!-- Badge health status -->
<span
:class="getHealthBadgeClass(source.healthStatus)"
class="px-2 py-1 text-xs font-medium"
:title="source.healthLastError || ''">
{{ getHealthLabel(source.healthStatus) }}
</span>
</div> </div>
@@ -39,6 +47,7 @@
<script setup> <script setup>
import { ArrowTopRightOnSquareIcon } from '@heroicons/vue/24/outline'; import { ArrowTopRightOnSquareIcon } from '@heroicons/vue/24/outline';
import { ScraperHealthStatus } from '../../domain/model/ScraperHealthStatus';
defineProps({ defineProps({
source: { source: {
@@ -86,4 +95,26 @@ const getOrientationBadgeClass = (source) => {
return 'bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-gray-300'; return 'bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-gray-300';
} }
}; };
const getHealthLabel = (status) => {
switch (status) {
case ScraperHealthStatus.OK: return '✓ ok';
case ScraperHealthStatus.KO: return '✗ ko';
case ScraperHealthStatus.TESTING: return '⟳ test';
default: return '? unknown';
}
};
const getHealthBadgeClass = (status) => {
switch (status) {
case ScraperHealthStatus.OK:
return 'bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-300';
case ScraperHealthStatus.KO:
return 'bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-300';
case ScraperHealthStatus.TESTING:
return 'bg-yellow-100 text-yellow-800 dark:bg-yellow-900 dark:text-yellow-300';
default:
return 'bg-gray-100 text-gray-600 dark:bg-gray-700 dark:text-gray-400';
}
};
</script> </script>

View File

@@ -1,17 +1,7 @@
<template> <template>
<div class="bg-white dark:bg-gray-800 rounded-lg shadow-lg border border-gray-200 dark:border-gray-700"> <div>
<!-- Header -->
<div class="bg-gray-50 dark:bg-gray-700 px-6 py-4 border-b border-gray-200 dark:border-gray-600 rounded-t-lg">
<div class="flex items-center space-x-2">
<Cog6ToothIcon class="w-5 h-5 text-gray-600 dark:text-gray-400" />
<h2 class="text-lg font-semibold text-gray-900 dark:text-white">
{{ isEditing ? 'Edit Scrapper Configuration' : 'New Scrapper Configuration' }}
</h2>
</div>
</div>
<!-- Form --> <!-- Form -->
<form @submit.prevent="handleSubmit" class="p-6 space-y-6"> <form @submit.prevent="handleSubmit" class="space-y-6">
<!-- Base URL --> <!-- Base URL -->
<div> <div>
<label for="baseUrl" class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2"> <label for="baseUrl" class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2">
@@ -22,25 +12,12 @@
v-model="form.baseUrl" v-model="form.baseUrl"
type="url" type="url"
required required
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-white" class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-white"
placeholder="https://example.com" /> placeholder="https://example.com" />
</div> </div>
<!-- Image Selector -->
<div>
<label for="imageSelector" class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2">
Image Selector
</label>
<input
id="imageSelector"
v-model="form.imageSelector"
type="text"
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-white"
placeholder=".reading-content .page-break img" />
</div>
<!-- Chapter URL Format --> <!-- Chapter URL Format -->
<div> <div class="border-t border-gray-200 dark:border-gray-700 pt-6">
<label for="chapterUrlFormat" class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2"> <label for="chapterUrlFormat" class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2">
Chapter URL Format <span class="text-gray-500">({slug}, {chapterNumber})</span> Chapter URL Format <span class="text-gray-500">({slug}, {chapterNumber})</span>
</label> </label>
@@ -49,37 +26,51 @@
v-model="form.chapterUrlFormat" v-model="form.chapterUrlFormat"
type="text" type="text"
required required
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-white" class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-white"
placeholder="https://example.com/manga/{slug}-{chapterNumber}/" /> placeholder="https://example.com/manga/{slug}-{chapterNumber}/" />
</div> </div>
<!-- Next Page Selector --> <!-- Selectors -->
<div class="border-t border-gray-200 dark:border-gray-700 pt-6 space-y-4">
<div>
<label for="imageSelector" class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2">
Image Selector
</label>
<input
id="imageSelector"
v-model="form.imageSelector"
type="text"
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-white"
placeholder=".reading-content .page-break img" />
</div>
<div> <div>
<label for="nextPageSelector" class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2"> <label for="nextPageSelector" class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2">
Next Page Selector <span class="text-gray-500">(let empty if vertical reader)</span> Next Page Selector <span class="text-gray-500">(laisser vide si lecteur vertical)</span>
</label> </label>
<input <input
id="nextPageSelector" id="nextPageSelector"
v-model="form.nextPageSelector" v-model="form.nextPageSelector"
type="text" type="text"
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-white" class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-white"
placeholder=".next-page" /> placeholder=".next-page" />
</div> </div>
<!-- Chapter Selector -->
<div> <div>
<label for="chapterSelector" class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2"> <label for="chapterSelector" class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2">
Chapter Selector <span class="text-gray-500">(required for Javascript scraping)</span> Chapter Selector <span class="text-gray-500">(requis pour le scraping Javascript)</span>
</label> </label>
<input <input
id="chapterSelector" id="chapterSelector"
v-model="form.chapterSelector" v-model="form.chapterSelector"
type="text" type="text"
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-white" class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-white"
placeholder=".chapter-selector" /> placeholder=".chapter-selector" />
</div> </div>
</div>
<!-- Scraping Type --> <!-- Scraping Type + Token -->
<div class="border-t border-gray-200 dark:border-gray-700 pt-6 space-y-4">
<div> <div>
<label for="scrapingType" class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2"> <label for="scrapingType" class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2">
Scraping Type Scraping Type
@@ -88,13 +79,12 @@
id="scrapingType" id="scrapingType"
v-model="form.scrapingType" v-model="form.scrapingType"
required required
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-white"> class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-white">
<option value="html">HTML</option> <option value="html">HTML</option>
<option value="javascript">Javascript</option> <option value="javascript">Javascript</option>
</select> </select>
</div> </div>
<!-- Token (optionnel) -->
<div> <div>
<label for="token" class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2"> <label for="token" class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2">
Token Token
@@ -103,78 +93,65 @@
id="token" id="token"
v-model="form.token" v-model="form.token"
type="text" type="text"
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-white" class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-white"
placeholder="Optional authentication token" /> placeholder="Optional authentication token" />
</div> </div>
<!-- Submit Button -->
<div class="flex justify-end">
<button
type="submit"
:disabled="saving"
class="px-6 py-2 bg-green-600 hover:bg-green-700 disabled:bg-green-400 text-white font-medium rounded-md transition-colors duration-200 flex items-center space-x-2">
<ArrowPathIcon v-if="saving" class="w-4 h-4 animate-spin" />
<span>{{ isEditing ? 'Update Configuration' : 'Create Configuration' }}</span>
<PencilSquareIcon v-if="!saving" class="w-4 h-4" />
</button>
</div> </div>
<!-- Error message --> <!-- Error message -->
<div v-if="error" class="text-red-600 dark:text-red-400 text-sm"> <div v-if="error" class="border-t border-gray-200 dark:border-gray-700 pt-6 text-red-600 dark:text-red-400 text-sm">
{{ error }} {{ error }}
</div> </div>
</form> </form>
<!-- Test Configuration Section --> <!-- Test Configuration Section -->
<div class="border-t border-gray-200 dark:border-gray-600 p-6 bg-gray-50 dark:bg-gray-700 rounded-b-lg"> <div class="border-t border-gray-200 dark:border-gray-700 pt-6 mt-6">
<div class="flex items-center space-x-2 mb-4"> <div class="flex items-center space-x-2 mb-6">
<WrenchScrewdriverIcon class="w-5 h-5 text-gray-600 dark:text-gray-400" /> <WrenchScrewdriverIcon class="w-5 h-5 text-gray-600 dark:text-gray-400" />
<h3 class="text-lg font-medium text-gray-900 dark:text-white">Test Configuration</h3> <h3 class="text-sm font-medium text-gray-900 dark:text-white">Configuration de test (health check)</h3>
</div> </div>
<div class="grid grid-cols-1 md:grid-cols-2 gap-4 mb-4"> <div class="grid grid-cols-1 md:grid-cols-2 gap-4 mb-4">
<div> <div>
<label for="testMangaSlug" class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2"> <label for="testSlug" class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2">
Manga Slug Manga Slug <span class="text-gray-500">(enregistré)</span>
</label> </label>
<input <input
id="testMangaSlug" id="testSlug"
v-model="testData.mangaSlug" v-model="form.testSlug"
type="text" type="text"
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-white" class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-white"
placeholder="manga-slug" /> placeholder="manga-slug" />
</div> </div>
<div> <div>
<label for="testChapterNumber" class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2"> <label for="testChapterNumber" class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2">
Chapter Number Numéro de chapitre <span class="text-gray-500">(enregistré)</span>
</label> </label>
<input <input
id="testChapterNumber" id="testChapterNumber"
v-model="testData.chapterNumber" v-model="form.testChapterNumber"
type="number" type="number"
step="0.1" step="0.1"
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-white" class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-white"
placeholder="1" /> placeholder="1" />
</div> </div>
</div> </div>
<!-- Preview de l'URL qui sera testée --> <!-- Preview URL -->
<div v-if="generatedTestUrl" class="mb-4 p-3 bg-blue-50 dark:bg-blue-900 border border-blue-200 dark:border-blue-700 rounded-md"> <div v-if="generatedTestUrl" class="mb-4 border-t border-gray-200 dark:border-gray-700 pt-4">
<div class="text-sm text-blue-800 dark:text-blue-200"> <p class="text-xs text-gray-500 dark:text-gray-400 mb-1">URL qui sera testée</p>
<strong>URL qui sera testée :</strong> <code class="text-xs text-gray-700 dark:text-gray-300 break-all">{{ generatedTestUrl }}</code>
<div class="mt-1 font-mono text-xs break-all">{{ generatedTestUrl }}</div>
</div>
</div> </div>
<button <button
type="button" type="button"
@click="testConfiguration" @click="testConfiguration"
:disabled="testing || !canTest" :disabled="testing || !canTest"
class="w-full px-4 py-2 bg-blue-600 hover:bg-blue-700 disabled:bg-blue-400 text-white font-medium rounded-md transition-colors duration-200 flex items-center justify-center space-x-2"> class="w-full px-4 py-2 bg-blue-600 hover:bg-blue-700 disabled:bg-blue-400 text-white font-medium transition-colors duration-200 flex items-center justify-center space-x-2">
<ArrowPathIcon v-if="testing" class="w-4 h-4 animate-spin" /> <ArrowPathIcon v-if="testing" class="w-4 h-4 animate-spin" />
<PlayIcon v-else class="w-4 h-4" /> <PlayIcon v-else class="w-4 h-4" />
<span>Test Configuration</span> <span>Tester maintenant</span>
</button> </button>
</div> </div>
</div> </div>
@@ -183,8 +160,6 @@
<script setup> <script setup>
import { import {
ArrowPathIcon, ArrowPathIcon,
Cog6ToothIcon,
PencilSquareIcon,
PlayIcon, PlayIcon,
WrenchScrewdriverIcon WrenchScrewdriverIcon
} from '@heroicons/vue/24/outline'; } from '@heroicons/vue/24/outline';
@@ -216,12 +191,9 @@ const form = ref({
nextPageSelector: '', nextPageSelector: '',
chapterSelector: '', chapterSelector: '',
scrapingType: 'html', scrapingType: 'html',
token: '' token: '',
}); testSlug: '',
testChapterNumber: '',
const testData = ref({
mangaSlug: '',
chapterNumber: ''
}); });
const testing = ref(false); const testing = ref(false);
@@ -229,20 +201,19 @@ const testing = ref(false);
const canTest = computed(() => { const canTest = computed(() => {
return form.value.baseUrl && return form.value.baseUrl &&
form.value.chapterUrlFormat && form.value.chapterUrlFormat &&
testData.value.mangaSlug && form.value.testSlug &&
testData.value.chapterNumber; form.value.testChapterNumber;
}); });
const generatedTestUrl = computed(() => { const generatedTestUrl = computed(() => {
if (!form.value.chapterUrlFormat || !testData.value.mangaSlug || !testData.value.chapterNumber) { if (!form.value.chapterUrlFormat || !form.value.testSlug || !form.value.testChapterNumber) {
return ''; return '';
} }
return form.value.chapterUrlFormat return form.value.chapterUrlFormat
.replace('{slug}', testData.value.mangaSlug) .replace('{slug}', form.value.testSlug)
.replace('{chapterNumber}', testData.value.chapterNumber); .replace('{chapterNumber}', form.value.testChapterNumber);
}); });
// Initialize form with source data if editing, clear if creating new
watch(() => props.source, (newSource) => { watch(() => props.source, (newSource) => {
if (newSource) { if (newSource) {
form.value = { form.value = {
@@ -252,10 +223,11 @@ watch(() => props.source, (newSource) => {
nextPageSelector: newSource.nextPageSelector || '', nextPageSelector: newSource.nextPageSelector || '',
chapterSelector: newSource.chapterSelector || '', chapterSelector: newSource.chapterSelector || '',
scrapingType: (newSource.scrapingType || 'html').toLowerCase(), scrapingType: (newSource.scrapingType || 'html').toLowerCase(),
token: newSource.token || '' token: newSource.token || '',
testSlug: newSource.testSlug || '',
testChapterNumber: newSource.testChapterNumber ?? '',
}; };
} else { } else {
// Reset form when no source (creating new)
form.value = { form.value = {
baseUrl: '', baseUrl: '',
imageSelector: '', imageSelector: '',
@@ -263,7 +235,9 @@ watch(() => props.source, (newSource) => {
nextPageSelector: '', nextPageSelector: '',
chapterSelector: '', chapterSelector: '',
scrapingType: 'html', scrapingType: 'html',
token: '' token: '',
testSlug: '',
testChapterNumber: '',
}; };
} }
}, { immediate: true }); }, { immediate: true });
@@ -272,14 +246,17 @@ const handleSubmit = () => {
emit('submit', { ...form.value }); emit('submit', { ...form.value });
}; };
defineExpose({ submitForm: handleSubmit });
const testConfiguration = async () => { const testConfiguration = async () => {
testing.value = true; testing.value = true;
try { try {
await emit('test', { await emit('test', {
configuration: { ...form.value }, configuration: { ...form.value },
testData: { testData: {
...testData.value, mangaSlug: form.value.testSlug,
testUrl: generatedTestUrl.value chapterNumber: form.value.testChapterNumber,
testUrl: generatedTestUrl.value,
} }
}); });
} finally { } finally {

View File

@@ -3,47 +3,28 @@
<Toolbar :config="toolbarConfig" /> <Toolbar :config="toolbarConfig" />
<div class="overflow-y-auto flex-1"> <div class="overflow-y-auto flex-1">
<div class="container mx-auto px-4 py-6"> <div class="px-6 py-8">
<!-- Header -->
<div class="mb-8">
<h1 class="text-3xl font-bold text-gray-900 dark:text-white mb-2">
Scrapper Configurations
</h1>
<p class="text-gray-600 dark:text-gray-400">
Gérez les configurations de scraping pour les différentes sources de manga
</p>
</div>
<!-- Loading State --> <!-- Loading State -->
<div v-if="loadingSources" class="flex justify-center py-12"> <div v-if="loadingSources" class="flex justify-center py-12">
<div class="animate-spin rounded-full h-12 w-12 border-b-2 border-blue-500"></div> <div class="animate-spin h-12 w-12 border-b-2 border-blue-500"></div>
</div> </div>
<!-- Error State --> <!-- Error State -->
<div v-else-if="sourcesError" class="bg-red-50 dark:bg-red-900 border border-red-200 dark:border-red-700 rounded-lg p-4 mb-6"> <div v-else-if="sourcesError" class="bg-red-50 dark:bg-red-900 border border-red-200 dark:border-red-700 p-4 mb-6">
<div class="flex items-center"> <div class="flex items-center">
<ExclamationTriangleIcon class="w-5 h-5 text-red-400 mr-2" /> <ExclamationTriangleIcon class="w-5 h-5 text-red-400 mr-2" />
<p class="text-red-800 dark:text-red-200">{{ sourcesError }}</p> <p class="text-red-800 dark:text-red-200">{{ sourcesError }}</p>
</div> </div>
<button <button
@click="contentSourceStore.loadSources()" @click="contentSourceStore.loadSources()"
class="mt-3 px-4 py-2 bg-red-600 text-white rounded hover:bg-red-700"> class="mt-3 px-4 py-2 bg-red-600 text-white hover:bg-red-700">
Réessayer Réessayer
</button> </button>
</div> </div>
<!-- Debug Info (temporary) -->
<div v-if="!loadingSources && !sourcesError && sources.length === 0" class="bg-blue-50 dark:bg-blue-900 border border-blue-200 dark:border-blue-700 rounded-lg p-4 mb-6">
<p class="text-blue-800 dark:text-blue-200">Aucune source trouvée. Rechargement en cours...</p>
<button
@click="contentSourceStore.loadSources()"
class="mt-2 px-4 py-2 bg-blue-600 text-white rounded hover:bg-blue-700">
Actualiser
</button>
</div>
<!-- Sources Grid --> <!-- Sources Grid -->
<div v-else class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-6"> <section v-else class="border-t border-gray-200 dark:border-gray-700 pt-6">
<div class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-6">
<!-- Existing Sources --> <!-- Existing Sources -->
<ContentSourceCard <ContentSourceCard
v-for="source in sources" v-for="source in sources"
@@ -55,20 +36,21 @@
<!-- Add New Configuration Card --> <!-- Add New Configuration Card -->
<div <div
@click="addNewSource" @click="addNewSource"
class="bg-gray-50 dark:bg-gray-700 border-2 border-dashed border-gray-300 dark:border-gray-600 rounded-lg p-6 hover:border-gray-400 dark:hover:border-gray-500 transition-colors cursor-pointer flex flex-col items-center justify-center h-full"> class="bg-gray-50 dark:bg-gray-700 border-2 border-dashed border-gray-300 dark:border-gray-600 p-6 hover:border-gray-400 dark:hover:border-gray-500 transition-colors cursor-pointer flex flex-col items-center justify-center h-full">
<PlusIcon class="w-8 h-8 text-gray-400 dark:text-gray-500 mb-3" /> <PlusIcon class="w-8 h-8 text-gray-400 dark:text-gray-500 mb-3" />
<span class="text-lg font-medium text-gray-600 dark:text-gray-400 mb-2"> <span class="text-lg font-medium text-gray-600 dark:text-gray-400 mb-2">
Add New Configuration Add New Configuration
</span> </span>
</div> </div>
</div> </div>
</section>
<!-- Import/Export Success Messages --> <!-- Import/Export Success Messages -->
<div v-if="showImportSuccess" class="fixed bottom-4 right-4 bg-green-500 text-white px-4 py-2 rounded-lg shadow-lg"> <div v-if="showImportSuccess" class="fixed bottom-4 right-4 bg-green-500 text-white px-4 py-2 shadow-lg">
Configuration importée avec succès ! Configuration importée avec succès !
</div> </div>
<div v-if="showExportSuccess" class="fixed bottom-4 right-4 bg-blue-500 text-white px-4 py-2 rounded-lg shadow-lg"> <div v-if="showExportSuccess" class="fixed bottom-4 right-4 bg-blue-500 text-white px-4 py-2 shadow-lg">
Configuration exportée ! Configuration exportée !
</div> </div>
</div> </div>
@@ -76,12 +58,12 @@
<!-- Import Modal --> <!-- Import Modal -->
<div v-if="showImportModal" class="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50 p-4"> <div v-if="showImportModal" class="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50 p-4">
<div class="bg-white dark:bg-gray-800 rounded-lg shadow-xl w-full max-w-md"> <div class="bg-white dark:bg-gray-800 shadow-xl w-full max-w-md">
<div class="p-6"> <div class="p-6">
<h3 class="text-lg font-semibold mb-4">Importer des configurations</h3> <h3 class="text-lg font-semibold mb-4">Importer des configurations</h3>
<textarea <textarea
v-model="importData" v-model="importData"
class="w-full h-40 p-3 border border-gray-300 dark:border-gray-600 rounded-md dark:bg-gray-700 dark:text-white" class="w-full h-40 p-3 border border-gray-300 dark:border-gray-600 dark:bg-gray-700 dark:text-white"
placeholder="Collez ici le JSON des configurations à importer..."></textarea> placeholder="Collez ici le JSON des configurations à importer..."></textarea>
<div class="flex justify-end space-x-3 mt-4"> <div class="flex justify-end space-x-3 mt-4">
@@ -93,7 +75,7 @@
<button <button
@click="handleImport" @click="handleImport"
:disabled="importing || !importData.trim()" :disabled="importing || !importData.trim()"
class="px-4 py-2 bg-blue-600 hover:bg-blue-700 disabled:bg-blue-400 text-white rounded-md"> class="px-4 py-2 bg-blue-600 hover:bg-blue-700 disabled:bg-blue-400 text-white">
{{ importing ? 'Import...' : 'Importer' }} {{ importing ? 'Import...' : 'Importer' }}
</button> </button>
</div> </div>
@@ -109,10 +91,11 @@ import {
ArrowPathIcon, ArrowPathIcon,
ArrowUpTrayIcon, ArrowUpTrayIcon,
ExclamationTriangleIcon, ExclamationTriangleIcon,
HeartIcon,
PlusIcon PlusIcon
} from '@heroicons/vue/24/outline'; } from '@heroicons/vue/24/outline';
import { storeToRefs } from 'pinia'; import { storeToRefs } from 'pinia';
import { computed, onMounted, ref } from 'vue'; import { computed, onMounted, onUnmounted, ref } from 'vue';
import { useRouter } from 'vue-router'; import { useRouter } from 'vue-router';
import Toolbar from '../../../../shared/components/ui/Toolbar.vue'; import Toolbar from '../../../../shared/components/ui/Toolbar.vue';
import { useContentSourceStore } from '../../application/store/contentSourceStore'; import { useContentSourceStore } from '../../application/store/contentSourceStore';
@@ -126,9 +109,13 @@ const {
loadingSources, loadingSources,
sourcesError, sourcesError,
importing, importing,
exporting exporting,
checkingHealth,
} = storeToRefs(contentSourceStore); } = storeToRefs(contentSourceStore);
// Mercure — écoute des mises à jour health
let mercureEventSource = null;
// Local state // Local state
const showImportModal = ref(false); const showImportModal = ref(false);
const showExportSuccess = ref(false); const showExportSuccess = ref(false);
@@ -138,40 +125,45 @@ const importData = ref('');
// Load sources on mount and clear current source // Load sources on mount and clear current source
onMounted(async () => { onMounted(async () => {
try { try {
contentSourceStore.clearCurrentSource(); // Clear any previously loaded source contentSourceStore.clearCurrentSource();
contentSourceStore.clearErrors(); // Clear any previous errors contentSourceStore.clearErrors();
await contentSourceStore.loadSources(); await contentSourceStore.loadSources();
} catch (error) { } catch (error) {
console.error('Erreur lors du chargement des sources:', error); console.error('Erreur lors du chargement des sources:', error);
} }
// Écoute Mercure pour les mises à jour de health status
const url = new URL('/.well-known/mercure', window.location.href);
sources.value.forEach(source => {
url.searchParams.append('topic', `scrapers/health/${source.id}`);
});
mercureEventSource = new EventSource(url.toString());
mercureEventSource.onmessage = (event) => {
try {
const data = JSON.parse(event.data);
contentSourceStore.updateSourceHealth(data.sourceId, data.status, data.error);
} catch (e) {
console.error('Erreur parsing Mercure event:', e);
}
};
});
onUnmounted(() => {
mercureEventSource?.close();
}); });
// Toolbar configuration // Toolbar configuration
const toolbarConfig = computed(() => ({ const toolbarConfig = computed(() => ({
leftSection: [ leftSection: [
{ { type: 'label', text: 'Scrapers', class: 'text-sm font-medium' },
icon: ArrowPathIcon,
label: 'Actualiser',
type: 'button',
onClick: () => contentSourceStore.loadSources(),
active: loadingSources.value
}
], ],
rightSection: [ rightSection: [
{ { type: 'button', icon: ArrowPathIcon, label: 'Actualiser', onClick: () => contentSourceStore.loadSources(), disabled: loadingSources.value },
icon: ArrowDownTrayIcon, { type: 'button', icon: HeartIcon, label: 'Tester tous', onClick: handleCheckAllHealth, disabled: checkingHealth.value },
label: 'Exporter', { type: 'button', icon: ArrowDownTrayIcon, label: 'Exporter', onClick: handleExport, disabled: exporting.value },
type: 'button', { type: 'button', icon: ArrowUpTrayIcon, label: 'Importer', onClick: () => showImportModal.value = true },
onClick: handleExport, ],
disabled: exporting.value
},
{
icon: ArrowUpTrayIcon,
label: 'Importer',
type: 'button',
onClick: () => showImportModal.value = true
}
]
})); }));
// Actions // Actions
@@ -190,6 +182,14 @@ const openSourceLink = (url) => {
window.open(url, '_blank'); window.open(url, '_blank');
}; };
async function handleCheckAllHealth() {
try {
await contentSourceStore.checkAllHealth();
} catch (error) {
console.error('Erreur lors du health check:', error);
}
}
async function handleExport() { async function handleExport() {
try { try {
const exportData = await contentSourceStore.exportSources(); const exportData = await contentSourceStore.exportSources();

View File

@@ -3,24 +3,15 @@
<Toolbar :config="toolbarConfig" /> <Toolbar :config="toolbarConfig" />
<div class="overflow-y-auto flex-1"> <div class="overflow-y-auto flex-1">
<div class="container mx-auto px-4 py-6"> <div class="px-6 py-8">
<!-- Back Navigation --> <section class="border-t border-gray-200 dark:border-gray-700 pt-6">
<div class="mb-6">
<button
@click="goBack"
class="flex items-center space-x-2 text-gray-600 dark:text-gray-400 hover:text-gray-900 dark:hover:text-gray-200 transition-colors">
<ArrowLeftIcon class="w-5 h-5" />
<span>Retour aux configurations</span>
</button>
</div>
<!-- Loading State --> <!-- Loading State -->
<div v-if="loadingCurrentSource" class="flex justify-center py-12"> <div v-if="loadingCurrentSource" class="flex justify-center py-12">
<div class="animate-spin rounded-full h-12 w-12 border-b-2 border-blue-500"></div> <div class="animate-spin h-12 w-12 border-b-2 border-blue-500"></div>
</div> </div>
<!-- Error State --> <!-- Error State -->
<div v-else-if="currentSourceError" class="bg-red-50 dark:bg-red-900 border border-red-200 dark:border-red-700 rounded-lg p-4 mb-6"> <div v-else-if="currentSourceError" class="bg-red-50 dark:bg-red-900 border border-red-200 dark:border-red-700 p-4 mb-6">
<div class="flex items-center"> <div class="flex items-center">
<ExclamationTriangleIcon class="w-5 h-5 text-red-400 mr-2" /> <ExclamationTriangleIcon class="w-5 h-5 text-red-400 mr-2" />
<p class="text-red-800 dark:text-red-200">{{ currentSourceError }}</p> <p class="text-red-800 dark:text-red-200">{{ currentSourceError }}</p>
@@ -28,18 +19,20 @@
</div> </div>
<!-- Form --> <!-- Form -->
<div v-else class="max-w-4xl mx-auto"> <div v-else>
<ContentSourceForm <ContentSourceForm
ref="formRef"
:source="currentSource" :source="currentSource"
:saving="saving" :saving="saving"
:error="saveError" :error="saveError"
@submit="handleSubmit" @submit="handleSubmit"
@test="handleTest" /> @test="handleTest" />
</div> </div>
</section>
<!-- Test Results Modal --> <!-- Test Results Modal -->
<div v-if="showTestResults" class="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50 p-4"> <div v-if="showTestResults" class="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50 p-4">
<div class="bg-white dark:bg-gray-800 rounded-lg shadow-xl w-full max-w-4xl max-h-[80vh] overflow-hidden"> <div class="bg-white dark:bg-gray-800 shadow-xl w-full max-w-4xl max-h-[80vh] overflow-hidden">
<div class="p-6 border-b border-gray-200 dark:border-gray-600"> <div class="p-6 border-b border-gray-200 dark:border-gray-600">
<div class="flex justify-between items-center"> <div class="flex justify-between items-center">
<h3 class="text-lg font-semibold">Résultats du test</h3> <h3 class="text-lg font-semibold">Résultats du test</h3>
@@ -54,7 +47,7 @@
<div class="p-6 overflow-y-auto"> <div class="p-6 overflow-y-auto">
<!-- Loading state during test --> <!-- Loading state during test -->
<div v-if="testingConfiguration" class="flex items-center justify-center py-8"> <div v-if="testingConfiguration" class="flex items-center justify-center py-8">
<div class="animate-spin rounded-full h-8 w-8 border-b-2 border-blue-500 mr-3"></div> <div class="animate-spin h-8 w-8 border-b-2 border-blue-500 mr-3"></div>
<span class="text-gray-600">Test en cours...</span> <span class="text-gray-600">Test en cours...</span>
</div> </div>
@@ -65,7 +58,7 @@
<span class="font-medium">Test réussi !</span> <span class="font-medium">Test réussi !</span>
</div> </div>
<div class="bg-green-50 dark:bg-green-900 border border-green-200 dark:border-green-700 rounded-lg p-4"> <div class="bg-green-50 dark:bg-green-900 border border-green-200 dark:border-green-700 p-4">
<div class="grid grid-cols-2 gap-4 text-sm"> <div class="grid grid-cols-2 gap-4 text-sm">
<div> <div>
<span class="font-medium text-green-800 dark:text-green-200">URL testée:</span> <span class="font-medium text-green-800 dark:text-green-200">URL testée:</span>
@@ -92,10 +85,11 @@
<img <img
:src="imageUrl" :src="imageUrl"
:alt="`Image ${index + 1}`" :alt="`Image ${index + 1}`"
class="w-full h-32 object-cover rounded border border-gray-200 dark:border-gray-600" class="w-full h-32 object-cover border border-gray-200 dark:border-gray-600"
referrerpolicy="no-referrer"
@error="handleImageError" @error="handleImageError"
@load="handleImageLoad" /> @load="handleImageLoad" />
<div class="absolute inset-0 bg-black bg-opacity-0 group-hover:bg-opacity-30 transition-opacity rounded flex items-center justify-center"> <div class="absolute inset-0 bg-black bg-opacity-0 group-hover:bg-opacity-30 transition-opacity flex items-center justify-center">
<span class="text-white opacity-0 group-hover:opacity-100 text-sm font-medium"> <span class="text-white opacity-0 group-hover:opacity-100 text-sm font-medium">
Page {{ index + 1 }} Page {{ index + 1 }}
</span> </span>
@@ -107,7 +101,7 @@
</p> </p>
</div> </div>
<div v-else class="bg-yellow-50 dark:bg-yellow-900 border border-yellow-200 dark:border-yellow-700 rounded-lg p-4"> <div v-else class="bg-yellow-50 dark:bg-yellow-900 border border-yellow-200 dark:border-yellow-700 p-4">
<div class="flex items-center"> <div class="flex items-center">
<ExclamationTriangleIcon class="w-5 h-5 text-yellow-400 mr-2" /> <ExclamationTriangleIcon class="w-5 h-5 text-yellow-400 mr-2" />
<p class="text-yellow-800 dark:text-yellow-200"> <p class="text-yellow-800 dark:text-yellow-200">
@@ -125,7 +119,7 @@
<span class="font-medium">Test échoué</span> <span class="font-medium">Test échoué</span>
</div> </div>
<div class="bg-red-50 dark:bg-red-900 border border-red-200 dark:border-red-700 rounded-lg p-4 mb-4"> <div class="bg-red-50 dark:bg-red-900 border border-red-200 dark:border-red-700 p-4 mb-4">
<div class="text-sm text-red-800 dark:text-red-200"> <div class="text-sm text-red-800 dark:text-red-200">
<div><strong>URL testée:</strong> {{ testResults.testedUrl || 'N/A' }}</div> <div><strong>URL testée:</strong> {{ testResults.testedUrl || 'N/A' }}</div>
<div><strong>Type de scraping:</strong> {{ testResults.scrapingType || 'N/A' }}</div> <div><strong>Type de scraping:</strong> {{ testResults.scrapingType || 'N/A' }}</div>
@@ -138,14 +132,14 @@
<div <div
v-for="(error, index) in testResults.errors" v-for="(error, index) in testResults.errors"
:key="index" :key="index"
class="bg-red-100 dark:bg-red-800 border-l-4 border-red-400 p-4 rounded"> class="bg-red-100 dark:bg-red-800 border-l-4 border-red-400 p-4">
<div class="flex items-start"> <div class="flex items-start">
<div class="flex-shrink-0"> <div class="flex-shrink-0">
<ExclamationTriangleIcon class="w-5 h-5 text-red-400" /> <ExclamationTriangleIcon class="w-5 h-5 text-red-400" />
</div> </div>
<div class="ml-3 flex-1"> <div class="ml-3 flex-1">
<div class="flex items-center mb-1"> <div class="flex items-center mb-1">
<span class="inline-flex items-center px-2 py-1 rounded-full text-xs font-medium bg-red-200 text-red-800 dark:bg-red-700 dark:text-red-200 mr-2"> <span class="inline-flex items-center px-2 py-1 text-xs font-medium bg-red-200 text-red-800 dark:bg-red-700 dark:text-red-200 mr-2">
{{ formatErrorType(error.type) }} {{ formatErrorType(error.type) }}
</span> </span>
<span class="text-sm font-medium text-red-800 dark:text-red-200"> <span class="text-sm font-medium text-red-800 dark:text-red-200">
@@ -155,7 +149,7 @@
<p class="text-sm text-red-700 dark:text-red-300 mb-2"> <p class="text-sm text-red-700 dark:text-red-300 mb-2">
{{ error.message }} {{ error.message }}
</p> </p>
<div class="bg-red-50 dark:bg-red-900 rounded p-2"> <div class="bg-red-50 dark:bg-red-900 p-2">
<p class="text-xs text-red-600 dark:text-red-400"> <p class="text-xs text-red-600 dark:text-red-400">
<strong>Suggestion :</strong> {{ error.suggestion }} <strong>Suggestion :</strong> {{ error.suggestion }}
</p> </p>
@@ -166,7 +160,7 @@
</div> </div>
<!-- Generic Error --> <!-- Generic Error -->
<div v-else-if="testResults.error" class="bg-red-50 dark:bg-red-900 border border-red-200 dark:border-red-700 rounded p-3"> <div v-else-if="testResults.error" class="bg-red-50 dark:bg-red-900 border border-red-200 dark:border-red-700 p-3">
<code class="text-sm text-red-800 dark:text-red-200"> <code class="text-sm text-red-800 dark:text-red-200">
{{ testResults.error }} {{ testResults.error }}
</code> </code>
@@ -177,7 +171,7 @@
</div> </div>
<!-- Success Message --> <!-- Success Message -->
<div v-if="showSuccessMessage" class="fixed bottom-4 right-4 bg-green-500 text-white px-4 py-2 rounded-lg shadow-lg"> <div v-if="showSuccessMessage" class="fixed bottom-4 right-4 bg-green-500 text-white px-4 py-2 shadow-lg">
Configuration {{ isEditing ? 'mise à jour' : 'créée' }} avec succès ! Configuration {{ isEditing ? 'mise à jour' : 'créée' }} avec succès !
</div> </div>
</div> </div>
@@ -190,6 +184,7 @@ import {
ArrowLeftIcon, ArrowLeftIcon,
CheckCircleIcon, CheckCircleIcon,
ExclamationTriangleIcon, ExclamationTriangleIcon,
PencilSquareIcon,
XCircleIcon, XCircleIcon,
XMarkIcon XMarkIcon
} from '@heroicons/vue/24/outline'; } from '@heroicons/vue/24/outline';
@@ -214,6 +209,9 @@ const {
saveError saveError
} = storeToRefs(contentSourceStore); } = storeToRefs(contentSourceStore);
// Form ref
const formRef = ref(null);
// Local state // Local state
const showTestResults = ref(false); const showTestResults = ref(false);
const showSuccessMessage = ref(false); const showSuccessMessage = ref(false);
@@ -233,16 +231,18 @@ onMounted(async () => {
}); });
// Toolbar configuration // Toolbar configuration
const toolbarConfig = { const toolbarConfig = computed(() => ({
leftSection: [], leftSection: [
rightSection: [] { type: 'button', icon: ArrowLeftIcon, label: 'Retour', onClick: () => router.push({ name: 'scrapper-configurations' }) },
}; { type: 'divider' },
{ type: 'label', text: isEditing.value ? 'Modifier la configuration' : 'Nouvelle configuration', class: 'text-sm font-medium' },
],
rightSection: [
{ type: 'button', icon: PencilSquareIcon, label: isEditing.value ? 'Mettre à jour' : 'Créer', onClick: () => formRef.value?.submitForm(), disabled: saving.value },
],
}));
// Actions // Actions
const goBack = () => {
router.push({ name: 'scrapper-configurations' });
};
const handleSubmit = async (formData) => { const handleSubmit = async (formData) => {
try { try {
if (isEditing.value) { if (isEditing.value) {
@@ -279,6 +279,11 @@ const handleTest = async ({ configuration, testData }) => {
testResults.value = {}; testResults.value = {};
try { try {
// Persister testSlug + testChapterNumber avant de lancer le test
if (isEditing.value) {
await contentSourceStore.updateSource(route.params.id, configuration);
}
// Préparer les données selon le format de l'API // Préparer les données selon le format de l'API
const testConfiguration = { const testConfiguration = {
baseUrl: configuration.baseUrl, baseUrl: configuration.baseUrl,

View File

@@ -180,6 +180,13 @@ services:
tags: tags:
- { name: messenger.message_handler, bus: command.bus } - { name: messenger.message_handler, bus: command.bus }
# Scraper Health Check
App\Domain\Scraping\Domain\Contract\Repository\ContentSourceForHealthCheckInterface:
alias: App\Domain\Setting\Infrastructure\Persistence\Repository\DoctrineContentSourceForHealthCheckRepository
App\Domain\Scraping\Domain\Contract\Repository\ContentSourceHealthRepositoryInterface:
alias: App\Domain\Setting\Infrastructure\Persistence\Repository\DoctrineContentSourceForHealthCheckRepository
# Import Domain Services # Import Domain Services
App\Domain\Import\Infrastructure\Service\FilenameAnalyzer: ~ App\Domain\Import\Infrastructure\Service\FilenameAnalyzer: ~

View File

@@ -0,0 +1,41 @@
<?php
declare(strict_types=1);
namespace DoctrineMigrations;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\Migrations\AbstractMigration;
/**
* Auto-generated Migration: Please modify to your needs!
*/
final class Version20260315221706 extends AbstractMigration
{
public function getDescription(): string
{
return '';
}
public function up(Schema $schema): void
{
// this up() migration is auto-generated, please modify it to your needs
$this->addSql('ALTER TABLE content_source ADD test_slug VARCHAR(255) DEFAULT NULL');
$this->addSql('ALTER TABLE content_source ADD test_chapter_number DOUBLE PRECISION DEFAULT NULL');
$this->addSql('ALTER TABLE content_source ADD health_status VARCHAR(20) DEFAULT \'unknown\' NOT NULL');
$this->addSql('ALTER TABLE content_source ADD health_last_tested_at TIMESTAMP(0) WITHOUT TIME ZONE DEFAULT NULL');
$this->addSql('ALTER TABLE content_source ADD health_last_error TEXT DEFAULT NULL');
$this->addSql('COMMENT ON COLUMN content_source.health_last_tested_at IS \'(DC2Type:datetime_immutable)\'');
}
public function down(Schema $schema): void
{
// this down() migration is auto-generated, please modify it to your needs
$this->addSql('CREATE SCHEMA public');
$this->addSql('ALTER TABLE content_source DROP test_slug');
$this->addSql('ALTER TABLE content_source DROP test_chapter_number');
$this->addSql('ALTER TABLE content_source DROP health_status');
$this->addSql('ALTER TABLE content_source DROP health_last_tested_at');
$this->addSql('ALTER TABLE content_source DROP health_last_error');
}
}

View File

@@ -0,0 +1,7 @@
<?php
namespace App\Domain\Scraping\Application\Command;
readonly class CheckAllScrapersHealth
{
}

View File

@@ -0,0 +1,64 @@
<?php
namespace App\Domain\Scraping\Application\CommandHandler;
use App\Domain\Scraping\Application\Command\CheckAllScrapersHealth;
use App\Domain\Scraping\Application\Command\TestScraperConfiguration;
use App\Domain\Scraping\Domain\Contract\Repository\ContentSourceForHealthCheckInterface;
use App\Domain\Scraping\Domain\Contract\Repository\ContentSourceHealthRepositoryInterface;
use Psr\Log\LoggerInterface;
readonly class CheckAllScrapersHealthHandler
{
public function __construct(
private ContentSourceForHealthCheckInterface $contentSourceForHealthCheckRepo,
private ContentSourceHealthRepositoryInterface $contentSourceHealthRepo,
private TestScraperConfigurationHandler $testScraperConfigurationHandler,
private LoggerInterface $logger,
) {
}
public function handle(CheckAllScrapersHealth $command): void
{
$sources = $this->contentSourceForHealthCheckRepo->getAll();
foreach ($sources as $source) {
if ($source->testSlug === null || $source->testChapterNumber === null) {
$this->logger->warning('ContentSource {id} has no test config, skipping health check.', ['id' => $source->id]);
continue;
}
try {
$this->contentSourceHealthRepo->markAsTesting($source->id);
$testUrl = str_replace(
['{slug}', '{chapterNumber}'],
[$source->testSlug, $source->testChapterNumber],
$source->chapterUrlFormat
);
$testCommand = new TestScraperConfiguration(
baseUrl: $source->baseUrl,
chapterUrlFormat: $source->chapterUrlFormat,
scrapingType: $source->scrapingType,
testUrl: $testUrl,
mangaSlug: $source->testSlug,
chapterNumber: $source->testChapterNumber,
imageSelector: $source->imageSelector,
nextPageSelector: $source->nextPageSelector,
chapterSelector: $source->chapterSelector,
);
$response = $this->testScraperConfigurationHandler->handle($testCommand);
if ($response->success) {
$this->contentSourceHealthRepo->markAsHealthy($source->id, new \DateTimeImmutable());
} else {
$firstError = $response->errors[0]['message'] ?? 'Erreur inconnue';
$this->contentSourceHealthRepo->markAsUnhealthy($source->id, new \DateTimeImmutable(), $firstError);
}
} catch (\Exception $e) {
$this->contentSourceHealthRepo->markAsUnhealthy($source->id, new \DateTimeImmutable(), $e->getMessage());
}
}
}
}

View File

@@ -0,0 +1,11 @@
<?php
namespace App\Domain\Scraping\Domain\Contract\Repository;
use App\Domain\Scraping\Domain\Model\ValueObject\ContentSourceHealthCheckData;
interface ContentSourceForHealthCheckInterface
{
/** @return ContentSourceHealthCheckData[] */
public function getAll(): array;
}

View File

@@ -0,0 +1,12 @@
<?php
namespace App\Domain\Scraping\Domain\Contract\Repository;
interface ContentSourceHealthRepositoryInterface
{
public function markAsTesting(int $sourceId): void;
public function markAsHealthy(int $sourceId, \DateTimeImmutable $testedAt): void;
public function markAsUnhealthy(int $sourceId, \DateTimeImmutable $testedAt, string $error): void;
}

View File

@@ -0,0 +1,19 @@
<?php
namespace App\Domain\Scraping\Domain\Model\ValueObject;
readonly class ContentSourceHealthCheckData
{
public function __construct(
public int $id,
public string $baseUrl,
public string $chapterUrlFormat,
public string $scrapingType,
public ?string $imageSelector,
public ?string $nextPageSelector,
public ?string $chapterSelector,
public ?string $testSlug,
public ?float $testChapterNumber,
) {
}
}

View File

@@ -0,0 +1,23 @@
<?php
namespace App\Domain\Scraping\Infrastructure\ApiPlatform\Resource;
use ApiPlatform\Metadata\ApiResource;
use ApiPlatform\Metadata\Post;
use App\Domain\Scraping\Infrastructure\ApiPlatform\State\Processor\CheckAllScrapersHealthStateProcessor;
#[ApiResource(
shortName: 'Scraping',
operations: [
new Post(
uriTemplate: '/scraping/check-all-health',
processor: CheckAllScrapersHealthStateProcessor::class,
output: false,
status: 202,
description: 'Déclenche le test de santé de tous les scrapers configurés avec testSlug',
),
]
)]
class CheckAllScrapersHealthResource
{
}

View File

@@ -0,0 +1,23 @@
<?php
namespace App\Domain\Scraping\Infrastructure\ApiPlatform\State\Processor;
use ApiPlatform\Metadata\Operation;
use ApiPlatform\State\ProcessorInterface;
use App\Domain\Scraping\Application\Command\CheckAllScrapersHealth;
use App\Domain\Scraping\Application\CommandHandler\CheckAllScrapersHealthHandler;
readonly class CheckAllScrapersHealthStateProcessor implements ProcessorInterface
{
public function __construct(
private CheckAllScrapersHealthHandler $handler,
) {
}
public function process(mixed $data, Operation $operation, array $uriVariables = [], array $context = []): null
{
$this->handler->handle(new CheckAllScrapersHealth());
return null;
}
}

View File

@@ -20,7 +20,14 @@ readonly class ImageDownloader implements ImageDownloaderInterface
public function download(string $url, string $destination): void public function download(string $url, string $destination): void
{ {
$response = $this->httpClient->request('GET', $url); $urlParts = parse_url($url);
$referer = ($urlParts['scheme'] ?? 'https') . '://' . ($urlParts['host'] ?? '');
$response = $this->httpClient->request('GET', $url, [
'headers' => [
'Referer' => $referer,
],
]);
$contentType = $response->getHeaders()['content-type'][0] ?? ''; $contentType = $response->getHeaders()['content-type'][0] ?? '';
if (!str_starts_with($contentType, 'image/')) { if (!str_starts_with($contentType, 'image/')) {

View File

@@ -12,6 +12,8 @@ readonly class UpsertContentSourceCommand
public ?string $imageSelector = null, public ?string $imageSelector = null,
public ?string $nextPageSelector = null, public ?string $nextPageSelector = null,
public ?string $chapterSelector = null, public ?string $chapterSelector = null,
public ?string $testSlug = null,
public ?float $testChapterNumber = null,
) { ) {
} }
} }

View File

@@ -26,6 +26,8 @@ readonly class UpsertContentSourceCommandHandler
imageSelector: $command->imageSelector, imageSelector: $command->imageSelector,
nextPageSelector: $command->nextPageSelector, nextPageSelector: $command->nextPageSelector,
chapterSelector: $command->chapterSelector, chapterSelector: $command->chapterSelector,
testSlug: $command->testSlug,
testChapterNumber: $command->testChapterNumber,
); );
$this->contentSourceRepository->save($contentSource); $this->contentSourceRepository->save($contentSource);
} }
@@ -38,6 +40,8 @@ readonly class UpsertContentSourceCommandHandler
imageSelector: $command->imageSelector, imageSelector: $command->imageSelector,
nextPageSelector: $command->nextPageSelector, nextPageSelector: $command->nextPageSelector,
chapterSelector: $command->chapterSelector, chapterSelector: $command->chapterSelector,
testSlug: $command->testSlug,
testChapterNumber: $command->testChapterNumber,
); );
$this->contentSourceRepository->save($contentSource); $this->contentSourceRepository->save($contentSource);
} }

View File

@@ -15,6 +15,11 @@ readonly class ContentSourceResponse
public ?string $nextPageSelector, public ?string $nextPageSelector,
public ?string $chapterSelector, public ?string $chapterSelector,
public string $cleanBaseUrl, public string $cleanBaseUrl,
public ?string $testSlug = null,
public ?float $testChapterNumber = null,
public string $healthStatus = 'unknown',
public ?\DateTimeImmutable $healthLastTestedAt = null,
public ?string $healthLastError = null,
) { ) {
} }
@@ -29,6 +34,11 @@ readonly class ContentSourceResponse
nextPageSelector: $contentSource->getNextPageSelector(), nextPageSelector: $contentSource->getNextPageSelector(),
chapterSelector: $contentSource->getChapterSelector(), chapterSelector: $contentSource->getChapterSelector(),
cleanBaseUrl: $contentSource->getCleanBaseUrl(), cleanBaseUrl: $contentSource->getCleanBaseUrl(),
testSlug: $contentSource->getTestSlug(),
testChapterNumber: $contentSource->getTestChapterNumber(),
healthStatus: $contentSource->getHealthStatus(),
healthLastTestedAt: $contentSource->getHealthLastTestedAt(),
healthLastError: $contentSource->getHealthLastError(),
); );
} }
} }

View File

@@ -12,6 +12,11 @@ final class ContentSource
private ?string $imageSelector = null, private ?string $imageSelector = null,
private ?string $nextPageSelector = null, private ?string $nextPageSelector = null,
private ?string $chapterSelector = null, private ?string $chapterSelector = null,
private ?string $testSlug = null,
private ?float $testChapterNumber = null,
private string $healthStatus = 'unknown',
private ?\DateTimeImmutable $healthLastTestedAt = null,
private ?string $healthLastError = null,
) { ) {
} }
@@ -50,6 +55,44 @@ final class ContentSource
return $this->chapterSelector; return $this->chapterSelector;
} }
public function getTestSlug(): ?string
{
return $this->testSlug;
}
public function getTestChapterNumber(): ?float
{
return $this->testChapterNumber;
}
public function getHealthStatus(): string
{
return $this->healthStatus;
}
public function getHealthLastTestedAt(): ?\DateTimeImmutable
{
return $this->healthLastTestedAt;
}
public function getHealthLastError(): ?string
{
return $this->healthLastError;
}
public function updateTestConfig(?string $testSlug, ?float $testChapterNumber): void
{
$this->testSlug = $testSlug;
$this->testChapterNumber = $testChapterNumber;
}
public function updateHealthStatus(string $status, ?\DateTimeImmutable $testedAt = null, ?string $error = null): void
{
$this->healthStatus = $status;
$this->healthLastTestedAt = $testedAt;
$this->healthLastError = $error;
}
public function updateId(int $id): void public function updateId(int $id): void
{ {
$this->id = $id; $this->id = $id;
@@ -71,6 +114,8 @@ final class ContentSource
?string $imageSelector = null, ?string $imageSelector = null,
?string $nextPageSelector = null, ?string $nextPageSelector = null,
?string $chapterSelector = null, ?string $chapterSelector = null,
?string $testSlug = null,
?float $testChapterNumber = null,
): self { ): self {
return new self( return new self(
id: null, id: null,
@@ -80,6 +125,8 @@ final class ContentSource
imageSelector: $imageSelector, imageSelector: $imageSelector,
nextPageSelector: $nextPageSelector, nextPageSelector: $nextPageSelector,
chapterSelector: $chapterSelector, chapterSelector: $chapterSelector,
testSlug: $testSlug,
testChapterNumber: $testChapterNumber,
); );
} }
@@ -90,6 +137,8 @@ final class ContentSource
?string $imageSelector = null, ?string $imageSelector = null,
?string $nextPageSelector = null, ?string $nextPageSelector = null,
?string $chapterSelector = null, ?string $chapterSelector = null,
?string $testSlug = null,
?float $testChapterNumber = null,
): void { ): void {
$this->baseUrl = $baseUrl; $this->baseUrl = $baseUrl;
$this->chapterUrlFormat = $chapterUrlFormat; $this->chapterUrlFormat = $chapterUrlFormat;
@@ -97,5 +146,7 @@ final class ContentSource
$this->imageSelector = $imageSelector; $this->imageSelector = $imageSelector;
$this->nextPageSelector = $nextPageSelector; $this->nextPageSelector = $nextPageSelector;
$this->chapterSelector = $chapterSelector; $this->chapterSelector = $chapterSelector;
$this->testSlug = $testSlug;
$this->testChapterNumber = $testChapterNumber;
} }
} }

View File

@@ -30,6 +30,11 @@ class GetContentSourceResource
public readonly ?string $nextPageSelector, public readonly ?string $nextPageSelector,
public readonly ?string $chapterSelector, public readonly ?string $chapterSelector,
public readonly string $cleanBaseUrl, public readonly string $cleanBaseUrl,
public readonly ?string $testSlug = null,
public readonly ?float $testChapterNumber = null,
public readonly string $healthStatus = 'unknown',
public readonly ?\DateTimeImmutable $healthLastTestedAt = null,
public readonly ?string $healthLastError = null,
) { ) {
} }
} }

View File

@@ -43,6 +43,8 @@ class UpsertContentSourceResource
public readonly ?string $imageSelector = null, public readonly ?string $imageSelector = null,
public readonly ?string $nextPageSelector = null, public readonly ?string $nextPageSelector = null,
public readonly ?string $chapterSelector = null, public readonly ?string $chapterSelector = null,
public readonly ?string $testSlug = null,
public readonly ?float $testChapterNumber = null,
) { ) {
} }
} }

View File

@@ -30,6 +30,8 @@ readonly class UpsertContentSourceStateProcessor implements ProcessorInterface
imageSelector: $data->imageSelector, imageSelector: $data->imageSelector,
nextPageSelector: $data->nextPageSelector, nextPageSelector: $data->nextPageSelector,
chapterSelector: $data->chapterSelector, chapterSelector: $data->chapterSelector,
testSlug: $data->testSlug,
testChapterNumber: $data->testChapterNumber,
); );
$this->handler->handle($command); $this->handler->handle($command);

View File

@@ -32,6 +32,11 @@ readonly class GetContentSourceStateProvider implements ProviderInterface
nextPageSelector: $response->nextPageSelector, nextPageSelector: $response->nextPageSelector,
chapterSelector: $response->chapterSelector, chapterSelector: $response->chapterSelector,
cleanBaseUrl: $response->cleanBaseUrl, cleanBaseUrl: $response->cleanBaseUrl,
testSlug: $response->testSlug,
testChapterNumber: $response->testChapterNumber,
healthStatus: $response->healthStatus,
healthLastTestedAt: $response->healthLastTestedAt,
healthLastError: $response->healthLastError,
); );
} catch (ContentSourceNotFoundException $e) { } catch (ContentSourceNotFoundException $e) {
throw new NotFoundHttpException($e->getMessage()); throw new NotFoundHttpException($e->getMessage());

View File

@@ -30,6 +30,11 @@ readonly class ListContentSourceStateProvider implements ProviderInterface
nextPageSelector: $contentSourceResponse->nextPageSelector, nextPageSelector: $contentSourceResponse->nextPageSelector,
chapterSelector: $contentSourceResponse->chapterSelector, chapterSelector: $contentSourceResponse->chapterSelector,
cleanBaseUrl: $contentSourceResponse->cleanBaseUrl, cleanBaseUrl: $contentSourceResponse->cleanBaseUrl,
testSlug: $contentSourceResponse->testSlug,
testChapterNumber: $contentSourceResponse->testChapterNumber,
healthStatus: $contentSourceResponse->healthStatus,
healthLastTestedAt: $contentSourceResponse->healthLastTestedAt,
healthLastError: $contentSourceResponse->healthLastError,
), ),
$response->contentSources $response->contentSources
); );

View File

@@ -17,6 +17,11 @@ readonly class ContentSourceMapper
imageSelector: $entity->getImageSelector(), imageSelector: $entity->getImageSelector(),
nextPageSelector: $entity->getNextPageSelector(), nextPageSelector: $entity->getNextPageSelector(),
chapterSelector: $entity->getChapterSelector(), chapterSelector: $entity->getChapterSelector(),
testSlug: $entity->getTestSlug(),
testChapterNumber: $entity->getTestChapterNumber(),
healthStatus: $entity->getHealthStatus(),
healthLastTestedAt: $entity->getHealthLastTestedAt(),
healthLastError: $entity->getHealthLastError(),
); );
} }
@@ -29,7 +34,12 @@ readonly class ContentSourceMapper
->setScrapingType($contentSource->getScrapingType()) ->setScrapingType($contentSource->getScrapingType())
->setImageSelector($contentSource->getImageSelector()) ->setImageSelector($contentSource->getImageSelector())
->setNextPageSelector($contentSource->getNextPageSelector()) ->setNextPageSelector($contentSource->getNextPageSelector())
->setChapterSelector($contentSource->getChapterSelector()); ->setChapterSelector($contentSource->getChapterSelector())
->setTestSlug($contentSource->getTestSlug())
->setTestChapterNumber($contentSource->getTestChapterNumber())
->setHealthStatus($contentSource->getHealthStatus())
->setHealthLastTestedAt($contentSource->getHealthLastTestedAt())
->setHealthLastError($contentSource->getHealthLastError());
return $entity; return $entity;
} }
@@ -41,7 +51,12 @@ readonly class ContentSourceMapper
->setScrapingType($contentSource->getScrapingType()) ->setScrapingType($contentSource->getScrapingType())
->setImageSelector($contentSource->getImageSelector()) ->setImageSelector($contentSource->getImageSelector())
->setNextPageSelector($contentSource->getNextPageSelector()) ->setNextPageSelector($contentSource->getNextPageSelector())
->setChapterSelector($contentSource->getChapterSelector()); ->setChapterSelector($contentSource->getChapterSelector())
->setTestSlug($contentSource->getTestSlug())
->setTestChapterNumber($contentSource->getTestChapterNumber())
->setHealthStatus($contentSource->getHealthStatus())
->setHealthLastTestedAt($contentSource->getHealthLastTestedAt())
->setHealthLastError($contentSource->getHealthLastError());
return $entity; return $entity;
} }

View File

@@ -0,0 +1,102 @@
<?php
namespace App\Domain\Setting\Infrastructure\Persistence\Repository;
use App\Domain\Scraping\Domain\Contract\Repository\ContentSourceForHealthCheckInterface;
use App\Domain\Scraping\Domain\Contract\Repository\ContentSourceHealthRepositoryInterface;
use App\Domain\Scraping\Domain\Model\ValueObject\ContentSourceHealthCheckData;
use App\Entity\ContentSource as ContentSourceEntity;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Log\LoggerInterface;
use Symfony\Component\Mercure\HubInterface;
use Symfony\Component\Mercure\Update;
readonly class DoctrineContentSourceForHealthCheckRepository implements ContentSourceForHealthCheckInterface, ContentSourceHealthRepositoryInterface
{
public function __construct(
private EntityManagerInterface $entityManager,
private HubInterface $hub,
private LoggerInterface $logger,
) {
}
public function getAll(): array
{
$entities = $this->entityManager->getRepository(ContentSourceEntity::class)->findAll();
return array_map(
fn (ContentSourceEntity $entity) => new ContentSourceHealthCheckData(
id: $entity->getId(),
baseUrl: $entity->getBaseUrl(),
chapterUrlFormat: $entity->getChapterUrlFormat(),
scrapingType: $entity->getScrapingType(),
imageSelector: $entity->getImageSelector(),
nextPageSelector: $entity->getNextPageSelector(),
chapterSelector: $entity->getChapterSelector(),
testSlug: $entity->getTestSlug(),
testChapterNumber: $entity->getTestChapterNumber(),
),
$entities
);
}
public function markAsTesting(int $sourceId): void
{
$entity = $this->entityManager->find(ContentSourceEntity::class, $sourceId);
if (!$entity) {
return;
}
$entity->setHealthStatus('testing')
->setHealthLastError(null);
$this->entityManager->flush();
$this->publishUpdate($sourceId, 'testing', null);
}
public function markAsHealthy(int $sourceId, \DateTimeImmutable $testedAt): void
{
$entity = $this->entityManager->find(ContentSourceEntity::class, $sourceId);
if (!$entity) {
return;
}
$entity->setHealthStatus('ok')
->setHealthLastTestedAt($testedAt)
->setHealthLastError(null);
$this->entityManager->flush();
$this->publishUpdate($sourceId, 'ok', null);
}
public function markAsUnhealthy(int $sourceId, \DateTimeImmutable $testedAt, string $error): void
{
$entity = $this->entityManager->find(ContentSourceEntity::class, $sourceId);
if (!$entity) {
return;
}
$entity->setHealthStatus('ko')
->setHealthLastTestedAt($testedAt)
->setHealthLastError($error);
$this->entityManager->flush();
$this->publishUpdate($sourceId, 'ko', $error);
}
private function publishUpdate(int $sourceId, string $status, ?string $error): void
{
try {
$this->hub->publish(new Update(
"scrapers/health/{$sourceId}",
json_encode(['sourceId' => $sourceId, 'status' => $status, 'error' => $error])
));
} catch (\Throwable $e) {
$this->logger->warning('Mercure publish failed for scraper health update', [
'sourceId' => $sourceId,
'status' => $status,
'error' => $e->getMessage(),
]);
}
}
}

View File

@@ -36,6 +36,21 @@ class ContentSource
#[ORM\Column(length: 255, nullable: true)] #[ORM\Column(length: 255, nullable: true)]
private ?string $ChapterSelector = null; private ?string $ChapterSelector = null;
#[ORM\Column(length: 255, nullable: true)]
private ?string $testSlug = null;
#[ORM\Column(nullable: true)]
private ?float $testChapterNumber = null;
#[ORM\Column(length: 20, options: ['default' => 'unknown'])]
private string $healthStatus = 'unknown';
#[ORM\Column(nullable: true)]
private ?\DateTimeImmutable $healthLastTestedAt = null;
#[ORM\Column(type: 'text', nullable: true)]
private ?string $healthLastError = null;
public function getId(): ?int public function getId(): ?int
{ {
return $this->id; return $this->id;
@@ -119,6 +134,66 @@ class ContentSource
return $this; return $this;
} }
public function getTestSlug(): ?string
{
return $this->testSlug;
}
public function setTestSlug(?string $testSlug): static
{
$this->testSlug = $testSlug;
return $this;
}
public function getTestChapterNumber(): ?float
{
return $this->testChapterNumber;
}
public function setTestChapterNumber(?float $testChapterNumber): static
{
$this->testChapterNumber = $testChapterNumber;
return $this;
}
public function getHealthStatus(): string
{
return $this->healthStatus;
}
public function setHealthStatus(string $healthStatus): static
{
$this->healthStatus = $healthStatus;
return $this;
}
public function getHealthLastTestedAt(): ?\DateTimeImmutable
{
return $this->healthLastTestedAt;
}
public function setHealthLastTestedAt(?\DateTimeImmutable $healthLastTestedAt): static
{
$this->healthLastTestedAt = $healthLastTestedAt;
return $this;
}
public function getHealthLastError(): ?string
{
return $this->healthLastError;
}
public function setHealthLastError(?string $healthLastError): static
{
$this->healthLastError = $healthLastError;
return $this;
}
public function getCleanBaseUrl(): string public function getCleanBaseUrl(): string
{ {
return preg_replace( return preg_replace(

View File

@@ -0,0 +1,27 @@
<?php
namespace App\Tests\Domain\Scraping\Adapter;
use App\Domain\Scraping\Domain\Contract\Repository\ContentSourceForHealthCheckInterface;
use App\Domain\Scraping\Domain\Model\ValueObject\ContentSourceHealthCheckData;
class InMemoryContentSourceForHealthCheckRepository implements ContentSourceForHealthCheckInterface
{
/** @var ContentSourceHealthCheckData[] */
private array $sources = [];
public function add(ContentSourceHealthCheckData $data): void
{
$this->sources[] = $data;
}
public function getAll(): array
{
return $this->sources;
}
public function clear(): void
{
$this->sources = [];
}
}

View File

@@ -0,0 +1,41 @@
<?php
namespace App\Tests\Domain\Scraping\Adapter;
use App\Domain\Scraping\Domain\Contract\Repository\ContentSourceHealthRepositoryInterface;
class InMemoryContentSourceHealthRepository implements ContentSourceHealthRepositoryInterface
{
/** @var array<int, array{status: string, testedAt: ?\DateTimeImmutable, error: ?string}> */
private array $statuses = [];
public function markAsTesting(int $sourceId): void
{
$this->statuses[$sourceId] = ['status' => 'testing', 'testedAt' => null, 'error' => null];
}
public function markAsHealthy(int $sourceId, \DateTimeImmutable $testedAt): void
{
$this->statuses[$sourceId] = ['status' => 'ok', 'testedAt' => $testedAt, 'error' => null];
}
public function markAsUnhealthy(int $sourceId, \DateTimeImmutable $testedAt, string $error): void
{
$this->statuses[$sourceId] = ['status' => 'ko', 'testedAt' => $testedAt, 'error' => $error];
}
public function getStatus(int $sourceId): ?string
{
return $this->statuses[$sourceId]['status'] ?? null;
}
public function getError(int $sourceId): ?string
{
return $this->statuses[$sourceId]['error'] ?? null;
}
public function clear(): void
{
$this->statuses = [];
}
}

View File

@@ -0,0 +1,154 @@
<?php
namespace App\Tests\Domain\Scraping\Application\CommandHandler;
use App\Domain\Scraping\Application\Command\CheckAllScrapersHealth;
use App\Domain\Scraping\Application\CommandHandler\CheckAllScrapersHealthHandler;
use App\Domain\Scraping\Application\CommandHandler\TestScraperConfigurationHandler;
use App\Domain\Scraping\Domain\Model\ValueObject\ContentSourceHealthCheckData;
use App\Tests\Domain\Scraping\Adapter\InMemoryContentSourceForHealthCheckRepository;
use App\Tests\Domain\Scraping\Adapter\InMemoryContentSourceHealthRepository;
use App\Tests\Domain\Scraping\Adapter\InMemoryScraperAdapter;
use App\Tests\Domain\Scraping\Adapter\InMemoryScraperFactory;
use PHPUnit\Framework\TestCase;
use Psr\Log\NullLogger;
class CheckAllScrapersHealthHandlerTest extends TestCase
{
private InMemoryContentSourceForHealthCheckRepository $sourceRepo;
private InMemoryContentSourceHealthRepository $healthRepo;
private InMemoryScraperFactory $scraperFactory;
private CheckAllScrapersHealthHandler $handler;
protected function setUp(): void
{
$this->sourceRepo = new InMemoryContentSourceForHealthCheckRepository();
$this->healthRepo = new InMemoryContentSourceHealthRepository();
$this->scraperFactory = new InMemoryScraperFactory();
$this->scraperFactory->addScraper('html', new InMemoryScraperAdapter());
$testScraperHandler = new TestScraperConfigurationHandler($this->scraperFactory);
$this->handler = new CheckAllScrapersHealthHandler(
$this->sourceRepo,
$this->healthRepo,
$testScraperHandler,
new NullLogger(),
);
}
public function testSourceWithoutTestSlugIsSkipped(): void
{
$this->sourceRepo->add(new ContentSourceHealthCheckData(
id: 1,
baseUrl: 'https://example.com',
chapterUrlFormat: 'https://example.com/{slug}/{chapterNumber}',
scrapingType: 'html',
imageSelector: 'img',
nextPageSelector: null,
chapterSelector: null,
testSlug: null,
testChapterNumber: null,
));
$this->handler->handle(new CheckAllScrapersHealth());
$this->assertNull($this->healthRepo->getStatus(1));
}
public function testSourceWithTestSlugIsMarkedAsHealthyOnSuccess(): void
{
$this->sourceRepo->add(new ContentSourceHealthCheckData(
id: 2,
baseUrl: 'https://example.com',
chapterUrlFormat: 'https://example.com/{slug}/{chapterNumber}',
scrapingType: 'html',
imageSelector: 'img',
nextPageSelector: null,
chapterSelector: null,
testSlug: 'one-piece',
testChapterNumber: 1.0,
));
$this->handler->handle(new CheckAllScrapersHealth());
$this->assertSame('ok', $this->healthRepo->getStatus(2));
$this->assertNull($this->healthRepo->getError(2));
}
public function testSourceIsMarkedAsUnhealthyWhenScraperThrows(): void
{
$failingScraper = new InMemoryScraperAdapter();
$failingScraper->simulateError(new \RuntimeException('Connexion refusée'));
$this->scraperFactory->addScraper('html', $failingScraper);
$this->sourceRepo->add(new ContentSourceHealthCheckData(
id: 3,
baseUrl: 'https://example.com',
chapterUrlFormat: 'https://example.com/{slug}/{chapterNumber}',
scrapingType: 'html',
imageSelector: 'img',
nextPageSelector: null,
chapterSelector: null,
testSlug: 'one-piece',
testChapterNumber: 1.0,
));
$this->handler->handle(new CheckAllScrapersHealth());
$this->assertSame('ko', $this->healthRepo->getStatus(3));
$this->assertNotNull($this->healthRepo->getError(3));
}
public function testMultipleSourcesAreAllProcessed(): void
{
$this->sourceRepo->add(new ContentSourceHealthCheckData(
id: 10,
baseUrl: 'https://siteA.com',
chapterUrlFormat: 'https://siteA.com/{slug}/{chapterNumber}',
scrapingType: 'html',
imageSelector: 'img',
nextPageSelector: null,
chapterSelector: null,
testSlug: 'manga-a',
testChapterNumber: 1.0,
));
$this->sourceRepo->add(new ContentSourceHealthCheckData(
id: 11,
baseUrl: 'https://siteB.com',
chapterUrlFormat: 'https://siteB.com/{slug}/{chapterNumber}',
scrapingType: 'html',
imageSelector: 'img',
nextPageSelector: null,
chapterSelector: null,
testSlug: null,
testChapterNumber: null,
));
$this->sourceRepo->add(new ContentSourceHealthCheckData(
id: 12,
baseUrl: 'https://siteC.com',
chapterUrlFormat: 'https://siteC.com/{slug}/{chapterNumber}',
scrapingType: 'html',
imageSelector: 'img',
nextPageSelector: null,
chapterSelector: null,
testSlug: 'manga-c',
testChapterNumber: 3.0,
));
$this->handler->handle(new CheckAllScrapersHealth());
$this->assertSame('ok', $this->healthRepo->getStatus(10));
$this->assertNull($this->healthRepo->getStatus(11)); // skippée
$this->assertSame('ok', $this->healthRepo->getStatus(12));
}
protected function tearDown(): void
{
$this->sourceRepo->clear();
$this->healthRepo->clear();
$this->scraperFactory->clear();
}
}

View File

@@ -0,0 +1,72 @@
<?php
declare(strict_types=1);
namespace App\Tests\Feature\Scraping;
use App\Entity\ContentSource;
use App\Tests\Feature\AbstractApiTestCase;
use Symfony\Component\HttpFoundation\Response;
use Zenstruck\Foundry\Test\ResetDatabase;
final class CheckAllScrapersHealthTest extends AbstractApiTestCase
{
use ResetDatabase;
private function post(): void
{
static::createClient()->request('POST', '/api/scraping/check-all-health', [
'json' => new \stdClass(),
]);
}
public function testItReturns202WithNoSources(): void
{
$this->post();
$this->assertResponseStatusCodeSame(Response::HTTP_ACCEPTED);
}
public function testItReturns202WithSourcesHavingNoTestConfig(): void
{
$source = new ContentSource();
$source->setBaseUrl('https://example.com')
->setChapterUrlFormat('https://example.com/{slug}/{chapterNumber}')
->setScrapingType('html');
$this->entityManager->persist($source);
$this->entityManager->flush();
$this->post();
$this->assertResponseStatusCodeSame(Response::HTTP_ACCEPTED);
// La source sans testSlug ne doit pas avoir son statut modifié
$this->entityManager->clear();
$reloaded = $this->entityManager->find(ContentSource::class, $source->getId());
$this->assertSame('unknown', $reloaded->getHealthStatus());
}
public function testHealthStatusIsUpdatedForSourcesWithTestConfig(): void
{
$source = new ContentSource();
$source->setBaseUrl('https://example.com')
->setChapterUrlFormat('https://example.com/{slug}/{chapterNumber}')
->setScrapingType('html')
->setTestSlug('one-piece')
->setTestChapterNumber(1.0);
$this->entityManager->persist($source);
$this->entityManager->flush();
$this->post();
$this->assertResponseStatusCodeSame(Response::HTTP_ACCEPTED);
// Le statut ne doit plus être 'unknown' après le test
$this->entityManager->clear();
$reloaded = $this->entityManager->find(ContentSource::class, $source->getId());
$this->assertNotSame('unknown', $reloaded->getHealthStatus());
$this->assertNotSame('testing', $reloaded->getHealthStatus()); // doit être terminé
}
}