refactor(scraping): job PENDING dès le POST HTTP, handler sans Doctrine #27

Merged
colgora merged 1 commits from refactor/scraping-ddd-pending-job into main 2026-03-17 15:38:56 +01:00
10 changed files with 252 additions and 356 deletions

View File

@@ -3,11 +3,14 @@ import { ApiJobRepository } from '../../infrastructure/api/ApiJobRepository';
const jobRepository = new ApiJobRepository(); const jobRepository = new ApiJobRepository();
const ACTIVE_STATUSES = ['pending', 'in_progress'];
export const useActivityStore = defineStore('activity', { export const useActivityStore = defineStore('activity', {
state: () => ({ state: () => ({
jobs: [], jobs: [],
loading: false, loading: false,
error: null, error: null,
mercureEventSource: null,
// Pagination // Pagination
currentPage: 1, currentPage: 1,
totalPages: 0, totalPages: 0,
@@ -15,21 +18,15 @@ export const useActivityStore = defineStore('activity', {
limit: 20, limit: 20,
hasNextPage: false, hasNextPage: false,
hasPreviousPage: false, hasPreviousPage: false,
// Filtres // Tri
filter: { sortBy: 'createdAt',
status: ['pending', 'in_progress'], // Par défaut, ne montrer que les actifs sortOrder: 'DESC',
sortBy: 'createdAt',
sortOrder: 'DESC'
}
}), }),
getters: { getters: {
activeJobs: state => state.jobs.filter(job => job.isActive()), activeJobs: state => state.jobs.filter(job => job.isActive()),
completedJobs: state => state.jobs.filter(job => job.isCompleted()),
failedJobs: state => state.jobs.filter(job => job.hasError()),
isLoading: state => state.loading, isLoading: state => state.loading,
hasError: state => !!state.error, hasError: state => !!state.error,
// Getters pour la pagination
paginationInfo: state => ({ paginationInfo: state => ({
currentPage: state.currentPage, currentPage: state.currentPage,
totalPages: state.totalPages, totalPages: state.totalPages,
@@ -41,44 +38,25 @@ export const useActivityStore = defineStore('activity', {
}, },
actions: { actions: {
/**
* Charge la liste des jobs selon les filtres actuels
* @param {number} page - Numéro de page optionnel
*/
async loadJobs(page = null) { async loadJobs(page = null) {
this.loading = true; this.loading = true;
this.error = null; this.error = null;
try { try {
const options = { const jobCollection = await jobRepository.getJobs({
page: page || this.currentPage, page: page || this.currentPage,
limit: this.limit, limit: this.limit,
sortBy: this.filter.sortBy, sortBy: this.sortBy,
sortOrder: this.filter.sortOrder, sortOrder: this.sortOrder,
status: this.filter.status status: ACTIVE_STATUSES,
}; });
const jobCollection = await jobRepository.getJobs(options);
// Mettre à jour les données
this.jobs = jobCollection.items; this.jobs = jobCollection.items;
this.currentPage = jobCollection.page; this.currentPage = jobCollection.page;
this.total = jobCollection.total; this.total = jobCollection.total;
this.hasNextPage = jobCollection.hasNextPage; this.hasNextPage = jobCollection.hasNextPage;
this.hasPreviousPage = jobCollection.hasPreviousPage; this.hasPreviousPage = jobCollection.hasPreviousPage;
// Calculer le nombre total de pages
this.totalPages = Math.ceil(this.total / this.limit); this.totalPages = Math.ceil(this.total / this.limit);
console.log('Store updated with:', {
jobs: this.jobs.length,
currentPage: this.currentPage,
total: this.total,
limit: this.limit,
totalPages: this.totalPages,
hasNextPage: this.hasNextPage,
hasPreviousPage: this.hasPreviousPage
});
} catch (error) { } catch (error) {
this.error = error.message; this.error = error.message;
console.error('Error loading jobs:', error); console.error('Error loading jobs:', error);
@@ -87,10 +65,6 @@ export const useActivityStore = defineStore('activity', {
} }
}, },
/**
* Va à une page spécifique
* @param {number} page
*/
async goToPage(page) { async goToPage(page) {
if (page >= 1 && page <= this.totalPages && page !== this.currentPage) { if (page >= 1 && page <= this.totalPages && page !== this.currentPage) {
this.currentPage = page; this.currentPage = page;
@@ -98,39 +72,26 @@ export const useActivityStore = defineStore('activity', {
} }
}, },
/** async updateSort(sortBy, sortOrder) {
* Met à jour les filtres et recharge la liste this.sortBy = sortBy;
* @param {Object} filter this.sortOrder = sortOrder;
*/ this.currentPage = 1;
async updateFilter(filter) {
this.filter = { ...this.filter, ...filter };
this.currentPage = 1; // Retourner à la première page lors du changement de filtre
await this.loadJobs(1); await this.loadJobs(1);
}, },
/**
* Met à jour la limite par page
* @param {number} limit
*/
async updateLimit(limit) { async updateLimit(limit) {
this.limit = limit; this.limit = limit;
this.currentPage = 1; // Retourner à la première page this.currentPage = 1;
await this.loadJobs(1); await this.loadJobs(1);
}, },
/**
* Supprime un job par son ID
* @param {string} id
*/
async deleteJob(id) { async deleteJob(id) {
this.loading = true; this.loading = true;
this.error = null; this.error = null;
try { try {
await jobRepository.deleteJob(id); await jobRepository.deleteJob(id);
// Supprimer le job de la liste locale
this.jobs = this.jobs.filter(job => job.id !== id); this.jobs = this.jobs.filter(job => job.id !== id);
// Recharger la page courante pour avoir les bons totaux
await this.loadJobs(this.currentPage); await this.loadJobs(this.currentPage);
} catch (error) { } catch (error) {
this.error = error.message; this.error = error.message;
@@ -140,17 +101,37 @@ export const useActivityStore = defineStore('activity', {
} }
}, },
/** updateJobProgress(jobId, progress) {
* Supprime tous les jobs correspondant aux critères const job = this.jobs.find(j => j.id === jobId);
* @param {Object} criteria if (job) job.progress = progress;
*/ },
subscribeMercure() {
if (this.mercureEventSource) return;
const url = new URL('/.well-known/mercure', window.location.origin);
url.searchParams.append('topic', 'jobs/activity');
this.mercureEventSource = new EventSource(url.toString());
this.mercureEventSource.onmessage = (event) => {
const data = JSON.parse(event.data);
if (data.type === 'job.progress_updated') {
this.updateJobProgress(data.jobId, data.progress);
}
};
},
unsubscribeMercure() {
if (this.mercureEventSource) {
this.mercureEventSource.close();
this.mercureEventSource = null;
}
},
async deleteJobs(criteria = {}) { async deleteJobs(criteria = {}) {
this.loading = true; this.loading = true;
this.error = null; this.error = null;
try { try {
const deleted = await jobRepository.deleteJobs(criteria); const deleted = await jobRepository.deleteJobs(criteria);
// Recharger la liste après suppression
await this.loadJobs(this.currentPage); await this.loadJobs(this.currentPage);
return deleted; return deleted;
} catch (error) { } catch (error) {
@@ -160,26 +141,5 @@ export const useActivityStore = defineStore('activity', {
this.loading = false; this.loading = false;
} }
}, },
/**
* Supprime tous les jobs terminés
*/
async deleteCompletedJobs() {
return this.deleteJobs({ status: ['COMPLETED'] });
},
/**
* Supprime tous les jobs en erreur
*/
async deleteFailedJobs() {
return this.deleteJobs({ status: ['ERROR'] });
},
/**
* Supprime tous les jobs
*/
async deleteAllJobs() {
return this.deleteJobs({});
}
} }
}); });

View File

@@ -1,169 +1,153 @@
<template> <template>
<div class="overflow-y-auto h-full"> <div class="flex flex-col h-full">
<Toolbar :config="toolbarConfig" class="mb-6" /> <Toolbar :config="toolbarConfig" />
<div v-if="activityStore.loading" class="flex justify-center py-8"> <div class="overflow-y-auto flex-1">
<div class="animate-spin rounded-full h-10 w-10 border-t-2 border-b-2 border-indigo-500"></div> <!-- Loading -->
</div> <div v-if="loading" class="flex justify-center py-12">
<div class="animate-spin h-10 w-10 border-b-2 border-indigo-500 rounded-full"></div>
</div>
<div v-else-if="activityStore.error" class="bg-red-100 dark:bg-red-900/20 border-l-4 border-red-500 text-red-700 dark:text-red-400 p-4 mb-6"> <!-- Error -->
<p>{{ activityStore.error }}</p> <div v-else-if="activityStore.error" class="px-6 py-8">
</div> <div class="bg-red-50 dark:bg-red-900 border border-red-200 dark:border-red-700 p-4">
<p class="text-red-800 dark:text-red-200">{{ activityStore.error }}</p>
</div>
</div>
<div v-else class="container mx-auto p-2"> <!-- Content -->
<div class="bg-white dark:bg-gray-800 overflow-hidden shadow rounded-lg"> <section v-else class="border-t border-gray-200 dark:border-gray-700">
<div class="overflow-x-auto"> <!-- Empty -->
<table class="min-w-full bg-white dark:bg-gray-800"> <div v-if="activityStore.jobs.length === 0" class="flex flex-col items-center justify-center py-20 text-gray-400 dark:text-gray-500">
<ClockIcon class="w-12 h-12 mb-3" />
<p class="text-base">Aucun job en cours ou en attente.</p>
</div>
<!-- Table -->
<div v-else class="overflow-x-auto">
<table class="min-w-full">
<thead> <thead>
<tr class="bg-gray-200 dark:bg-gray-700 text-gray-800 dark:text-gray-200"> <tr class="border-b border-gray-200 dark:border-gray-700 text-xs font-semibold text-gray-400 dark:text-gray-500 uppercase tracking-wider">
<th class="w-1/12 py-3 px-4 text-left"> <th class="w-2/11 py-3 px-6 text-left">Type</th>
<input <th class="w-2/11 py-3 px-4 text-left">Statut</th>
type="checkbox" <th class="w-3/11 py-3 px-4 text-left">Informations</th>
class="form-checkbox h-5 w-5 text-green-600" <th class="w-3/11 py-3 px-4 text-left">Progression</th>
@change="toggleSelectAll" /> <th class="w-1/11 py-3 px-4 text-left">Actions</th>
</th>
<th class="w-2/12 py-3 px-4 text-left">Type</th>
<th class="w-2/12 py-3 px-4 text-left">Statut</th>
<th class="w-3/12 py-3 px-4 text-left">Informations</th>
<th class="w-3/12 py-3 px-4 text-left">Progression</th>
<th class="w-1/12 py-3 px-4 text-left">Actions</th>
</tr> </tr>
</thead> </thead>
<tbody class="text-gray-700 dark:text-gray-300"> <tbody class="divide-y divide-gray-100 dark:divide-gray-700/50 text-gray-700 dark:text-gray-300">
<template v-if="activityStore.jobs.length === 0"> <JobItem
<tr> v-for="job in activityStore.jobs"
<td colspan="6" class="py-8 px-4 text-center text-gray-500"> :key="job.id"
<div class="flex flex-col items-center"> :job="job"
<ClockIcon class="h-12 w-12 text-gray-300 dark:text-gray-600 mb-4" /> @delete="deleteJob" />
<p class="text-lg font-medium dark:text-gray-300">Aucune activité trouvée</p>
<p class="text-sm dark:text-gray-400">Aucune activité ne correspond aux filtres actuels.</p>
</div>
</td>
</tr>
</template>
<template v-else>
<JobItem
v-for="job in activityStore.jobs"
:key="job.id"
:job="job"
@delete="deleteJob" />
</template>
</tbody> </tbody>
</table> </table>
</div> </div>
<!-- Pagination --> <!-- Pagination -->
<Pagination <Pagination
v-if="activityStore.total > activityStore.limit" v-if="total > activityStore.limit"
:current-page="activityStore.currentPage" :current-page="activityStore.currentPage"
:total-pages="activityStore.totalPages" :total-pages="activityStore.totalPages"
:total="activityStore.total" :total="total"
:limit="activityStore.limit" :limit="activityStore.limit"
:has-next-page="activityStore.hasNextPage" :has-next-page="activityStore.hasNextPage"
:has-previous-page="activityStore.hasPreviousPage" :has-previous-page="activityStore.hasPreviousPage"
@page-change="changePage" /> @page-change="changePage" />
</div> </section>
</div> </div>
</div> </div>
</template> </template>
<script setup> <script setup>
import { ArrowPathIcon, ClockIcon, FunnelIcon, TrashIcon } from '@heroicons/vue/24/outline'; import { ArrowPathIcon, BarsArrowDownIcon, ClockIcon, TrashIcon } from '@heroicons/vue/24/outline';
import { computed, onMounted, ref } from 'vue'; import { storeToRefs } from 'pinia';
import { computed, onMounted, onUnmounted } from 'vue';
import Pagination from '../../../../shared/components/ui/Pagination.vue'; import Pagination from '../../../../shared/components/ui/Pagination.vue';
import Toolbar from '../../../../shared/components/ui/Toolbar.vue'; import Toolbar from '../../../../shared/components/ui/Toolbar.vue';
import { useActivityStore } from '../../application/store/activityStore'; import { useActivityStore } from '../../application/store/activityStore';
import JobItem from '../components/JobItem.vue'; import JobItem from '../components/JobItem.vue';
const activityStore = useActivityStore(); const activityStore = useActivityStore();
const selectedAll = ref(false);
// Statuts disponibles pour le filtre const { sortBy, sortOrder, total, loading } = storeToRefs(activityStore);
const statusOptions = [
{ value: ['pending', 'in_progress'], label: 'Actifs' },
{ value: ['pending', 'in_progress', 'completed', 'failed'], label: 'Tous' },
{ value: ['completed'], label: 'Terminés' },
{ value: ['failed'], label: 'En erreur' },
{ value: ['pending'], label: 'En attente' },
{ value: ['in_progress'], label: 'En cours' }
];
// Index du statut actif (par défaut "Actifs") const isSortSelected = (by, order) => sortBy.value === by && sortOrder.value === order;
const activeStatusIndex = ref(0);
// Configuration de la toolbar réactive const toolbarConfig = computed(() => ({
const toolbarConfig = computed(() => ({ leftSection: [
leftSection: [ { type: 'label', text: 'Activité', class: 'text-sm font-medium' },
{ { type: 'label', text: `(${total.value})`, class: 'text-sm text-gray-400' },
icon: FunnelIcon, ],
type: 'dropdown', rightSection: [
label: statusOptions[activeStatusIndex.value].label, {
active: false, type: 'dropdown',
items: statusOptions.map((option, index) => ({ icon: BarsArrowDownIcon,
label: option.label, label: 'Trier',
isSelected: index === activeStatusIndex.value, items: [
onClick: () => setStatusFilter(index) {
})) label: 'Plus récent',
} isSelected: isSortSelected('createdAt', 'DESC'),
], onClick: () => activityStore.updateSort('createdAt', 'DESC'),
rightSection: [ },
{ {
icon: ArrowPathIcon, label: 'Plus ancien',
type: 'button', isSelected: isSortSelected('createdAt', 'ASC'),
label: 'Rafraîchir', onClick: () => activityStore.updateSort('createdAt', 'ASC'),
onClick: refreshJobs },
}, {
{ label: 'Par type',
icon: TrashIcon, isSelected: isSortSelected('type', 'ASC'),
type: 'button', onClick: () => activityStore.updateSort('type', 'ASC'),
label: 'Supprimer visibles', },
onClick: deleteVisibleJobs {
} label: 'Par statut',
] isSelected: isSortSelected('status', 'ASC'),
})); onClick: () => activityStore.updateSort('status', 'ASC'),
},
],
},
{
type: 'button',
icon: ArrowPathIcon,
label: 'Rafraîchir',
disabled: loading.value,
onClick: () => activityStore.loadJobs(),
},
{
type: 'button',
icon: TrashIcon,
label: 'Supprimer visibles',
disabled: loading.value || total.value === 0,
onClick: deleteVisibleJobs,
},
],
}));
onMounted(() => { onMounted(() => {
loadJobs(); activityStore.loadJobs();
}); activityStore.subscribeMercure();
});
function loadJobs() { onUnmounted(() => {
activityStore.loadJobs(); activityStore.unsubscribeMercure();
});
function changePage(page) {
activityStore.goToPage(page);
}
function deleteJob(id) {
if (confirm('Voulez-vous vraiment supprimer ce job ?')) {
activityStore.deleteJob(id);
} }
}
function refreshJobs() { function deleteVisibleJobs() {
loadJobs(); if (activityStore.jobs.length === 0) return;
} if (confirm('Voulez-vous vraiment supprimer tous les jobs visibles ?')) {
activityStore.deleteJobs({ status: ['pending', 'in_progress'] });
function changePage(page) {
activityStore.goToPage(page);
}
function toggleSelectAll() {
selectedAll.value = !selectedAll.value;
// La logique pour sélectionner tous les jobs serait ajoutée ici
}
function setStatusFilter(index) {
if (index >= 0 && index < statusOptions.length) {
activeStatusIndex.value = index;
activityStore.updateFilter({ status: statusOptions[index].value });
}
}
function deleteJob(id) {
if (confirm('Voulez-vous vraiment supprimer ce job ?')) {
activityStore.deleteJob(id);
}
}
function deleteVisibleJobs() {
if (activityStore.jobs.length === 0) {
return;
}
const statusLabel = statusOptions[activeStatusIndex.value].label.toLowerCase();
if (confirm(`Voulez-vous vraiment supprimer tous les jobs ${statusLabel} ?`)) {
activityStore.deleteJobs({ status: activityStore.filter.status });
}
} }
}
</script> </script>

View File

@@ -17,7 +17,6 @@ framework:
command.bus: command.bus:
middleware: middleware:
- validation - validation
- doctrine_transaction
event.bus: event.bus:
default_middleware: allow_no_handlers default_middleware: allow_no_handlers

View File

@@ -5,7 +5,8 @@ namespace App\Domain\Scraping\Application\Command;
readonly class ScrapeChapter readonly class ScrapeChapter
{ {
public function __construct( public function __construct(
public string $chapterId public string $chapterId,
public string $jobId
) { ) {
} }
} }

View File

@@ -13,14 +13,11 @@ use App\Domain\Shared\Domain\Event\ChapterScraped;
use App\Domain\Scraping\Domain\Event\ChapterScrapingFailed; use App\Domain\Scraping\Domain\Event\ChapterScrapingFailed;
use App\Domain\Scraping\Domain\Event\ChapterScrapingStarted; use App\Domain\Scraping\Domain\Event\ChapterScrapingStarted;
use App\Domain\Scraping\Domain\Model\Chapter; use App\Domain\Scraping\Domain\Model\Chapter;
use App\Domain\Scraping\Domain\Model\ScrapingJob;
use App\Domain\Scraping\Domain\Model\Source; use App\Domain\Scraping\Domain\Model\Source;
use App\Domain\Scraping\Domain\Model\ValueObject\ScrapingRequest; use App\Domain\Scraping\Domain\Model\ValueObject\ScrapingRequest;
use App\Domain\Scraping\Domain\Model\ValueObject\TempDirectory; use App\Domain\Scraping\Domain\Model\ValueObject\TempDirectory;
use App\Domain\Shared\Domain\Contract\JobRepositoryInterface; use App\Domain\Shared\Domain\Contract\JobRepositoryInterface;
use Ramsey\Uuid\Uuid;
use Symfony\Component\Messenger\MessageBusInterface; use Symfony\Component\Messenger\MessageBusInterface;
use Doctrine\ORM\EntityManagerInterface;
readonly class ScrapeChapterHandler readonly class ScrapeChapterHandler
{ {
@@ -33,151 +30,92 @@ readonly class ScrapeChapterHandler
private MangaRepositoryInterface $mangaRepository, private MangaRepositoryInterface $mangaRepository,
private SourceRepositoryInterface $sourceRepository, private SourceRepositoryInterface $sourceRepository,
private MessageBusInterface $eventBus, private MessageBusInterface $eventBus,
private EntityManagerInterface $entityManager
) { ) {
} }
public function handle(ScrapeChapter $command): void public function handle(ScrapeChapter $command): void
{ {
$job = null; /** @var Chapter $chapter */
try { $chapter = $this->chapterRepository->getById($command->chapterId);
// 1. Récupération du chapitre $manga = $this->mangaRepository->getById($chapter->mangaId);
/**@var Chapter $chapter */
$chapter = $this->chapterRepository->getById($command->chapterId);
if (!$chapter) {
throw new \InvalidArgumentException("Chapter not found with ID: {$command->chapterId}");
}
// 2. Récupération du manga $job = $this->jobRepository->get($command->jobId);
$manga = $this->mangaRepository->getById($chapter->mangaId); $job->context['chapterId'] = $command->chapterId;
if (!$manga) { $job->context['mangaTitle'] = $manga->getTitle();
throw new \InvalidArgumentException("Manga not found with ID: {$chapter->mangaId}"); $job->start();
} $this->jobRepository->save($job);
// 3. Dispatch de l'événement de démarrage $this->eventBus->dispatch(new ChapterScrapingStarted($manga->getTitle(), $chapter->chapterNumber));
$this->eventBus->dispatch(new ChapterScrapingStarted($manga->getTitle(), $chapter->chapterNumber));
// 4. Détermination des sources à utiliser $sources = $this->getSourcesToTry($manga);
$sources = $this->getSourcesToTry($manga); $slugsToTry = array_merge([$manga->getSlug()], $manga->getAlternativeSlugs());
if (empty($sources)) { $success = false;
throw new \InvalidArgumentException("No sources available for scraping"); $lastException = null;
}
// 5. Essai de scraping sur chaque source jusqu'à succès foreach ($sources as $source) {
$success = false; foreach ($slugsToTry as $slug) {
$lastException = null; try {
$job->context['sourceId'] = $source->getId()->getValue();
foreach ($sources as $source) {
// Préparer la liste des slugs à essayer : slug principal + slugs alternatifs
$slugsToTry = array_merge([$manga->getSlug()], $manga->getAlternativeSlugs());
foreach ($slugsToTry as $slug) {
$job = new ScrapingJob(
Uuid::uuid4()->toString(),
$chapter->mangaId,
$chapter->chapterNumber,
$source->getId()->getValue()
);
// Ajout de l'ID du chapitre et du slug dans le contexte du job
$job->context['chapterId'] = $command->chapterId;
$job->context['slug'] = $slug; $job->context['slug'] = $slug;
$job->context['mangaTitle'] = $manga->getTitle();
$job->start();
$this->jobRepository->save($job); $this->jobRepository->save($job);
try { $scrapingParameters = $source->getScrappingParameters();
$this->entityManager->beginTransaction(); $scrapingParameters['chapterNumber'] = $chapter->chapterNumber;
$scrapingType = $scrapingParameters['scrapingType'] ?? 'html';
// 5. Scraping des URLs avec le slug courant $scrapingRequest = new ScrapingRequest(
$scrapingParameters = $source->getScrappingParameters(); $scrapingType,
$scrapingParameters['chapterNumber'] = $chapter->chapterNumber; $source->buildChapterUrl($slug, $chapter->chapterNumber),
$scrapingType = $scrapingParameters['scrapingType'] ?? 'html'; $scrapingParameters
);
$scrapingRequest = new ScrapingRequest( $scraper = $this->scraperFactory->getScraperWithFallback($scrapingType);
$scrapingType, $scrapingResult = $scraper->scrape($scrapingRequest);
$source->buildChapterUrl($slug, $chapter->chapterNumber),
$scrapingParameters
);
// Sélection du scraper approprié selon le type $tempDir = new TempDirectory();
$scraper = $this->scraperFactory->getScraperWithFallback($scrapingType); $downloadResults = $this->imageDownloader->downloadBatch(
$scrapingResult = $scraper->scrape($scrapingRequest); $scrapingResult->getImageUrls(),
$tempDir,
$job->id
);
// 6. Téléchargement des images $localPaths = array_map(fn ($r) => $r->getLocalPath(), $downloadResults);
$tempDir = new TempDirectory(); $pagesDirectory = $this->imageStorage->storeChapterImages($command->chapterId, $localPaths);
$downloadResults = $this->imageDownloader->downloadBatch( $pageCount = count($downloadResults);
$scrapingResult->getImageUrls(),
$tempDir,
$job->id
);
// 7. Stockage des images individuelles $job->complete();
$localPaths = array_map(fn ($r) => $r->getLocalPath(), $downloadResults); $this->jobRepository->save($job);
$pagesDirectory = $this->imageStorage->storeChapterImages($command->chapterId, $localPaths);
$pageCount = count($downloadResults);
$job->complete(); $this->eventBus->dispatch(new ChapterScraped($job->id, $command->chapterId, $pagesDirectory, $pageCount));
$this->jobRepository->save($job); $tempDir->cleanup();
$this->entityManager->commit(); $success = true;
$this->eventBus->dispatch(new ChapterScraped($job->id, $command->chapterId, $pagesDirectory, $pageCount));
// 8. Nettoyage
$tempDir->cleanup();
// Scraping réussi, pas besoin d'essayer d'autres slugs ni d'autres sources
$success = true;
break;
} catch (\Exception $e) {
dump('EXCEPTION for source ' . $source->getName() . ' with slug ' . $slug . ': ' . $e->getMessage());
$this->entityManager->rollback();
if (isset($job)) {
$job->fail($e->getMessage());
$this->jobRepository->save($job);
}
$lastException = $e;
// Continuer avec le slug suivant pour cette source
}
}
// Si le scraping a réussi avec un des slugs, sortir de la boucle des sources
if ($success) {
break; break;
} catch (\Exception $e) {
$lastException = $e;
} }
} }
// Si toutes les sources ont échoué if ($success) {
if (!$success) { break;
$errorMessage = $lastException ? $lastException->getMessage() : "Failed to scrape chapter from all available sources";
$this->eventBus->dispatch(new ChapterScrapingFailed($chapter->mangaId, $chapter->chapterNumber, $errorMessage));
} }
}
} catch (\Exception $e) { if (!$success) {
if (isset($job)) { $errorMessage = $lastException?->getMessage() ?? 'Failed to scrape chapter from all available sources';
$job->fail($e->getMessage()); $job->fail($errorMessage);
$this->jobRepository->save($job); $this->jobRepository->save($job);
} $this->eventBus->dispatch(new ChapterScrapingFailed($chapter->mangaId, $chapter->chapterNumber, $errorMessage));
$this->eventBus->dispatch(new ChapterScrapingFailed($chapter->mangaId ?? 'unknown', $chapter->chapterNumber ?? 'unknown', $e->getMessage()));
} }
} }
/** /**
* Détermine les sources à utiliser pour le scraping en fonction des préférences du manga
*
* @param \App\Domain\Scraping\Domain\Model\Manga $manga * @param \App\Domain\Scraping\Domain\Model\Manga $manga
* @return Source[] * @return Source[]
*/ */
private function getSourcesToTry(\App\Domain\Scraping\Domain\Model\Manga $manga): array private function getSourcesToTry(\App\Domain\Scraping\Domain\Model\Manga $manga): array
{ {
// Si le manga a des sources préférées, les utiliser
if ($manga->hasPreferredSources()) { if ($manga->hasPreferredSources()) {
$preferredSources = []; $preferredSources = [];
foreach ($manga->getPreferredSources() as $sourceId) { foreach ($manga->getPreferredSources() as $sourceId) {
@@ -186,7 +124,6 @@ readonly class ScrapeChapterHandler
$preferredSources[] = $source; $preferredSources[] = $source;
} }
// Limiter à 3 sources préférées maximum
if (count($preferredSources) >= 3) { if (count($preferredSources) >= 3) {
break; break;
} }
@@ -197,7 +134,6 @@ readonly class ScrapeChapterHandler
} }
} }
// Sinon, utiliser toutes les sources disponibles
return $this->sourceRepository->getAll(); return $this->sourceRepository->getAll();
} }
} }

View File

@@ -8,9 +8,9 @@ class ScrapingJob extends Job
{ {
public function __construct( public function __construct(
string $id, string $id,
string $mangaId, ?string $mangaId = null,
float $chapterNumber, ?float $chapterNumber = null,
string $sourceId ?string $sourceId = null
) { ) {
parent::__construct($id, 'scraping_job'); parent::__construct($id, 'scraping_job');
$this->maxAttempts = 1; $this->maxAttempts = 1;

View File

@@ -5,13 +5,17 @@ namespace App\Domain\Scraping\Infrastructure\ApiPlatform\State\Processor;
use ApiPlatform\Metadata\Operation; use ApiPlatform\Metadata\Operation;
use ApiPlatform\State\ProcessorInterface; use ApiPlatform\State\ProcessorInterface;
use App\Domain\Scraping\Application\Command\ScrapeChapter; use App\Domain\Scraping\Application\Command\ScrapeChapter;
use App\Domain\Scraping\Domain\Model\ScrapingJob;
use App\Domain\Scraping\Infrastructure\ApiPlatform\Dto\ScrapeChapterRequest; use App\Domain\Scraping\Infrastructure\ApiPlatform\Dto\ScrapeChapterRequest;
use App\Domain\Shared\Domain\Contract\JobRepositoryInterface;
use Ramsey\Uuid\Uuid;
use Symfony\Component\Messenger\MessageBusInterface; use Symfony\Component\Messenger\MessageBusInterface;
final class ScrapeChapterStateProcessor implements ProcessorInterface final class ScrapeChapterStateProcessor implements ProcessorInterface
{ {
public function __construct( public function __construct(
private readonly MessageBusInterface $commandBus private readonly MessageBusInterface $commandBus,
private readonly JobRepositoryInterface $jobRepository,
) { ) {
} }
@@ -20,10 +24,11 @@ final class ScrapeChapterStateProcessor implements ProcessorInterface
*/ */
public function process(mixed $data, Operation $operation, array $uriVariables = [], array $context = []): void public function process(mixed $data, Operation $operation, array $uriVariables = [], array $context = []): void
{ {
$this->commandBus->dispatch( $jobId = Uuid::uuid4()->toString();
new ScrapeChapter( $job = new ScrapingJob($jobId);
$data->chapterId $job->context['chapterId'] = $data->chapterId;
) $this->jobRepository->save($job);
);
$this->commandBus->dispatch(new ScrapeChapter($data->chapterId, $jobId));
} }
} }

View File

@@ -5,6 +5,7 @@ namespace App\Domain\Scraping\Infrastructure\EventSubscriber;
use App\Domain\Shared\Domain\Event\ChapterScraped; use App\Domain\Shared\Domain\Event\ChapterScraped;
use App\Domain\Scraping\Domain\Event\ChapterScrapingFailed; use App\Domain\Scraping\Domain\Event\ChapterScrapingFailed;
use App\Domain\Scraping\Domain\Event\ChapterScrapingStarted; use App\Domain\Scraping\Domain\Event\ChapterScrapingStarted;
use App\Domain\Scraping\Domain\Event\PageScrapingProgressed;
use App\Domain\Scraping\Domain\Contract\Repository\ChapterRepositoryInterface; use App\Domain\Scraping\Domain\Contract\Repository\ChapterRepositoryInterface;
use App\Domain\Shared\Domain\Contract\JobRepositoryInterface; use App\Domain\Shared\Domain\Contract\JobRepositoryInterface;
use App\Domain\Shared\Domain\Contract\NotificationInterface; use App\Domain\Shared\Domain\Contract\NotificationInterface;
@@ -30,6 +31,22 @@ class ScrapingEventSubscriber implements EventSubscriberInterface
return []; return [];
} }
#[AsMessageHandler]
public function onPageScrapingProgressed(PageScrapingProgressed $event): void
{
$progress = (int) round($event->getProgress()->getPercentage());
$update = new Update(
'jobs/activity',
json_encode([
'type' => 'job.progress_updated',
'jobId' => $event->getJobId(),
'progress' => $progress,
])
);
$this->hub->publish($update);
}
#[AsMessageHandler] #[AsMessageHandler]
public function onChapterScrapingStarted(ChapterScrapingStarted $event): void public function onChapterScrapingStarted(ChapterScrapingStarted $event): void
{ {

View File

@@ -7,6 +7,7 @@ use App\Domain\Scraping\Application\CommandHandler\ScrapeChapterHandler;
use App\Domain\Scraping\Domain\Event\ChapterScrapingFailed; use App\Domain\Scraping\Domain\Event\ChapterScrapingFailed;
use App\Domain\Scraping\Domain\Event\ChapterScrapingStarted; use App\Domain\Scraping\Domain\Event\ChapterScrapingStarted;
use App\Domain\Scraping\Domain\Model\Chapter; use App\Domain\Scraping\Domain\Model\Chapter;
use App\Domain\Scraping\Domain\Model\ScrapingJob;
use App\Domain\Shared\Domain\Event\ChapterScraped; use App\Domain\Shared\Domain\Event\ChapterScraped;
use App\Tests\Domain\Scraping\Adapter\InMemoryChapterRepository; use App\Tests\Domain\Scraping\Adapter\InMemoryChapterRepository;
use App\Tests\Domain\Scraping\Adapter\InMemoryEventBus; use App\Tests\Domain\Scraping\Adapter\InMemoryEventBus;
@@ -16,8 +17,6 @@ use App\Tests\Domain\Scraping\Adapter\InMemoryMangaRepository;
use App\Tests\Domain\Scraping\Adapter\InMemoryScraperFactory; use App\Tests\Domain\Scraping\Adapter\InMemoryScraperFactory;
use App\Tests\Domain\Scraping\Adapter\InMemorySourceRepository; use App\Tests\Domain\Scraping\Adapter\InMemorySourceRepository;
use App\Tests\Domain\Shared\Adapter\InMemoryJobRepository; use App\Tests\Domain\Shared\Adapter\InMemoryJobRepository;
use Doctrine\ORM\EntityManagerInterface;
use PHPUnit\Framework\MockObject\MockObject;
use PHPUnit\Framework\TestCase; use PHPUnit\Framework\TestCase;
class ScrapeChapterHandlerTest extends TestCase class ScrapeChapterHandlerTest extends TestCase
@@ -30,7 +29,6 @@ class ScrapeChapterHandlerTest extends TestCase
private InMemoryMangaRepository $mangaRepository; private InMemoryMangaRepository $mangaRepository;
private InMemorySourceRepository $sourceRepository; private InMemorySourceRepository $sourceRepository;
private InMemoryEventBus $eventBus; private InMemoryEventBus $eventBus;
private EntityManagerInterface|MockObject $entityManager;
private ScrapeChapterHandler $handler; private ScrapeChapterHandler $handler;
protected function setUp(): void protected function setUp(): void
@@ -43,11 +41,6 @@ class ScrapeChapterHandlerTest extends TestCase
$this->mangaRepository = new InMemoryMangaRepository(); $this->mangaRepository = new InMemoryMangaRepository();
$this->sourceRepository = new InMemorySourceRepository(); $this->sourceRepository = new InMemorySourceRepository();
$this->eventBus = new InMemoryEventBus(); $this->eventBus = new InMemoryEventBus();
$this->entityManager = $this->createMock(EntityManagerInterface::class);
$this->entityManager->method('beginTransaction')->willReturn(null);
$this->entityManager->method('commit')->willReturn(null);
$this->entityManager->method('rollback')->willReturn(null);
$this->chapterRepository->save(new Chapter( $this->chapterRepository->save(new Chapter(
id: '1', id: '1',
@@ -65,21 +58,21 @@ class ScrapeChapterHandlerTest extends TestCase
$this->mangaRepository, $this->mangaRepository,
$this->sourceRepository, $this->sourceRepository,
$this->eventBus, $this->eventBus,
$this->entityManager
); );
} }
public function testHandleSuccessfully(): void public function testHandleSuccessfully(): void
{ {
$command = new ScrapeChapter( $jobId = 'test-job-id';
chapterId: '1' $job = new ScrapingJob($jobId, 'test-manga', 2);
); $this->jobRepository->save($job);
$command = new ScrapeChapter(chapterId: '1', jobId: $jobId);
$this->handler->handle($command); $this->handler->handle($command);
$job = $this->jobRepository->findByType('scraping_job'); $jobs = $this->jobRepository->findByType('scraping_job');
$this->assertCount(1, $job); $this->assertCount(1, $jobs);
$job = array_values($job)[0]; $job = array_values($jobs)[0];
$dispatchedMessages = $this->eventBus->getDispatchedMessages(); $dispatchedMessages = $this->eventBus->getDispatchedMessages();
$this->assertCount(2, $dispatchedMessages); $this->assertCount(2, $dispatchedMessages);

View File

@@ -35,13 +35,14 @@ class ScrapeChapterTest extends AbstractApiTestCase
// Then // Then
$this->assertResponseStatusCodeSame(202); $this->assertResponseStatusCodeSame(202);
$messages = $this->messageBus->getDispatchedMessages(); $messages = InMemoryMessageBus::$messages;
$this->assertCount(1, $messages, 'Un message devrait être dispatché'); $this->assertCount(1, $messages, 'Un message devrait être dispatché');
/** @var ScrapeChapter $message */ /** @var ScrapeChapter $message */
$message = $messages[0]; $message = $messages[0];
$this->assertInstanceOf(ScrapeChapter::class, $message); $this->assertInstanceOf(ScrapeChapter::class, $message);
$this->assertEquals('chapter-123', $message->chapterId); $this->assertEquals('chapter-123', $message->chapterId);
$this->assertNotEmpty($message->jobId);
} }
public function testInitiateChapterScrapingWithInvalidPayload(): void public function testInitiateChapterScrapingWithInvalidPayload(): void
@@ -72,6 +73,6 @@ class ScrapeChapterTest extends AbstractApiTestCase
protected function tearDown(): void protected function tearDown(): void
{ {
parent::tearDown(); parent::tearDown();
$this->messageBus->clear(); InMemoryMessageBus::$messages = [];
} }
} }