Compare commits
2 Commits
fix/reader
...
6065eb0eb1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6065eb0eb1 | ||
|
|
84c4557abf |
@@ -1,13 +1,17 @@
|
||||
import { defineStore } from 'pinia';
|
||||
import { Job } from '../../domain/entities/job';
|
||||
import { ApiJobRepository } from '../../infrastructure/api/ApiJobRepository';
|
||||
|
||||
const jobRepository = new ApiJobRepository();
|
||||
|
||||
const ACTIVE_STATUSES = ['pending', 'in_progress'];
|
||||
|
||||
export const useActivityStore = defineStore('activity', {
|
||||
state: () => ({
|
||||
jobs: [],
|
||||
loading: false,
|
||||
error: null,
|
||||
mercureEventSource: null,
|
||||
// Pagination
|
||||
currentPage: 1,
|
||||
totalPages: 0,
|
||||
@@ -15,21 +19,15 @@ export const useActivityStore = defineStore('activity', {
|
||||
limit: 20,
|
||||
hasNextPage: false,
|
||||
hasPreviousPage: false,
|
||||
// Filtres
|
||||
filter: {
|
||||
status: ['pending', 'in_progress'], // Par défaut, ne montrer que les actifs
|
||||
sortBy: 'createdAt',
|
||||
sortOrder: 'DESC'
|
||||
}
|
||||
// Tri
|
||||
sortBy: 'createdAt',
|
||||
sortOrder: 'DESC',
|
||||
}),
|
||||
|
||||
getters: {
|
||||
activeJobs: state => state.jobs.filter(job => job.isActive()),
|
||||
completedJobs: state => state.jobs.filter(job => job.isCompleted()),
|
||||
failedJobs: state => state.jobs.filter(job => job.hasError()),
|
||||
isLoading: state => state.loading,
|
||||
hasError: state => !!state.error,
|
||||
// Getters pour la pagination
|
||||
paginationInfo: state => ({
|
||||
currentPage: state.currentPage,
|
||||
totalPages: state.totalPages,
|
||||
@@ -41,44 +39,25 @@ export const useActivityStore = defineStore('activity', {
|
||||
},
|
||||
|
||||
actions: {
|
||||
/**
|
||||
* Charge la liste des jobs selon les filtres actuels
|
||||
* @param {number} page - Numéro de page optionnel
|
||||
*/
|
||||
async loadJobs(page = null) {
|
||||
this.loading = true;
|
||||
this.error = null;
|
||||
|
||||
try {
|
||||
const options = {
|
||||
const jobCollection = await jobRepository.getJobs({
|
||||
page: page || this.currentPage,
|
||||
limit: this.limit,
|
||||
sortBy: this.filter.sortBy,
|
||||
sortOrder: this.filter.sortOrder,
|
||||
status: this.filter.status
|
||||
};
|
||||
sortBy: this.sortBy,
|
||||
sortOrder: this.sortOrder,
|
||||
status: ACTIVE_STATUSES,
|
||||
});
|
||||
|
||||
const jobCollection = await jobRepository.getJobs(options);
|
||||
|
||||
// Mettre à jour les données
|
||||
this.jobs = jobCollection.items;
|
||||
this.currentPage = jobCollection.page;
|
||||
this.total = jobCollection.total;
|
||||
this.hasNextPage = jobCollection.hasNextPage;
|
||||
this.hasPreviousPage = jobCollection.hasPreviousPage;
|
||||
|
||||
// Calculer le nombre total de pages
|
||||
this.totalPages = Math.ceil(this.total / this.limit);
|
||||
|
||||
console.log('Store updated with:', {
|
||||
jobs: this.jobs.length,
|
||||
currentPage: this.currentPage,
|
||||
total: this.total,
|
||||
limit: this.limit,
|
||||
totalPages: this.totalPages,
|
||||
hasNextPage: this.hasNextPage,
|
||||
hasPreviousPage: this.hasPreviousPage
|
||||
});
|
||||
} catch (error) {
|
||||
this.error = error.message;
|
||||
console.error('Error loading jobs:', error);
|
||||
@@ -87,10 +66,6 @@ export const useActivityStore = defineStore('activity', {
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Va à une page spécifique
|
||||
* @param {number} page
|
||||
*/
|
||||
async goToPage(page) {
|
||||
if (page >= 1 && page <= this.totalPages && page !== this.currentPage) {
|
||||
this.currentPage = page;
|
||||
@@ -98,39 +73,26 @@ export const useActivityStore = defineStore('activity', {
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Met à jour les filtres et recharge la liste
|
||||
* @param {Object} filter
|
||||
*/
|
||||
async updateFilter(filter) {
|
||||
this.filter = { ...this.filter, ...filter };
|
||||
this.currentPage = 1; // Retourner à la première page lors du changement de filtre
|
||||
async updateSort(sortBy, sortOrder) {
|
||||
this.sortBy = sortBy;
|
||||
this.sortOrder = sortOrder;
|
||||
this.currentPage = 1;
|
||||
await this.loadJobs(1);
|
||||
},
|
||||
|
||||
/**
|
||||
* Met à jour la limite par page
|
||||
* @param {number} limit
|
||||
*/
|
||||
async updateLimit(limit) {
|
||||
this.limit = limit;
|
||||
this.currentPage = 1; // Retourner à la première page
|
||||
this.currentPage = 1;
|
||||
await this.loadJobs(1);
|
||||
},
|
||||
|
||||
/**
|
||||
* Supprime un job par son ID
|
||||
* @param {string} id
|
||||
*/
|
||||
async deleteJob(id) {
|
||||
this.loading = true;
|
||||
this.error = null;
|
||||
|
||||
try {
|
||||
await jobRepository.deleteJob(id);
|
||||
// Supprimer le job de la liste locale
|
||||
this.jobs = this.jobs.filter(job => job.id !== id);
|
||||
// Recharger la page courante pour avoir les bons totaux
|
||||
await this.loadJobs(this.currentPage);
|
||||
} catch (error) {
|
||||
this.error = error.message;
|
||||
@@ -140,17 +102,75 @@ export const useActivityStore = defineStore('activity', {
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Supprime tous les jobs correspondant aux critères
|
||||
* @param {Object} criteria
|
||||
*/
|
||||
updateJobProgress(jobId, progress) {
|
||||
const job = this.jobs.find(j => j.id === jobId);
|
||||
if (job) job.progress = progress;
|
||||
},
|
||||
|
||||
handleJobCreated(data) {
|
||||
const alreadyExists = this.jobs.some(j => j.id === data.id);
|
||||
if (alreadyExists) return;
|
||||
|
||||
const job = Job.create({
|
||||
id: data.id,
|
||||
type: data.type_job,
|
||||
status: data.status,
|
||||
createdAt: data.createdAt,
|
||||
context: data.context,
|
||||
attempts: data.attempts,
|
||||
maxAttempts: data.maxAttempts,
|
||||
});
|
||||
|
||||
this.jobs.unshift(job);
|
||||
this.total += 1;
|
||||
this.totalPages = Math.ceil(this.total / this.limit);
|
||||
},
|
||||
|
||||
handleJobStatusChange(jobId, newStatus) {
|
||||
const job = this.jobs.find(j => j.id === jobId);
|
||||
if (!job) return;
|
||||
|
||||
if (newStatus === 'in_progress') {
|
||||
job.status = 'in_progress';
|
||||
} else {
|
||||
setTimeout(() => {
|
||||
this.jobs = this.jobs.filter(j => j.id !== jobId);
|
||||
this.total = Math.max(0, this.total - 1);
|
||||
this.totalPages = Math.ceil(this.total / this.limit);
|
||||
}, 1500);
|
||||
}
|
||||
},
|
||||
|
||||
subscribeMercure() {
|
||||
if (this.mercureEventSource) return;
|
||||
const url = new URL('/.well-known/mercure', window.location.origin);
|
||||
url.searchParams.append('topic', 'jobs/activity');
|
||||
this.mercureEventSource = new EventSource(url.toString());
|
||||
this.mercureEventSource.onmessage = (event) => {
|
||||
const data = JSON.parse(event.data);
|
||||
if (data.type === 'job.created') {
|
||||
this.handleJobCreated(data);
|
||||
} else if (data.type === 'job.progress_updated') {
|
||||
this.updateJobProgress(data.jobId, data.progress);
|
||||
} else if (data.type === 'job.status_changed') {
|
||||
this.handleJobStatusChange(data.jobId, data.status);
|
||||
}
|
||||
};
|
||||
},
|
||||
|
||||
unsubscribeMercure() {
|
||||
if (this.mercureEventSource) {
|
||||
this.mercureEventSource.close();
|
||||
this.mercureEventSource = null;
|
||||
}
|
||||
},
|
||||
|
||||
async deleteJobs(criteria = {}) {
|
||||
this.loading = true;
|
||||
this.error = null;
|
||||
|
||||
try {
|
||||
const deleted = await jobRepository.deleteJobs(criteria);
|
||||
// Recharger la liste après suppression
|
||||
await this.loadJobs(this.currentPage);
|
||||
return deleted;
|
||||
} catch (error) {
|
||||
@@ -160,26 +180,5 @@ export const useActivityStore = defineStore('activity', {
|
||||
this.loading = false;
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Supprime tous les jobs terminés
|
||||
*/
|
||||
async deleteCompletedJobs() {
|
||||
return this.deleteJobs({ status: ['COMPLETED'] });
|
||||
},
|
||||
|
||||
/**
|
||||
* Supprime tous les jobs en erreur
|
||||
*/
|
||||
async deleteFailedJobs() {
|
||||
return this.deleteJobs({ status: ['ERROR'] });
|
||||
},
|
||||
|
||||
/**
|
||||
* Supprime tous les jobs
|
||||
*/
|
||||
async deleteAllJobs() {
|
||||
return this.deleteJobs({});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1,169 +1,153 @@
|
||||
<template>
|
||||
<div class="overflow-y-auto h-full">
|
||||
<Toolbar :config="toolbarConfig" class="mb-6" />
|
||||
<div class="flex flex-col h-full">
|
||||
<Toolbar :config="toolbarConfig" />
|
||||
|
||||
<div v-if="activityStore.loading" class="flex justify-center py-8">
|
||||
<div class="animate-spin rounded-full h-10 w-10 border-t-2 border-b-2 border-indigo-500"></div>
|
||||
</div>
|
||||
<div class="overflow-y-auto flex-1">
|
||||
<!-- Loading -->
|
||||
<div v-if="loading" class="flex justify-center py-12">
|
||||
<div class="animate-spin h-10 w-10 border-b-2 border-indigo-500 rounded-full"></div>
|
||||
</div>
|
||||
|
||||
<div v-else-if="activityStore.error" class="bg-red-100 dark:bg-red-900/20 border-l-4 border-red-500 text-red-700 dark:text-red-400 p-4 mb-6">
|
||||
<p>{{ activityStore.error }}</p>
|
||||
</div>
|
||||
<!-- Error -->
|
||||
<div v-else-if="activityStore.error" class="px-6 py-8">
|
||||
<div class="bg-red-50 dark:bg-red-900 border border-red-200 dark:border-red-700 p-4">
|
||||
<p class="text-red-800 dark:text-red-200">{{ activityStore.error }}</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-else class="container mx-auto p-2">
|
||||
<div class="bg-white dark:bg-gray-800 overflow-hidden shadow rounded-lg">
|
||||
<div class="overflow-x-auto">
|
||||
<table class="min-w-full bg-white dark:bg-gray-800">
|
||||
<!-- Content -->
|
||||
<section v-else class="border-t border-gray-200 dark:border-gray-700">
|
||||
<!-- Empty -->
|
||||
<div v-if="activityStore.jobs.length === 0" class="flex flex-col items-center justify-center py-20 text-gray-400 dark:text-gray-500">
|
||||
<ClockIcon class="w-12 h-12 mb-3" />
|
||||
<p class="text-base">Aucun job en cours ou en attente.</p>
|
||||
</div>
|
||||
|
||||
<!-- Table -->
|
||||
<div v-else class="overflow-x-auto">
|
||||
<table class="min-w-full">
|
||||
<thead>
|
||||
<tr class="bg-gray-200 dark:bg-gray-700 text-gray-800 dark:text-gray-200">
|
||||
<th class="w-1/12 py-3 px-4 text-left">
|
||||
<input
|
||||
type="checkbox"
|
||||
class="form-checkbox h-5 w-5 text-green-600"
|
||||
@change="toggleSelectAll" />
|
||||
</th>
|
||||
<th class="w-2/12 py-3 px-4 text-left">Type</th>
|
||||
<th class="w-2/12 py-3 px-4 text-left">Statut</th>
|
||||
<th class="w-3/12 py-3 px-4 text-left">Informations</th>
|
||||
<th class="w-3/12 py-3 px-4 text-left">Progression</th>
|
||||
<th class="w-1/12 py-3 px-4 text-left">Actions</th>
|
||||
<tr class="border-b border-gray-200 dark:border-gray-700 text-xs font-semibold text-gray-400 dark:text-gray-500 uppercase tracking-wider">
|
||||
<th class="w-2/11 py-3 px-6 text-left">Type</th>
|
||||
<th class="w-2/11 py-3 px-4 text-left">Statut</th>
|
||||
<th class="w-3/11 py-3 px-4 text-left">Informations</th>
|
||||
<th class="w-3/11 py-3 px-4 text-left">Progression</th>
|
||||
<th class="w-1/11 py-3 px-4 text-left">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody class="text-gray-700 dark:text-gray-300">
|
||||
<template v-if="activityStore.jobs.length === 0">
|
||||
<tr>
|
||||
<td colspan="6" class="py-8 px-4 text-center text-gray-500">
|
||||
<div class="flex flex-col items-center">
|
||||
<ClockIcon class="h-12 w-12 text-gray-300 dark:text-gray-600 mb-4" />
|
||||
<p class="text-lg font-medium dark:text-gray-300">Aucune activité trouvée</p>
|
||||
<p class="text-sm dark:text-gray-400">Aucune activité ne correspond aux filtres actuels.</p>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
</template>
|
||||
<template v-else>
|
||||
<JobItem
|
||||
v-for="job in activityStore.jobs"
|
||||
:key="job.id"
|
||||
:job="job"
|
||||
@delete="deleteJob" />
|
||||
</template>
|
||||
<tbody class="divide-y divide-gray-100 dark:divide-gray-700/50 text-gray-700 dark:text-gray-300">
|
||||
<JobItem
|
||||
v-for="job in activityStore.jobs"
|
||||
:key="job.id"
|
||||
:job="job"
|
||||
@delete="deleteJob" />
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<!-- Pagination -->
|
||||
<Pagination
|
||||
v-if="activityStore.total > activityStore.limit"
|
||||
v-if="total > activityStore.limit"
|
||||
:current-page="activityStore.currentPage"
|
||||
:total-pages="activityStore.totalPages"
|
||||
:total="activityStore.total"
|
||||
:total="total"
|
||||
:limit="activityStore.limit"
|
||||
:has-next-page="activityStore.hasNextPage"
|
||||
:has-previous-page="activityStore.hasPreviousPage"
|
||||
@page-change="changePage" />
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { ArrowPathIcon, ClockIcon, FunnelIcon, TrashIcon } from '@heroicons/vue/24/outline';
|
||||
import { computed, onMounted, ref } from 'vue';
|
||||
import { ArrowPathIcon, BarsArrowDownIcon, ClockIcon, TrashIcon } from '@heroicons/vue/24/outline';
|
||||
import { storeToRefs } from 'pinia';
|
||||
import { computed, onMounted, onUnmounted } from 'vue';
|
||||
import Pagination from '../../../../shared/components/ui/Pagination.vue';
|
||||
import Toolbar from '../../../../shared/components/ui/Toolbar.vue';
|
||||
import { useActivityStore } from '../../application/store/activityStore';
|
||||
import JobItem from '../components/JobItem.vue';
|
||||
|
||||
const activityStore = useActivityStore();
|
||||
const selectedAll = ref(false);
|
||||
const activityStore = useActivityStore();
|
||||
|
||||
// Statuts disponibles pour le filtre
|
||||
const statusOptions = [
|
||||
{ value: ['pending', 'in_progress'], label: 'Actifs' },
|
||||
{ value: ['pending', 'in_progress', 'completed', 'failed'], label: 'Tous' },
|
||||
{ value: ['completed'], label: 'Terminés' },
|
||||
{ value: ['failed'], label: 'En erreur' },
|
||||
{ value: ['pending'], label: 'En attente' },
|
||||
{ value: ['in_progress'], label: 'En cours' }
|
||||
];
|
||||
const { sortBy, sortOrder, total, loading } = storeToRefs(activityStore);
|
||||
|
||||
// Index du statut actif (par défaut "Actifs")
|
||||
const activeStatusIndex = ref(0);
|
||||
const isSortSelected = (by, order) => sortBy.value === by && sortOrder.value === order;
|
||||
|
||||
// Configuration de la toolbar réactive
|
||||
const toolbarConfig = computed(() => ({
|
||||
leftSection: [
|
||||
{
|
||||
icon: FunnelIcon,
|
||||
type: 'dropdown',
|
||||
label: statusOptions[activeStatusIndex.value].label,
|
||||
active: false,
|
||||
items: statusOptions.map((option, index) => ({
|
||||
label: option.label,
|
||||
isSelected: index === activeStatusIndex.value,
|
||||
onClick: () => setStatusFilter(index)
|
||||
}))
|
||||
}
|
||||
],
|
||||
rightSection: [
|
||||
{
|
||||
icon: ArrowPathIcon,
|
||||
type: 'button',
|
||||
label: 'Rafraîchir',
|
||||
onClick: refreshJobs
|
||||
},
|
||||
{
|
||||
icon: TrashIcon,
|
||||
type: 'button',
|
||||
label: 'Supprimer visibles',
|
||||
onClick: deleteVisibleJobs
|
||||
}
|
||||
]
|
||||
}));
|
||||
const toolbarConfig = computed(() => ({
|
||||
leftSection: [
|
||||
{ type: 'label', text: 'Activité', class: 'text-sm font-medium' },
|
||||
{ type: 'label', text: `(${total.value})`, class: 'text-sm text-gray-400' },
|
||||
],
|
||||
rightSection: [
|
||||
{
|
||||
type: 'dropdown',
|
||||
icon: BarsArrowDownIcon,
|
||||
label: 'Trier',
|
||||
items: [
|
||||
{
|
||||
label: 'Plus récent',
|
||||
isSelected: isSortSelected('createdAt', 'DESC'),
|
||||
onClick: () => activityStore.updateSort('createdAt', 'DESC'),
|
||||
},
|
||||
{
|
||||
label: 'Plus ancien',
|
||||
isSelected: isSortSelected('createdAt', 'ASC'),
|
||||
onClick: () => activityStore.updateSort('createdAt', 'ASC'),
|
||||
},
|
||||
{
|
||||
label: 'Par type',
|
||||
isSelected: isSortSelected('type', 'ASC'),
|
||||
onClick: () => activityStore.updateSort('type', 'ASC'),
|
||||
},
|
||||
{
|
||||
label: 'Par statut',
|
||||
isSelected: isSortSelected('status', 'ASC'),
|
||||
onClick: () => activityStore.updateSort('status', 'ASC'),
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'button',
|
||||
icon: ArrowPathIcon,
|
||||
label: 'Rafraîchir',
|
||||
disabled: loading.value,
|
||||
onClick: () => activityStore.loadJobs(),
|
||||
},
|
||||
{
|
||||
type: 'button',
|
||||
icon: TrashIcon,
|
||||
label: 'Supprimer visibles',
|
||||
disabled: loading.value || total.value === 0,
|
||||
onClick: deleteVisibleJobs,
|
||||
},
|
||||
],
|
||||
}));
|
||||
|
||||
onMounted(() => {
|
||||
loadJobs();
|
||||
});
|
||||
onMounted(() => {
|
||||
activityStore.loadJobs();
|
||||
activityStore.subscribeMercure();
|
||||
});
|
||||
|
||||
function loadJobs() {
|
||||
activityStore.loadJobs();
|
||||
onUnmounted(() => {
|
||||
activityStore.unsubscribeMercure();
|
||||
});
|
||||
|
||||
function changePage(page) {
|
||||
activityStore.goToPage(page);
|
||||
}
|
||||
|
||||
function deleteJob(id) {
|
||||
if (confirm('Voulez-vous vraiment supprimer ce job ?')) {
|
||||
activityStore.deleteJob(id);
|
||||
}
|
||||
}
|
||||
|
||||
function refreshJobs() {
|
||||
loadJobs();
|
||||
}
|
||||
|
||||
function changePage(page) {
|
||||
activityStore.goToPage(page);
|
||||
}
|
||||
|
||||
function toggleSelectAll() {
|
||||
selectedAll.value = !selectedAll.value;
|
||||
// La logique pour sélectionner tous les jobs serait ajoutée ici
|
||||
}
|
||||
|
||||
function setStatusFilter(index) {
|
||||
if (index >= 0 && index < statusOptions.length) {
|
||||
activeStatusIndex.value = index;
|
||||
activityStore.updateFilter({ status: statusOptions[index].value });
|
||||
}
|
||||
}
|
||||
|
||||
function deleteJob(id) {
|
||||
if (confirm('Voulez-vous vraiment supprimer ce job ?')) {
|
||||
activityStore.deleteJob(id);
|
||||
}
|
||||
}
|
||||
|
||||
function deleteVisibleJobs() {
|
||||
if (activityStore.jobs.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const statusLabel = statusOptions[activeStatusIndex.value].label.toLowerCase();
|
||||
if (confirm(`Voulez-vous vraiment supprimer tous les jobs ${statusLabel} ?`)) {
|
||||
activityStore.deleteJobs({ status: activityStore.filter.status });
|
||||
}
|
||||
function deleteVisibleJobs() {
|
||||
if (activityStore.jobs.length === 0) return;
|
||||
if (confirm('Voulez-vous vraiment supprimer tous les jobs visibles ?')) {
|
||||
activityStore.deleteJobs({ status: ['pending', 'in_progress'] });
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -17,7 +17,6 @@ framework:
|
||||
command.bus:
|
||||
middleware:
|
||||
- validation
|
||||
- doctrine_transaction
|
||||
event.bus:
|
||||
default_middleware: allow_no_handlers
|
||||
|
||||
|
||||
@@ -5,7 +5,8 @@ namespace App\Domain\Scraping\Application\Command;
|
||||
readonly class ScrapeChapter
|
||||
{
|
||||
public function __construct(
|
||||
public string $chapterId
|
||||
public string $chapterId,
|
||||
public string $jobId
|
||||
) {
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,14 +13,11 @@ use App\Domain\Shared\Domain\Event\ChapterScraped;
|
||||
use App\Domain\Scraping\Domain\Event\ChapterScrapingFailed;
|
||||
use App\Domain\Scraping\Domain\Event\ChapterScrapingStarted;
|
||||
use App\Domain\Scraping\Domain\Model\Chapter;
|
||||
use App\Domain\Scraping\Domain\Model\ScrapingJob;
|
||||
use App\Domain\Scraping\Domain\Model\Source;
|
||||
use App\Domain\Scraping\Domain\Model\ValueObject\ScrapingRequest;
|
||||
use App\Domain\Scraping\Domain\Model\ValueObject\TempDirectory;
|
||||
use App\Domain\Shared\Domain\Contract\JobRepositoryInterface;
|
||||
use Ramsey\Uuid\Uuid;
|
||||
use Symfony\Component\Messenger\MessageBusInterface;
|
||||
use Doctrine\ORM\EntityManagerInterface;
|
||||
|
||||
readonly class ScrapeChapterHandler
|
||||
{
|
||||
@@ -33,151 +30,92 @@ readonly class ScrapeChapterHandler
|
||||
private MangaRepositoryInterface $mangaRepository,
|
||||
private SourceRepositoryInterface $sourceRepository,
|
||||
private MessageBusInterface $eventBus,
|
||||
private EntityManagerInterface $entityManager
|
||||
) {
|
||||
}
|
||||
|
||||
public function handle(ScrapeChapter $command): void
|
||||
{
|
||||
$job = null;
|
||||
try {
|
||||
// 1. Récupération du chapitre
|
||||
/**@var Chapter $chapter */
|
||||
$chapter = $this->chapterRepository->getById($command->chapterId);
|
||||
if (!$chapter) {
|
||||
throw new \InvalidArgumentException("Chapter not found with ID: {$command->chapterId}");
|
||||
}
|
||||
/** @var Chapter $chapter */
|
||||
$chapter = $this->chapterRepository->getById($command->chapterId);
|
||||
$manga = $this->mangaRepository->getById($chapter->mangaId);
|
||||
|
||||
// 2. Récupération du manga
|
||||
$manga = $this->mangaRepository->getById($chapter->mangaId);
|
||||
if (!$manga) {
|
||||
throw new \InvalidArgumentException("Manga not found with ID: {$chapter->mangaId}");
|
||||
}
|
||||
$job = $this->jobRepository->get($command->jobId);
|
||||
$job->context['chapterId'] = $command->chapterId;
|
||||
$job->context['mangaTitle'] = $manga->getTitle();
|
||||
$job->start();
|
||||
$this->jobRepository->save($job);
|
||||
|
||||
// 3. Dispatch de l'événement de démarrage
|
||||
$this->eventBus->dispatch(new ChapterScrapingStarted($manga->getTitle(), $chapter->chapterNumber));
|
||||
$this->eventBus->dispatch(new ChapterScrapingStarted($job->id, $manga->getTitle(), $chapter->chapterNumber));
|
||||
|
||||
// 4. Détermination des sources à utiliser
|
||||
$sources = $this->getSourcesToTry($manga);
|
||||
if (empty($sources)) {
|
||||
throw new \InvalidArgumentException("No sources available for scraping");
|
||||
}
|
||||
$sources = $this->getSourcesToTry($manga);
|
||||
$slugsToTry = array_merge([$manga->getSlug()], $manga->getAlternativeSlugs());
|
||||
$success = false;
|
||||
$lastException = null;
|
||||
|
||||
// 5. Essai de scraping sur chaque source jusqu'à succès
|
||||
$success = false;
|
||||
$lastException = null;
|
||||
|
||||
foreach ($sources as $source) {
|
||||
// Préparer la liste des slugs à essayer : slug principal + slugs alternatifs
|
||||
$slugsToTry = array_merge([$manga->getSlug()], $manga->getAlternativeSlugs());
|
||||
|
||||
foreach ($slugsToTry as $slug) {
|
||||
$job = new ScrapingJob(
|
||||
Uuid::uuid4()->toString(),
|
||||
$chapter->mangaId,
|
||||
$chapter->chapterNumber,
|
||||
$source->getId()->getValue()
|
||||
);
|
||||
|
||||
// Ajout de l'ID du chapitre et du slug dans le contexte du job
|
||||
$job->context['chapterId'] = $command->chapterId;
|
||||
foreach ($sources as $source) {
|
||||
foreach ($slugsToTry as $slug) {
|
||||
try {
|
||||
$job->context['sourceId'] = $source->getId()->getValue();
|
||||
$job->context['slug'] = $slug;
|
||||
$job->context['mangaTitle'] = $manga->getTitle();
|
||||
|
||||
$job->start();
|
||||
$this->jobRepository->save($job);
|
||||
|
||||
try {
|
||||
$this->entityManager->beginTransaction();
|
||||
$scrapingParameters = $source->getScrappingParameters();
|
||||
$scrapingParameters['chapterNumber'] = $chapter->chapterNumber;
|
||||
$scrapingType = $scrapingParameters['scrapingType'] ?? 'html';
|
||||
|
||||
// 5. Scraping des URLs avec le slug courant
|
||||
$scrapingParameters = $source->getScrappingParameters();
|
||||
$scrapingParameters['chapterNumber'] = $chapter->chapterNumber;
|
||||
$scrapingType = $scrapingParameters['scrapingType'] ?? 'html';
|
||||
$scrapingRequest = new ScrapingRequest(
|
||||
$scrapingType,
|
||||
$source->buildChapterUrl($slug, $chapter->chapterNumber),
|
||||
$scrapingParameters
|
||||
);
|
||||
|
||||
$scrapingRequest = new ScrapingRequest(
|
||||
$scrapingType,
|
||||
$source->buildChapterUrl($slug, $chapter->chapterNumber),
|
||||
$scrapingParameters
|
||||
);
|
||||
$scraper = $this->scraperFactory->getScraperWithFallback($scrapingType);
|
||||
$scrapingResult = $scraper->scrape($scrapingRequest);
|
||||
|
||||
// Sélection du scraper approprié selon le type
|
||||
$scraper = $this->scraperFactory->getScraperWithFallback($scrapingType);
|
||||
$scrapingResult = $scraper->scrape($scrapingRequest);
|
||||
$tempDir = new TempDirectory();
|
||||
$downloadResults = $this->imageDownloader->downloadBatch(
|
||||
$scrapingResult->getImageUrls(),
|
||||
$tempDir,
|
||||
$job->id
|
||||
);
|
||||
|
||||
// 6. Téléchargement des images
|
||||
$tempDir = new TempDirectory();
|
||||
$downloadResults = $this->imageDownloader->downloadBatch(
|
||||
$scrapingResult->getImageUrls(),
|
||||
$tempDir,
|
||||
$job->id
|
||||
);
|
||||
$localPaths = array_map(fn ($r) => $r->getLocalPath(), $downloadResults);
|
||||
$pagesDirectory = $this->imageStorage->storeChapterImages($command->chapterId, $localPaths);
|
||||
$pageCount = count($downloadResults);
|
||||
|
||||
// 7. Stockage des images individuelles
|
||||
$localPaths = array_map(fn ($r) => $r->getLocalPath(), $downloadResults);
|
||||
$pagesDirectory = $this->imageStorage->storeChapterImages($command->chapterId, $localPaths);
|
||||
$pageCount = count($downloadResults);
|
||||
$job->complete();
|
||||
$this->jobRepository->save($job);
|
||||
|
||||
$job->complete();
|
||||
$this->jobRepository->save($job);
|
||||
$this->eventBus->dispatch(new ChapterScraped($job->id, $command->chapterId, $pagesDirectory, $pageCount));
|
||||
$tempDir->cleanup();
|
||||
|
||||
$this->entityManager->commit();
|
||||
|
||||
$this->eventBus->dispatch(new ChapterScraped($job->id, $command->chapterId, $pagesDirectory, $pageCount));
|
||||
|
||||
// 8. Nettoyage
|
||||
$tempDir->cleanup();
|
||||
|
||||
// Scraping réussi, pas besoin d'essayer d'autres slugs ni d'autres sources
|
||||
$success = true;
|
||||
break;
|
||||
|
||||
} catch (\Exception $e) {
|
||||
dump('EXCEPTION for source ' . $source->getName() . ' with slug ' . $slug . ': ' . $e->getMessage());
|
||||
|
||||
$this->entityManager->rollback();
|
||||
|
||||
if (isset($job)) {
|
||||
$job->fail($e->getMessage());
|
||||
$this->jobRepository->save($job);
|
||||
}
|
||||
|
||||
$lastException = $e;
|
||||
|
||||
// Continuer avec le slug suivant pour cette source
|
||||
}
|
||||
}
|
||||
|
||||
// Si le scraping a réussi avec un des slugs, sortir de la boucle des sources
|
||||
if ($success) {
|
||||
$success = true;
|
||||
break;
|
||||
|
||||
} catch (\Exception $e) {
|
||||
$lastException = $e;
|
||||
}
|
||||
}
|
||||
|
||||
// Si toutes les sources ont échoué
|
||||
if (!$success) {
|
||||
$errorMessage = $lastException ? $lastException->getMessage() : "Failed to scrape chapter from all available sources";
|
||||
$this->eventBus->dispatch(new ChapterScrapingFailed($chapter->mangaId, $chapter->chapterNumber, $errorMessage));
|
||||
if ($success) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
} catch (\Exception $e) {
|
||||
if (isset($job)) {
|
||||
$job->fail($e->getMessage());
|
||||
$this->jobRepository->save($job);
|
||||
}
|
||||
$this->eventBus->dispatch(new ChapterScrapingFailed($chapter->mangaId ?? 'unknown', $chapter->chapterNumber ?? 'unknown', $e->getMessage()));
|
||||
if (!$success) {
|
||||
$errorMessage = $lastException?->getMessage() ?? 'Failed to scrape chapter from all available sources';
|
||||
$job->fail($errorMessage);
|
||||
$this->jobRepository->save($job);
|
||||
$this->eventBus->dispatch(new ChapterScrapingFailed($job->id, $chapter->mangaId, $chapter->chapterNumber, $errorMessage));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Détermine les sources à utiliser pour le scraping en fonction des préférences du manga
|
||||
*
|
||||
* @param \App\Domain\Scraping\Domain\Model\Manga $manga
|
||||
* @return Source[]
|
||||
*/
|
||||
private function getSourcesToTry(\App\Domain\Scraping\Domain\Model\Manga $manga): array
|
||||
{
|
||||
// Si le manga a des sources préférées, les utiliser
|
||||
if ($manga->hasPreferredSources()) {
|
||||
$preferredSources = [];
|
||||
foreach ($manga->getPreferredSources() as $sourceId) {
|
||||
@@ -186,7 +124,6 @@ readonly class ScrapeChapterHandler
|
||||
$preferredSources[] = $source;
|
||||
}
|
||||
|
||||
// Limiter à 3 sources préférées maximum
|
||||
if (count($preferredSources) >= 3) {
|
||||
break;
|
||||
}
|
||||
@@ -197,7 +134,6 @@ readonly class ScrapeChapterHandler
|
||||
}
|
||||
}
|
||||
|
||||
// Sinon, utiliser toutes les sources disponibles
|
||||
return $this->sourceRepository->getAll();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,12 +5,18 @@ namespace App\Domain\Scraping\Domain\Event;
|
||||
readonly class ChapterScrapingFailed
|
||||
{
|
||||
public function __construct(
|
||||
private string $jobId,
|
||||
private string $mangaId,
|
||||
private string $chapterNumber,
|
||||
private string $reason
|
||||
) {
|
||||
}
|
||||
|
||||
public function getJobId(): string
|
||||
{
|
||||
return $this->jobId;
|
||||
}
|
||||
|
||||
public function getMangaId(): string
|
||||
{
|
||||
return $this->mangaId;
|
||||
|
||||
@@ -5,11 +5,17 @@ namespace App\Domain\Scraping\Domain\Event;
|
||||
class ChapterScrapingStarted
|
||||
{
|
||||
public function __construct(
|
||||
private readonly string $jobId,
|
||||
private readonly string $mangaTitle,
|
||||
private readonly float $chapterNumber,
|
||||
) {
|
||||
}
|
||||
|
||||
public function getJobId(): string
|
||||
{
|
||||
return $this->jobId;
|
||||
}
|
||||
|
||||
public function getMangaTitle(): string
|
||||
{
|
||||
return $this->mangaTitle;
|
||||
|
||||
@@ -8,9 +8,9 @@ class ScrapingJob extends Job
|
||||
{
|
||||
public function __construct(
|
||||
string $id,
|
||||
string $mangaId,
|
||||
float $chapterNumber,
|
||||
string $sourceId
|
||||
?string $mangaId = null,
|
||||
?float $chapterNumber = null,
|
||||
?string $sourceId = null
|
||||
) {
|
||||
parent::__construct($id, 'scraping_job');
|
||||
$this->maxAttempts = 1;
|
||||
|
||||
@@ -5,13 +5,20 @@ namespace App\Domain\Scraping\Infrastructure\ApiPlatform\State\Processor;
|
||||
use ApiPlatform\Metadata\Operation;
|
||||
use ApiPlatform\State\ProcessorInterface;
|
||||
use App\Domain\Scraping\Application\Command\ScrapeChapter;
|
||||
use App\Domain\Scraping\Domain\Model\ScrapingJob;
|
||||
use App\Domain\Scraping\Infrastructure\ApiPlatform\Dto\ScrapeChapterRequest;
|
||||
use App\Domain\Shared\Domain\Contract\JobRepositoryInterface;
|
||||
use Ramsey\Uuid\Uuid;
|
||||
use Symfony\Component\Mercure\HubInterface;
|
||||
use Symfony\Component\Mercure\Update;
|
||||
use Symfony\Component\Messenger\MessageBusInterface;
|
||||
|
||||
final class ScrapeChapterStateProcessor implements ProcessorInterface
|
||||
{
|
||||
public function __construct(
|
||||
private readonly MessageBusInterface $commandBus
|
||||
private readonly MessageBusInterface $commandBus,
|
||||
private readonly JobRepositoryInterface $jobRepository,
|
||||
private readonly HubInterface $hub,
|
||||
) {
|
||||
}
|
||||
|
||||
@@ -20,10 +27,25 @@ final class ScrapeChapterStateProcessor implements ProcessorInterface
|
||||
*/
|
||||
public function process(mixed $data, Operation $operation, array $uriVariables = [], array $context = []): void
|
||||
{
|
||||
$this->commandBus->dispatch(
|
||||
new ScrapeChapter(
|
||||
$data->chapterId
|
||||
)
|
||||
);
|
||||
$jobId = Uuid::uuid4()->toString();
|
||||
$job = new ScrapingJob($jobId);
|
||||
$job->context['chapterId'] = $data->chapterId;
|
||||
$this->jobRepository->save($job);
|
||||
|
||||
$this->hub->publish(new Update(
|
||||
'jobs/activity',
|
||||
json_encode([
|
||||
'type' => 'job.created',
|
||||
'id' => $job->id,
|
||||
'type_job' => $job->type,
|
||||
'status' => $job->status->value,
|
||||
'createdAt' => $job->createdAt->format('c'),
|
||||
'context' => $job->context,
|
||||
'attempts' => $job->attempts,
|
||||
'maxAttempts' => $job->maxAttempts,
|
||||
])
|
||||
));
|
||||
|
||||
$this->commandBus->dispatch(new ScrapeChapter($data->chapterId, $jobId));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ namespace App\Domain\Scraping\Infrastructure\EventSubscriber;
|
||||
use App\Domain\Shared\Domain\Event\ChapterScraped;
|
||||
use App\Domain\Scraping\Domain\Event\ChapterScrapingFailed;
|
||||
use App\Domain\Scraping\Domain\Event\ChapterScrapingStarted;
|
||||
use App\Domain\Scraping\Domain\Event\PageScrapingProgressed;
|
||||
use App\Domain\Scraping\Domain\Contract\Repository\ChapterRepositoryInterface;
|
||||
use App\Domain\Shared\Domain\Contract\JobRepositoryInterface;
|
||||
use App\Domain\Shared\Domain\Contract\NotificationInterface;
|
||||
@@ -30,14 +31,32 @@ class ScrapingEventSubscriber implements EventSubscriberInterface
|
||||
return [];
|
||||
}
|
||||
|
||||
#[AsMessageHandler]
|
||||
public function onPageScrapingProgressed(PageScrapingProgressed $event): void
|
||||
{
|
||||
$progress = (int) round($event->getProgress()->getPercentage());
|
||||
|
||||
$update = new Update(
|
||||
'jobs/activity',
|
||||
json_encode([
|
||||
'type' => 'job.progress_updated',
|
||||
'jobId' => $event->getJobId(),
|
||||
'progress' => $progress,
|
||||
])
|
||||
);
|
||||
$this->hub->publish($update);
|
||||
}
|
||||
|
||||
#[AsMessageHandler]
|
||||
public function onChapterScrapingStarted(ChapterScrapingStarted $event): void
|
||||
{
|
||||
$chapterNumber = $event->getChapterNumber();
|
||||
$mangaTitle = $event->getMangaTitle();
|
||||
$this->hub->publish(new Update(
|
||||
'jobs/activity',
|
||||
json_encode(['type' => 'job.status_changed', 'jobId' => $event->getJobId(), 'status' => 'in_progress'])
|
||||
));
|
||||
|
||||
$this->notification->sendInfo(
|
||||
sprintf('Scraping du chapitre %s de "%s" démarré', $chapterNumber, $mangaTitle)
|
||||
sprintf('Scraping du chapitre %s de "%s" démarré', $event->getChapterNumber(), $event->getMangaTitle())
|
||||
);
|
||||
}
|
||||
|
||||
@@ -84,6 +103,11 @@ class ScrapingEventSubscriber implements EventSubscriberInterface
|
||||
$update = new Update($topics, json_encode($data));
|
||||
$this->hub->publish($update);
|
||||
|
||||
$this->hub->publish(new Update(
|
||||
'jobs/activity',
|
||||
json_encode(['type' => 'job.status_changed', 'jobId' => $jobId, 'status' => 'completed'])
|
||||
));
|
||||
|
||||
$mangaTitle = $job->context['mangaTitle'] ?? 'manga inconnu';
|
||||
$this->notification->sendSuccess(
|
||||
sprintf('Chapitre %s de "%s" scrappé avec succès', $chapter->chapterNumber, $mangaTitle)
|
||||
@@ -93,6 +117,11 @@ class ScrapingEventSubscriber implements EventSubscriberInterface
|
||||
#[AsMessageHandler]
|
||||
public function onChapterScrapingFailed(ChapterScrapingFailed $event): void
|
||||
{
|
||||
$this->hub->publish(new Update(
|
||||
'jobs/activity',
|
||||
json_encode(['type' => 'job.status_changed', 'jobId' => $event->getJobId(), 'status' => 'failed'])
|
||||
));
|
||||
|
||||
$this->logger->info('ChapterScrapingFailed reçu pour mangaId: ' . $event->getMangaId() . ', chapter: ' . $event->getChapterNumber());
|
||||
|
||||
$data = [
|
||||
|
||||
@@ -7,6 +7,7 @@ use App\Domain\Scraping\Application\CommandHandler\ScrapeChapterHandler;
|
||||
use App\Domain\Scraping\Domain\Event\ChapterScrapingFailed;
|
||||
use App\Domain\Scraping\Domain\Event\ChapterScrapingStarted;
|
||||
use App\Domain\Scraping\Domain\Model\Chapter;
|
||||
use App\Domain\Scraping\Domain\Model\ScrapingJob;
|
||||
use App\Domain\Shared\Domain\Event\ChapterScraped;
|
||||
use App\Tests\Domain\Scraping\Adapter\InMemoryChapterRepository;
|
||||
use App\Tests\Domain\Scraping\Adapter\InMemoryEventBus;
|
||||
@@ -16,8 +17,6 @@ use App\Tests\Domain\Scraping\Adapter\InMemoryMangaRepository;
|
||||
use App\Tests\Domain\Scraping\Adapter\InMemoryScraperFactory;
|
||||
use App\Tests\Domain\Scraping\Adapter\InMemorySourceRepository;
|
||||
use App\Tests\Domain\Shared\Adapter\InMemoryJobRepository;
|
||||
use Doctrine\ORM\EntityManagerInterface;
|
||||
use PHPUnit\Framework\MockObject\MockObject;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
class ScrapeChapterHandlerTest extends TestCase
|
||||
@@ -30,7 +29,6 @@ class ScrapeChapterHandlerTest extends TestCase
|
||||
private InMemoryMangaRepository $mangaRepository;
|
||||
private InMemorySourceRepository $sourceRepository;
|
||||
private InMemoryEventBus $eventBus;
|
||||
private EntityManagerInterface|MockObject $entityManager;
|
||||
private ScrapeChapterHandler $handler;
|
||||
|
||||
protected function setUp(): void
|
||||
@@ -43,11 +41,6 @@ class ScrapeChapterHandlerTest extends TestCase
|
||||
$this->mangaRepository = new InMemoryMangaRepository();
|
||||
$this->sourceRepository = new InMemorySourceRepository();
|
||||
$this->eventBus = new InMemoryEventBus();
|
||||
$this->entityManager = $this->createMock(EntityManagerInterface::class);
|
||||
|
||||
$this->entityManager->method('beginTransaction')->willReturn(null);
|
||||
$this->entityManager->method('commit')->willReturn(null);
|
||||
$this->entityManager->method('rollback')->willReturn(null);
|
||||
|
||||
$this->chapterRepository->save(new Chapter(
|
||||
id: '1',
|
||||
@@ -65,26 +58,27 @@ class ScrapeChapterHandlerTest extends TestCase
|
||||
$this->mangaRepository,
|
||||
$this->sourceRepository,
|
||||
$this->eventBus,
|
||||
$this->entityManager
|
||||
);
|
||||
}
|
||||
|
||||
public function testHandleSuccessfully(): void
|
||||
{
|
||||
$command = new ScrapeChapter(
|
||||
chapterId: '1'
|
||||
);
|
||||
$jobId = 'test-job-id';
|
||||
$job = new ScrapingJob($jobId, 'test-manga', 2);
|
||||
$this->jobRepository->save($job);
|
||||
|
||||
$command = new ScrapeChapter(chapterId: '1', jobId: $jobId);
|
||||
$this->handler->handle($command);
|
||||
|
||||
$job = $this->jobRepository->findByType('scraping_job');
|
||||
$this->assertCount(1, $job);
|
||||
$job = array_values($job)[0];
|
||||
$jobs = $this->jobRepository->findByType('scraping_job');
|
||||
$this->assertCount(1, $jobs);
|
||||
$job = array_values($jobs)[0];
|
||||
|
||||
$dispatchedMessages = $this->eventBus->getDispatchedMessages();
|
||||
$this->assertCount(2, $dispatchedMessages);
|
||||
|
||||
$this->assertInstanceOf(ChapterScrapingStarted::class, $dispatchedMessages[0]);
|
||||
$this->assertSame($jobId, $dispatchedMessages[0]->getJobId());
|
||||
$this->assertSame(2.0, $dispatchedMessages[0]->getChapterNumber());
|
||||
|
||||
$this->assertInstanceOf(ChapterScraped::class, $dispatchedMessages[1]);
|
||||
|
||||
@@ -35,13 +35,14 @@ class ScrapeChapterTest extends AbstractApiTestCase
|
||||
// Then
|
||||
$this->assertResponseStatusCodeSame(202);
|
||||
|
||||
$messages = $this->messageBus->getDispatchedMessages();
|
||||
$messages = InMemoryMessageBus::$messages;
|
||||
$this->assertCount(1, $messages, 'Un message devrait être dispatché');
|
||||
|
||||
/** @var ScrapeChapter $message */
|
||||
$message = $messages[0];
|
||||
$this->assertInstanceOf(ScrapeChapter::class, $message);
|
||||
$this->assertEquals('chapter-123', $message->chapterId);
|
||||
$this->assertNotEmpty($message->jobId);
|
||||
}
|
||||
|
||||
public function testInitiateChapterScrapingWithInvalidPayload(): void
|
||||
@@ -72,6 +73,6 @@ class ScrapeChapterTest extends AbstractApiTestCase
|
||||
protected function tearDown(): void
|
||||
{
|
||||
parent::tearDown();
|
||||
$this->messageBus->clear();
|
||||
InMemoryMessageBus::$messages = [];
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user