colgora/Mangarr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: ajout de la gestion des chapitres de manga, incluant la récupération et la sauvegarde des chapitres en français et en anglais, ainsi que l'optimisation de la logique de sauvegarde pour éviter les doublons

Browse Source
This commit is contained in:
ext.jeremy.guillot@maxicoffee.domains
2025-04-01 16:01:55 +02:00
parent 34dfa57dc0
commit 0111f1b5f1
12 changed files with 254 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
src/Domain/Manga/Application/CommandHandler/FetchMangaChaptersHandler.php
View File

@@ -28,6 +28,8 @@ readonly class FetchMangaChaptersHandler
$offset = 0;
$limit = 500;
$hasMore = true;
$chaptersByNumber = [];
$chapterNumbers = [];
while ($hasMore) {
$feed = $this->mangadexClient->getMangaFeed(
@@ -37,10 +39,20 @@ readonly class FetchMangaChaptersHandler
);
foreach ($feed['data'] as $chapterData) {
$chapterNumber = (float) $chapterData['attributes']['chapter'];
$language = $chapterData['attributes']['translatedLanguage'];
// On ne traite que les chapitres en français ou en anglais
if (!in_array($language, ['fr', 'en'])) {
continue;
}
// Si le chapitre n'existe pas encore ou si c'est une version française
if (!isset($chaptersByNumber[$chapterNumber]) || $language === 'fr') {
$chapter = new Chapter(
new ChapterId((string) Uuid::uuid4()),
$manga->getId()->getValue(),
(float) $chapterData['attributes']['chapter'],
$chapterNumber,
$chapterData['attributes']['title'],
isset($chapterData['attributes']['volume']) ? (int) $chapterData['attributes']['volume'] : null,
true,
@@ -48,11 +60,26 @@ readonly class FetchMangaChaptersHandler
new \DateTimeImmutable()
);
$this->mangaRepository->saveChapter($chapter);
$chaptersByNumber[$chapterNumber] = $chapter;
$chapterNumbers[] = $chapterNumber;
}
}
$offset += $limit;
$hasMore = count($feed['data']) === $limit;
}
// Récupère les chapitres existants
$existingChapters = $this->mangaRepository->findExistingChaptersByNumbers(
$manga->getId()->getValue(),
$chapterNumbers
);
// Sauvegarde uniquement les nouveaux chapitres
foreach ($chaptersByNumber as $chapterNumber => $chapter) {
if (!isset($existingChapters[$chapterNumber])) {
$this->mangaRepository->saveChapter($chapter);
}
}
}
}

1
src/Domain/Manga/Domain/Contract/Repository/MangaRepositoryInterface.php
View File

@@ -21,4 +21,5 @@ interface MangaRepositoryInterface
public function findBySlug(MangaSlug $slug): ?Manga;
public function search(string $query, int $page = 1, int $limit = 20): array;
public function countSearch(string $query): int;
public function findExistingChaptersByNumbers(string $mangaId, array $chapterNumbers): array;
}

20
src/Domain/Manga/Infrastructure/Persistence/LegacyMangaRepository.php
View File

@@ -200,6 +200,26 @@ readonly class LegacyMangaRepository implements MangaRepositoryInterface
->getSingleScalarResult();
}
public function findExistingChaptersByNumbers(string $mangaId, array $chapterNumbers): array
{
$queryBuilder = $this->entityManager->createQueryBuilder()
->select('c')
->from(EntityChapter::class, 'c')
->where('c.manga = :mangaId')
->andWhere('c.number IN (:chapterNumbers)')
->setParameter('mangaId', $mangaId)
->setParameter('chapterNumbers', $chapterNumbers);
$chapters = $queryBuilder->getQuery()->getResult();
$chaptersByNumber = [];
foreach ($chapters as $chapter) {
$chaptersByNumber[$chapter->getNumber()] = $this->toChapterDomain($chapter);
}
return $chaptersByNumber;
}
private function toDomain(EntityManga $entity): DomainManga
{
return new DomainManga(

2
src/Domain/Scraping/Application/CommandHandler/ScrapeChapterHandler.php
View File

@@ -5,14 +5,12 @@ namespace App\Domain\Scraping\Application\CommandHandler;
use App\Domain\Scraping\Application\Command\ScrapeChapter;
use App\Domain\Scraping\Domain\Contract\Repository\ChapterRepositoryInterface;
use App\Domain\Scraping\Domain\Contract\Repository\MangaRepositoryInterface;
use App\Domain\Scraping\Domain\Contract\Repository\ScrapingJobRepositoryInterface;
use App\Domain\Scraping\Domain\Contract\Repository\SourceRepositoryInterface;
use App\Domain\Scraping\Domain\Contract\Service\CbzGeneratorInterface;
use App\Domain\Scraping\Domain\Contract\Service\ImageDownloaderInterface;
use App\Domain\Scraping\Domain\Contract\Service\ScraperInterface;
use App\Domain\Scraping\Domain\Event\ChapterScraped;
use App\Domain\Scraping\Domain\Event\ChapterScrapingFailed;
use App\Domain\Scraping\Domain\Event\ChapterScrapingStarted;
use App\Domain\Scraping\Domain\Model\ScrapingJob;
use App\Domain\Scraping\Domain\Model\ValueObject\CbzGenerationRequest;
use App\Domain\Scraping\Domain\Model\ValueObject\ScrapingRequest;

12
src/Domain/Scraping/Domain/Event/ChapterScraped.php
View File

@@ -2,13 +2,15 @@
namespace App\Domain\Scraping\Domain\Event;
class ChapterScraped
readonly class ChapterScraped
{
public function __construct(
private string $jobId
) {
}
/**
* @param string $getId
*/
public function __construct(string $getId)
public function getJobId(): string
{
return $this->jobId;
}
}

15
src/Domain/Shared/Infrastructure/ApiPlatform/State/Provider/GetJobListStateProvider.php
View File

@@ -10,6 +10,7 @@ use App\Domain\Shared\Application\Query\ListJobsQuery;
use App\Domain\Shared\Application\QueryHandler\ListJobsQueryHandler;
use App\Domain\Shared\Domain\Model\JobStatus;
use App\Domain\Shared\Infrastructure\ApiPlatform\Resource\GetJobListResource;
use ApiPlatform\State\Pagination\ArrayPaginator;
readonly class GetJobListStateProvider implements ProviderInterface
{
@@ -17,7 +18,7 @@ readonly class GetJobListStateProvider implements ProviderInterface
private ListJobsQueryHandler $handler
) {}
public function provide(Operation $operation, array $uriVariables = [], array $context = []): array
public function provide(Operation $operation, array $uriVariables = [], array $context = []): ArrayPaginator
{
$filters = $context['filters'] ?? [];
@@ -60,15 +61,13 @@ readonly class GetJobListStateProvider implements ProviderInterface
$response = $this->handler->handle($query);
return [
'items' => array_map(
return new ArrayPaginator(
array_map(
fn($job) => GetJobListResource::fromJob($job),
$response->items
),
'total' => $response->total,
'page' => $response->page,
'limit' => $response->limit,
'pages' => $response->pages
];
0,
$response->total
);
}
}

5
src/Domain/Shared/Infrastructure/Persistence/Repository/DoctrineJobRepository.php
View File

@@ -20,12 +20,10 @@ readonly class DoctrineJobRepository implements JobRepositoryInterface
public function save(Job $job): void
{
dump('save', $job);
/** @var JobEntity|null $existingJobEntity */
$existingJobEntity = $this->entityManager->find(JobEntity::class, $job->id);
if ($existingJobEntity) {
dump('existingJobEntity', $existingJobEntity);
$existingJobEntity->setStatus($job->status->value);
$existingJobEntity->setStartedAt($job->startedAt);
$existingJobEntity->setCompletedAt($job->completedAt);
@@ -33,14 +31,11 @@ readonly class DoctrineJobRepository implements JobRepositoryInterface
$existingJobEntity->setAttempts($job->attempts);
$existingJobEntity->setContext($job->context);
$this->entityManager->persist($existingJobEntity);
dump('updated', $existingJobEntity);
} else {
$entity = $this->mapper->toEntity($job);
$this->entityManager->persist($entity);
dump('created', $entity);
}
$this->entityManager->flush();
dump('flushed');
}
public function get(string $id): Job

11
tests/Domain/Manga/Adapter/InMemoryMangaRepository.php
View File

@@ -179,4 +179,15 @@ class InMemoryMangaRepository implements MangaRepositoryInterface
{
return count($this->search($query, 1, PHP_INT_MAX));
}
public function findExistingChaptersByNumbers(string $mangaId, array $chapterNumbers): array
{
if (!isset($this->chapters[$mangaId])) {
return [];
}
return array_filter(
$this->chapters[$mangaId],
fn (Chapter $chapter) => in_array($chapter->getNumber(), $chapterNumbers)
);
}
}

3
tests/Domain/Manga/Application/CommandHandler/FetchMangaChaptersHandlerTest.php
View File

@@ -52,7 +52,8 @@ class FetchMangaChaptersHandlerTest extends TestCase
'attributes' => [
'chapter' => '1',
'title' => 'Chapter 1',
'volume' => '1'
'volume' => '1',
'translatedLanguage' => 'fr'
]
]
]);

57
tests/Domain/Scraping/Application/CommandHandler/ScrapeChapterHandlerTest.php
View File

@@ -8,27 +8,30 @@ use App\Domain\Scraping\Domain\Event\ChapterScraped;
use App\Domain\Scraping\Domain\Event\ChapterScrapingFailed;
use App\Domain\Scraping\Domain\Event\ChapterScrapingStarted;
use App\Domain\Scraping\Domain\Model\Chapter;
use App\Domain\Scraping\Domain\Model\ScrapingStatus;
use App\Domain\Shared\Domain\Model\JobStatus;
use App\Tests\Domain\Scraping\Adapter\InMemoryChapterRepository;
use App\Tests\Domain\Scraping\Adapter\InMemoryCbzGenerator;
use App\Tests\Domain\Scraping\Adapter\InMemoryEventBus;
use App\Tests\Domain\Scraping\Adapter\InMemoryImageDownloader;
use App\Tests\Domain\Scraping\Adapter\InMemoryMangaRepository;
use App\Tests\Domain\Scraping\Adapter\InMemoryScraperAdapter;
use App\Tests\Domain\Scraping\Adapter\InMemoryScrapingJobRepository;
use App\Tests\Domain\Scraping\Adapter\InMemorySourceRepository;
use App\Tests\Domain\Shared\Adapter\InMemoryJobRepository;
use Doctrine\ORM\EntityManagerInterface;
use PHPUnit\Framework\TestCase;
use PHPUnit\Framework\MockObject\MockObject;
class ScrapeChapterHandlerTest extends TestCase
{
private InMemoryScraperAdapter $scraper;
private InMemoryImageDownloader $imageDownloader;
private InMemoryCbzGenerator $cbzGenerator;
private InMemoryScrapingJobRepository $scrapingJobRepository;
private InMemoryJobRepository $jobRepository;
private InMemoryChapterRepository $chapterRepository;
private InMemoryMangaRepository $mangaRepository;
private InMemorySourceRepository $sourceRepository;
private InMemoryEventBus $eventBus;
private EntityManagerInterface|MockObject $entityManager;
private ScrapeChapterHandler $handler;
protected function setUp(): void
@@ -36,11 +39,16 @@ class ScrapeChapterHandlerTest extends TestCase
$this->scraper = new InMemoryScraperAdapter();
$this->imageDownloader = new InMemoryImageDownloader();
$this->cbzGenerator = new InMemoryCbzGenerator('/test/project/dir');
$this->scrapingJobRepository = new InMemoryScrapingJobRepository();
$this->jobRepository = new InMemoryJobRepository();
$this->chapterRepository = new InMemoryChapterRepository();
$this->mangaRepository = new InMemoryMangaRepository();
$this->sourceRepository = new InMemorySourceRepository();
$this->eventBus = new InMemoryEventBus();
$this->entityManager = $this->createMock(EntityManagerInterface::class);
$this->entityManager->method('beginTransaction')->willReturn(null);
$this->entityManager->method('commit')->willReturn(null);
$this->entityManager->method('rollback')->willReturn(null);
$this->chapterRepository->save(new Chapter(
id: '1',
@@ -54,11 +62,12 @@ class ScrapeChapterHandlerTest extends TestCase
$this->scraper,
$this->imageDownloader,
$this->cbzGenerator,
$this->scrapingJobRepository,
$this->jobRepository,
$this->chapterRepository,
$this->mangaRepository,
$this->sourceRepository,
$this->eventBus
$this->eventBus,
$this->entityManager
);
}
@@ -72,15 +81,14 @@ class ScrapeChapterHandlerTest extends TestCase
$this->handler->handle($command);
$scrapingJobs = $this->scrapingJobRepository->getJobs();
$this->assertCount(1, $scrapingJobs);
$job = $scrapingJobs[0];
$job = $this->jobRepository->findByType('scraping_job');
$this->assertCount(1, $job);
$job = array_values($job)[0];
$dispatchedMessages = $this->eventBus->getDispatchedMessages();
$this->assertCount(2, $dispatchedMessages);
$this->assertInstanceOf(ChapterScrapingStarted::class, $dispatchedMessages[0]);
$this->assertInstanceOf(ChapterScraped::class, $dispatchedMessages[1]);
$this->assertEquals($job->getId(), $dispatchedMessages[0]->getJobId());
$this->assertCount(1, $dispatchedMessages);
$this->assertInstanceOf(ChapterScraped::class, $dispatchedMessages[0]);
$this->assertEquals($job->id, $dispatchedMessages[0]->getJobId());
$chapter = $this->chapterRepository->getByMangaIdAndChapterNumber('test-manga', 2);
$this->assertNotNull($chapter->cbzPath);
@@ -97,28 +105,25 @@ class ScrapeChapterHandlerTest extends TestCase
$exception = new \Exception('Scraping failed');
$this->scraper->simulateError($exception);
$this->expectException(\Exception::class);
$this->expectExceptionMessage('Scraping failed');
$this->handler->handle($command);
$dispatchedMessages = $this->eventBus->getDispatchedMessages();
$this->assertCount(2, $dispatchedMessages);
$this->assertInstanceOf(ChapterScrapingStarted::class, $dispatchedMessages[0]);
$this->assertInstanceOf(ChapterScrapingFailed::class, $dispatchedMessages[1]);
$this->assertEquals('test-manga', $dispatchedMessages[1]->getMangaId());
$this->assertEquals('2', $dispatchedMessages[1]->getChapterNumber());
$this->assertEquals('Scraping failed', $dispatchedMessages[1]->getReason());
$this->assertCount(1, $dispatchedMessages);
$this->assertInstanceOf(ChapterScrapingFailed::class, $dispatchedMessages[0]);
$this->assertEquals('test-manga', $dispatchedMessages[0]->getMangaId());
$this->assertEquals('2', $dispatchedMessages[0]->getChapterNumber());
$this->assertEquals('Scraping failed', $dispatchedMessages[0]->getReason());
$jobs = $this->scrapingJobRepository->getJobs();
$jobs = $this->jobRepository->findByType('scraping_job');
$job = array_values($jobs)[0];
$this->assertCount(1, $jobs);
$this->assertEquals(ScrapingStatus::FAILED, $jobs[0]->status);
$this->assertEquals('Scraping failed', $jobs[0]->failureReason);
$this->assertEquals(JobStatus::FAILED, $job->status);
$this->assertEquals('Scraping failed', $job->failureReason);
}
protected function tearDown(): void
{
$this->scrapingJobRepository->clear();
$this->jobRepository->clear();
$this->chapterRepository->clear();
$this->mangaRepository->clear();
$this->sourceRepository->clear();

104
tests/Domain/Shared/Adapter/InMemoryJobRepository.php Normal file
View File

@@ -0,0 +1,104 @@
<?php
namespace App\Tests\Domain\Shared\Adapter;
use App\Domain\Shared\Domain\Contract\JobRepositoryInterface;
use App\Domain\Shared\Domain\Model\Job;
use App\Domain\Shared\Domain\Model\JobStatus;
class InMemoryJobRepository implements JobRepositoryInterface
{
/** @var Job[] */
private array $jobs = [];
public function save(Job $job): void
{
$this->jobs[$job->id] = $job;
}
public function get(string $id): ?Job
{
return $this->jobs[$id] ?? null;
}
public function findByStatus(JobStatus $status): array
{
return array_filter($this->jobs, fn(Job $job) => $job->status === $status);
}
public function findByType(string $type): array
{
return array_filter($this->jobs, fn(Job $job) => $job->type === $type);
}
public function findPendingJobs(): array
{
return $this->findByStatus(JobStatus::PENDING);
}
public function findInProgressJobs(): array
{
return $this->findByStatus(JobStatus::IN_PROGRESS);
}
public function findFailedJobs(): array
{
return $this->findByStatus(JobStatus::FAILED);
}
public function findByCriteria(array $criteria): array
{
$jobs = $this->jobs;
if (isset($criteria['statuses']) && is_array($criteria['statuses']) && !empty($criteria['statuses'])) {
$jobs = array_filter($jobs, fn(Job $job) => in_array($job->status, $criteria['statuses']));
} elseif (isset($criteria['status'])) {
$jobs = array_filter($jobs, fn(Job $job) => $job->status === $criteria['status']);
}
if (isset($criteria['type'])) {
$jobs = array_filter($jobs, fn(Job $job) => $job->type === $criteria['type']);
}
if (isset($criteria['createdAfter'])) {
$jobs = array_filter($jobs, fn(Job $job) => $job->createdAt >= $criteria['createdAfter']);
}
if (isset($criteria['createdBefore'])) {
$jobs = array_filter($jobs, fn(Job $job) => $job->createdAt <= $criteria['createdBefore']);
}
if (isset($criteria['sortBy'])) {
usort($jobs, function(Job $a, Job $b) use ($criteria) {
$sortOrder = $criteria['sortOrder'] ?? 'ASC';
$comparison = match($criteria['sortBy']) {
'createdAt' => $a->createdAt <=> $b->createdAt,
'startedAt' => ($a->startedAt ?? new \DateTimeImmutable()) <=> ($b->startedAt ?? new \DateTimeImmutable()),
'completedAt' => ($a->completedAt ?? new \DateTimeImmutable()) <=> ($b->completedAt ?? new \DateTimeImmutable()),
default => 0
};
return $sortOrder === 'ASC' ? $comparison : -$comparison;
});
}
if (isset($criteria['offset'])) {
$jobs = array_slice($jobs, $criteria['offset']);
}
if (isset($criteria['limit'])) {
$jobs = array_slice($jobs, 0, $criteria['limit']);
}
return array_values($jobs);
}
public function countByCriteria(array $criteria): int
{
return count($this->findByCriteria($criteria));
}
public function clear(): void
{
$this->jobs = [];
}
}

37
tests/Feature/Scraping/ScrapingStatusTest.php
View File

@@ -9,12 +9,13 @@ use App\Domain\Scraping\Domain\Model\ValueObject\ImageUrl;
use App\Domain\Scraping\Domain\Model\ValueObject\PageNumber;
use Ramsey\Uuid\Uuid;
use Symfony\Component\Messenger\MessageBusInterface;
use App\Domain\Scraping\Domain\Contract\Repository\ScrapingJobRepositoryInterface;
use App\Domain\Shared\Domain\Contract\JobRepositoryInterface;
use App\Domain\Shared\Domain\Model\JobStatus;
class ScrapingStatusTest extends ApiTestCase
{
private MessageBusInterface $messageBus;
private ScrapingJobRepositoryInterface $repository;
private JobRepositoryInterface $repository;
protected function setUp(): void
{
@@ -23,7 +24,7 @@ class ScrapingStatusTest extends ApiTestCase
self::bootKernel();
$this->messageBus = self::getContainer()->get(MessageBusInterface::class);
$this->repository = self::getContainer()->get(ScrapingJobRepositoryInterface::class);
$this->repository = self::getContainer()->get(JobRepositoryInterface::class);
}
public function testGetScrapingStatus(): void
@@ -32,27 +33,39 @@ class ScrapingStatusTest extends ApiTestCase
$jobId = Uuid::uuid4()->toString();
$job = new ScrapingJob($jobId, 'manga-123', 1, 'source-789');
$job->addPage(new PageNumber(1), new ImageUrl('http://example.com/page1.jpg'));
$job->addPage(new PageNumber(2), new ImageUrl('http://example.com/page2.jpg'));
$job->start();
$this->repository->save($job);
// When
$response = static::createClient()->request('GET', '/api/scraping/jobs/'.$jobId.'/status');
$response = static::createClient()->request('GET', '/api/jobs?status=in_progress');
// Then
$this->assertResponseIsSuccessful();
$this->assertJsonContains([
'jobId' => $jobId,
'status' => ScrapingStatus::IN_PROGRESS->value,
'progress' => 0
]);
$responseData = $response->toArray();
$this->assertArrayHasKey('hydra:member', $responseData);
$this->assertIsArray($responseData['hydra:member']);
$this->assertCount(1, $responseData['hydra:member']);
$jobData = $responseData['hydra:member'][0];
$this->assertEquals('/api/jobs/' . $jobId, $jobData['@id']);
$this->assertEquals('Job', $jobData['@type']);
$this->assertEquals($jobId, $jobData['id']);
$this->assertEquals('scraping_job', $jobData['type']);
$this->assertEquals(JobStatus::IN_PROGRESS->value, $jobData['status']);
$this->assertEquals([
'mangaId' => 'manga-123',
'chapterNumber' => 1,
'sourceId' => 'source-789'
], $jobData['context']);
}
public function testGetScrapingStatusForNonExistentJob(): void
{
// When
$response = static::createClient()->request('GET', '/api/scraping/jobs/non-existent-id/status', [
$response = static::createClient()->request('GET', '/api/jobs/non-existent-id?status=in_progress', [
'headers' => ['Accept' => 'application/json']
]);