colgora/Mangarr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added:

Browse Source 
- Messenger, Mercure
- chapter download flow (lelscan only)
This commit is contained in:
Jérémy Guillot
2024-06-13 18:11:11 +02:00
parent f88fa2c232
commit bc85649789
24 changed files with 744 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
src/Controller/MangaController.php
View File

@@ -3,17 +3,22 @@
namespace App\Controller;
use App\Entity\Manga;
use App\Message\DownloadChapter;
use App\Repository\ChapterRepository;
use App\Repository\MangaRepository;
use App\Service\MangaExportService;
use App\Service\LelScansProviderService;
use App\Service\MangaScraperServiceOld;
use App\Service\MangaUpdatesMetadataProvider;
use Doctrine\ORM\NonUniqueResultException;
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
use Symfony\Component\HttpFoundation\BinaryFileResponse;
use Symfony\Component\HttpFoundation\JsonResponse;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\HttpFoundation\ResponseHeaderBag;
use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
use Symfony\Component\Messenger\MessageBusInterface;
use Symfony\Component\Routing\Annotation\Route;
use Symfony\Component\String\Slugger\AsciiSlugger;
@@ -24,7 +29,9 @@ class MangaController extends AbstractController
private readonly MangaExportService $mangaExportService,
private readonly LelScansProviderService $mangaProviderService,
private readonly MangaRepository $mangaRepository,
private MangaUpdatesMetadataProvider $mangaUpdatesDbProvider
private ChapterRepository $chapterRepository,
private MangaUpdatesMetadataProvider $mangaUpdatesDbProvider,
private MessageBusInterface $bus
)
{
}
@@ -39,9 +46,13 @@ class MangaController extends AbstractController
]);
}
/**
* @throws NonUniqueResultException
*/
#[Route('/manga/{mangaSlug}', name: 'manga_show')]
public function showChapters(string $mangaSlug): Response
{
// $manga = $this->mangaRepository->findOneWithChapterBy(['slug' => $mangaSlug]);
$manga = $this->mangaRepository->findOneBy(['slug' => $mangaSlug]);
if (!$manga) {
@@ -52,6 +63,9 @@ class MangaController extends AbstractController
foreach ($manga->getChapters() as $chapter) {
$volume = $chapter->getVolume() ?? 'Not Found';
$chaptersByVolume[$volume][] = $chapter;
usort($chaptersByVolume[$volume], function ($a, $b) {
return $a->getNumber() <=> $b->getNumber();
});
}
$chaptersByVolume = array_map('array_reverse', array_reverse($chaptersByVolume, true));
@@ -105,6 +119,21 @@ class MangaController extends AbstractController
]);
}
#[Route('/addChapter/{id}', name: 'add_chapter')]
public function addChapterMessenger(int $id): JsonResponse
{
$chapter = $this->chapterRepository->find($id);
if (!$chapter) {
return new JsonResponse(['error' => 'Chapter Not Found.'], 400);
}elseif ($chapter->getLocalPath() !== null){
return new JsonResponse(['error' => 'Chapter already scraped.'], 400);
}
$this->bus->dispatch(new DownloadChapter($id));
return new JsonResponse(['success' => 'Scrapping started...'], 200);
}
#[Route('/manga/{mangaSlug}/chapter/{chapterNumber}/download', name: 'download_chapter')]
public function downloadChapter(string $mangaSlug, float $chapterNumber): BinaryFileResponse
{

2
src/Entity/Manga.php
View File

@@ -19,7 +19,7 @@ class Manga
#[ORM\Column(length: 255)]
private ?string $title = null;
#[ORM\OneToMany(mappedBy: 'manga', targetEntity: Chapter::class, orphanRemoval: true)]
#[ORM\OneToMany(mappedBy: 'manga', targetEntity: Chapter::class, fetch: 'EAGER', orphanRemoval: true)]
private Collection $chapters;
#[ORM\Column(length: 255, unique: true)]

2
src/EventListener/MangaScrapedListener.php
View File

@@ -35,6 +35,8 @@ class MangaScrapedListener
$this->entityManager->persist($manga);
}
$chapter->setLocalPath($mangaData['directory']);
foreach ($mangaData['pages'] as $pageData) {
$page = $chapter->getPageByNumber($pageData['page_number']);
if (!$page) {

7
src/EventSubscriber/MangaScrapedEvent.php
View File

@@ -11,12 +11,14 @@ class MangaScrapedEvent extends Event
private string $mangaTitle;
private float $chapterNumber;
private array $pagesData;
private string $chapterDirectory;
public function __construct(string $mangaTitle, float $chapterNumber, array $pagesData)
public function __construct(string $mangaTitle, float $chapterNumber, array $pagesData, string $chapterDirectory)
{
$this->mangaTitle = $mangaTitle;
$this->chapterNumber = $chapterNumber;
$this->pagesData = $pagesData;
$this->chapterDirectory = $chapterDirectory;
}
public function getMangaData(): array
@@ -24,7 +26,8 @@ class MangaScrapedEvent extends Event
return [
'title' => $this->mangaTitle,
'chapter' => $this->chapterNumber,
'pages' => $this->pagesData
'pages' => $this->pagesData,
'directory' => $this->chapterDirectory
];
}
}

18
src/Message/DownloadChapter.php Normal file
View File

@@ -0,0 +1,18 @@
<?php
namespace App\Message;
class DownloadChapter
{
private int $chapterId;
public function __construct(int $chapterId)
{
$this->chapterId = $chapterId;
}
public function getChapterId(): int
{
return $this->chapterId;
}
}

57
src/MessageHandler/DownloadChapterHandler.php Normal file
View File

@@ -0,0 +1,57 @@
<?php
namespace App\MessageHandler;
use App\Entity\ContentSource;
use App\Message\DownloadChapter;
use App\Repository\ChapterRepository;
use App\Repository\MangaRepository;
use App\Service\LelScansProviderService;
use App\Service\MangaScraperService;
use App\Service\NotificationService;
use Exception;
use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;
use Symfony\Component\Messenger\Attribute\AsMessageHandler;
#[AsMessageHandler]
readonly class DownloadChapterHandler
{
public function __construct(
private ChapterRepository $chapterRepository,
private MangaScraperService $mangaScraperService,
private NotificationService $notificationService
)
{
}
/**
* @throws Exception
*/
public function __invoke(DownloadChapter $message): void
{
$chapter = $this->chapterRepository->find($message->getChapterId());
if (!$chapter) {
$this->notificationService->sendUpdate('notification', ['status' => 'error', 'message' => 'Chapter not found.']);
throw new BadRequestHttpException('Chapter not found');
}elseif ($chapter->getLocalPath() !== null){
$this->notificationService->sendUpdate('notification', ['status' => 'error', 'message' => 'Chapter already scraped.']);
throw new BadRequestHttpException('Chapter already downloaded');
}
$lelScanSource = new ContentSource();
$lelScanSource->setBaseUrl('https://lelscans.net')
->setImageSelector('#image img')
->setChapterUrlFormat('https://lelscans.net/scan-%s/%s')
->setNextPageSelector('a[title="Suivant"]')
->setScrapingType('html');
try {
$this->mangaScraperService->scrapeChapter($chapter, $lelScanSource);
} catch (Exception $e) {
$this->notificationService->sendUpdate('notification', ['status' => 'error', 'message' => 'An error occurred while scraping the chapter.']);
throw new Exception('Error scraping chapter: ' . $e->getMessage());
}
$this->notificationService->sendUpdate('notification', ['status' => 'success', 'message' => 'Chapter scraped successfully.']);
}
}

17
src/Repository/MangaRepository.php
View File

@@ -4,6 +4,7 @@ namespace App\Repository;
use App\Entity\Manga;
use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository;
use Doctrine\ORM\NonUniqueResultException;
use Doctrine\Persistence\ManagerRegistry;
/**
@@ -48,6 +49,22 @@ class MangaRepository extends ServiceEntityRepository
->getResult();
}
/**
* @throws NonUniqueResultException
*/
public function findOneWithChapterBy(array $params): ?Manga
{
$query = $this->createQueryBuilder('m');
foreach ($params as $key => $value) {
$query->andWhere("m.$key = :$key")
->setParameter($key, $value);
}
$query->leftJoin('m.chapters', 'c')
->addSelect('c');
return $query->getQuery()->getOneOrNullResult();
}
// /**
// * @return Manga[] Returns an array of Manga objects
// */

110
src/Service/MangaScraperService.php
View File

@@ -8,7 +8,11 @@ use App\Entity\ContentSource;
use App\EventSubscriber\MangaScrapedEvent;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use GuzzleHttp\Exception\RequestException;
use Symfony\Component\DomCrawler\Crawler;
use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;
use Symfony\Component\HttpKernel\Exception\HttpException;
use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
use Symfony\Component\Routing\Matcher\UrlMatcher;
use Symfony\Component\Routing\RequestContext;
use Symfony\Component\Routing\Route;
@@ -27,42 +31,31 @@ class MangaScraperService
$this->eventDispatcher = $eventDispatcher;
}
public function extractMangaPageData(string $html, ContentSource $mangaSource): array
private function extractMangaPageData(string $html, ContentSource $mangaSource): array
{
$crawler = new Crawler($html);
$imgUrls = [];
$imgUrl = $crawler->filter($mangaSource->getImageSelector())->attr('src')
?? $crawler->filter($mangaSource->getImageSelector())->attr('data-src');
// Search for images with different extensions
foreach (['img[src$=".jpg"]', 'img[src$=".jpeg"]', 'img[src$=".png"]', 'img'] as $selector) {
$crawler->filter($selector)->each(function (Crawler $node) use (&$imgUrls) {
$src = $node->attr('src') ?? $node->attr('data-src');
if ($src) {
$imgUrls[] = $src;
}
});
}
// dd($imgUrl);
if (empty($imgUrls)) {
throw new \Exception('No valid image found on the page.');
}
// if (empty($imgUrl)) {
// throw new \Exception('No valid image found on the page.');
// }
$nextLink = $crawler->filter($mangaSource->getNextPageSelector());
$nextUrl = $nextLink->count() > 0 ? $nextLink->attr('href') : null;
// Convert relative URLs to absolute URLs
$baseUrl = $mangaSource->getBaseUrl();
$imgUrls = array_map(function ($imgUrl) use ($baseUrl) {
if (!preg_match('/^https?:\/\//', $imgUrl)) {
$urlComponents = parse_url($baseUrl);
$scheme = $urlComponents['scheme'];
$host = $urlComponents['host'];
$imgUrl = $scheme . '://' . $host . '/' . ltrim($imgUrl, '/');
}
return $imgUrl;
}, $imgUrls);
if (!preg_match('/^https?:\/\//', $imgUrl)) {
$urlComponents = parse_url($mangaSource->getBaseUrl());
$scheme = $urlComponents['scheme'];
$host = $urlComponents['host'];
$imgUrl = $scheme . '://' . $host . '/' . ltrim($imgUrl, '/');
}
return [
'image_urls' => $imgUrls,
'image_url' => $imgUrl,
'next_page_url' => $nextUrl,
];
}
@@ -75,7 +68,7 @@ class MangaScraperService
$allChaptersData = [];
foreach ($manga->getChapters() as $chapter) {
$chapterData = $this->scrapeChapter($manga, $chapter, $mangaSource);
$chapterData = $this->scrapeChapter($chapter, $mangaSource);
if ($chapterData !== false) {
$allChaptersData[$chapter->getNumber()] = $chapterData;
}
@@ -84,13 +77,13 @@ class MangaScraperService
return $allChaptersData;
}
private function scrapeChapter(Manga $manga, Chapter $chapter, ContentSource $mangaSource): array|bool
public function scrapeChapter(Chapter $chapter, ContentSource $mangaSource): array|bool
{
switch ($mangaSource->getScrapingType()) {
case 'html':
return $this->scrapeChapterHtml($manga, $chapter, $mangaSource);
return $this->scrapeChapterHtml($chapter->getManga(), $chapter, $mangaSource);
case 'javascript':
return $this->scrapeChapterJavaScript($manga, $chapter, $mangaSource);
return $this->scrapeChapterJavaScript($chapter->getManga(), $chapter, $mangaSource);
// case 'api':
// // Implémentez la méthode de scraping par API si nécessaire
// return $this->scrapeChapterApi($manga, $chapter, $mangaSource);
@@ -121,10 +114,10 @@ class MangaScraperService
// Appeler le script Puppeteer avec les paramètres nécessaires
$output = [];
$command = sprintf('node puppeteer-script.js "%s" "%s" "%s" 2>&1', $url, $imageSelector, $nextButtonSelector); // Redirect stderr to stdout
dump($command);
// dump($command);
// exec($command, $output, $return_var);
dd($command, $output);
// dd($command, $output);
// Convertir la sortie JSON en tableau PHP
return json_decode(implode("", $output), true);
@@ -156,34 +149,25 @@ class MangaScraperService
$html = $this->fetchHtml($currentPageUrl);
$page = $this->extractMangaPageData($html, $mangaSource);
foreach ($page['image_urls'] as $imgUrl) {
dump($imgUrl);
dump(base64_decode($imgUrl));
// Déterminer l'extension de l'image
$imageExtension = pathinfo(parse_url($imgUrl, PHP_URL_PATH), PATHINFO_EXTENSION);
// Déterminer l'extension de l'image
$imageExtension = pathinfo(parse_url($page['image_url'], PHP_URL_PATH), PATHINFO_EXTENSION);
// Construire le nom de fichier de l'image
$imageName = sprintf('%03d.%s', count($pageData) + 1, $imageExtension);
$imagePath = sprintf('%s/%s', $chapterDir, $imageName);
// Construire le nom de fichier de l'image
$imageName = sprintf('%03d.%s', count($pageData) + 1, $imageExtension);
$imagePath = sprintf('%s/%s', $chapterDir, $imageName);
$this->downloadAndSaveImage($imgUrl, $imagePath);
$this->downloadAndSaveImage($page['image_url'], $imagePath);
$pageData[] = [
'image_url' => $imgUrl,
'local_image_url' => sprintf('/manga-images/%s/%s/%s', $mangaTitle, $chapterNumber, $imageName),
'page_number' => count($pageData) + 1,
];
}
// Si plus d'une image a été trouvée, ne pas chercher la page suivante
if (count($page['image_urls']) > 1) {
break;
}
$pageData[] = [
'image_url' => $page['image_url'],
'local_image_url' => sprintf('/manga-images/%s/%s/%s', $mangaTitle, $chapterNumber, $imageName),
'page_number' => count($pageData) + 1,
];
$currentPageUrl = $page['next_page_url'];
} while ($currentPageUrl);
$event = new MangaScrapedEvent($mangaTitle, $chapterNumber, $pageData);
$event = new MangaScrapedEvent($mangaTitle, $chapterNumber, $pageData, $chapterDir);
$this->eventDispatcher->dispatch($event, MangaScrapedEvent::NAME);
return $pageData;
@@ -195,9 +179,25 @@ class MangaScraperService
private function fetchHtml(string $url): string
{
$client = new Client();
$response = $client->get($url);
return (string)$response->getBody();
try {
$response = $client->get($url, [
'http_errors' => true,
'allow_redirects' => false
]);
$statusCode = $response->getStatusCode();
if ($statusCode >= 300 && $statusCode < 400) {
throw new NotFoundHttpException('Chapter Not Found at ' . $url);
} elseif ($statusCode == 404) {
throw new NotFoundHttpException('Chapter Not Found at ' . $url);
}
return (string)$response->getBody();
} catch (HttpException $e) {
throw new BadRequestHttpException('Bad Request: ' . $e->getMessage());
}
}
/**
@@ -240,7 +240,7 @@ class MangaScraperService
];
}
$event = new MangaScrapedEvent($mangaTitle, $chapterNumber, $pageData);
$event = new MangaScrapedEvent($mangaTitle, $chapterNumber, $pageData, $chapterDir);
$this->eventDispatcher->dispatch($event, MangaScrapedEvent::NAME);
return $pageData;

8
src/Service/MangadexProvider.php
View File

@@ -24,8 +24,9 @@ readonly class MangadexProvider implements MetadataProviderInterface
$results = $this->client->get('/manga', [
'title' => $title,
'contentRating' => ['safe'],
'includes' => ['cover_art', 'author']
'contentRating' => ['safe', 'suggestive'],
'includes' => ['cover_art', 'author'],
'limit' => 25
]);
$mangas = [];
@@ -112,7 +113,8 @@ readonly class MangadexProvider implements MetadataProviderInterface
return $manga;
}
private function getFeedWithPagination(string $externalId, int $page){
private function getFeedWithPagination(string $externalId, int $page): array
{
return $this->client->get('/manga/' . $externalId . '/feed', [
'limit' => 500,
'translatedLanguage' =>['en'],

20
src/Service/NotificationService.php Normal file
View File

@@ -0,0 +1,20 @@
<?php
namespace App\Service;
use Symfony\Component\Mercure\HubInterface;
use Symfony\Component\Mercure\Update;
class NotificationService
{
public function __construct(private HubInterface $hub)
{
}
public function sendUpdate(string $topic, mixed $data): void
{
$update = new Update($topic, json_encode($data));
$this->hub->publish($update);
}
}

12
src/Twig/Components/MangaSearch.php
View File

@@ -30,12 +30,12 @@ class MangaSearch
*/
public function getMangas(): Collection|null
{
return new ArrayCollection($this->mangaRepository->findAll());
// return new ArrayCollection($this->mangaRepository->findAll());
// if ($this->query === null || $this->query === '') {
// return null;
// }
//
// return $this->mangadexProvider->search($this->query);
if ($this->query === null || $this->query === '') {
return null;
}
return $this->mangadexProvider->search($this->query);
}
}