colgora/Mangarr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added:

Browse Source 
- ContentSource handling in message
- ContentSource list, add/update ui
- nextPageSelector and imageSelector can be null
- cleanup
This commit is contained in:
Jérémy Guillot
2024-06-30 20:47:27 +02:00
parent ba30d3102d
commit 3012adfee7
24 changed files with 762 additions and 707 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/Controller/MangaController.php
View File

@@ -10,13 +10,13 @@ use App\Repository\ChapterRepository;
use App\Repository\MangaRepository;
use App\Service\CbzService;
use App\Service\MangadexProvider;
use App\Service\NotificationService;
use Doctrine\DBAL\Exception\UniqueConstraintViolationException;
use Doctrine\ORM\EntityManagerInterface;
use Doctrine\ORM\NonUniqueResultException;
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
use Symfony\Component\HttpFoundation\BinaryFileResponse;
use Symfony\Component\HttpFoundation\JsonResponse;
use Symfony\Component\HttpFoundation\RedirectResponse;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\HttpFoundation\ResponseHeaderBag;
@@ -27,13 +27,14 @@ use Symfony\Component\Routing\Attribute\Route;
class MangaController extends AbstractController
{
public function __construct(
private readonly MangaRepository $mangaRepository,
private readonly ChapterRepository $chapterRepository,
private readonly MessageBusInterface $bus,
private readonly CbzService $cbzService,
private readonly ToolbarFactory $toolbarFactory,
private MangadexProvider $mangadexProvider,
private EntityManagerInterface $entityManager
private readonly MangaRepository $mangaRepository,
private readonly ChapterRepository $chapterRepository,
private readonly MessageBusInterface $bus,
private readonly CbzService $cbzService,
private readonly ToolbarFactory $toolbarFactory,
private readonly MangadexProvider $mangadexProvider,
private readonly EntityManagerInterface $entityManager,
private readonly NotificationService $notificationService
)
{
}
@@ -171,6 +172,14 @@ class MangaController extends AbstractController
$allChapters = array_merge($mangaFeed, $mangaAggregate);
if (empty($allChapters)) {
$this->notificationService->sendUpdate([
'status' => 'error',
'message' => 'No chapters found for this manga.'
]);
return $this->redirectToRoute('app_manga_search', ['query' => $manga->getTitle()]);
}
$mergedChapters = [];
foreach ($allChapters as $chapter) {
$number = $chapter->getNumber();
@@ -187,7 +196,7 @@ class MangaController extends AbstractController
}
}
foreach($mergedChapters as $chapter) {
foreach ($mergedChapters as $chapter) {
$manga->addChapter($chapter);
}

47
src/Controller/MenuController.php
View File

@@ -1,47 +0,0 @@
<?php
namespace App\Controller;
use App\Repository\MangaRepository;
use App\Service\LelScansProviderService;
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\String\Slugger\AsciiSlugger;
class MenuController extends AbstractController
{
private MangaRepository $mangaRepository;
private LelScansProviderService $mangaProviderService;
public function __construct(MangaRepository $mangaRepository, LelScansProviderService $mangaProviderService)
{
$this->mangaRepository = $mangaRepository;
$this->mangaProviderService = $mangaProviderService;
}
public function menu(): Response
{
$availableManga = $this->mangaProviderService->getMangaList();
foreach($availableManga as $key => $manga) {
$availableManga[$key]['slug'] = $this->titleToSlug($manga['name']);
}
$mangas = $this->mangaRepository->findAll();
return $this->render('menu/menu_old.html.twig', [
'availableManga' => $availableManga,
'mangas' => $mangas,
]);
}
private function slugToTitle(string $slug): string
{
$slugger = new AsciiSlugger();
return $slugger->slug($slug)->replace('-', ' ')->title(true)->toString();
}
private function titleToSlug(string $title): string
{
$slugger = new AsciiSlugger();
return $slugger->slug($title)->lower()->toString();
}
}

87
src/Controller/SettingsController.php
View File

@@ -2,12 +2,28 @@
namespace App\Controller;
use App\Entity\ContentSource;
use App\Form\ContentSourceType;
use App\Repository\ContentSourceRepository;
use App\Service\MangaScraperService;
use Doctrine\ORM\EntityManagerInterface;
use GuzzleHttp\Exception\GuzzleException;
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
use Symfony\Component\HttpFoundation\JsonResponse;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\Routing\Attribute\Route;
class SettingsController extends AbstractController
{
public function __construct(
private MangaScraperService $mangaScraperService,
private EntityManagerInterface $entityManager
)
{
}
#[Route('/settings', name: 'app_settings')]
public function index(): Response
{
@@ -32,14 +48,77 @@ class SettingsController extends AbstractController
]);
}
#[Route('/settings/scrappers', name: 'app_settings_scrappers')]
public function scrappers(): Response
#[Route('/settings/scrappers/list', name: 'app_settings_scrappers_list')]
public function list(ContentSourceRepository $repository): Response
{
return $this->render('settings/index.html.twig', [
'controller_name' => 'SettingsController',
$contentSources = $repository->findAll();
return $this->render('settings/scrapper_list.html.twig', [
'contentSources' => $contentSources,
]);
}
#[Route('/settings/scrappers/{id}', name: 'app_settings_scrappers', defaults: ['id' => null])]
public function scrappers(Request $request, ?ContentSource $contentSource): Response
{
$isNew = $contentSource === null;
$contentSource = $contentSource ?? new ContentSource();
$form = $this->createForm(ContentSourceType::class, $contentSource);
$form->handleRequest($request);
if ($form->isSubmitted() && $form->isValid()) {
$this->entityManager->persist($contentSource);
$this->entityManager->flush();
$this->addFlash('success', ($isNew ? 'New scrapper configuration saved' : 'Scrapper configuration updated') . ' successfully.');
return $this->redirectToRoute('app_settings_scrappers_list');
}
return $this->render('settings/scrappers.html.twig', [
'form' => $form->createView(),
'isNew' => $isNew,
]);
}
/**
* @throws GuzzleException
*/
#[Route('/settings/scrappers_test', name: 'app_settings_scrappers_test', methods: ['POST'])]
public function scrapperTest(Request $request): JsonResponse
{
$contentSource = new ContentSource();
$form = $this->createForm(ContentSourceType::class, $contentSource);
$form->submit($request->request->all()['content_source']);
if ($form->isValid()) {
$mangaSlug = $request->request->get('mangaSlug');
$chapterNumber = $request->request->get('chapterNumber');
$scrapedData = $this->mangaScraperService->testScrapingHtml($mangaSlug, $chapterNumber, $contentSource);
return new JsonResponse([
'success' => true,
'message' => 'Test successful',
'data' => $scrapedData
]);
} else {
return new JsonResponse([
'success' => false,
'message' => 'Invalid form submission',
'errors' => $this->getFormErrors($form)
]);
}
}
private function getFormErrors($form): array
{
$errors = [];
foreach ($form->getErrors(true) as $error) {
$errors[] = $error->getMessage();
}
return $errors;
}
#[Route('/settings/ui', name: 'app_settings_ui')]
public function ui(): Response
{

2
src/Entity/ContentSource.php
View File

@@ -62,7 +62,7 @@ class ContentSource
return $this->NextPageSelector;
}
public function setNextPageSelector(string $NextPageSelector): static
public function setNextPageSelector(?string $NextPageSelector): static
{
$this->NextPageSelector = $NextPageSelector;

46
src/Form/ContentSourceType.php Normal file
View File

@@ -0,0 +1,46 @@
<?php
namespace App\Form;
use App\Entity\ContentSource;
use Symfony\Component\Form\AbstractType;
use Symfony\Component\Form\Extension\Core\Type\ChoiceType;
use Symfony\Component\Form\Extension\Core\Type\TextType;
use Symfony\Component\Form\Extension\Core\Type\UrlType;
use Symfony\Component\Form\FormBuilderInterface;
use Symfony\Component\OptionsResolver\OptionsResolver;
class ContentSourceType extends AbstractType
{
public function buildForm(FormBuilderInterface $builder, array $options): void
{
$builder
->add('baseUrl', UrlType::class, [
'label' => 'Base URL',
])
->add('imageSelector', TextType::class, [
'label' => 'Image Selector',
])
->add('chapterUrlFormat', TextType::class, [
'label' => 'Chapter URL Format',
])
->add('nextPageSelector', TextType::class, [
'label' => 'Next Page Selector (let empty if vertical reader)',
'required' => false,
])
->add('scrapingType', ChoiceType::class, [
'label' => 'Scraping Type',
'choices' => [
'HTML' => 'html',
'JavaScript' => 'javascript'
],
]);
}
public function configureOptions(OptionsResolver $resolver): void
{
$resolver->setDefaults([
'data_class' => ContentSource::class,
]);
}
}

30
src/MessageHandler/DownloadChapterHandler.php
View File

@@ -5,8 +5,7 @@ namespace App\MessageHandler;
use App\Entity\ContentSource;
use App\Message\DownloadChapter;
use App\Repository\ChapterRepository;
use App\Repository\MangaRepository;
use App\Service\LelScansProviderService;
use App\Repository\ContentSourceRepository;
use App\Service\MangaScraperService;
use App\Service\NotificationService;
use Exception;
@@ -20,7 +19,8 @@ readonly class DownloadChapterHandler
public function __construct(
private ChapterRepository $chapterRepository,
private MangaScraperService $mangaScraperService,
private NotificationService $notificationService
private NotificationService $notificationService,
private ContentSourceRepository $contentSourceRepository
)
{
@@ -40,21 +40,27 @@ readonly class DownloadChapterHandler
throw new BadRequestHttpException('Chapter already downloaded');
}
$sources = [
(new ContentSource())
->setBaseUrl('https://lelscans.net')
->setImageSelector('#image img')
->setChapterUrlFormat('https://lelscans.net/scan-%s/%s')
->setNextPageSelector('a[title="Suivant"]')
->setScrapingType('html'),
$sources = $this->contentSourceRepository->findAll();
$sources[] =
(new ContentSource())
->setBaseUrl('https://api.mangadex.org/')
->setImageSelector('img')
->setChapterUrlFormat('at-home/server/%s')
->setScrapingType('mangadex')
];
;
// (new ContentSource())
// ->setBaseUrl('https://lelscans.net')
// ->setImageSelector('#image img')
// ->setChapterUrlFormat('https://lelscans.net/scan-%s/%s')
// ->setNextPageSelector('a[title="Suivant"]')
// ->setScrapingType('html'),
// (new ContentSource())
// ->setBaseUrl('https://darkscans.net/')
// ->setImageSelector('.reading-content img')
// ->setChapterUrlFormat('https://darkscans.net/mangas/%s/chapter-%s/')
// ->setNextPageSelector(null)
// ->setScrapingType('html')
$scrapedSuccessfully = false;

67
src/Service/LelScansProviderService.php
View File

@@ -1,67 +0,0 @@
<?php
namespace App\Service;
use App\Entity\Manga;
use App\Interface\ContentProviderInterface;
use Symfony\Component\BrowserKit\HttpBrowser as Client;
use Symfony\Component\DomCrawler\Crawler;
class LelScansProviderService implements ContentProviderInterface
{
const PROVIDER_URL = 'https://lelscans.net/';
const MANGA_SLUG = '/{manga}/{chapter}/{page}';
private Client $client;
public function __construct()
{
$this->client = new Client();
}
public function getMangaList(): array
{
$crawler = $this->client->request('GET', self::PROVIDER_URL);
$mangaList = [];
$crawler->filter('select > option')->each(function (Crawler $node) use (&$mangaList) {
$mangaName = $node->text();
$mangaUrl = $node->attr('value');
if ($mangaName && $mangaUrl && !preg_match('/^\d+(\.\d+)?$/', $mangaName)) {
$mangaList[] = [
'name' => $mangaName,
'url' => $mangaUrl,
];
}
});
return $mangaList;
}
public function getChapterList($mangaSlug): array
{
$crawler = $this->client->request('GET', self::PROVIDER_URL . 'lecture-en-ligne-' . $mangaSlug . '.php');
$chapterList = [];
$crawler->filter('select > option')->each(function (Crawler $node) use (&$chapterList) {
$chapterName = $node->text();
$chapterUrl = $node->attr('value');
if ($chapterName && $chapterUrl && preg_match('/^\d+(\.\d+)?$/', $chapterName)) {
$chapterList[] = [
'number' => $chapterName,
];
}
});
return $chapterList;
}
#[\Override] public function getAvailableContent(Manga $manga): array
{
// TODO: Implement getAvailableContent() method.
}
#[\Override] public function getContent(Manga $manga): array
{
// TODO: Implement getContent() method.
}
}

100
src/Service/MangaExportService.php
View File

@@ -1,100 +0,0 @@
<?php
namespace App\Service;
use Symfony\Component\Filesystem\Filesystem;
use Symfony\Component\HttpFoundation\BinaryFileResponse;
use Symfony\Component\HttpFoundation\ResponseHeaderBag;
use ZipArchive;
use RecursiveDirectoryIterator;
use RecursiveIteratorIterator;
class MangaExportService
{
const IMG_BASE_DIR = '/public/manga-images';
const EXPORT_BASE_DIR = '/public/manga-export';
private string $projectDir;
public function __construct($projectDir)
{
$this->projectDir = $projectDir;
}
public function exportMangaChapter(string $mangaTitle, int $chapterNumber): bool|string
{
$chapterDir = $this->getMangaDir($mangaTitle, $chapterNumber);
$cbzFilePath = $this->getExportDir($mangaTitle, $chapterNumber);
if(!is_dir($chapterDir)){
return false;
}
$cbzDirectory = dirname($cbzFilePath);
if (!is_dir($cbzDirectory)) {
mkdir($cbzDirectory, 0755, true);
}
$fileSystem = new Filesystem();
if($fileSystem->exists($cbzFilePath)){
return 'already_exported';
}
return $this->createCbzFromDirectory($chapterDir, $cbzFilePath);
}
public function downloadCbz(string $mangaTitle, int $chapterNumber): BinaryFileResponse|bool
{
$filePathCbz = $this->getExportDir($mangaTitle, $chapterNumber);
$fileSystem = new Filesystem();
if($fileSystem->exists($filePathCbz)){
return new BinaryFileResponse($filePathCbz);
}
$chapterDir = $this->getMangaDir($mangaTitle, $chapterNumber);
if(is_dir($chapterDir)){
if($this->exportMangaChapter($mangaTitle, $chapterNumber)){
return new BinaryFileResponse($filePathCbz);
}
}
return false;
}
private function createCbzFromDirectory(string $sourceDirectory, string $cbzFilePath): bool
{
$zip = new ZipArchive();
// Ouvre le fichier .cbz en écriture
if ($zip->open($cbzFilePath, ZipArchive::CREATE | ZipArchive::OVERWRITE) !== true) {
return false;
}
$files = new RecursiveIteratorIterator(
new RecursiveDirectoryIterator($sourceDirectory),
RecursiveIteratorIterator::LEAVES_ONLY
);
// Ajoute les fichiers d'image au fichier .cbz
foreach ($files as $file) {
if (!$file->isDir()) {
$filePath = $file->getRealPath();
$relativePath = substr($filePath, strlen($sourceDirectory) + 1);
$zip->addFile($filePath, $relativePath);
}
}
$zip->close();
return true;
}
private function getMangaDir(string $mangaTitle, int $chapterNumber): string
{
return sprintf('%s/%s/%d', $this->projectDir . self::IMG_BASE_DIR, $mangaTitle, $chapterNumber);
}
private function getExportDir(string $mangaTitle, int $chapterNumber): string
{
return sprintf('%s/%s/%d', $this->projectDir . self::EXPORT_BASE_DIR, $mangaTitle, $chapterNumber) . '.cbz';
}
}

17
src/Service/MangaProviderFactory.php
View File

@@ -1,17 +0,0 @@
<?php
namespace App\Service;
use App\Interface\ContentProviderInterface;
class MangaProviderFactory
{
public static function create($providerName): ContentProviderInterface
{
return match ($providerName) {
'LelScans' => new LelScansProviderService(),
'AutreManga' => new AutreMangaProviderService(),
default => throw new \Exception("Provider {$providerName} non supporté."),
};
}
}

120
src/Service/MangaScraperService.php
View File

@@ -166,6 +166,21 @@ class MangaScraperService
return json_decode(implode("", $output), true);
}
/**
* @throws GuzzleException
*/
public function testScrapingHtml(string $mangaSlug, string $chapterNumber, ContentSource $contentSource): array
{
$chapterUrl = $contentSource->getChapterUrl($mangaSlug, $chapterNumber);
$html = $this->fetchHtml($chapterUrl);
if ($contentSource->getNextPageSelector() === null) {
return $this->scrapeVerticalReader($html, $contentSource);
} else {
return $this->scrapeHorizontalReader($chapterUrl, $contentSource);
}
}
/**
* @throws GuzzleException
*/
@@ -173,32 +188,32 @@ class MangaScraperService
{
$chapterUrl = $mangaSource->getChapterUrl($manga->getSlug(), $chapter->getNumber());
$pageData = [];
$currentPageUrl = $chapterUrl;
$tempDir = sys_get_temp_dir() . '/' . uniqid('manga_scraper_');
mkdir($tempDir);
do {
$html = $this->fetchHtml($currentPageUrl);
$page = $this->extractMangaPageData($html, $mangaSource);
$pageData = [];
$imageName = sprintf('%03d.%s', count($pageData) + 1, pathinfo(parse_url($page['image_url'], PHP_URL_PATH), PATHINFO_EXTENSION));
if ($mangaSource->getNextPageSelector() === null) {
// Lecteur vertical
$html = $this->fetchHtml($chapterUrl);
$pageData = $this->scrapeVerticalReader($html, $mangaSource);
} else {
// Lecteur horizontal (paginé)
$pageData = $this->scrapeHorizontalReader($chapterUrl, $mangaSource);
}
// Télécharger et sauvegarder les images
foreach ($pageData as $index => &$page) {
$imageName = sprintf('%03d.%s', $index + 1, pathinfo(parse_url($page['image_url'], PHP_URL_PATH), PATHINFO_EXTENSION));
$imagePath = $tempDir . '/' . $imageName;
$this->downloadAndSaveImage($page['image_url'], $imagePath);
$event = new PageScrappingProgressEvent($chapter->getId(), count($pageData) + 1, 0);
$event = new PageScrappingProgressEvent($chapter->getId(), $index + 1, count($pageData));
$this->eventDispatcher->dispatch($event, PageScrappingProgressEvent::NAME);
$pageData[] = [
'image_url' => $page['image_url'],
'local_image_url' => $imagePath,
'page_number' => count($pageData) + 1,
];
$currentPageUrl = $page['next_page_url'];
} while ($currentPageUrl);
$page['local_image_url'] = $imagePath;
}
$cbzFilePath = $this->generateCbzPath($manga, $chapter);
$this->createCbzFile($tempDir, $pageData, $cbzFilePath);
@@ -210,7 +225,78 @@ class MangaScraperService
// Nettoyage du répertoire temporaire
$this->cleanupTempFiles($tempDir);
return true;
return $pageData;
}
private function scrapeVerticalReader(string $html, ContentSource $contentSource): array
{
$crawler = new Crawler($html);
$images = $crawler->filter($contentSource->getImageSelector());
$pageData = [];
foreach ($images as $index => $image) {
if($image->getAttribute('src') === ''){
$imgUrl = $image->getAttribute('data-src');
}else{
$imgUrl = $image->getAttribute('src');
}
$pageData[] = [
'image_url' => $this->cleanImageUrl($imgUrl),
'page_number' => $index + 1,
];
}
return $pageData;
}
/**
* @throws GuzzleException
*/
private function scrapeHorizontalReader(string $chapterUrl, ContentSource $contentSource): array
{
$pageData = [];
$currentPageUrl = $chapterUrl;
do {
$html = $this->fetchHtml($currentPageUrl);
$page = $this->extractMangaPageData($html, $contentSource);
$pageData[] = [
'image_url' => $this->cleanImageUrl($page['image_url']),
'page_number' => count($pageData) + 1,
];
$currentPageUrl = $page['next_page_url'];
} while ($currentPageUrl);
return $pageData;
}
/**
* Processes a single image
* @throws GuzzleException
*/
private function processImage(string $imgUrl, string $tempDir, array &$pageData, int $index, Chapter $chapter): void
{
$imgUrl = $this->cleanImageUrl($imgUrl);
$imageName = sprintf('%03d.%s', $index + 1, pathinfo(parse_url($imgUrl, PHP_URL_PATH), PATHINFO_EXTENSION));
$imagePath = $tempDir . '/' . $imageName;
$this->downloadAndSaveImage($imgUrl, $imagePath);
// $event = new PageScrappingProgressEvent($chapter->getId(), $index + 1, 0);
// $this->eventDispatcher->dispatch($event, PageScrappingProgressEvent::NAME);
$pageData[] = [
'image_url' => $imgUrl,
'local_image_url' => $imagePath,
'page_number' => $index + 1,
];
}
private function cleanImageUrl(string $url): string
{
return preg_replace('/[\x00-\x1F\x7F]/', '', trim($url));
}
/**

157
src/Service/MangaScraperServiceOld.php
View File

@@ -1,157 +0,0 @@
<?php
namespace App\Service;
use App\EventSubscriber\MangaScrapedEvent;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use Symfony\Component\DomCrawler\Crawler;
use Symfony\Component\Routing\Matcher\UrlMatcher;
use Symfony\Component\Routing\RequestContext;
use Symfony\Component\Routing\Route;
use Symfony\Component\Routing\RouteCollection;
use Symfony\Contracts\EventDispatcher\EventDispatcherInterface;
class MangaScraperServiceOld
{
const string IMG_BASE_DIR = '/public/manga-images';
private string $projectDir;
private EventDispatcherInterface $eventDispatcher;
public function __construct($projectDir, EventDispatcherInterface $eventDispatcher)
{
$this->projectDir = $projectDir;
$this->eventDispatcher = $eventDispatcher;
}
public function extractMangaPageData(string $html): array
{
$baseUrl = 'https://lelscans.net';
//pour éviter à PhpStorm de gueuler...
$selector = 'img';
$crawler = new Crawler($html);
$imgUrl = $crawler->filter($selector)->attr('src');
$nextLink = $crawler->filter('a[title="Suivant"]');
if (!preg_match('/^https?:\/\//', $imgUrl)) {
$urlComponents = parse_url($baseUrl);
$scheme = $urlComponents['scheme'];
$host = $urlComponents['host'];
// Construit l'URL absolue de l'image
$imgUrl = $scheme . '://' . $host . '/' . ltrim($imgUrl, '/');
}
if($nextLink->count() > 0){
$nextUrl = $nextLink->attr('href');
}else{
$nextUrl = null;
}
return [
'image_url' => $imgUrl,
'next_page_url' => $nextUrl,
];
}
/**
* @throws GuzzleException
*/
public function scrapeMangaChapter(string $chapterUrl, string $mangaTitle, float $chapterNumber): array|bool
{
if(!$this->isChapterAvailable($chapterUrl, $chapterNumber)){
return false;
}
$pageData = [];
$currentPageUrl = $chapterUrl;
$mangaDir = sprintf('%s/%s', $this->projectDir . self::IMG_BASE_DIR, $mangaTitle);
if (!is_dir($mangaDir)) {
mkdir($mangaDir, 0755, true);
}
// Créez le dossier du chapitre s'il n'existe pas
$chapterDir = sprintf('%s/%s', $mangaDir, $chapterNumber);
if (!is_dir($chapterDir)) {
mkdir($chapterDir, 0755, true);
}
do {
$html = $this->fetchHtml($currentPageUrl);
$page = $this->extractMangaPageData($html);
$pageData[] = $page;
$currentPageUrl = $page['next_page_url'];
// Construisez le nom de fichier de l'image
$imageName = sprintf('%03d.jpg', count($pageData));
// Construisez le chemin du fichier de l'image
$imagePath = sprintf('%s/%s', $chapterDir, $imageName);
// Téléchargez et enregistrez l'image
$this->downloadAndSaveImage($page['image_url'], $imagePath);
// Modifiez les données de la page pour inclure l'URL de l'image stockée localement
$pageData[count($pageData) - 1]['local_image_url'] = sprintf('/manga-images/%s/%s/%s', $mangaTitle, $chapterNumber, $imageName);
$pageData[count($pageData) - 1]['page_number'] = count($pageData);
} while ($currentPageUrl);
$event = new MangaScrapedEvent($mangaTitle, $chapterNumber, $pageData);
$this->eventDispatcher->dispatch($event, MangaScrapedEvent::NAME);
return $pageData;
}
/**
* @throws GuzzleException
*/
private function fetchHtml(string $url): string
{
$client = new Client();
$response = $client->get($url);
return (string) $response->getBody();
}
/**
* @throws GuzzleException
*/
private function downloadAndSaveImage(string $imageUrl, string $destinationPath): void
{
$client = new Client();
$response = $client->get($imageUrl);
file_put_contents($destinationPath, $response->getBody()->getContents());
}
/**
* @throws GuzzleException
*/
private function isChapterAvailable(string $chapterUrl, float $chapterNumber): bool
{
$html = $this->fetchHtml($chapterUrl);
$crawler = new Crawler($html);
$nextLink = $crawler->filter('a[title="Suivant"]');
if($nextLink->count() === 0){
return false;
}else{
$nextUrl = $nextLink->attr('href');
}
$routeCollection = new RouteCollection();
$routeCollection->add('manga_chapter', new Route('/scan-{manga}/{chapter}/{page}'));
$context = new RequestContext('/');
$matcher = new UrlMatcher($routeCollection, $context);
$path = parse_url($nextUrl, PHP_URL_PATH);
$parameters = $matcher->match($path);
if((float) $parameters['chapter'] !== $chapterNumber){
return false;
}
return true;
}
}

26
src/Service/MangadexProvider.php
View File

@@ -125,12 +125,19 @@ readonly class MangadexProvider implements MetadataProviderInterface
private function getFeedWithPagination(string $externalId, int $page): array
{
return $this->client->get('/manga/' . $externalId . '/feed', [
'limit' => 500,
'translatedLanguage' =>['en', 'fr'],
'order' => ['chapter' => 'asc'],
'offset' => $page * 500
]);
try {
$response = $this->client->get('/manga/' . $externalId . '/feed', [
'limit' => 500,
'translatedLanguage' =>['en', 'fr'],
'order' => ['chapter' => 'asc'],
'offset' => $page * 500
]);
}catch(\Exception $e){
$this->notificationService->sendUpdate(['status' => 'error', 'message' => 'An error occurred while fetching data from Mangadex.']);
return [];
}
return $response;
}
public function getMangaAggregate(Manga $manga): array
@@ -139,7 +146,12 @@ readonly class MangadexProvider implements MetadataProviderInterface
return [];
}
$response = $this->client->get('/manga/' . $manga->getExternalId() . '/aggregate');
try {
$response = $this->client->get('/manga/' . $manga->getExternalId() . '/aggregate');
}catch(\Exception $e){
// $this->notificationService->sendUpdate(['status' => 'error', 'message' => 'An error occurred while fetching data from Mangadex.']);
return [];
}
$chapterEntities = [];
if($response['result'] === 'ok'){

73
src/Service/SushiScanProviderService.php
View File

@@ -1,73 +0,0 @@
<?php
namespace App\Service;
use App\Entity\Manga;
use App\Interface\ContentProviderInterface;
use Symfony\Component\BrowserKit\HttpBrowser;
use Symfony\Component\BrowserKit\HttpBrowser as Client;
//use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use Symfony\Component\DomCrawler\Crawler;
use Symfony\Component\HttpClient\HttpClient;
class SushiScanProviderService
{
const PROVIDER_URL = 'https://sushiscan.net/catalogue/';
const MANGA_SLUG = '/{manga}/{chapter}/{page}';
const CONTENT_TYPE = ['volume', 'chapitre'];
private Client $client;
public function __construct()
{
$httpClient = HttpClient::create(['timeout' => 60]);
$this->client = new HttpBrowser($httpClient);
}
public function getAvailableContent(Manga $manga)
{
$url = 'http://flaresolverr:8191/v1';
$jsonContent = json_encode([
'cmd' => 'request.get',
'url' => self::PROVIDER_URL . $manga->getSlug(),
'maxTimeout' => 90000,
]);
try{
$crawler = $this->client->request('POST', $url, [], [], [
'HTTP_CONTENT_TYPE' => 'application/json',
], $jsonContent);
}catch (\Exception $e) {
dd($e);
}
$contentList = [];
dd($crawler);
$crawler->filter('#chapterList ul > li')->each(function (Crawler $node) use (&$contentList) {
dump($node);
// $contentName = $node->text();
// $contentUrl = $node->attr('href');
// if ($contentName && $contentUrl) {
// $contentList[] = [
// 'name' => $contentName,
// 'url' => $contentUrl,
// ];
// }
});
return $contentList;
}
/**
* @param string $mangaSlug
* @return array
*/
public function getChapterList(string $mangaSlug): array
{
// TODO: Implement getChapterList() method.
}
}

111
src/Twig/Components/NewMangaForm.php
View File

@@ -1,111 +0,0 @@
<?php
namespace App\Twig\Components;
use App\Entity\Manga;
use App\Service\MangadexProvider;
use Doctrine\DBAL\Exception\UniqueConstraintViolationException;
use Doctrine\ORM\EntityManagerInterface;
use Symfony\Component\HttpFoundation\RedirectResponse;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\Routing\Generator\UrlGeneratorInterface;
use Symfony\UX\LiveComponent\Attribute\AsLiveComponent;
use Symfony\UX\LiveComponent\Attribute\LiveAction;
use Symfony\UX\LiveComponent\Attribute\LiveProp;
use Symfony\UX\LiveComponent\ComponentToolsTrait;
use Symfony\UX\LiveComponent\DefaultActionTrait;
#[AsLiveComponent]
class NewMangaForm
{
use ComponentToolsTrait;
use DefaultActionTrait;
public ?Manga $manga = null;
#[LiveProp(writable: true)]
public array $mangaData = [];
#[LiveProp(writable: true)]
public ?int $index = 0;
public function __construct(private UrlGeneratorInterface $urlGenerator)
{
}
public function mount(Manga $manga): void
{
$this->manga = $manga;
$this->mangaData = [
'title' => $manga->getTitle(),
'slug' => $manga->getSlug(),
'description' => $manga->getDescription(),
'imageUrl' => $manga->getImageUrl(),
'status' => $manga->getStatus(),
'genres' => $manga->getGenres(),
'author' => $manga->getAuthor(),
'publicationYear' => $manga->getPublicationYear(),
'rating' => $manga->getRating(),
'externalId' => $manga->getExternalId(),
];
}
#[LiveAction]
public function saveManga(EntityManagerInterface $entityManager, MangadexProvider $mangadexProvider): Response
{
$manga = new Manga();
$manga->setTitle($this->mangaData['title'])
->setSlug($this->mangaData['slug'])
->setDescription($this->mangaData['description'])
->setImageUrl($this->mangaData['imageUrl'])
->setStatus($this->mangaData['status'])
->setGenres($this->mangaData['genres'])
->setAuthor($this->mangaData['author'])
->setPublicationYear($this->mangaData['publicationYear'])
->setRating($this->mangaData['rating'])
->setExternalId($this->mangaData['externalId']);
$mangaFeed = $mangadexProvider->getFeed($manga);
$mangaAggregate = $mangadexProvider->getMangaAggregate($manga);
$allChapters = array_merge($mangaFeed, $mangaAggregate);
$mergedChapters = [];
foreach ($allChapters as $chapter) {
$number = $chapter->getNumber();
if (isset($mergedChapters[$number])) {
$existingChapter = $mergedChapters[$number];
if (!empty($chapter->getExternalId()) ||
(empty($existingChapter->getExternalId()) && !strpos($chapter->getTitle(), 'Chapter ') == 0)) {
$mergedChapters[$number] = $chapter;
}
} else {
$mergedChapters[$number] = $chapter;
}
}
foreach($mergedChapters as $chapter) {
$manga->addChapter($chapter);
}
$mangaChapterUrl = $this->urlGenerator->generate('app_manga_show', ['mangaSlug' => $manga->getSlug()]);
try {
foreach ($manga->getChapters() as $chapter) {
$entityManager->persist($chapter);
}
$entityManager->persist($manga);
$entityManager->flush();
} catch (\Exception $e) {
if ($e instanceof UniqueConstraintViolationException) {
return new RedirectResponse($mangaChapterUrl);
}
throw $e;
}
return new RedirectResponse($mangaChapterUrl);
}
}

28
src/Twig/Extension/AppExtension.php Normal file
View File

@@ -0,0 +1,28 @@
<?php
namespace App\Twig\Extension;
use Twig\Extension\AbstractExtension;
use Twig\TwigFunction;
class AppExtension extends AbstractExtension
{
public function getFunctions(): array
{
return [
new TwigFunction('get_placeholder', [$this, 'getPlaceholder']),
];
}
public function getPlaceholder(string $fieldName): string
{
return match ($fieldName) {
'baseUrl' => 'https://example.com',
'imageSelector' => '.manga-image img',
'chapterUrlFormat' => 'https://example.com/manga/{slug}/chapter-{number}',
'nextPageSelector' => '.next-page',
'scrapingType' => 'Select scraping type',
default => '',
};
}
}