diff --git a/.gitignore b/.gitignore index 978541b..5f1456d 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ yarn-error.log ###< symfony/webpack-encore-bundle ### /public/manga-export/ /public/manga-images/ +/public/cbz/ diff --git a/assets/controllers/scrapper_configure_controller.js b/assets/controllers/scrapper_configure_controller.js new file mode 100644 index 0000000..c86c690 --- /dev/null +++ b/assets/controllers/scrapper_configure_controller.js @@ -0,0 +1,78 @@ +import { Controller } from '@hotwired/stimulus'; + +export default class extends Controller { + static targets = ['form', 'testForm', 'imageSelector', 'nextPageSelector', 'testResults', 'scrapingType'] + + connect() { + } + + async saveConfiguration(event) { + console.log('saveConfiguration called'); + event.preventDefault(); + this.formTarget.submit(); + } + + async testConfiguration(event) { + console.log('testConfiguration called'); + event.preventDefault(); + const formData = new FormData(this.formTarget); + const testFormData = new FormData(this.testFormTarget); + + for (let [key, value] of formData.entries()) { + const cleanKey = key.replace(/^content_source\[(.+)]$/, '$1'); + testFormData.append(`content_source[${cleanKey}]`, value); + } + + try { + const response = await fetch(this.testFormTarget.action, { + method: 'POST', + body: testFormData + }); + + const result = await response.json(); + + if (result.success) { + this.displayTestResults(result.data); + } else { + this.displayError(result.message, result.errors); + } + } catch (error) { + console.log(error) + this.displayError('An error occurred while testing the configuration'); + } + } + + displayTestResults(data) { + let html = '

Test Results

'; + html += '
'; + data.forEach(page => { + html += ` +
+ Page ${page.page_number} +

Page ${page.page_number}

+
+ `; + }); + html += '
'; + this.testResultsTarget.innerHTML = html; + } + + displayError(message, errors = []) { + let errorHtml = ` + '; + this.testResultsTarget.innerHTML = errorHtml; + } +} diff --git a/composer.json b/composer.json index d1e9d64..e31efb2 100644 --- a/composer.json +++ b/composer.json @@ -10,6 +10,7 @@ "ext-ctype": "*", "ext-curl": "*", "ext-iconv": "*", + "ext-zip": "*", "api-platform/core": "^3.2", "doctrine/dbal": "^3", "doctrine/doctrine-bundle": "^2.11", @@ -26,6 +27,7 @@ "symfony/dotenv": "7.0.*", "symfony/expression-language": "7.0.*", "symfony/flex": "^2", + "symfony/form": "7.0.*", "symfony/framework-bundle": "7.0.*", "symfony/http-client": "7.0.*", "symfony/mercure-bundle": "^0.3.9", @@ -44,8 +46,7 @@ "symfony/webpack-encore-bundle": "^2.1", "symfony/yaml": "7.0.*", "twig/extra-bundle": "^2.12|^3.0", - "twig/twig": "^2.12|^3.0", - "ext-zip": "*" + "twig/twig": "^2.12|^3.0" }, "config": { "allow-plugins": { diff --git a/composer.lock b/composer.lock index 8304c75..763ceb8 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "e06fcb8d122d322f8406d9ab78787ddf", + "content-hash": "d52c83bad4e4c116ba33e0f33b9cfd7b", "packages": [ { "name": "api-platform/core", @@ -4201,6 +4201,102 @@ ], "time": "2024-01-02T11:08:32+00:00" }, + { + "name": "symfony/form", + "version": "v7.0.8", + "source": { + "type": "git", + "url": "https://github.com/symfony/form.git", + "reference": "1d0128e2f7e80c346ec51fa4d1ce4fec0d435eeb" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/form/zipball/1d0128e2f7e80c346ec51fa4d1ce4fec0d435eeb", + "reference": "1d0128e2f7e80c346ec51fa4d1ce4fec0d435eeb", + "shasum": "" + }, + "require": { + "php": ">=8.2", + "symfony/event-dispatcher": "^6.4|^7.0", + "symfony/options-resolver": "^6.4|^7.0", + "symfony/polyfill-ctype": "~1.8", + "symfony/polyfill-intl-icu": "^1.21", + "symfony/polyfill-mbstring": "~1.0", + "symfony/property-access": "^6.4|^7.0", + "symfony/service-contracts": "^2.5|^3" + }, + "conflict": { + "symfony/console": "<6.4", + "symfony/dependency-injection": "<6.4", + "symfony/doctrine-bridge": "<6.4", + "symfony/error-handler": "<6.4", + "symfony/framework-bundle": "<6.4", + "symfony/http-kernel": "<6.4", + "symfony/translation": "<6.4.3|>=7.0,<7.0.3", + "symfony/translation-contracts": "<2.5", + "symfony/twig-bridge": "<6.4" + }, + "require-dev": { + "doctrine/collections": "^1.0|^2.0", + "symfony/config": "^6.4|^7.0", + "symfony/console": "^6.4|^7.0", + "symfony/dependency-injection": "^6.4|^7.0", + "symfony/expression-language": "^6.4|^7.0", + "symfony/html-sanitizer": "^6.4|^7.0", + "symfony/http-foundation": "^6.4|^7.0", + "symfony/http-kernel": "^6.4|^7.0", + "symfony/intl": "^6.4|^7.0", + "symfony/security-core": "^6.4|^7.0", + "symfony/security-csrf": "^6.4|^7.0", + "symfony/translation": "^6.4.3|^7.0.3", + "symfony/uid": "^6.4|^7.0", + "symfony/validator": "^6.4|^7.0", + "symfony/var-dumper": "^6.4|^7.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\Form\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Allows to easily create, process and reuse HTML forms", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/form/tree/v7.0.8" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2024-05-31T14:55:39+00:00" + }, { "name": "symfony/framework-bundle", "version": "v7.0.2", @@ -5202,6 +5298,73 @@ ], "time": "2023-11-06T17:08:13+00:00" }, + { + "name": "symfony/options-resolver", + "version": "v7.0.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/options-resolver.git", + "reference": "700ff4096e346f54cb628ea650767c8130f1001f" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/options-resolver/zipball/700ff4096e346f54cb628ea650767c8130f1001f", + "reference": "700ff4096e346f54cb628ea650767c8130f1001f", + "shasum": "" + }, + "require": { + "php": ">=8.2", + "symfony/deprecation-contracts": "^2.5|^3" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\OptionsResolver\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Provides an improved replacement for the array_replace PHP function", + "homepage": "https://symfony.com", + "keywords": [ + "config", + "configuration", + "options" + ], + "support": { + "source": "https://github.com/symfony/options-resolver/tree/v7.0.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2023-08-08T10:20:21+00:00" + }, { "name": "symfony/password-hasher", "version": "v7.0.3", @@ -5355,6 +5518,90 @@ ], "time": "2023-01-26T09:26:14+00:00" }, + { + "name": "symfony/polyfill-intl-icu", + "version": "v1.30.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-intl-icu.git", + "reference": "e76343c631b453088e2260ac41dfebe21954de81" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-intl-icu/zipball/e76343c631b453088e2260ac41dfebe21954de81", + "reference": "e76343c631b453088e2260ac41dfebe21954de81", + "shasum": "" + }, + "require": { + "php": ">=7.1" + }, + "suggest": { + "ext-intl": "For best performance and support of other locales than \"en\"" + }, + "type": "library", + "extra": { + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Intl\\Icu\\": "" + }, + "classmap": [ + "Resources/stubs" + ], + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill for intl's ICU-related data and classes", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "icu", + "intl", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-intl-icu/tree/v1.30.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2024-05-31T15:07:36+00:00" + }, { "name": "symfony/polyfill-intl-idn", "version": "v1.29.0", @@ -10676,73 +10923,6 @@ ], "time": "2023-10-31T18:23:49+00:00" }, - { - "name": "symfony/options-resolver", - "version": "v7.0.0", - "source": { - "type": "git", - "url": "https://github.com/symfony/options-resolver.git", - "reference": "700ff4096e346f54cb628ea650767c8130f1001f" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/options-resolver/zipball/700ff4096e346f54cb628ea650767c8130f1001f", - "reference": "700ff4096e346f54cb628ea650767c8130f1001f", - "shasum": "" - }, - "require": { - "php": ">=8.2", - "symfony/deprecation-contracts": "^2.5|^3" - }, - "type": "library", - "autoload": { - "psr-4": { - "Symfony\\Component\\OptionsResolver\\": "" - }, - "exclude-from-classmap": [ - "/Tests/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Fabien Potencier", - "email": "fabien@symfony.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Provides an improved replacement for the array_replace PHP function", - "homepage": "https://symfony.com", - "keywords": [ - "config", - "configuration", - "options" - ], - "support": { - "source": "https://github.com/symfony/options-resolver/tree/v7.0.0" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2023-08-08T10:20:21+00:00" - }, { "name": "symfony/phpunit-bridge", "version": "v7.0.4", @@ -11308,7 +11488,8 @@ "php": ">=8.3.1", "ext-ctype": "*", "ext-curl": "*", - "ext-iconv": "*" + "ext-iconv": "*", + "ext-zip": "*" }, "platform-dev": [], "plugin-api-version": "2.6.0" diff --git a/config/services.yaml b/config/services.yaml index 68e7b42..02dbb01 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -39,19 +39,10 @@ services: referer: true protocols: [ 'http', 'https' ] track_redirects: true - - - App\Service\MangaScraperServiceOld: - arguments: - $projectDir: '%kernel.project_dir%' App\Service\MangaScraperService: arguments: $projectDir: '%kernel.project_dir%' - - App\Service\MangaExportService: - arguments: - $projectDir: '%kernel.project_dir%' App\Service\MangaImportService: arguments: @@ -69,9 +60,6 @@ services: tags: - { name: kernel.event_subscriber } - App\Controller\MenuController: - tags: [ 'controller.service_arguments' ] - App\Client\MangadexClient: arguments: $httpClient: '@GuzzleHttp\Client' diff --git a/migrations/Version20240610115931.php b/migrations/Version20240610115931.php index 0a1fd60..801536d 100644 --- a/migrations/Version20240610115931.php +++ b/migrations/Version20240610115931.php @@ -21,7 +21,7 @@ final class Version20240610115931 extends AbstractMigration { // this up() migration is auto-generated, please modify it to your needs $this->addSql('CREATE SEQUENCE content_source_id_seq INCREMENT BY 1 MINVALUE 1 START 1'); - $this->addSql('CREATE TABLE content_source (id INT NOT NULL, base_url VARCHAR(255) NOT NULL, image_selector VARCHAR(255) NOT NULL, next_page_selector VARCHAR(255) NOT NULL, chapter_url_format VARCHAR(255) NOT NULL, scraping_type VARCHAR(255) NOT NULL, PRIMARY KEY(id))'); + $this->addSql('CREATE TABLE content_source (id INT NOT NULL, base_url VARCHAR(255) NOT NULL, image_selector VARCHAR(255) DEFAULT NULL, next_page_selector VARCHAR(255) DEFAULT NULL, chapter_url_format VARCHAR(255) NOT NULL, scraping_type VARCHAR(255) NOT NULL, PRIMARY KEY(id))'); $this->addSql('ALTER TABLE chapter ADD volume INT DEFAULT NULL'); $this->addSql('ALTER TABLE chapter ADD title VARCHAR(255) DEFAULT NULL'); $this->addSql('ALTER TABLE chapter ADD local_path VARCHAR(255) DEFAULT NULL'); diff --git a/src/Controller/MangaController.php b/src/Controller/MangaController.php index e62a7aa..99ff61c 100644 --- a/src/Controller/MangaController.php +++ b/src/Controller/MangaController.php @@ -10,13 +10,13 @@ use App\Repository\ChapterRepository; use App\Repository\MangaRepository; use App\Service\CbzService; use App\Service\MangadexProvider; +use App\Service\NotificationService; use Doctrine\DBAL\Exception\UniqueConstraintViolationException; use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\NonUniqueResultException; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; use Symfony\Component\HttpFoundation\BinaryFileResponse; use Symfony\Component\HttpFoundation\JsonResponse; -use Symfony\Component\HttpFoundation\RedirectResponse; use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; use Symfony\Component\HttpFoundation\ResponseHeaderBag; @@ -27,13 +27,14 @@ use Symfony\Component\Routing\Attribute\Route; class MangaController extends AbstractController { public function __construct( - private readonly MangaRepository $mangaRepository, - private readonly ChapterRepository $chapterRepository, - private readonly MessageBusInterface $bus, - private readonly CbzService $cbzService, - private readonly ToolbarFactory $toolbarFactory, - private MangadexProvider $mangadexProvider, - private EntityManagerInterface $entityManager + private readonly MangaRepository $mangaRepository, + private readonly ChapterRepository $chapterRepository, + private readonly MessageBusInterface $bus, + private readonly CbzService $cbzService, + private readonly ToolbarFactory $toolbarFactory, + private readonly MangadexProvider $mangadexProvider, + private readonly EntityManagerInterface $entityManager, + private readonly NotificationService $notificationService ) { } @@ -171,6 +172,14 @@ class MangaController extends AbstractController $allChapters = array_merge($mangaFeed, $mangaAggregate); + if (empty($allChapters)) { + $this->notificationService->sendUpdate([ + 'status' => 'error', + 'message' => 'No chapters found for this manga.' + ]); + return $this->redirectToRoute('app_manga_search', ['query' => $manga->getTitle()]); + } + $mergedChapters = []; foreach ($allChapters as $chapter) { $number = $chapter->getNumber(); @@ -187,7 +196,7 @@ class MangaController extends AbstractController } } - foreach($mergedChapters as $chapter) { + foreach ($mergedChapters as $chapter) { $manga->addChapter($chapter); } diff --git a/src/Controller/MenuController.php b/src/Controller/MenuController.php deleted file mode 100644 index eff2264..0000000 --- a/src/Controller/MenuController.php +++ /dev/null @@ -1,47 +0,0 @@ -mangaRepository = $mangaRepository; - $this->mangaProviderService = $mangaProviderService; - } - - public function menu(): Response - { - $availableManga = $this->mangaProviderService->getMangaList(); - - foreach($availableManga as $key => $manga) { - $availableManga[$key]['slug'] = $this->titleToSlug($manga['name']); - } - - $mangas = $this->mangaRepository->findAll(); - return $this->render('menu/menu_old.html.twig', [ - 'availableManga' => $availableManga, - 'mangas' => $mangas, - ]); - } - - private function slugToTitle(string $slug): string - { - $slugger = new AsciiSlugger(); - return $slugger->slug($slug)->replace('-', ' ')->title(true)->toString(); - } - - private function titleToSlug(string $title): string - { - $slugger = new AsciiSlugger(); - return $slugger->slug($title)->lower()->toString(); - } -} diff --git a/src/Controller/SettingsController.php b/src/Controller/SettingsController.php index e8b058e..8c4ec55 100644 --- a/src/Controller/SettingsController.php +++ b/src/Controller/SettingsController.php @@ -2,12 +2,28 @@ namespace App\Controller; +use App\Entity\ContentSource; +use App\Form\ContentSourceType; +use App\Repository\ContentSourceRepository; +use App\Service\MangaScraperService; +use Doctrine\ORM\EntityManagerInterface; +use GuzzleHttp\Exception\GuzzleException; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; +use Symfony\Component\HttpFoundation\JsonResponse; +use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; use Symfony\Component\Routing\Attribute\Route; class SettingsController extends AbstractController { + public function __construct( + private MangaScraperService $mangaScraperService, + private EntityManagerInterface $entityManager + ) + { + + } + #[Route('/settings', name: 'app_settings')] public function index(): Response { @@ -32,14 +48,77 @@ class SettingsController extends AbstractController ]); } - #[Route('/settings/scrappers', name: 'app_settings_scrappers')] - public function scrappers(): Response + #[Route('/settings/scrappers/list', name: 'app_settings_scrappers_list')] + public function list(ContentSourceRepository $repository): Response { - return $this->render('settings/index.html.twig', [ - 'controller_name' => 'SettingsController', + $contentSources = $repository->findAll(); + + return $this->render('settings/scrapper_list.html.twig', [ + 'contentSources' => $contentSources, ]); } + #[Route('/settings/scrappers/{id}', name: 'app_settings_scrappers', defaults: ['id' => null])] + public function scrappers(Request $request, ?ContentSource $contentSource): Response + { + $isNew = $contentSource === null; + $contentSource = $contentSource ?? new ContentSource(); + + $form = $this->createForm(ContentSourceType::class, $contentSource); + $form->handleRequest($request); + + if ($form->isSubmitted() && $form->isValid()) { + $this->entityManager->persist($contentSource); + $this->entityManager->flush(); + $this->addFlash('success', ($isNew ? 'New scrapper configuration saved' : 'Scrapper configuration updated') . ' successfully.'); + return $this->redirectToRoute('app_settings_scrappers_list'); + } + + return $this->render('settings/scrappers.html.twig', [ + 'form' => $form->createView(), + 'isNew' => $isNew, + ]); + } + + /** + * @throws GuzzleException + */ + #[Route('/settings/scrappers_test', name: 'app_settings_scrappers_test', methods: ['POST'])] + public function scrapperTest(Request $request): JsonResponse + { + $contentSource = new ContentSource(); + $form = $this->createForm(ContentSourceType::class, $contentSource); + $form->submit($request->request->all()['content_source']); + + if ($form->isValid()) { + $mangaSlug = $request->request->get('mangaSlug'); + $chapterNumber = $request->request->get('chapterNumber'); + + $scrapedData = $this->mangaScraperService->testScrapingHtml($mangaSlug, $chapterNumber, $contentSource); + + return new JsonResponse([ + 'success' => true, + 'message' => 'Test successful', + 'data' => $scrapedData + ]); + } else { + return new JsonResponse([ + 'success' => false, + 'message' => 'Invalid form submission', + 'errors' => $this->getFormErrors($form) + ]); + } + } + + private function getFormErrors($form): array + { + $errors = []; + foreach ($form->getErrors(true) as $error) { + $errors[] = $error->getMessage(); + } + return $errors; + } + #[Route('/settings/ui', name: 'app_settings_ui')] public function ui(): Response { diff --git a/src/Entity/ContentSource.php b/src/Entity/ContentSource.php index 5f90073..a55fe74 100644 --- a/src/Entity/ContentSource.php +++ b/src/Entity/ContentSource.php @@ -62,7 +62,7 @@ class ContentSource return $this->NextPageSelector; } - public function setNextPageSelector(string $NextPageSelector): static + public function setNextPageSelector(?string $NextPageSelector): static { $this->NextPageSelector = $NextPageSelector; diff --git a/src/Form/ContentSourceType.php b/src/Form/ContentSourceType.php new file mode 100644 index 0000000..1341c1c --- /dev/null +++ b/src/Form/ContentSourceType.php @@ -0,0 +1,46 @@ +add('baseUrl', UrlType::class, [ + 'label' => 'Base URL', + ]) + ->add('imageSelector', TextType::class, [ + 'label' => 'Image Selector', + ]) + ->add('chapterUrlFormat', TextType::class, [ + 'label' => 'Chapter URL Format', + ]) + ->add('nextPageSelector', TextType::class, [ + 'label' => 'Next Page Selector (let empty if vertical reader)', + 'required' => false, + ]) + ->add('scrapingType', ChoiceType::class, [ + 'label' => 'Scraping Type', + 'choices' => [ + 'HTML' => 'html', + 'JavaScript' => 'javascript' + ], + ]); + } + + public function configureOptions(OptionsResolver $resolver): void + { + $resolver->setDefaults([ + 'data_class' => ContentSource::class, + ]); + } +} diff --git a/src/MessageHandler/DownloadChapterHandler.php b/src/MessageHandler/DownloadChapterHandler.php index 69995f9..a7c95a7 100644 --- a/src/MessageHandler/DownloadChapterHandler.php +++ b/src/MessageHandler/DownloadChapterHandler.php @@ -5,8 +5,7 @@ namespace App\MessageHandler; use App\Entity\ContentSource; use App\Message\DownloadChapter; use App\Repository\ChapterRepository; -use App\Repository\MangaRepository; -use App\Service\LelScansProviderService; +use App\Repository\ContentSourceRepository; use App\Service\MangaScraperService; use App\Service\NotificationService; use Exception; @@ -20,7 +19,8 @@ readonly class DownloadChapterHandler public function __construct( private ChapterRepository $chapterRepository, private MangaScraperService $mangaScraperService, - private NotificationService $notificationService + private NotificationService $notificationService, + private ContentSourceRepository $contentSourceRepository ) { @@ -40,21 +40,27 @@ readonly class DownloadChapterHandler throw new BadRequestHttpException('Chapter already downloaded'); } - $sources = [ - (new ContentSource()) - ->setBaseUrl('https://lelscans.net') - ->setImageSelector('#image img') - ->setChapterUrlFormat('https://lelscans.net/scan-%s/%s') - ->setNextPageSelector('a[title="Suivant"]') - ->setScrapingType('html'), - + $sources = $this->contentSourceRepository->findAll(); + $sources[] = (new ContentSource()) ->setBaseUrl('https://api.mangadex.org/') ->setImageSelector('img') ->setChapterUrlFormat('at-home/server/%s') ->setScrapingType('mangadex') - ]; + ; +// (new ContentSource()) +// ->setBaseUrl('https://lelscans.net') +// ->setImageSelector('#image img') +// ->setChapterUrlFormat('https://lelscans.net/scan-%s/%s') +// ->setNextPageSelector('a[title="Suivant"]') +// ->setScrapingType('html'), +// (new ContentSource()) +// ->setBaseUrl('https://darkscans.net/') +// ->setImageSelector('.reading-content img') +// ->setChapterUrlFormat('https://darkscans.net/mangas/%s/chapter-%s/') +// ->setNextPageSelector(null) +// ->setScrapingType('html') $scrapedSuccessfully = false; diff --git a/src/Service/LelScansProviderService.php b/src/Service/LelScansProviderService.php deleted file mode 100644 index cdeca56..0000000 --- a/src/Service/LelScansProviderService.php +++ /dev/null @@ -1,67 +0,0 @@ -client = new Client(); - } - - public function getMangaList(): array - { - $crawler = $this->client->request('GET', self::PROVIDER_URL); - $mangaList = []; - - $crawler->filter('select > option')->each(function (Crawler $node) use (&$mangaList) { - $mangaName = $node->text(); - $mangaUrl = $node->attr('value'); - if ($mangaName && $mangaUrl && !preg_match('/^\d+(\.\d+)?$/', $mangaName)) { - $mangaList[] = [ - 'name' => $mangaName, - 'url' => $mangaUrl, - ]; - } - }); - - return $mangaList; - } - - public function getChapterList($mangaSlug): array - { - $crawler = $this->client->request('GET', self::PROVIDER_URL . 'lecture-en-ligne-' . $mangaSlug . '.php'); - $chapterList = []; - - $crawler->filter('select > option')->each(function (Crawler $node) use (&$chapterList) { - $chapterName = $node->text(); - $chapterUrl = $node->attr('value'); - if ($chapterName && $chapterUrl && preg_match('/^\d+(\.\d+)?$/', $chapterName)) { - $chapterList[] = [ - 'number' => $chapterName, - ]; - } - }); - - return $chapterList; - } - - #[\Override] public function getAvailableContent(Manga $manga): array - { - // TODO: Implement getAvailableContent() method. - } - - #[\Override] public function getContent(Manga $manga): array - { - // TODO: Implement getContent() method. - } -} diff --git a/src/Service/MangaExportService.php b/src/Service/MangaExportService.php deleted file mode 100644 index 02d9cf4..0000000 --- a/src/Service/MangaExportService.php +++ /dev/null @@ -1,100 +0,0 @@ -projectDir = $projectDir; - } - - public function exportMangaChapter(string $mangaTitle, int $chapterNumber): bool|string - { - $chapterDir = $this->getMangaDir($mangaTitle, $chapterNumber); - $cbzFilePath = $this->getExportDir($mangaTitle, $chapterNumber); - - if(!is_dir($chapterDir)){ - return false; - } - - $cbzDirectory = dirname($cbzFilePath); - if (!is_dir($cbzDirectory)) { - mkdir($cbzDirectory, 0755, true); - } - - $fileSystem = new Filesystem(); - if($fileSystem->exists($cbzFilePath)){ - return 'already_exported'; - } - - return $this->createCbzFromDirectory($chapterDir, $cbzFilePath); - } - - public function downloadCbz(string $mangaTitle, int $chapterNumber): BinaryFileResponse|bool - { - $filePathCbz = $this->getExportDir($mangaTitle, $chapterNumber); - - $fileSystem = new Filesystem(); - if($fileSystem->exists($filePathCbz)){ - return new BinaryFileResponse($filePathCbz); - } - - $chapterDir = $this->getMangaDir($mangaTitle, $chapterNumber); - if(is_dir($chapterDir)){ - if($this->exportMangaChapter($mangaTitle, $chapterNumber)){ - return new BinaryFileResponse($filePathCbz); - } - } - - return false; - } - - private function createCbzFromDirectory(string $sourceDirectory, string $cbzFilePath): bool - { - $zip = new ZipArchive(); - - // Ouvre le fichier .cbz en écriture - if ($zip->open($cbzFilePath, ZipArchive::CREATE | ZipArchive::OVERWRITE) !== true) { - return false; - } - - $files = new RecursiveIteratorIterator( - new RecursiveDirectoryIterator($sourceDirectory), - RecursiveIteratorIterator::LEAVES_ONLY - ); - - // Ajoute les fichiers d'image au fichier .cbz - foreach ($files as $file) { - if (!$file->isDir()) { - $filePath = $file->getRealPath(); - $relativePath = substr($filePath, strlen($sourceDirectory) + 1); - $zip->addFile($filePath, $relativePath); - } - } - - $zip->close(); - - return true; - } - - private function getMangaDir(string $mangaTitle, int $chapterNumber): string - { - return sprintf('%s/%s/%d', $this->projectDir . self::IMG_BASE_DIR, $mangaTitle, $chapterNumber); - } - - private function getExportDir(string $mangaTitle, int $chapterNumber): string - { - return sprintf('%s/%s/%d', $this->projectDir . self::EXPORT_BASE_DIR, $mangaTitle, $chapterNumber) . '.cbz'; - } -} \ No newline at end of file diff --git a/src/Service/MangaProviderFactory.php b/src/Service/MangaProviderFactory.php deleted file mode 100644 index 01706e9..0000000 --- a/src/Service/MangaProviderFactory.php +++ /dev/null @@ -1,17 +0,0 @@ - new LelScansProviderService(), - 'AutreManga' => new AutreMangaProviderService(), - default => throw new \Exception("Provider {$providerName} non supporté."), - }; - } -} diff --git a/src/Service/MangaScraperService.php b/src/Service/MangaScraperService.php index 4f77858..d29907c 100644 --- a/src/Service/MangaScraperService.php +++ b/src/Service/MangaScraperService.php @@ -166,6 +166,21 @@ class MangaScraperService return json_decode(implode("", $output), true); } + /** + * @throws GuzzleException + */ + public function testScrapingHtml(string $mangaSlug, string $chapterNumber, ContentSource $contentSource): array + { + $chapterUrl = $contentSource->getChapterUrl($mangaSlug, $chapterNumber); + $html = $this->fetchHtml($chapterUrl); + + if ($contentSource->getNextPageSelector() === null) { + return $this->scrapeVerticalReader($html, $contentSource); + } else { + return $this->scrapeHorizontalReader($chapterUrl, $contentSource); + } + } + /** * @throws GuzzleException */ @@ -173,32 +188,32 @@ class MangaScraperService { $chapterUrl = $mangaSource->getChapterUrl($manga->getSlug(), $chapter->getNumber()); - $pageData = []; - $currentPageUrl = $chapterUrl; - $tempDir = sys_get_temp_dir() . '/' . uniqid('manga_scraper_'); mkdir($tempDir); - do { - $html = $this->fetchHtml($currentPageUrl); - $page = $this->extractMangaPageData($html, $mangaSource); + $pageData = []; - $imageName = sprintf('%03d.%s', count($pageData) + 1, pathinfo(parse_url($page['image_url'], PHP_URL_PATH), PATHINFO_EXTENSION)); + if ($mangaSource->getNextPageSelector() === null) { + // Lecteur vertical + $html = $this->fetchHtml($chapterUrl); + $pageData = $this->scrapeVerticalReader($html, $mangaSource); + } else { + // Lecteur horizontal (paginé) + $pageData = $this->scrapeHorizontalReader($chapterUrl, $mangaSource); + } + + // Télécharger et sauvegarder les images + foreach ($pageData as $index => &$page) { + $imageName = sprintf('%03d.%s', $index + 1, pathinfo(parse_url($page['image_url'], PHP_URL_PATH), PATHINFO_EXTENSION)); $imagePath = $tempDir . '/' . $imageName; $this->downloadAndSaveImage($page['image_url'], $imagePath); - $event = new PageScrappingProgressEvent($chapter->getId(), count($pageData) + 1, 0); + $event = new PageScrappingProgressEvent($chapter->getId(), $index + 1, count($pageData)); $this->eventDispatcher->dispatch($event, PageScrappingProgressEvent::NAME); - $pageData[] = [ - 'image_url' => $page['image_url'], - 'local_image_url' => $imagePath, - 'page_number' => count($pageData) + 1, - ]; - - $currentPageUrl = $page['next_page_url']; - } while ($currentPageUrl); + $page['local_image_url'] = $imagePath; + } $cbzFilePath = $this->generateCbzPath($manga, $chapter); $this->createCbzFile($tempDir, $pageData, $cbzFilePath); @@ -210,7 +225,78 @@ class MangaScraperService // Nettoyage du répertoire temporaire $this->cleanupTempFiles($tempDir); - return true; + return $pageData; + } + + private function scrapeVerticalReader(string $html, ContentSource $contentSource): array + { + $crawler = new Crawler($html); + $images = $crawler->filter($contentSource->getImageSelector()); + + $pageData = []; + foreach ($images as $index => $image) { + if($image->getAttribute('src') === ''){ + $imgUrl = $image->getAttribute('data-src'); + }else{ + $imgUrl = $image->getAttribute('src'); + } + $pageData[] = [ + 'image_url' => $this->cleanImageUrl($imgUrl), + 'page_number' => $index + 1, + ]; + } + + return $pageData; + } + + /** + * @throws GuzzleException + */ + private function scrapeHorizontalReader(string $chapterUrl, ContentSource $contentSource): array + { + $pageData = []; + $currentPageUrl = $chapterUrl; + + do { + $html = $this->fetchHtml($currentPageUrl); + $page = $this->extractMangaPageData($html, $contentSource); + + $pageData[] = [ + 'image_url' => $this->cleanImageUrl($page['image_url']), + 'page_number' => count($pageData) + 1, + ]; + + $currentPageUrl = $page['next_page_url']; + } while ($currentPageUrl); + + return $pageData; + } + + /** + * Processes a single image + * @throws GuzzleException + */ + private function processImage(string $imgUrl, string $tempDir, array &$pageData, int $index, Chapter $chapter): void + { + $imgUrl = $this->cleanImageUrl($imgUrl); + $imageName = sprintf('%03d.%s', $index + 1, pathinfo(parse_url($imgUrl, PHP_URL_PATH), PATHINFO_EXTENSION)); + $imagePath = $tempDir . '/' . $imageName; + + $this->downloadAndSaveImage($imgUrl, $imagePath); + +// $event = new PageScrappingProgressEvent($chapter->getId(), $index + 1, 0); +// $this->eventDispatcher->dispatch($event, PageScrappingProgressEvent::NAME); + + $pageData[] = [ + 'image_url' => $imgUrl, + 'local_image_url' => $imagePath, + 'page_number' => $index + 1, + ]; + } + + private function cleanImageUrl(string $url): string + { + return preg_replace('/[\x00-\x1F\x7F]/', '', trim($url)); } /** diff --git a/src/Service/MangaScraperServiceOld.php b/src/Service/MangaScraperServiceOld.php deleted file mode 100644 index bd06161..0000000 --- a/src/Service/MangaScraperServiceOld.php +++ /dev/null @@ -1,157 +0,0 @@ -projectDir = $projectDir; - $this->eventDispatcher = $eventDispatcher; - } - - public function extractMangaPageData(string $html): array - { - $baseUrl = 'https://lelscans.net'; - //pour éviter à PhpStorm de gueuler... - $selector = 'img'; - $crawler = new Crawler($html); - $imgUrl = $crawler->filter($selector)->attr('src'); - $nextLink = $crawler->filter('a[title="Suivant"]'); - - if (!preg_match('/^https?:\/\//', $imgUrl)) { - $urlComponents = parse_url($baseUrl); - $scheme = $urlComponents['scheme']; - $host = $urlComponents['host']; - - // Construit l'URL absolue de l'image - $imgUrl = $scheme . '://' . $host . '/' . ltrim($imgUrl, '/'); - } - - if($nextLink->count() > 0){ - $nextUrl = $nextLink->attr('href'); - }else{ - $nextUrl = null; - } - - return [ - 'image_url' => $imgUrl, - 'next_page_url' => $nextUrl, - ]; - } - - /** - * @throws GuzzleException - */ - public function scrapeMangaChapter(string $chapterUrl, string $mangaTitle, float $chapterNumber): array|bool - { - if(!$this->isChapterAvailable($chapterUrl, $chapterNumber)){ - return false; - } - - $pageData = []; - $currentPageUrl = $chapterUrl; - - $mangaDir = sprintf('%s/%s', $this->projectDir . self::IMG_BASE_DIR, $mangaTitle); - if (!is_dir($mangaDir)) { - mkdir($mangaDir, 0755, true); - } - - // Créez le dossier du chapitre s'il n'existe pas - $chapterDir = sprintf('%s/%s', $mangaDir, $chapterNumber); - if (!is_dir($chapterDir)) { - mkdir($chapterDir, 0755, true); - } - - do { - $html = $this->fetchHtml($currentPageUrl); - $page = $this->extractMangaPageData($html); - $pageData[] = $page; - $currentPageUrl = $page['next_page_url']; - - // Construisez le nom de fichier de l'image - $imageName = sprintf('%03d.jpg', count($pageData)); - - // Construisez le chemin du fichier de l'image - $imagePath = sprintf('%s/%s', $chapterDir, $imageName); - - // Téléchargez et enregistrez l'image - $this->downloadAndSaveImage($page['image_url'], $imagePath); - - // Modifiez les données de la page pour inclure l'URL de l'image stockée localement - $pageData[count($pageData) - 1]['local_image_url'] = sprintf('/manga-images/%s/%s/%s', $mangaTitle, $chapterNumber, $imageName); - $pageData[count($pageData) - 1]['page_number'] = count($pageData); - - } while ($currentPageUrl); - - $event = new MangaScrapedEvent($mangaTitle, $chapterNumber, $pageData); - $this->eventDispatcher->dispatch($event, MangaScrapedEvent::NAME); - - return $pageData; - } - - /** - * @throws GuzzleException - */ - private function fetchHtml(string $url): string - { - $client = new Client(); - $response = $client->get($url); - - return (string) $response->getBody(); - } - - /** - * @throws GuzzleException - */ - private function downloadAndSaveImage(string $imageUrl, string $destinationPath): void - { - $client = new Client(); - $response = $client->get($imageUrl); - - file_put_contents($destinationPath, $response->getBody()->getContents()); - } - - /** - * @throws GuzzleException - */ - private function isChapterAvailable(string $chapterUrl, float $chapterNumber): bool - { - $html = $this->fetchHtml($chapterUrl); - $crawler = new Crawler($html); - $nextLink = $crawler->filter('a[title="Suivant"]'); - - if($nextLink->count() === 0){ - return false; - }else{ - $nextUrl = $nextLink->attr('href'); - } - - $routeCollection = new RouteCollection(); - $routeCollection->add('manga_chapter', new Route('/scan-{manga}/{chapter}/{page}')); - $context = new RequestContext('/'); - $matcher = new UrlMatcher($routeCollection, $context); - $path = parse_url($nextUrl, PHP_URL_PATH); - $parameters = $matcher->match($path); - - if((float) $parameters['chapter'] !== $chapterNumber){ - return false; - } - - return true; - } -} diff --git a/src/Service/MangadexProvider.php b/src/Service/MangadexProvider.php index 7994b27..3e089ce 100644 --- a/src/Service/MangadexProvider.php +++ b/src/Service/MangadexProvider.php @@ -125,12 +125,19 @@ readonly class MangadexProvider implements MetadataProviderInterface private function getFeedWithPagination(string $externalId, int $page): array { - return $this->client->get('/manga/' . $externalId . '/feed', [ - 'limit' => 500, - 'translatedLanguage' =>['en', 'fr'], - 'order' => ['chapter' => 'asc'], - 'offset' => $page * 500 - ]); + try { + $response = $this->client->get('/manga/' . $externalId . '/feed', [ + 'limit' => 500, + 'translatedLanguage' =>['en', 'fr'], + 'order' => ['chapter' => 'asc'], + 'offset' => $page * 500 + ]); + }catch(\Exception $e){ + $this->notificationService->sendUpdate(['status' => 'error', 'message' => 'An error occurred while fetching data from Mangadex.']); + return []; + } + + return $response; } public function getMangaAggregate(Manga $manga): array @@ -139,7 +146,12 @@ readonly class MangadexProvider implements MetadataProviderInterface return []; } - $response = $this->client->get('/manga/' . $manga->getExternalId() . '/aggregate'); + try { + $response = $this->client->get('/manga/' . $manga->getExternalId() . '/aggregate'); + }catch(\Exception $e){ +// $this->notificationService->sendUpdate(['status' => 'error', 'message' => 'An error occurred while fetching data from Mangadex.']); + return []; + } $chapterEntities = []; if($response['result'] === 'ok'){ diff --git a/src/Service/SushiScanProviderService.php b/src/Service/SushiScanProviderService.php deleted file mode 100644 index dccc42a..0000000 --- a/src/Service/SushiScanProviderService.php +++ /dev/null @@ -1,73 +0,0 @@ - 60]); - $this->client = new HttpBrowser($httpClient); - } - - public function getAvailableContent(Manga $manga) - { - $url = 'http://flaresolverr:8191/v1'; - $jsonContent = json_encode([ - 'cmd' => 'request.get', - 'url' => self::PROVIDER_URL . $manga->getSlug(), - 'maxTimeout' => 90000, - ]); - - - try{ - $crawler = $this->client->request('POST', $url, [], [], [ - 'HTTP_CONTENT_TYPE' => 'application/json', - ], $jsonContent); - - }catch (\Exception $e) { - dd($e); - } - $contentList = []; - - dd($crawler); - - $crawler->filter('#chapterList ul > li')->each(function (Crawler $node) use (&$contentList) { - dump($node); -// $contentName = $node->text(); -// $contentUrl = $node->attr('href'); -// if ($contentName && $contentUrl) { -// $contentList[] = [ -// 'name' => $contentName, -// 'url' => $contentUrl, -// ]; -// } - }); - - return $contentList; - } - - /** - * @param string $mangaSlug - * @return array - */ - public function getChapterList(string $mangaSlug): array - { - // TODO: Implement getChapterList() method. - } -} diff --git a/src/Twig/Components/NewMangaForm.php b/src/Twig/Components/NewMangaForm.php deleted file mode 100644 index 9dc2660..0000000 --- a/src/Twig/Components/NewMangaForm.php +++ /dev/null @@ -1,111 +0,0 @@ -manga = $manga; - $this->mangaData = [ - 'title' => $manga->getTitle(), - 'slug' => $manga->getSlug(), - 'description' => $manga->getDescription(), - 'imageUrl' => $manga->getImageUrl(), - 'status' => $manga->getStatus(), - 'genres' => $manga->getGenres(), - 'author' => $manga->getAuthor(), - 'publicationYear' => $manga->getPublicationYear(), - 'rating' => $manga->getRating(), - 'externalId' => $manga->getExternalId(), - ]; - } - - #[LiveAction] - public function saveManga(EntityManagerInterface $entityManager, MangadexProvider $mangadexProvider): Response - { - $manga = new Manga(); - $manga->setTitle($this->mangaData['title']) - ->setSlug($this->mangaData['slug']) - ->setDescription($this->mangaData['description']) - ->setImageUrl($this->mangaData['imageUrl']) - ->setStatus($this->mangaData['status']) - ->setGenres($this->mangaData['genres']) - ->setAuthor($this->mangaData['author']) - ->setPublicationYear($this->mangaData['publicationYear']) - ->setRating($this->mangaData['rating']) - ->setExternalId($this->mangaData['externalId']); - - $mangaFeed = $mangadexProvider->getFeed($manga); - $mangaAggregate = $mangadexProvider->getMangaAggregate($manga); - - $allChapters = array_merge($mangaFeed, $mangaAggregate); - - $mergedChapters = []; - foreach ($allChapters as $chapter) { - $number = $chapter->getNumber(); - - if (isset($mergedChapters[$number])) { - $existingChapter = $mergedChapters[$number]; - - if (!empty($chapter->getExternalId()) || - (empty($existingChapter->getExternalId()) && !strpos($chapter->getTitle(), 'Chapter ') == 0)) { - $mergedChapters[$number] = $chapter; - } - } else { - $mergedChapters[$number] = $chapter; - } - } - - foreach($mergedChapters as $chapter) { - $manga->addChapter($chapter); - } - - $mangaChapterUrl = $this->urlGenerator->generate('app_manga_show', ['mangaSlug' => $manga->getSlug()]); - - try { - foreach ($manga->getChapters() as $chapter) { - $entityManager->persist($chapter); - } - - $entityManager->persist($manga); - $entityManager->flush(); - } catch (\Exception $e) { - if ($e instanceof UniqueConstraintViolationException) { - return new RedirectResponse($mangaChapterUrl); - } - throw $e; - } - - return new RedirectResponse($mangaChapterUrl); - } -} diff --git a/src/Twig/Extension/AppExtension.php b/src/Twig/Extension/AppExtension.php new file mode 100644 index 0000000..8f373f7 --- /dev/null +++ b/src/Twig/Extension/AppExtension.php @@ -0,0 +1,28 @@ + 'https://example.com', + 'imageSelector' => '.manga-image img', + 'chapterUrlFormat' => 'https://example.com/manga/{slug}/chapter-{number}', + 'nextPageSelector' => '.next-page', + 'scrapingType' => 'Select scraping type', + default => '', + }; + } +} diff --git a/templates/menu/menu.html.twig b/templates/menu/menu.html.twig index 5b44651..1b7103d 100644 --- a/templates/menu/menu.html.twig +++ b/templates/menu/menu.html.twig @@ -38,7 +38,7 @@ {% endif %} diff --git a/templates/settings/scrapper_list.html.twig b/templates/settings/scrapper_list.html.twig new file mode 100644 index 0000000..0685f03 --- /dev/null +++ b/templates/settings/scrapper_list.html.twig @@ -0,0 +1,46 @@ +{% extends 'base.html.twig' %} + +{% block title %}Scrapper Configurations{% endblock %} + +{% block body %} +
+

Scrapper Configurations

+ +
+ {% for contentSource in contentSources %} +
+
+
+
+ {{ contentSource.baseUrl|replace({'http://': '', 'https://': ''})|trim('/', 'right') }} +
+ + + +
+
+
+
+ + {{ contentSource.scrapingType }} + + + {{ contentSource.nextPageSelector ? 'Horizontal' : 'Vertical' }} + +
+
+ + Edit configuration + +
+ {% endfor %} + + +
+ +

Add New Configuration

+
+
+
+
+{% endblock %} diff --git a/templates/settings/scrappers.html.twig b/templates/settings/scrappers.html.twig new file mode 100644 index 0000000..242259c --- /dev/null +++ b/templates/settings/scrappers.html.twig @@ -0,0 +1,66 @@ +{% extends 'base.html.twig' %} + +{% block title %}{{ isNew ? 'Create' : 'Edit' }} Scrapper Configuration{% endblock %} + +{% block body %} +
+
+
+

+ {{ isNew ? 'Create' : 'Edit' }} Scrapper Configuration +

+
+
+ {{ form_start(form, {'attr': {'class': 'space-y-6', 'data-scrapper-configure-target': 'form', 'data-action': 'submit->scrapper-configure#saveConfiguration'}}) }} + + {% for field in form.children %} +
+ {{ form_label(field, null, {'label_attr': {'class': 'block text-sm font-medium text-gray-700 mb-2'}}) }} + {{ form_widget(field, {'attr': { + 'class': 'mt-1 block w-full border border-gray-300 rounded-md shadow-sm py-2 px-3 focus:outline-none focus:ring-green-500 focus:border-green-500 sm:text-sm', + 'placeholder': get_placeholder(field.vars.name) + }}) }} +
+ {% endfor %} + +
+ +
+ + {{ form_end(form) }} + +
+

+ Test Configuration +

+
+
+ + +
+
+ + +
+
+ +
+
+
+ +
+
+
+
+{% endblock %}