colgora/Mangarr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added:

Browse Source 
- Messenger, Mercure
- chapter download flow (lelscan only)
This commit is contained in:
Jérémy Guillot
2024-06-13 18:11:11 +02:00
parent f88fa2c232
commit bc85649789
24 changed files with 744 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
.env
View File

@@ -32,3 +32,20 @@ MANGADEX_CLIENT_ID='personal-client-c6ea0ee7-8d48-41cd-8813-51b874177332-627526e
MANGADEX_CLIENT_SECRET='abMpCrSDYMWPjd24Pitl14t6RFqTs0cy'
MANGADEX_USERNAME='Colgora'
MANGADEX_PASSWORD='Hagaren666!'
###> symfony/messenger ###
# Choose one of the transports below
# MESSENGER_TRANSPORT_DSN=amqp://guest:guest@localhost:5672/%2f/messages
# MESSENGER_TRANSPORT_DSN=redis://localhost:6379/messages
MESSENGER_TRANSPORT_DSN=doctrine://default?auto_setup=0
###< symfony/messenger ###
###> symfony/mercure-bundle ###
# See https://symfony.com/doc/current/mercure.html#configuration
# The URL of the Mercure hub, used by the app to publish updates (can be a local URL)
MERCURE_URL=https://localhost/.well-known/mercure
# The public URL of the Mercure hub, used by the browser to connect
MERCURE_PUBLIC_URL=https://localhost/.well-known/mercure
# The secret used to sign the JWTs
MERCURE_JWT_SECRET="Mangarr-JWT-Secret"
###< symfony/mercure-bundle ###

3
Makefile
View File

@@ -135,6 +135,9 @@ twig-extension: ## Create a new twig extension
stimulus: ## Create a new stimulus controller
@$(SYMFONY) make:stimulus-controller
consume: ## Consume messages
@$(SYMFONY) messenger:consume async -vv
## —— Webpack Encore —————————————————————————————————————————————————————————————
npm-install: ## Install npm dependencies
@$(DOCKER_COMP) exec php npm install

2
assets/styles/app.scss
View File

@@ -1,4 +1,4 @@
//@import "bootstrap/scss/bootstrap";
@import '@fortawesome/fontawesome-free/scss/fontawesome.scss';
@import "tailwindcss/base";
@import "tailwindcss/components";
@import "tailwindcss/utilities";

6
compose.override.yaml
View File

@@ -25,3 +25,9 @@ services:
ports:
- "5432"
###< doctrine/doctrine-bundle ###
###> symfony/mercure-bundle ###
mercure:
ports:
- "80"
###< symfony/mercure-bundle ###

29
compose.yaml
View File

@@ -64,12 +64,41 @@ services:
depends_on:
- database
###> symfony/mercure-bundle ###
mercure:
image: dunglas/mercure
restart: unless-stopped
environment:
# Uncomment the following line to disable HTTPS,
#SERVER_NAME: ':80'
MERCURE_PUBLISHER_JWT_KEY: '!ChangeThisMercureHubJWTSecretKey!'
MERCURE_SUBSCRIBER_JWT_KEY: '!ChangeThisMercureHubJWTSecretKey!'
# Set the URL of your Symfony project (without trailing slash!) as value of the cors_origins directive
MERCURE_EXTRA_DIRECTIVES: |
cors_origins http://127.0.0.1:8000 http://localhost:8000 https://127.0.0.1:8000 https://localhost:8000 https://localhost http://localhost
# Comment the following line to disable the development mode
# command: /usr/bin/caddy run --config /etc/caddy/Caddyfile.dev
healthcheck:
test: ["CMD", "curl", "-f", "https://localhost/healthz"]
timeout: 5s
retries: 5
start_period: 60s
volumes:
- mercure_data:/data
- mercure_config:/config
###< symfony/mercure-bundle ###
volumes:
caddy_data:
caddy_config:
###> doctrine/doctrine-bundle ###
database_data:
###> symfony/mercure-bundle ###
mercure_data:
mercure_config:
###< symfony/mercure-bundle ###
###< doctrine/doctrine-bundle ###
networks:
mangarr_network:

3
composer.json
View File

@@ -22,11 +22,14 @@
"runtime/frankenphp-symfony": "^0.2.0",
"symfony/asset": "7.0.*",
"symfony/console": "7.0.*",
"symfony/doctrine-messenger": "7.0.*",
"symfony/dotenv": "7.0.*",
"symfony/expression-language": "7.0.*",
"symfony/flex": "^2",
"symfony/framework-bundle": "7.0.*",
"symfony/http-client": "7.0.*",
"symfony/mercure-bundle": "^0.3.9",
"symfony/messenger": "7.0.*",
"symfony/mime": "7.0.*",
"symfony/monolog-bundle": "^3.10",
"symfony/property-access": "7.0.*",

400
composer.lock generated
View File

@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "08c76de0049c9ace64fab221e185979c",
"content-hash": "e06fcb8d122d322f8406d9ab78787ddf",
"packages": [
{
"name": "api-platform/core",
@@ -1820,6 +1820,79 @@
],
"time": "2023-12-03T20:05:35+00:00"
},
{
"name": "lcobucci/jwt",
"version": "5.3.0",
"source": {
"type": "git",
"url": "https://github.com/lcobucci/jwt.git",
"reference": "08071d8d2c7f4b00222cc4b1fb6aa46990a80f83"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/lcobucci/jwt/zipball/08071d8d2c7f4b00222cc4b1fb6aa46990a80f83",
"reference": "08071d8d2c7f4b00222cc4b1fb6aa46990a80f83",
"shasum": ""
},
"require": {
"ext-openssl": "*",
"ext-sodium": "*",
"php": "~8.1.0 || ~8.2.0 || ~8.3.0",
"psr/clock": "^1.0"
},
"require-dev": {
"infection/infection": "^0.27.0",
"lcobucci/clock": "^3.0",
"lcobucci/coding-standard": "^11.0",
"phpbench/phpbench": "^1.2.9",
"phpstan/extension-installer": "^1.2",
"phpstan/phpstan": "^1.10.7",
"phpstan/phpstan-deprecation-rules": "^1.1.3",
"phpstan/phpstan-phpunit": "^1.3.10",
"phpstan/phpstan-strict-rules": "^1.5.0",
"phpunit/phpunit": "^10.2.6"
},
"suggest": {
"lcobucci/clock": ">= 3.0"
},
"type": "library",
"autoload": {
"psr-4": {
"Lcobucci\\JWT\\": "src"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"BSD-3-Clause"
],
"authors": [
{
"name": "Luís Cobucci",
"email": "lcobucci@gmail.com",
"role": "Developer"
}
],
"description": "A simple library to work with JSON Web Token and JSON Web Signature",
"keywords": [
"JWS",
"jwt"
],
"support": {
"issues": "https://github.com/lcobucci/jwt/issues",
"source": "https://github.com/lcobucci/jwt/tree/5.3.0"
},
"funding": [
{
"url": "https://github.com/lcobucci",
"type": "github"
},
{
"url": "https://www.patreon.com/lcobucci",
"type": "patreon"
}
],
"time": "2024-04-11T23:07:54+00:00"
},
{
"name": "monolog/monolog",
"version": "3.5.0",
@@ -3496,6 +3569,78 @@
],
"time": "2023-12-27T08:42:13+00:00"
},
{
"name": "symfony/doctrine-messenger",
"version": "v7.0.8",
"source": {
"type": "git",
"url": "https://github.com/symfony/doctrine-messenger.git",
"reference": "8b86a71fafa47c17cf631a737e76c5220624a2dc"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/doctrine-messenger/zipball/8b86a71fafa47c17cf631a737e76c5220624a2dc",
"reference": "8b86a71fafa47c17cf631a737e76c5220624a2dc",
"shasum": ""
},
"require": {
"doctrine/dbal": "^3.6|^4",
"php": ">=8.2",
"symfony/messenger": "^6.4|^7.0",
"symfony/service-contracts": "^2.5|^3"
},
"conflict": {
"doctrine/persistence": "<1.3"
},
"require-dev": {
"doctrine/persistence": "^1.3|^2|^3",
"symfony/property-access": "^6.4|^7.0",
"symfony/serializer": "^6.4|^7.0"
},
"type": "symfony-messenger-bridge",
"autoload": {
"psr-4": {
"Symfony\\Component\\Messenger\\Bridge\\Doctrine\\": ""
},
"exclude-from-classmap": [
"/Tests/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Fabien Potencier",
"email": "fabien@symfony.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Symfony Doctrine Messenger Bridge",
"homepage": "https://symfony.com",
"support": {
"source": "https://github.com/symfony/doctrine-messenger/tree/v7.0.8"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2024-05-31T14:55:39+00:00"
},
{
"name": "symfony/dotenv",
"version": "v7.0.2",
@@ -4561,6 +4706,259 @@
],
"time": "2023-12-30T15:41:17+00:00"
},
{
"name": "symfony/mercure",
"version": "v0.6.5",
"source": {
"type": "git",
"url": "https://github.com/symfony/mercure.git",
"reference": "304cf84609ef645d63adc65fc6250292909a461b"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/mercure/zipball/304cf84609ef645d63adc65fc6250292909a461b",
"reference": "304cf84609ef645d63adc65fc6250292909a461b",
"shasum": ""
},
"require": {
"php": ">=7.1.3",
"symfony/deprecation-contracts": "^2.0|^3.0|^4.0",
"symfony/http-client": "^4.4|^5.0|^6.0|^7.0",
"symfony/http-foundation": "^4.4|^5.0|^6.0|^7.0",
"symfony/polyfill-php80": "^1.22",
"symfony/web-link": "^4.4|^5.0|^6.0|^7.0"
},
"require-dev": {
"lcobucci/jwt": "^3.4|^4.0|^5.0",
"symfony/event-dispatcher": "^4.4|^5.0|^6.0|^7.0",
"symfony/http-kernel": "^4.4|^5.0|^6.0|^7.0",
"symfony/phpunit-bridge": "^5.2|^6.0|^7.0",
"symfony/stopwatch": "^4.4|^5.0|^6.0|^7.0",
"twig/twig": "^2.0|^3.0|^4.0"
},
"suggest": {
"symfony/stopwatch": "Integration with the profiler performances"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-main": "0.6.x-dev"
},
"thanks": {
"name": "dunglas/mercure",
"url": "https://github.com/dunglas/mercure"
}
},
"autoload": {
"psr-4": {
"Symfony\\Component\\Mercure\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Kévin Dunglas",
"email": "dunglas@gmail.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Symfony Mercure Component",
"homepage": "https://symfony.com",
"keywords": [
"mercure",
"push",
"sse",
"updates"
],
"support": {
"issues": "https://github.com/symfony/mercure/issues",
"source": "https://github.com/symfony/mercure/tree/v0.6.5"
},
"funding": [
{
"url": "https://github.com/dunglas",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/mercure",
"type": "tidelift"
}
],
"time": "2024-04-08T12:51:34+00:00"
},
{
"name": "symfony/mercure-bundle",
"version": "v0.3.9",
"source": {
"type": "git",
"url": "https://github.com/symfony/mercure-bundle.git",
"reference": "77435d740b228e9f5f3f065b6db564f85f2cdb64"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/mercure-bundle/zipball/77435d740b228e9f5f3f065b6db564f85f2cdb64",
"reference": "77435d740b228e9f5f3f065b6db564f85f2cdb64",
"shasum": ""
},
"require": {
"lcobucci/jwt": "^3.4|^4.0|^5.0",
"php": ">=7.1.3",
"symfony/config": "^4.4|^5.0|^6.0|^7.0",
"symfony/dependency-injection": "^4.4|^5.4|^6.0|^7.0",
"symfony/http-kernel": "^4.4|^5.0|^6.0|^7.0",
"symfony/mercure": "^0.6.1",
"symfony/web-link": "^4.4|^5.0|^6.0|^7.0"
},
"require-dev": {
"symfony/phpunit-bridge": "^4.3.7|^5.0|^6.0|^7.0",
"symfony/stopwatch": "^4.3.7|^5.0|^6.0|^7.0",
"symfony/ux-turbo": "*",
"symfony/var-dumper": "^4.3.7|^5.0|^6.0|^7.0"
},
"suggest": {
"symfony/messenger": "To use the Messenger integration"
},
"type": "symfony-bundle",
"extra": {
"branch-alias": {
"dev-main": "0.3.x-dev"
}
},
"autoload": {
"psr-4": {
"Symfony\\Bundle\\MercureBundle\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Kévin Dunglas",
"email": "dunglas@gmail.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Symfony MercureBundle",
"homepage": "https://symfony.com",
"keywords": [
"mercure",
"push",
"sse",
"updates"
],
"support": {
"issues": "https://github.com/symfony/mercure-bundle/issues",
"source": "https://github.com/symfony/mercure-bundle/tree/v0.3.9"
},
"funding": [
{
"url": "https://github.com/dunglas",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/mercure-bundle",
"type": "tidelift"
}
],
"time": "2024-05-31T09:07:18+00:00"
},
{
"name": "symfony/messenger",
"version": "v7.0.8",
"source": {
"type": "git",
"url": "https://github.com/symfony/messenger.git",
"reference": "ed7bccfe31e7f0bdb5b101f48b6027622a7a48cb"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/messenger/zipball/ed7bccfe31e7f0bdb5b101f48b6027622a7a48cb",
"reference": "ed7bccfe31e7f0bdb5b101f48b6027622a7a48cb",
"shasum": ""
},
"require": {
"php": ">=8.2",
"psr/log": "^1|^2|^3",
"symfony/clock": "^6.4|^7.0"
},
"conflict": {
"symfony/console": "<6.4",
"symfony/event-dispatcher": "<6.4",
"symfony/event-dispatcher-contracts": "<2.5",
"symfony/framework-bundle": "<6.4",
"symfony/http-kernel": "<6.4",
"symfony/serializer": "<6.4"
},
"require-dev": {
"psr/cache": "^1.0|^2.0|^3.0",
"symfony/console": "^6.4|^7.0",
"symfony/dependency-injection": "^6.4|^7.0",
"symfony/event-dispatcher": "^6.4|^7.0",
"symfony/http-kernel": "^6.4|^7.0",
"symfony/process": "^6.4|^7.0",
"symfony/property-access": "^6.4|^7.0",
"symfony/rate-limiter": "^6.4|^7.0",
"symfony/routing": "^6.4|^7.0",
"symfony/serializer": "^6.4|^7.0",
"symfony/service-contracts": "^2.5|^3",
"symfony/stopwatch": "^6.4|^7.0",
"symfony/validator": "^6.4|^7.0"
},
"type": "library",
"autoload": {
"psr-4": {
"Symfony\\Component\\Messenger\\": ""
},
"exclude-from-classmap": [
"/Tests/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Samuel Roze",
"email": "samuel.roze@gmail.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Helps applications send and receive messages to/from other applications or via message queues",
"homepage": "https://symfony.com",
"support": {
"source": "https://github.com/symfony/messenger/tree/v7.0.8"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2024-05-31T14:55:39+00:00"
},
{
"name": "symfony/mime",
"version": "v7.0.8",

1
config/bundles.php
View File

@@ -18,4 +18,5 @@ return [
Symfony\UX\LiveComponent\LiveComponentBundle::class => ['all' => true],
Symfony\UX\StimulusBundle\StimulusBundle::class => ['all' => true],
Twig\Extra\TwigExtraBundle\TwigExtraBundle::class => ['all' => true],
Symfony\Bundle\MercureBundle\MercureBundle::class => ['all' => true],
];

31
src/Controller/MangaController.php
View File

@@ -3,17 +3,22 @@
namespace App\Controller;
use App\Entity\Manga;
use App\Message\DownloadChapter;
use App\Repository\ChapterRepository;
use App\Repository\MangaRepository;
use App\Service\MangaExportService;
use App\Service\LelScansProviderService;
use App\Service\MangaScraperServiceOld;
use App\Service\MangaUpdatesMetadataProvider;
use Doctrine\ORM\NonUniqueResultException;
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
use Symfony\Component\HttpFoundation\BinaryFileResponse;
use Symfony\Component\HttpFoundation\JsonResponse;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\HttpFoundation\ResponseHeaderBag;
use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
use Symfony\Component\Messenger\MessageBusInterface;
use Symfony\Component\Routing\Annotation\Route;
use Symfony\Component\String\Slugger\AsciiSlugger;
@@ -24,7 +29,9 @@ class MangaController extends AbstractController
private readonly MangaExportService $mangaExportService,
private readonly LelScansProviderService $mangaProviderService,
private readonly MangaRepository $mangaRepository,
private MangaUpdatesMetadataProvider $mangaUpdatesDbProvider
private ChapterRepository $chapterRepository,
private MangaUpdatesMetadataProvider $mangaUpdatesDbProvider,
private MessageBusInterface $bus
)
{
}
@@ -39,9 +46,13 @@ class MangaController extends AbstractController
]);
}
/**
* @throws NonUniqueResultException
*/
#[Route('/manga/{mangaSlug}', name: 'manga_show')]
public function showChapters(string $mangaSlug): Response
{
// $manga = $this->mangaRepository->findOneWithChapterBy(['slug' => $mangaSlug]);
$manga = $this->mangaRepository->findOneBy(['slug' => $mangaSlug]);
if (!$manga) {
@@ -52,6 +63,9 @@ class MangaController extends AbstractController
foreach ($manga->getChapters() as $chapter) {
$volume = $chapter->getVolume() ?? 'Not Found';
$chaptersByVolume[$volume][] = $chapter;
usort($chaptersByVolume[$volume], function ($a, $b) {
return $a->getNumber() <=> $b->getNumber();
});
}
$chaptersByVolume = array_map('array_reverse', array_reverse($chaptersByVolume, true));
@@ -105,6 +119,21 @@ class MangaController extends AbstractController
]);
}
#[Route('/addChapter/{id}', name: 'add_chapter')]
public function addChapterMessenger(int $id): JsonResponse
{
$chapter = $this->chapterRepository->find($id);
if (!$chapter) {
return new JsonResponse(['error' => 'Chapter Not Found.'], 400);
}elseif ($chapter->getLocalPath() !== null){
return new JsonResponse(['error' => 'Chapter already scraped.'], 400);
}
$this->bus->dispatch(new DownloadChapter($id));
return new JsonResponse(['success' => 'Scrapping started...'], 200);
}
#[Route('/manga/{mangaSlug}/chapter/{chapterNumber}/download', name: 'download_chapter')]
public function downloadChapter(string $mangaSlug, float $chapterNumber): BinaryFileResponse
{

2
src/Entity/Manga.php
View File

@@ -19,7 +19,7 @@ class Manga
#[ORM\Column(length: 255)]
private ?string $title = null;
#[ORM\OneToMany(mappedBy: 'manga', targetEntity: Chapter::class, orphanRemoval: true)]
#[ORM\OneToMany(mappedBy: 'manga', targetEntity: Chapter::class, fetch: 'EAGER', orphanRemoval: true)]
private Collection $chapters;
#[ORM\Column(length: 255, unique: true)]

2
src/EventListener/MangaScrapedListener.php
View File

@@ -35,6 +35,8 @@ class MangaScrapedListener
$this->entityManager->persist($manga);
}
$chapter->setLocalPath($mangaData['directory']);
foreach ($mangaData['pages'] as $pageData) {
$page = $chapter->getPageByNumber($pageData['page_number']);
if (!$page) {

7
src/EventSubscriber/MangaScrapedEvent.php
View File

@@ -11,12 +11,14 @@ class MangaScrapedEvent extends Event
private string $mangaTitle;
private float $chapterNumber;
private array $pagesData;
private string $chapterDirectory;
public function __construct(string $mangaTitle, float $chapterNumber, array $pagesData)
public function __construct(string $mangaTitle, float $chapterNumber, array $pagesData, string $chapterDirectory)
{
$this->mangaTitle = $mangaTitle;
$this->chapterNumber = $chapterNumber;
$this->pagesData = $pagesData;
$this->chapterDirectory = $chapterDirectory;
}
public function getMangaData(): array
@@ -24,7 +26,8 @@ class MangaScrapedEvent extends Event
return [
'title' => $this->mangaTitle,
'chapter' => $this->chapterNumber,
'pages' => $this->pagesData
'pages' => $this->pagesData,
'directory' => $this->chapterDirectory
];
}
}

18
src/Message/DownloadChapter.php Normal file
View File

@@ -0,0 +1,18 @@
<?php
namespace App\Message;
class DownloadChapter
{
private int $chapterId;
public function __construct(int $chapterId)
{
$this->chapterId = $chapterId;
}
public function getChapterId(): int
{
return $this->chapterId;
}
}

57
src/MessageHandler/DownloadChapterHandler.php Normal file
View File

@@ -0,0 +1,57 @@
<?php
namespace App\MessageHandler;
use App\Entity\ContentSource;
use App\Message\DownloadChapter;
use App\Repository\ChapterRepository;
use App\Repository\MangaRepository;
use App\Service\LelScansProviderService;
use App\Service\MangaScraperService;
use App\Service\NotificationService;
use Exception;
use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;
use Symfony\Component\Messenger\Attribute\AsMessageHandler;
#[AsMessageHandler]
readonly class DownloadChapterHandler
{
public function __construct(
private ChapterRepository $chapterRepository,
private MangaScraperService $mangaScraperService,
private NotificationService $notificationService
)
{
}
/**
* @throws Exception
*/
public function __invoke(DownloadChapter $message): void
{
$chapter = $this->chapterRepository->find($message->getChapterId());
if (!$chapter) {
$this->notificationService->sendUpdate('notification', ['status' => 'error', 'message' => 'Chapter not found.']);
throw new BadRequestHttpException('Chapter not found');
}elseif ($chapter->getLocalPath() !== null){
$this->notificationService->sendUpdate('notification', ['status' => 'error', 'message' => 'Chapter already scraped.']);
throw new BadRequestHttpException('Chapter already downloaded');
}
$lelScanSource = new ContentSource();
$lelScanSource->setBaseUrl('https://lelscans.net')
->setImageSelector('#image img')
->setChapterUrlFormat('https://lelscans.net/scan-%s/%s')
->setNextPageSelector('a[title="Suivant"]')
->setScrapingType('html');
try {
$this->mangaScraperService->scrapeChapter($chapter, $lelScanSource);
} catch (Exception $e) {
$this->notificationService->sendUpdate('notification', ['status' => 'error', 'message' => 'An error occurred while scraping the chapter.']);
throw new Exception('Error scraping chapter: ' . $e->getMessage());
}
$this->notificationService->sendUpdate('notification', ['status' => 'success', 'message' => 'Chapter scraped successfully.']);
}
}

17
src/Repository/MangaRepository.php
View File

@@ -4,6 +4,7 @@ namespace App\Repository;
use App\Entity\Manga;
use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository;
use Doctrine\ORM\NonUniqueResultException;
use Doctrine\Persistence\ManagerRegistry;
/**
@@ -48,6 +49,22 @@ class MangaRepository extends ServiceEntityRepository
->getResult();
}
/**
* @throws NonUniqueResultException
*/
public function findOneWithChapterBy(array $params): ?Manga
{
$query = $this->createQueryBuilder('m');
foreach ($params as $key => $value) {
$query->andWhere("m.$key = :$key")
->setParameter($key, $value);
}
$query->leftJoin('m.chapters', 'c')
->addSelect('c');
return $query->getQuery()->getOneOrNullResult();
}
// /**
// * @return Manga[] Returns an array of Manga objects
// */

110
src/Service/MangaScraperService.php
View File

@@ -8,7 +8,11 @@ use App\Entity\ContentSource;
use App\EventSubscriber\MangaScrapedEvent;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use GuzzleHttp\Exception\RequestException;
use Symfony\Component\DomCrawler\Crawler;
use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;
use Symfony\Component\HttpKernel\Exception\HttpException;
use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
use Symfony\Component\Routing\Matcher\UrlMatcher;
use Symfony\Component\Routing\RequestContext;
use Symfony\Component\Routing\Route;
@@ -27,42 +31,31 @@ class MangaScraperService
$this->eventDispatcher = $eventDispatcher;
}
public function extractMangaPageData(string $html, ContentSource $mangaSource): array
private function extractMangaPageData(string $html, ContentSource $mangaSource): array
{
$crawler = new Crawler($html);
$imgUrls = [];
$imgUrl = $crawler->filter($mangaSource->getImageSelector())->attr('src')
?? $crawler->filter($mangaSource->getImageSelector())->attr('data-src');
// Search for images with different extensions
foreach (['img[src$=".jpg"]', 'img[src$=".jpeg"]', 'img[src$=".png"]', 'img'] as $selector) {
$crawler->filter($selector)->each(function (Crawler $node) use (&$imgUrls) {
$src = $node->attr('src') ?? $node->attr('data-src');
if ($src) {
$imgUrls[] = $src;
}
});
}
// dd($imgUrl);
if (empty($imgUrls)) {
throw new \Exception('No valid image found on the page.');
}
// if (empty($imgUrl)) {
// throw new \Exception('No valid image found on the page.');
// }
$nextLink = $crawler->filter($mangaSource->getNextPageSelector());
$nextUrl = $nextLink->count() > 0 ? $nextLink->attr('href') : null;
// Convert relative URLs to absolute URLs
$baseUrl = $mangaSource->getBaseUrl();
$imgUrls = array_map(function ($imgUrl) use ($baseUrl) {
if (!preg_match('/^https?:\/\//', $imgUrl)) {
$urlComponents = parse_url($baseUrl);
$scheme = $urlComponents['scheme'];
$host = $urlComponents['host'];
$imgUrl = $scheme . '://' . $host . '/' . ltrim($imgUrl, '/');
}
return $imgUrl;
}, $imgUrls);
if (!preg_match('/^https?:\/\//', $imgUrl)) {
$urlComponents = parse_url($mangaSource->getBaseUrl());
$scheme = $urlComponents['scheme'];
$host = $urlComponents['host'];
$imgUrl = $scheme . '://' . $host . '/' . ltrim($imgUrl, '/');
}
return [
'image_urls' => $imgUrls,
'image_url' => $imgUrl,
'next_page_url' => $nextUrl,
];
}
@@ -75,7 +68,7 @@ class MangaScraperService
$allChaptersData = [];
foreach ($manga->getChapters() as $chapter) {
$chapterData = $this->scrapeChapter($manga, $chapter, $mangaSource);
$chapterData = $this->scrapeChapter($chapter, $mangaSource);
if ($chapterData !== false) {
$allChaptersData[$chapter->getNumber()] = $chapterData;
}
@@ -84,13 +77,13 @@ class MangaScraperService
return $allChaptersData;
}
private function scrapeChapter(Manga $manga, Chapter $chapter, ContentSource $mangaSource): array|bool
public function scrapeChapter(Chapter $chapter, ContentSource $mangaSource): array|bool
{
switch ($mangaSource->getScrapingType()) {
case 'html':
return $this->scrapeChapterHtml($manga, $chapter, $mangaSource);
return $this->scrapeChapterHtml($chapter->getManga(), $chapter, $mangaSource);
case 'javascript':
return $this->scrapeChapterJavaScript($manga, $chapter, $mangaSource);
return $this->scrapeChapterJavaScript($chapter->getManga(), $chapter, $mangaSource);
// case 'api':
// // Implémentez la méthode de scraping par API si nécessaire
// return $this->scrapeChapterApi($manga, $chapter, $mangaSource);
@@ -121,10 +114,10 @@ class MangaScraperService
// Appeler le script Puppeteer avec les paramètres nécessaires
$output = [];
$command = sprintf('node puppeteer-script.js "%s" "%s" "%s" 2>&1', $url, $imageSelector, $nextButtonSelector); // Redirect stderr to stdout
dump($command);
// dump($command);
// exec($command, $output, $return_var);
dd($command, $output);
// dd($command, $output);
// Convertir la sortie JSON en tableau PHP
return json_decode(implode("", $output), true);
@@ -156,34 +149,25 @@ class MangaScraperService
$html = $this->fetchHtml($currentPageUrl);
$page = $this->extractMangaPageData($html, $mangaSource);
foreach ($page['image_urls'] as $imgUrl) {
dump($imgUrl);
dump(base64_decode($imgUrl));
// Déterminer l'extension de l'image
$imageExtension = pathinfo(parse_url($imgUrl, PHP_URL_PATH), PATHINFO_EXTENSION);
// Déterminer l'extension de l'image
$imageExtension = pathinfo(parse_url($page['image_url'], PHP_URL_PATH), PATHINFO_EXTENSION);
// Construire le nom de fichier de l'image
$imageName = sprintf('%03d.%s', count($pageData) + 1, $imageExtension);
$imagePath = sprintf('%s/%s', $chapterDir, $imageName);
// Construire le nom de fichier de l'image
$imageName = sprintf('%03d.%s', count($pageData) + 1, $imageExtension);
$imagePath = sprintf('%s/%s', $chapterDir, $imageName);
$this->downloadAndSaveImage($imgUrl, $imagePath);
$this->downloadAndSaveImage($page['image_url'], $imagePath);
$pageData[] = [
'image_url' => $imgUrl,
'local_image_url' => sprintf('/manga-images/%s/%s/%s', $mangaTitle, $chapterNumber, $imageName),
'page_number' => count($pageData) + 1,
];
}
// Si plus d'une image a été trouvée, ne pas chercher la page suivante
if (count($page['image_urls']) > 1) {
break;
}
$pageData[] = [
'image_url' => $page['image_url'],
'local_image_url' => sprintf('/manga-images/%s/%s/%s', $mangaTitle, $chapterNumber, $imageName),
'page_number' => count($pageData) + 1,
];
$currentPageUrl = $page['next_page_url'];
} while ($currentPageUrl);
$event = new MangaScrapedEvent($mangaTitle, $chapterNumber, $pageData);
$event = new MangaScrapedEvent($mangaTitle, $chapterNumber, $pageData, $chapterDir);
$this->eventDispatcher->dispatch($event, MangaScrapedEvent::NAME);
return $pageData;
@@ -195,9 +179,25 @@ class MangaScraperService
private function fetchHtml(string $url): string
{
$client = new Client();
$response = $client->get($url);
return (string)$response->getBody();
try {
$response = $client->get($url, [
'http_errors' => true,
'allow_redirects' => false
]);
$statusCode = $response->getStatusCode();
if ($statusCode >= 300 && $statusCode < 400) {
throw new NotFoundHttpException('Chapter Not Found at ' . $url);
} elseif ($statusCode == 404) {
throw new NotFoundHttpException('Chapter Not Found at ' . $url);
}
return (string)$response->getBody();
} catch (HttpException $e) {
throw new BadRequestHttpException('Bad Request: ' . $e->getMessage());
}
}
/**
@@ -240,7 +240,7 @@ class MangaScraperService
];
}
$event = new MangaScrapedEvent($mangaTitle, $chapterNumber, $pageData);
$event = new MangaScrapedEvent($mangaTitle, $chapterNumber, $pageData, $chapterDir);
$this->eventDispatcher->dispatch($event, MangaScrapedEvent::NAME);
return $pageData;

8
src/Service/MangadexProvider.php
View File

@@ -24,8 +24,9 @@ readonly class MangadexProvider implements MetadataProviderInterface
$results = $this->client->get('/manga', [
'title' => $title,
'contentRating' => ['safe'],
'includes' => ['cover_art', 'author']
'contentRating' => ['safe', 'suggestive'],
'includes' => ['cover_art', 'author'],
'limit' => 25
]);
$mangas = [];
@@ -112,7 +113,8 @@ readonly class MangadexProvider implements MetadataProviderInterface
return $manga;
}
private function getFeedWithPagination(string $externalId, int $page){
private function getFeedWithPagination(string $externalId, int $page): array
{
return $this->client->get('/manga/' . $externalId . '/feed', [
'limit' => 500,
'translatedLanguage' =>['en'],

20
src/Service/NotificationService.php Normal file
View File

@@ -0,0 +1,20 @@
<?php
namespace App\Service;
use Symfony\Component\Mercure\HubInterface;
use Symfony\Component\Mercure\Update;
class NotificationService
{
public function __construct(private HubInterface $hub)
{
}
public function sendUpdate(string $topic, mixed $data): void
{
$update = new Update($topic, json_encode($data));
$this->hub->publish($update);
}
}

12
src/Twig/Components/MangaSearch.php
View File

@@ -30,12 +30,12 @@ class MangaSearch
*/
public function getMangas(): Collection|null
{
return new ArrayCollection($this->mangaRepository->findAll());
// return new ArrayCollection($this->mangaRepository->findAll());
// if ($this->query === null || $this->query === '') {
// return null;
// }
//
// return $this->mangadexProvider->search($this->query);
if ($this->query === null || $this->query === '') {
return null;
}
return $this->mangadexProvider->search($this->query);
}
}

24
symfony.lock
View File

@@ -142,6 +142,30 @@
"ref": "fadbfe33303a76e25cb63401050439aa9b1a9c7f"
}
},
"symfony/mercure-bundle": {
"version": "0.3",
"recipe": {
"repo": "github.com/symfony/recipes",
"branch": "main",
"version": "0.3",
"ref": "d097c114aae82c5bc88d48ac164fe523f1003292"
},
"files": [
"config/packages/mercure.yaml"
]
},
"symfony/messenger": {
"version": "7.0",
"recipe": {
"repo": "github.com/symfony/recipes",
"branch": "main",
"version": "6.0",
"ref": "ba1ac4e919baba5644d31b57a3284d6ba12d52ee"
},
"files": [
"config/packages/messenger.yaml"
]
},
"symfony/monolog-bundle": {
"version": "3.10",
"recipe": {

6
tailwind.config.js
View File

@@ -9,4 +9,10 @@ module.exports = {
plugins: [
// require("daisyui"),
],
safelist: [
'bg-red-500',
'bg-blue-500',
'bg-yellow-500',
'bg-green-500',
],
}

16
templates/base.html.twig
View File

@@ -12,7 +12,17 @@
{{ encore_entry_script_tags('app') }}
{% endblock %}
</head>
<body>
<body class="bg-gray-50">
<div data-controller="mercure"
data-mercure-topic="notification">
</div>
<div data-controller="alert" class="fixed right-0 z-50 flex justify-center w-full">
<div data-alert-target="alert" style="display: none;"
class="mt-8 max-w-fit p-4 rounded-lg shadow-lg text-white border-2 border-gray-200 transition-opacity duration-1000 ease-out">
<i data-alert-target="icon" class="fa text-xl mr-2"></i>
<span data-alert-target="message"></span>
</div>
</div>
<div class="h-full">
<div class="fixed flex flex-row justify-start items-center w-full z-10 bg-green-600 h-16 z-30">
<div class="flex justify-center ml-10">
@@ -49,13 +59,13 @@
{% block toolbar %}
{% endblock %}
</div>
<div class="z-10">
<div class="z-10 bg-gray-50">
{% block body %}
{% endblock %}
</div>
</div>
</div>
</div>
<script src="https://kit.fontawesome.com/42e345444d.js" crossorigin="anonymous"></script>
{#<script src="https://kit.fontawesome.com/42e345444d.js" crossorigin="anonymous"></script>#}
</body>
</html>

27
templates/manga/show_chapters.html.twig
View File

@@ -97,7 +97,7 @@
<tbody>
{% for chapter in chapters %}
<tr class="border-t hover:bg-green-100">
{% if chapter.pagesLink|length > 0 %}
{% if chapter.localPath is not null %}
<td class="px-4 py-2 text-green-500">
<a href="{{ path('read_chapter_page', { mangaSlug: manga.slug, chapterNumber: chapter.number, pageNumber: 1 }) }}">
{{ chapter.number }}</a>
@@ -107,15 +107,32 @@
{% endif %}
<td class="px-4 py-2 w-full text-left">
{% if chapter.pagesLink|length > 0 %}
{% if chapter.localPath is not null %}
<a class=""
href="{{ path('read_chapter_page', { mangaSlug: manga.slug, chapterNumber: chapter.number }) }}">{{ chapter.title ?? 'No title' }}</a>
href="{{ path('read_chapter_page', { mangaSlug: manga.slug, chapterNumber: chapter.number, pageNumber: 1 }) }}">{{ chapter.title ?? 'No title' }}</a>
{% else %}
{{ chapter.title ?? 'No title'}}
{{ chapter.title ?? 'No title' }}
{% endif %}
</td>
<td class="px-4 py-2 flex justify-end gap-2">
{# <twig:DownloadChapter mangaSlug="{{ manga.slug }}" chapterNumber="{{ chapter.number }}" /> #}
{% if chapter.localPath is null %}
<button
data-controller="download-chapter"
data-action="click->download-chapter#handleClick"
data-url="{{ path('add_chapter', {id: chapter.id}) }}"
>
<span class="text-gray-500 hover:text-green-500">
<i data-download-chapter-target="icon" class="fas fa-search"></i>
</span>
</button>
{% else %}
<button>
<span class="text-gray-500" disabled="disabled">
<i data-download-chapter-target="icon" class="fas fa-search"></i>
</span>
</button>
{% endif %}
<a href="#" class="text-gray-500 hover:text-green-500">
<i class="fas fa-download"></i>
</a>

4
templates/menu/menu.html.twig
View File

@@ -21,6 +21,10 @@
<i class="fas fa-calendar-alt mr-2"></i>
<span>Calendrier</span>
</li>
<li class="mb-4 pl-8 flex items-center hover:text-green-600">
<i class="fas fa-clock mr-2"></i>
<span>Activité</span>
</li>
<li class="mb-4 pl-8 flex items-center hover:text-green-600">
<i class="fas fa-cog mr-2"></i>
<span>Paramètres</span>