Added:
- ContentSource handling in message - ContentSource list, add/update ui - nextPageSelector and imageSelector can be null - cleanup
This commit is contained in:
Jérémy Guillot
1
.gitignore
vendored
1
.gitignore
vendored
@@ -33,3 +33,4 @@ yarn-error.log
|
|||||||
###< symfony/webpack-encore-bundle ###
|
###< symfony/webpack-encore-bundle ###
|
||||||
/public/manga-export/
|
/public/manga-export/
|
||||||
/public/manga-images/
|
/public/manga-images/
|
||||||
|
/public/cbz/
|
||||||
|
|||||||
78
assets/controllers/scrapper_configure_controller.js
Normal file
78
assets/controllers/scrapper_configure_controller.js
Normal file
@@ -0,0 +1,78 @@
|
|||||||
|
import { Controller } from '@hotwired/stimulus';
|
||||||
|
|
||||||
|
export default class extends Controller {
|
||||||
|
static targets = ['form', 'testForm', 'imageSelector', 'nextPageSelector', 'testResults', 'scrapingType']
|
||||||
|
|
||||||
|
connect() {
|
||||||
|
}
|
||||||
|
|
||||||
|
async saveConfiguration(event) {
|
||||||
|
console.log('saveConfiguration called');
|
||||||
|
event.preventDefault();
|
||||||
|
this.formTarget.submit();
|
||||||
|
}
|
||||||
|
|
||||||
|
async testConfiguration(event) {
|
||||||
|
console.log('testConfiguration called');
|
||||||
|
event.preventDefault();
|
||||||
|
const formData = new FormData(this.formTarget);
|
||||||
|
const testFormData = new FormData(this.testFormTarget);
|
||||||
|
|
||||||
|
for (let [key, value] of formData.entries()) {
|
||||||
|
const cleanKey = key.replace(/^content_source\[(.+)]$/, '$1');
|
||||||
|
testFormData.append(`content_source[${cleanKey}]`, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(this.testFormTarget.action, {
|
||||||
|
method: 'POST',
|
||||||
|
body: testFormData
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await response.json();
|
||||||
|
|
||||||
|
if (result.success) {
|
||||||
|
this.displayTestResults(result.data);
|
||||||
|
} else {
|
||||||
|
this.displayError(result.message, result.errors);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.log(error)
|
||||||
|
this.displayError('An error occurred while testing the configuration');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
displayTestResults(data) {
|
||||||
|
let html = '<h3 class="text-xl font-semibold mb-4">Test Results</h3>';
|
||||||
|
html += '<div class="grid grid-cols-2 md:grid-cols-3 lg:grid-cols-4 gap-4">';
|
||||||
|
data.forEach(page => {
|
||||||
|
html += `
|
||||||
|
<div class="border rounded-lg p-2 flex flex-col items-center">
|
||||||
|
<img src="${page.image_url}" alt="Page ${page.page_number}" class="w-full h-48 object-cover mb-2">
|
||||||
|
<p class="text-sm font-medium">Page ${page.page_number}</p>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
});
|
||||||
|
html += '</div>';
|
||||||
|
this.testResultsTarget.innerHTML = html;
|
||||||
|
}
|
||||||
|
|
||||||
|
displayError(message, errors = []) {
|
||||||
|
let errorHtml = `
|
||||||
|
<div class="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded relative" role="alert">
|
||||||
|
<strong class="font-bold">Error:</strong>
|
||||||
|
<span class="block sm:inline">${message}</span>
|
||||||
|
`;
|
||||||
|
|
||||||
|
if (errors.length > 0) {
|
||||||
|
errorHtml += '<ul class="list-disc list-inside mt-2">';
|
||||||
|
errors.forEach(error => {
|
||||||
|
errorHtml += `<li>${error}</li>`;
|
||||||
|
});
|
||||||
|
errorHtml += '</ul>';
|
||||||
|
}
|
||||||
|
|
||||||
|
errorHtml += '</div>';
|
||||||
|
this.testResultsTarget.innerHTML = errorHtml;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -10,6 +10,7 @@
|
|||||||
"ext-ctype": "*",
|
"ext-ctype": "*",
|
||||||
"ext-curl": "*",
|
"ext-curl": "*",
|
||||||
"ext-iconv": "*",
|
"ext-iconv": "*",
|
||||||
|
"ext-zip": "*",
|
||||||
"api-platform/core": "^3.2",
|
"api-platform/core": "^3.2",
|
||||||
"doctrine/dbal": "^3",
|
"doctrine/dbal": "^3",
|
||||||
"doctrine/doctrine-bundle": "^2.11",
|
"doctrine/doctrine-bundle": "^2.11",
|
||||||
@@ -26,6 +27,7 @@
|
|||||||
"symfony/dotenv": "7.0.*",
|
"symfony/dotenv": "7.0.*",
|
||||||
"symfony/expression-language": "7.0.*",
|
"symfony/expression-language": "7.0.*",
|
||||||
"symfony/flex": "^2",
|
"symfony/flex": "^2",
|
||||||
|
"symfony/form": "7.0.*",
|
||||||
"symfony/framework-bundle": "7.0.*",
|
"symfony/framework-bundle": "7.0.*",
|
||||||
"symfony/http-client": "7.0.*",
|
"symfony/http-client": "7.0.*",
|
||||||
"symfony/mercure-bundle": "^0.3.9",
|
"symfony/mercure-bundle": "^0.3.9",
|
||||||
@@ -44,8 +46,7 @@
|
|||||||
"symfony/webpack-encore-bundle": "^2.1",
|
"symfony/webpack-encore-bundle": "^2.1",
|
||||||
"symfony/yaml": "7.0.*",
|
"symfony/yaml": "7.0.*",
|
||||||
"twig/extra-bundle": "^2.12|^3.0",
|
"twig/extra-bundle": "^2.12|^3.0",
|
||||||
"twig/twig": "^2.12|^3.0",
|
"twig/twig": "^2.12|^3.0"
|
||||||
"ext-zip": "*"
|
|
||||||
},
|
},
|
||||||
"config": {
|
"config": {
|
||||||
"allow-plugins": {
|
"allow-plugins": {
|
||||||
|
|||||||
319
composer.lock
generated
319
composer.lock
generated
@@ -4,7 +4,7 @@
|
|||||||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||||
"This file is @generated automatically"
|
"This file is @generated automatically"
|
||||||
],
|
],
|
||||||
"content-hash": "e06fcb8d122d322f8406d9ab78787ddf",
|
"content-hash": "d52c83bad4e4c116ba33e0f33b9cfd7b",
|
||||||
"packages": [
|
"packages": [
|
||||||
{
|
{
|
||||||
"name": "api-platform/core",
|
"name": "api-platform/core",
|
||||||
@@ -4201,6 +4201,102 @@
|
|||||||
],
|
],
|
||||||
"time": "2024-01-02T11:08:32+00:00"
|
"time": "2024-01-02T11:08:32+00:00"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "symfony/form",
|
||||||
|
"version": "v7.0.8",
|
||||||
|
"source": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/symfony/form.git",
|
||||||
|
"reference": "1d0128e2f7e80c346ec51fa4d1ce4fec0d435eeb"
|
||||||
|
},
|
||||||
|
"dist": {
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://api.github.com/repos/symfony/form/zipball/1d0128e2f7e80c346ec51fa4d1ce4fec0d435eeb",
|
||||||
|
"reference": "1d0128e2f7e80c346ec51fa4d1ce4fec0d435eeb",
|
||||||
|
"shasum": ""
|
||||||
|
},
|
||||||
|
"require": {
|
||||||
|
"php": ">=8.2",
|
||||||
|
"symfony/event-dispatcher": "^6.4|^7.0",
|
||||||
|
"symfony/options-resolver": "^6.4|^7.0",
|
||||||
|
"symfony/polyfill-ctype": "~1.8",
|
||||||
|
"symfony/polyfill-intl-icu": "^1.21",
|
||||||
|
"symfony/polyfill-mbstring": "~1.0",
|
||||||
|
"symfony/property-access": "^6.4|^7.0",
|
||||||
|
"symfony/service-contracts": "^2.5|^3"
|
||||||
|
},
|
||||||
|
"conflict": {
|
||||||
|
"symfony/console": "<6.4",
|
||||||
|
"symfony/dependency-injection": "<6.4",
|
||||||
|
"symfony/doctrine-bridge": "<6.4",
|
||||||
|
"symfony/error-handler": "<6.4",
|
||||||
|
"symfony/framework-bundle": "<6.4",
|
||||||
|
"symfony/http-kernel": "<6.4",
|
||||||
|
"symfony/translation": "<6.4.3|>=7.0,<7.0.3",
|
||||||
|
"symfony/translation-contracts": "<2.5",
|
||||||
|
"symfony/twig-bridge": "<6.4"
|
||||||
|
},
|
||||||
|
"require-dev": {
|
||||||
|
"doctrine/collections": "^1.0|^2.0",
|
||||||
|
"symfony/config": "^6.4|^7.0",
|
||||||
|
"symfony/console": "^6.4|^7.0",
|
||||||
|
"symfony/dependency-injection": "^6.4|^7.0",
|
||||||
|
"symfony/expression-language": "^6.4|^7.0",
|
||||||
|
"symfony/html-sanitizer": "^6.4|^7.0",
|
||||||
|
"symfony/http-foundation": "^6.4|^7.0",
|
||||||
|
"symfony/http-kernel": "^6.4|^7.0",
|
||||||
|
"symfony/intl": "^6.4|^7.0",
|
||||||
|
"symfony/security-core": "^6.4|^7.0",
|
||||||
|
"symfony/security-csrf": "^6.4|^7.0",
|
||||||
|
"symfony/translation": "^6.4.3|^7.0.3",
|
||||||
|
"symfony/uid": "^6.4|^7.0",
|
||||||
|
"symfony/validator": "^6.4|^7.0",
|
||||||
|
"symfony/var-dumper": "^6.4|^7.0"
|
||||||
|
},
|
||||||
|
"type": "library",
|
||||||
|
"autoload": {
|
||||||
|
"psr-4": {
|
||||||
|
"Symfony\\Component\\Form\\": ""
|
||||||
|
},
|
||||||
|
"exclude-from-classmap": [
|
||||||
|
"/Tests/"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"notification-url": "https://packagist.org/downloads/",
|
||||||
|
"license": [
|
||||||
|
"MIT"
|
||||||
|
],
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Fabien Potencier",
|
||||||
|
"email": "fabien@symfony.com"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Symfony Community",
|
||||||
|
"homepage": "https://symfony.com/contributors"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "Allows to easily create, process and reuse HTML forms",
|
||||||
|
"homepage": "https://symfony.com",
|
||||||
|
"support": {
|
||||||
|
"source": "https://github.com/symfony/form/tree/v7.0.8"
|
||||||
|
},
|
||||||
|
"funding": [
|
||||||
|
{
|
||||||
|
"url": "https://symfony.com/sponsor",
|
||||||
|
"type": "custom"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"url": "https://github.com/fabpot",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
|
||||||
|
"type": "tidelift"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"time": "2024-05-31T14:55:39+00:00"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "symfony/framework-bundle",
|
"name": "symfony/framework-bundle",
|
||||||
"version": "v7.0.2",
|
"version": "v7.0.2",
|
||||||
@@ -5202,6 +5298,73 @@
|
|||||||
],
|
],
|
||||||
"time": "2023-11-06T17:08:13+00:00"
|
"time": "2023-11-06T17:08:13+00:00"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "symfony/options-resolver",
|
||||||
|
"version": "v7.0.0",
|
||||||
|
"source": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/symfony/options-resolver.git",
|
||||||
|
"reference": "700ff4096e346f54cb628ea650767c8130f1001f"
|
||||||
|
},
|
||||||
|
"dist": {
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://api.github.com/repos/symfony/options-resolver/zipball/700ff4096e346f54cb628ea650767c8130f1001f",
|
||||||
|
"reference": "700ff4096e346f54cb628ea650767c8130f1001f",
|
||||||
|
"shasum": ""
|
||||||
|
},
|
||||||
|
"require": {
|
||||||
|
"php": ">=8.2",
|
||||||
|
"symfony/deprecation-contracts": "^2.5|^3"
|
||||||
|
},
|
||||||
|
"type": "library",
|
||||||
|
"autoload": {
|
||||||
|
"psr-4": {
|
||||||
|
"Symfony\\Component\\OptionsResolver\\": ""
|
||||||
|
},
|
||||||
|
"exclude-from-classmap": [
|
||||||
|
"/Tests/"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"notification-url": "https://packagist.org/downloads/",
|
||||||
|
"license": [
|
||||||
|
"MIT"
|
||||||
|
],
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Fabien Potencier",
|
||||||
|
"email": "fabien@symfony.com"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Symfony Community",
|
||||||
|
"homepage": "https://symfony.com/contributors"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "Provides an improved replacement for the array_replace PHP function",
|
||||||
|
"homepage": "https://symfony.com",
|
||||||
|
"keywords": [
|
||||||
|
"config",
|
||||||
|
"configuration",
|
||||||
|
"options"
|
||||||
|
],
|
||||||
|
"support": {
|
||||||
|
"source": "https://github.com/symfony/options-resolver/tree/v7.0.0"
|
||||||
|
},
|
||||||
|
"funding": [
|
||||||
|
{
|
||||||
|
"url": "https://symfony.com/sponsor",
|
||||||
|
"type": "custom"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"url": "https://github.com/fabpot",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
|
||||||
|
"type": "tidelift"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"time": "2023-08-08T10:20:21+00:00"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "symfony/password-hasher",
|
"name": "symfony/password-hasher",
|
||||||
"version": "v7.0.3",
|
"version": "v7.0.3",
|
||||||
@@ -5355,6 +5518,90 @@
|
|||||||
],
|
],
|
||||||
"time": "2023-01-26T09:26:14+00:00"
|
"time": "2023-01-26T09:26:14+00:00"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "symfony/polyfill-intl-icu",
|
||||||
|
"version": "v1.30.0",
|
||||||
|
"source": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/symfony/polyfill-intl-icu.git",
|
||||||
|
"reference": "e76343c631b453088e2260ac41dfebe21954de81"
|
||||||
|
},
|
||||||
|
"dist": {
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://api.github.com/repos/symfony/polyfill-intl-icu/zipball/e76343c631b453088e2260ac41dfebe21954de81",
|
||||||
|
"reference": "e76343c631b453088e2260ac41dfebe21954de81",
|
||||||
|
"shasum": ""
|
||||||
|
},
|
||||||
|
"require": {
|
||||||
|
"php": ">=7.1"
|
||||||
|
},
|
||||||
|
"suggest": {
|
||||||
|
"ext-intl": "For best performance and support of other locales than \"en\""
|
||||||
|
},
|
||||||
|
"type": "library",
|
||||||
|
"extra": {
|
||||||
|
"thanks": {
|
||||||
|
"name": "symfony/polyfill",
|
||||||
|
"url": "https://github.com/symfony/polyfill"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"autoload": {
|
||||||
|
"files": [
|
||||||
|
"bootstrap.php"
|
||||||
|
],
|
||||||
|
"psr-4": {
|
||||||
|
"Symfony\\Polyfill\\Intl\\Icu\\": ""
|
||||||
|
},
|
||||||
|
"classmap": [
|
||||||
|
"Resources/stubs"
|
||||||
|
],
|
||||||
|
"exclude-from-classmap": [
|
||||||
|
"/Tests/"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"notification-url": "https://packagist.org/downloads/",
|
||||||
|
"license": [
|
||||||
|
"MIT"
|
||||||
|
],
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Nicolas Grekas",
|
||||||
|
"email": "p@tchwork.com"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Symfony Community",
|
||||||
|
"homepage": "https://symfony.com/contributors"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "Symfony polyfill for intl's ICU-related data and classes",
|
||||||
|
"homepage": "https://symfony.com",
|
||||||
|
"keywords": [
|
||||||
|
"compatibility",
|
||||||
|
"icu",
|
||||||
|
"intl",
|
||||||
|
"polyfill",
|
||||||
|
"portable",
|
||||||
|
"shim"
|
||||||
|
],
|
||||||
|
"support": {
|
||||||
|
"source": "https://github.com/symfony/polyfill-intl-icu/tree/v1.30.0"
|
||||||
|
},
|
||||||
|
"funding": [
|
||||||
|
{
|
||||||
|
"url": "https://symfony.com/sponsor",
|
||||||
|
"type": "custom"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"url": "https://github.com/fabpot",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
|
||||||
|
"type": "tidelift"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"time": "2024-05-31T15:07:36+00:00"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "symfony/polyfill-intl-idn",
|
"name": "symfony/polyfill-intl-idn",
|
||||||
"version": "v1.29.0",
|
"version": "v1.29.0",
|
||||||
@@ -10676,73 +10923,6 @@
|
|||||||
],
|
],
|
||||||
"time": "2023-10-31T18:23:49+00:00"
|
"time": "2023-10-31T18:23:49+00:00"
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"name": "symfony/options-resolver",
|
|
||||||
"version": "v7.0.0",
|
|
||||||
"source": {
|
|
||||||
"type": "git",
|
|
||||||
"url": "https://github.com/symfony/options-resolver.git",
|
|
||||||
"reference": "700ff4096e346f54cb628ea650767c8130f1001f"
|
|
||||||
},
|
|
||||||
"dist": {
|
|
||||||
"type": "zip",
|
|
||||||
"url": "https://api.github.com/repos/symfony/options-resolver/zipball/700ff4096e346f54cb628ea650767c8130f1001f",
|
|
||||||
"reference": "700ff4096e346f54cb628ea650767c8130f1001f",
|
|
||||||
"shasum": ""
|
|
||||||
},
|
|
||||||
"require": {
|
|
||||||
"php": ">=8.2",
|
|
||||||
"symfony/deprecation-contracts": "^2.5|^3"
|
|
||||||
},
|
|
||||||
"type": "library",
|
|
||||||
"autoload": {
|
|
||||||
"psr-4": {
|
|
||||||
"Symfony\\Component\\OptionsResolver\\": ""
|
|
||||||
},
|
|
||||||
"exclude-from-classmap": [
|
|
||||||
"/Tests/"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"notification-url": "https://packagist.org/downloads/",
|
|
||||||
"license": [
|
|
||||||
"MIT"
|
|
||||||
],
|
|
||||||
"authors": [
|
|
||||||
{
|
|
||||||
"name": "Fabien Potencier",
|
|
||||||
"email": "fabien@symfony.com"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Symfony Community",
|
|
||||||
"homepage": "https://symfony.com/contributors"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"description": "Provides an improved replacement for the array_replace PHP function",
|
|
||||||
"homepage": "https://symfony.com",
|
|
||||||
"keywords": [
|
|
||||||
"config",
|
|
||||||
"configuration",
|
|
||||||
"options"
|
|
||||||
],
|
|
||||||
"support": {
|
|
||||||
"source": "https://github.com/symfony/options-resolver/tree/v7.0.0"
|
|
||||||
},
|
|
||||||
"funding": [
|
|
||||||
{
|
|
||||||
"url": "https://symfony.com/sponsor",
|
|
||||||
"type": "custom"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"url": "https://github.com/fabpot",
|
|
||||||
"type": "github"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
|
|
||||||
"type": "tidelift"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"time": "2023-08-08T10:20:21+00:00"
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"name": "symfony/phpunit-bridge",
|
"name": "symfony/phpunit-bridge",
|
||||||
"version": "v7.0.4",
|
"version": "v7.0.4",
|
||||||
@@ -11308,7 +11488,8 @@
|
|||||||
"php": ">=8.3.1",
|
"php": ">=8.3.1",
|
||||||
"ext-ctype": "*",
|
"ext-ctype": "*",
|
||||||
"ext-curl": "*",
|
"ext-curl": "*",
|
||||||
"ext-iconv": "*"
|
"ext-iconv": "*",
|
||||||
|
"ext-zip": "*"
|
||||||
},
|
},
|
||||||
"platform-dev": [],
|
"platform-dev": [],
|
||||||
"plugin-api-version": "2.6.0"
|
"plugin-api-version": "2.6.0"
|
||||||
|
|||||||
@@ -40,19 +40,10 @@ services:
|
|||||||
protocols: [ 'http', 'https' ]
|
protocols: [ 'http', 'https' ]
|
||||||
track_redirects: true
|
track_redirects: true
|
||||||
|
|
||||||
|
|
||||||
App\Service\MangaScraperServiceOld:
|
|
||||||
arguments:
|
|
||||||
$projectDir: '%kernel.project_dir%'
|
|
||||||
|
|
||||||
App\Service\MangaScraperService:
|
App\Service\MangaScraperService:
|
||||||
arguments:
|
arguments:
|
||||||
$projectDir: '%kernel.project_dir%'
|
$projectDir: '%kernel.project_dir%'
|
||||||
|
|
||||||
App\Service\MangaExportService:
|
|
||||||
arguments:
|
|
||||||
$projectDir: '%kernel.project_dir%'
|
|
||||||
|
|
||||||
App\Service\MangaImportService:
|
App\Service\MangaImportService:
|
||||||
arguments:
|
arguments:
|
||||||
$projectDir: '%kernel.project_dir%'
|
$projectDir: '%kernel.project_dir%'
|
||||||
@@ -69,9 +60,6 @@ services:
|
|||||||
tags:
|
tags:
|
||||||
- { name: kernel.event_subscriber }
|
- { name: kernel.event_subscriber }
|
||||||
|
|
||||||
App\Controller\MenuController:
|
|
||||||
tags: [ 'controller.service_arguments' ]
|
|
||||||
|
|
||||||
App\Client\MangadexClient:
|
App\Client\MangadexClient:
|
||||||
arguments:
|
arguments:
|
||||||
$httpClient: '@GuzzleHttp\Client'
|
$httpClient: '@GuzzleHttp\Client'
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ final class Version20240610115931 extends AbstractMigration
|
|||||||
{
|
{
|
||||||
// this up() migration is auto-generated, please modify it to your needs
|
// this up() migration is auto-generated, please modify it to your needs
|
||||||
$this->addSql('CREATE SEQUENCE content_source_id_seq INCREMENT BY 1 MINVALUE 1 START 1');
|
$this->addSql('CREATE SEQUENCE content_source_id_seq INCREMENT BY 1 MINVALUE 1 START 1');
|
||||||
$this->addSql('CREATE TABLE content_source (id INT NOT NULL, base_url VARCHAR(255) NOT NULL, image_selector VARCHAR(255) NOT NULL, next_page_selector VARCHAR(255) NOT NULL, chapter_url_format VARCHAR(255) NOT NULL, scraping_type VARCHAR(255) NOT NULL, PRIMARY KEY(id))');
|
$this->addSql('CREATE TABLE content_source (id INT NOT NULL, base_url VARCHAR(255) NOT NULL, image_selector VARCHAR(255) DEFAULT NULL, next_page_selector VARCHAR(255) DEFAULT NULL, chapter_url_format VARCHAR(255) NOT NULL, scraping_type VARCHAR(255) NOT NULL, PRIMARY KEY(id))');
|
||||||
$this->addSql('ALTER TABLE chapter ADD volume INT DEFAULT NULL');
|
$this->addSql('ALTER TABLE chapter ADD volume INT DEFAULT NULL');
|
||||||
$this->addSql('ALTER TABLE chapter ADD title VARCHAR(255) DEFAULT NULL');
|
$this->addSql('ALTER TABLE chapter ADD title VARCHAR(255) DEFAULT NULL');
|
||||||
$this->addSql('ALTER TABLE chapter ADD local_path VARCHAR(255) DEFAULT NULL');
|
$this->addSql('ALTER TABLE chapter ADD local_path VARCHAR(255) DEFAULT NULL');
|
||||||
|
|||||||
@@ -10,13 +10,13 @@ use App\Repository\ChapterRepository;
|
|||||||
use App\Repository\MangaRepository;
|
use App\Repository\MangaRepository;
|
||||||
use App\Service\CbzService;
|
use App\Service\CbzService;
|
||||||
use App\Service\MangadexProvider;
|
use App\Service\MangadexProvider;
|
||||||
|
use App\Service\NotificationService;
|
||||||
use Doctrine\DBAL\Exception\UniqueConstraintViolationException;
|
use Doctrine\DBAL\Exception\UniqueConstraintViolationException;
|
||||||
use Doctrine\ORM\EntityManagerInterface;
|
use Doctrine\ORM\EntityManagerInterface;
|
||||||
use Doctrine\ORM\NonUniqueResultException;
|
use Doctrine\ORM\NonUniqueResultException;
|
||||||
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
|
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
|
||||||
use Symfony\Component\HttpFoundation\BinaryFileResponse;
|
use Symfony\Component\HttpFoundation\BinaryFileResponse;
|
||||||
use Symfony\Component\HttpFoundation\JsonResponse;
|
use Symfony\Component\HttpFoundation\JsonResponse;
|
||||||
use Symfony\Component\HttpFoundation\RedirectResponse;
|
|
||||||
use Symfony\Component\HttpFoundation\Request;
|
use Symfony\Component\HttpFoundation\Request;
|
||||||
use Symfony\Component\HttpFoundation\Response;
|
use Symfony\Component\HttpFoundation\Response;
|
||||||
use Symfony\Component\HttpFoundation\ResponseHeaderBag;
|
use Symfony\Component\HttpFoundation\ResponseHeaderBag;
|
||||||
@@ -27,13 +27,14 @@ use Symfony\Component\Routing\Attribute\Route;
|
|||||||
class MangaController extends AbstractController
|
class MangaController extends AbstractController
|
||||||
{
|
{
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private readonly MangaRepository $mangaRepository,
|
private readonly MangaRepository $mangaRepository,
|
||||||
private readonly ChapterRepository $chapterRepository,
|
private readonly ChapterRepository $chapterRepository,
|
||||||
private readonly MessageBusInterface $bus,
|
private readonly MessageBusInterface $bus,
|
||||||
private readonly CbzService $cbzService,
|
private readonly CbzService $cbzService,
|
||||||
private readonly ToolbarFactory $toolbarFactory,
|
private readonly ToolbarFactory $toolbarFactory,
|
||||||
private MangadexProvider $mangadexProvider,
|
private readonly MangadexProvider $mangadexProvider,
|
||||||
private EntityManagerInterface $entityManager
|
private readonly EntityManagerInterface $entityManager,
|
||||||
|
private readonly NotificationService $notificationService
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
@@ -171,6 +172,14 @@ class MangaController extends AbstractController
|
|||||||
|
|
||||||
$allChapters = array_merge($mangaFeed, $mangaAggregate);
|
$allChapters = array_merge($mangaFeed, $mangaAggregate);
|
||||||
|
|
||||||
|
if (empty($allChapters)) {
|
||||||
|
$this->notificationService->sendUpdate([
|
||||||
|
'status' => 'error',
|
||||||
|
'message' => 'No chapters found for this manga.'
|
||||||
|
]);
|
||||||
|
return $this->redirectToRoute('app_manga_search', ['query' => $manga->getTitle()]);
|
||||||
|
}
|
||||||
|
|
||||||
$mergedChapters = [];
|
$mergedChapters = [];
|
||||||
foreach ($allChapters as $chapter) {
|
foreach ($allChapters as $chapter) {
|
||||||
$number = $chapter->getNumber();
|
$number = $chapter->getNumber();
|
||||||
@@ -187,7 +196,7 @@ class MangaController extends AbstractController
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach($mergedChapters as $chapter) {
|
foreach ($mergedChapters as $chapter) {
|
||||||
$manga->addChapter($chapter);
|
$manga->addChapter($chapter);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,47 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
namespace App\Controller;
|
|
||||||
|
|
||||||
use App\Repository\MangaRepository;
|
|
||||||
use App\Service\LelScansProviderService;
|
|
||||||
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
|
|
||||||
use Symfony\Component\HttpFoundation\Response;
|
|
||||||
use Symfony\Component\String\Slugger\AsciiSlugger;
|
|
||||||
|
|
||||||
class MenuController extends AbstractController
|
|
||||||
{
|
|
||||||
private MangaRepository $mangaRepository;
|
|
||||||
private LelScansProviderService $mangaProviderService;
|
|
||||||
public function __construct(MangaRepository $mangaRepository, LelScansProviderService $mangaProviderService)
|
|
||||||
{
|
|
||||||
$this->mangaRepository = $mangaRepository;
|
|
||||||
$this->mangaProviderService = $mangaProviderService;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function menu(): Response
|
|
||||||
{
|
|
||||||
$availableManga = $this->mangaProviderService->getMangaList();
|
|
||||||
|
|
||||||
foreach($availableManga as $key => $manga) {
|
|
||||||
$availableManga[$key]['slug'] = $this->titleToSlug($manga['name']);
|
|
||||||
}
|
|
||||||
|
|
||||||
$mangas = $this->mangaRepository->findAll();
|
|
||||||
return $this->render('menu/menu_old.html.twig', [
|
|
||||||
'availableManga' => $availableManga,
|
|
||||||
'mangas' => $mangas,
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
|
|
||||||
private function slugToTitle(string $slug): string
|
|
||||||
{
|
|
||||||
$slugger = new AsciiSlugger();
|
|
||||||
return $slugger->slug($slug)->replace('-', ' ')->title(true)->toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
private function titleToSlug(string $title): string
|
|
||||||
{
|
|
||||||
$slugger = new AsciiSlugger();
|
|
||||||
return $slugger->slug($title)->lower()->toString();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -2,12 +2,28 @@
|
|||||||
|
|
||||||
namespace App\Controller;
|
namespace App\Controller;
|
||||||
|
|
||||||
|
use App\Entity\ContentSource;
|
||||||
|
use App\Form\ContentSourceType;
|
||||||
|
use App\Repository\ContentSourceRepository;
|
||||||
|
use App\Service\MangaScraperService;
|
||||||
|
use Doctrine\ORM\EntityManagerInterface;
|
||||||
|
use GuzzleHttp\Exception\GuzzleException;
|
||||||
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
|
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
|
||||||
|
use Symfony\Component\HttpFoundation\JsonResponse;
|
||||||
|
use Symfony\Component\HttpFoundation\Request;
|
||||||
use Symfony\Component\HttpFoundation\Response;
|
use Symfony\Component\HttpFoundation\Response;
|
||||||
use Symfony\Component\Routing\Attribute\Route;
|
use Symfony\Component\Routing\Attribute\Route;
|
||||||
|
|
||||||
class SettingsController extends AbstractController
|
class SettingsController extends AbstractController
|
||||||
{
|
{
|
||||||
|
public function __construct(
|
||||||
|
private MangaScraperService $mangaScraperService,
|
||||||
|
private EntityManagerInterface $entityManager
|
||||||
|
)
|
||||||
|
{
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
#[Route('/settings', name: 'app_settings')]
|
#[Route('/settings', name: 'app_settings')]
|
||||||
public function index(): Response
|
public function index(): Response
|
||||||
{
|
{
|
||||||
@@ -32,14 +48,77 @@ class SettingsController extends AbstractController
|
|||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[Route('/settings/scrappers', name: 'app_settings_scrappers')]
|
#[Route('/settings/scrappers/list', name: 'app_settings_scrappers_list')]
|
||||||
public function scrappers(): Response
|
public function list(ContentSourceRepository $repository): Response
|
||||||
{
|
{
|
||||||
return $this->render('settings/index.html.twig', [
|
$contentSources = $repository->findAll();
|
||||||
'controller_name' => 'SettingsController',
|
|
||||||
|
return $this->render('settings/scrapper_list.html.twig', [
|
||||||
|
'contentSources' => $contentSources,
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[Route('/settings/scrappers/{id}', name: 'app_settings_scrappers', defaults: ['id' => null])]
|
||||||
|
public function scrappers(Request $request, ?ContentSource $contentSource): Response
|
||||||
|
{
|
||||||
|
$isNew = $contentSource === null;
|
||||||
|
$contentSource = $contentSource ?? new ContentSource();
|
||||||
|
|
||||||
|
$form = $this->createForm(ContentSourceType::class, $contentSource);
|
||||||
|
$form->handleRequest($request);
|
||||||
|
|
||||||
|
if ($form->isSubmitted() && $form->isValid()) {
|
||||||
|
$this->entityManager->persist($contentSource);
|
||||||
|
$this->entityManager->flush();
|
||||||
|
$this->addFlash('success', ($isNew ? 'New scrapper configuration saved' : 'Scrapper configuration updated') . ' successfully.');
|
||||||
|
return $this->redirectToRoute('app_settings_scrappers_list');
|
||||||
|
}
|
||||||
|
|
||||||
|
return $this->render('settings/scrappers.html.twig', [
|
||||||
|
'form' => $form->createView(),
|
||||||
|
'isNew' => $isNew,
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @throws GuzzleException
|
||||||
|
*/
|
||||||
|
#[Route('/settings/scrappers_test', name: 'app_settings_scrappers_test', methods: ['POST'])]
|
||||||
|
public function scrapperTest(Request $request): JsonResponse
|
||||||
|
{
|
||||||
|
$contentSource = new ContentSource();
|
||||||
|
$form = $this->createForm(ContentSourceType::class, $contentSource);
|
||||||
|
$form->submit($request->request->all()['content_source']);
|
||||||
|
|
||||||
|
if ($form->isValid()) {
|
||||||
|
$mangaSlug = $request->request->get('mangaSlug');
|
||||||
|
$chapterNumber = $request->request->get('chapterNumber');
|
||||||
|
|
||||||
|
$scrapedData = $this->mangaScraperService->testScrapingHtml($mangaSlug, $chapterNumber, $contentSource);
|
||||||
|
|
||||||
|
return new JsonResponse([
|
||||||
|
'success' => true,
|
||||||
|
'message' => 'Test successful',
|
||||||
|
'data' => $scrapedData
|
||||||
|
]);
|
||||||
|
} else {
|
||||||
|
return new JsonResponse([
|
||||||
|
'success' => false,
|
||||||
|
'message' => 'Invalid form submission',
|
||||||
|
'errors' => $this->getFormErrors($form)
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private function getFormErrors($form): array
|
||||||
|
{
|
||||||
|
$errors = [];
|
||||||
|
foreach ($form->getErrors(true) as $error) {
|
||||||
|
$errors[] = $error->getMessage();
|
||||||
|
}
|
||||||
|
return $errors;
|
||||||
|
}
|
||||||
|
|
||||||
#[Route('/settings/ui', name: 'app_settings_ui')]
|
#[Route('/settings/ui', name: 'app_settings_ui')]
|
||||||
public function ui(): Response
|
public function ui(): Response
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -62,7 +62,7 @@ class ContentSource
|
|||||||
return $this->NextPageSelector;
|
return $this->NextPageSelector;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function setNextPageSelector(string $NextPageSelector): static
|
public function setNextPageSelector(?string $NextPageSelector): static
|
||||||
{
|
{
|
||||||
$this->NextPageSelector = $NextPageSelector;
|
$this->NextPageSelector = $NextPageSelector;
|
||||||
|
|
||||||
|
|||||||
46
src/Form/ContentSourceType.php
Normal file
46
src/Form/ContentSourceType.php
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\Form;
|
||||||
|
|
||||||
|
use App\Entity\ContentSource;
|
||||||
|
use Symfony\Component\Form\AbstractType;
|
||||||
|
use Symfony\Component\Form\Extension\Core\Type\ChoiceType;
|
||||||
|
use Symfony\Component\Form\Extension\Core\Type\TextType;
|
||||||
|
use Symfony\Component\Form\Extension\Core\Type\UrlType;
|
||||||
|
use Symfony\Component\Form\FormBuilderInterface;
|
||||||
|
use Symfony\Component\OptionsResolver\OptionsResolver;
|
||||||
|
|
||||||
|
class ContentSourceType extends AbstractType
|
||||||
|
{
|
||||||
|
public function buildForm(FormBuilderInterface $builder, array $options): void
|
||||||
|
{
|
||||||
|
$builder
|
||||||
|
->add('baseUrl', UrlType::class, [
|
||||||
|
'label' => 'Base URL',
|
||||||
|
])
|
||||||
|
->add('imageSelector', TextType::class, [
|
||||||
|
'label' => 'Image Selector',
|
||||||
|
])
|
||||||
|
->add('chapterUrlFormat', TextType::class, [
|
||||||
|
'label' => 'Chapter URL Format',
|
||||||
|
])
|
||||||
|
->add('nextPageSelector', TextType::class, [
|
||||||
|
'label' => 'Next Page Selector (let empty if vertical reader)',
|
||||||
|
'required' => false,
|
||||||
|
])
|
||||||
|
->add('scrapingType', ChoiceType::class, [
|
||||||
|
'label' => 'Scraping Type',
|
||||||
|
'choices' => [
|
||||||
|
'HTML' => 'html',
|
||||||
|
'JavaScript' => 'javascript'
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function configureOptions(OptionsResolver $resolver): void
|
||||||
|
{
|
||||||
|
$resolver->setDefaults([
|
||||||
|
'data_class' => ContentSource::class,
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,8 +5,7 @@ namespace App\MessageHandler;
|
|||||||
use App\Entity\ContentSource;
|
use App\Entity\ContentSource;
|
||||||
use App\Message\DownloadChapter;
|
use App\Message\DownloadChapter;
|
||||||
use App\Repository\ChapterRepository;
|
use App\Repository\ChapterRepository;
|
||||||
use App\Repository\MangaRepository;
|
use App\Repository\ContentSourceRepository;
|
||||||
use App\Service\LelScansProviderService;
|
|
||||||
use App\Service\MangaScraperService;
|
use App\Service\MangaScraperService;
|
||||||
use App\Service\NotificationService;
|
use App\Service\NotificationService;
|
||||||
use Exception;
|
use Exception;
|
||||||
@@ -20,7 +19,8 @@ readonly class DownloadChapterHandler
|
|||||||
public function __construct(
|
public function __construct(
|
||||||
private ChapterRepository $chapterRepository,
|
private ChapterRepository $chapterRepository,
|
||||||
private MangaScraperService $mangaScraperService,
|
private MangaScraperService $mangaScraperService,
|
||||||
private NotificationService $notificationService
|
private NotificationService $notificationService,
|
||||||
|
private ContentSourceRepository $contentSourceRepository
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
|
|
||||||
@@ -40,21 +40,27 @@ readonly class DownloadChapterHandler
|
|||||||
throw new BadRequestHttpException('Chapter already downloaded');
|
throw new BadRequestHttpException('Chapter already downloaded');
|
||||||
}
|
}
|
||||||
|
|
||||||
$sources = [
|
$sources = $this->contentSourceRepository->findAll();
|
||||||
(new ContentSource())
|
$sources[] =
|
||||||
->setBaseUrl('https://lelscans.net')
|
|
||||||
->setImageSelector('#image img')
|
|
||||||
->setChapterUrlFormat('https://lelscans.net/scan-%s/%s')
|
|
||||||
->setNextPageSelector('a[title="Suivant"]')
|
|
||||||
->setScrapingType('html'),
|
|
||||||
|
|
||||||
(new ContentSource())
|
(new ContentSource())
|
||||||
->setBaseUrl('https://api.mangadex.org/')
|
->setBaseUrl('https://api.mangadex.org/')
|
||||||
->setImageSelector('img')
|
->setImageSelector('img')
|
||||||
->setChapterUrlFormat('at-home/server/%s')
|
->setChapterUrlFormat('at-home/server/%s')
|
||||||
->setScrapingType('mangadex')
|
->setScrapingType('mangadex')
|
||||||
];
|
;
|
||||||
|
|
||||||
|
// (new ContentSource())
|
||||||
|
// ->setBaseUrl('https://lelscans.net')
|
||||||
|
// ->setImageSelector('#image img')
|
||||||
|
// ->setChapterUrlFormat('https://lelscans.net/scan-%s/%s')
|
||||||
|
// ->setNextPageSelector('a[title="Suivant"]')
|
||||||
|
// ->setScrapingType('html'),
|
||||||
|
// (new ContentSource())
|
||||||
|
// ->setBaseUrl('https://darkscans.net/')
|
||||||
|
// ->setImageSelector('.reading-content img')
|
||||||
|
// ->setChapterUrlFormat('https://darkscans.net/mangas/%s/chapter-%s/')
|
||||||
|
// ->setNextPageSelector(null)
|
||||||
|
// ->setScrapingType('html')
|
||||||
|
|
||||||
$scrapedSuccessfully = false;
|
$scrapedSuccessfully = false;
|
||||||
|
|
||||||
|
|||||||
@@ -1,67 +0,0 @@
|
|||||||
<?php
|
|
||||||
namespace App\Service;
|
|
||||||
|
|
||||||
use App\Entity\Manga;
|
|
||||||
use App\Interface\ContentProviderInterface;
|
|
||||||
use Symfony\Component\BrowserKit\HttpBrowser as Client;
|
|
||||||
use Symfony\Component\DomCrawler\Crawler;
|
|
||||||
|
|
||||||
class LelScansProviderService implements ContentProviderInterface
|
|
||||||
{
|
|
||||||
const PROVIDER_URL = 'https://lelscans.net/';
|
|
||||||
const MANGA_SLUG = '/{manga}/{chapter}/{page}';
|
|
||||||
|
|
||||||
private Client $client;
|
|
||||||
|
|
||||||
public function __construct()
|
|
||||||
{
|
|
||||||
$this->client = new Client();
|
|
||||||
}
|
|
||||||
|
|
||||||
public function getMangaList(): array
|
|
||||||
{
|
|
||||||
$crawler = $this->client->request('GET', self::PROVIDER_URL);
|
|
||||||
$mangaList = [];
|
|
||||||
|
|
||||||
$crawler->filter('select > option')->each(function (Crawler $node) use (&$mangaList) {
|
|
||||||
$mangaName = $node->text();
|
|
||||||
$mangaUrl = $node->attr('value');
|
|
||||||
if ($mangaName && $mangaUrl && !preg_match('/^\d+(\.\d+)?$/', $mangaName)) {
|
|
||||||
$mangaList[] = [
|
|
||||||
'name' => $mangaName,
|
|
||||||
'url' => $mangaUrl,
|
|
||||||
];
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
return $mangaList;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function getChapterList($mangaSlug): array
|
|
||||||
{
|
|
||||||
$crawler = $this->client->request('GET', self::PROVIDER_URL . 'lecture-en-ligne-' . $mangaSlug . '.php');
|
|
||||||
$chapterList = [];
|
|
||||||
|
|
||||||
$crawler->filter('select > option')->each(function (Crawler $node) use (&$chapterList) {
|
|
||||||
$chapterName = $node->text();
|
|
||||||
$chapterUrl = $node->attr('value');
|
|
||||||
if ($chapterName && $chapterUrl && preg_match('/^\d+(\.\d+)?$/', $chapterName)) {
|
|
||||||
$chapterList[] = [
|
|
||||||
'number' => $chapterName,
|
|
||||||
];
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
return $chapterList;
|
|
||||||
}
|
|
||||||
|
|
||||||
#[\Override] public function getAvailableContent(Manga $manga): array
|
|
||||||
{
|
|
||||||
// TODO: Implement getAvailableContent() method.
|
|
||||||
}
|
|
||||||
|
|
||||||
#[\Override] public function getContent(Manga $manga): array
|
|
||||||
{
|
|
||||||
// TODO: Implement getContent() method.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,100 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
namespace App\Service;
|
|
||||||
|
|
||||||
use Symfony\Component\Filesystem\Filesystem;
|
|
||||||
use Symfony\Component\HttpFoundation\BinaryFileResponse;
|
|
||||||
use Symfony\Component\HttpFoundation\ResponseHeaderBag;
|
|
||||||
use ZipArchive;
|
|
||||||
use RecursiveDirectoryIterator;
|
|
||||||
use RecursiveIteratorIterator;
|
|
||||||
class MangaExportService
|
|
||||||
{
|
|
||||||
const IMG_BASE_DIR = '/public/manga-images';
|
|
||||||
const EXPORT_BASE_DIR = '/public/manga-export';
|
|
||||||
private string $projectDir;
|
|
||||||
|
|
||||||
public function __construct($projectDir)
|
|
||||||
{
|
|
||||||
$this->projectDir = $projectDir;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function exportMangaChapter(string $mangaTitle, int $chapterNumber): bool|string
|
|
||||||
{
|
|
||||||
$chapterDir = $this->getMangaDir($mangaTitle, $chapterNumber);
|
|
||||||
$cbzFilePath = $this->getExportDir($mangaTitle, $chapterNumber);
|
|
||||||
|
|
||||||
if(!is_dir($chapterDir)){
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
$cbzDirectory = dirname($cbzFilePath);
|
|
||||||
if (!is_dir($cbzDirectory)) {
|
|
||||||
mkdir($cbzDirectory, 0755, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
$fileSystem = new Filesystem();
|
|
||||||
if($fileSystem->exists($cbzFilePath)){
|
|
||||||
return 'already_exported';
|
|
||||||
}
|
|
||||||
|
|
||||||
return $this->createCbzFromDirectory($chapterDir, $cbzFilePath);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function downloadCbz(string $mangaTitle, int $chapterNumber): BinaryFileResponse|bool
|
|
||||||
{
|
|
||||||
$filePathCbz = $this->getExportDir($mangaTitle, $chapterNumber);
|
|
||||||
|
|
||||||
$fileSystem = new Filesystem();
|
|
||||||
if($fileSystem->exists($filePathCbz)){
|
|
||||||
return new BinaryFileResponse($filePathCbz);
|
|
||||||
}
|
|
||||||
|
|
||||||
$chapterDir = $this->getMangaDir($mangaTitle, $chapterNumber);
|
|
||||||
if(is_dir($chapterDir)){
|
|
||||||
if($this->exportMangaChapter($mangaTitle, $chapterNumber)){
|
|
||||||
return new BinaryFileResponse($filePathCbz);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function createCbzFromDirectory(string $sourceDirectory, string $cbzFilePath): bool
|
|
||||||
{
|
|
||||||
$zip = new ZipArchive();
|
|
||||||
|
|
||||||
// Ouvre le fichier .cbz en écriture
|
|
||||||
if ($zip->open($cbzFilePath, ZipArchive::CREATE | ZipArchive::OVERWRITE) !== true) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
$files = new RecursiveIteratorIterator(
|
|
||||||
new RecursiveDirectoryIterator($sourceDirectory),
|
|
||||||
RecursiveIteratorIterator::LEAVES_ONLY
|
|
||||||
);
|
|
||||||
|
|
||||||
// Ajoute les fichiers d'image au fichier .cbz
|
|
||||||
foreach ($files as $file) {
|
|
||||||
if (!$file->isDir()) {
|
|
||||||
$filePath = $file->getRealPath();
|
|
||||||
$relativePath = substr($filePath, strlen($sourceDirectory) + 1);
|
|
||||||
$zip->addFile($filePath, $relativePath);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$zip->close();
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function getMangaDir(string $mangaTitle, int $chapterNumber): string
|
|
||||||
{
|
|
||||||
return sprintf('%s/%s/%d', $this->projectDir . self::IMG_BASE_DIR, $mangaTitle, $chapterNumber);
|
|
||||||
}
|
|
||||||
|
|
||||||
private function getExportDir(string $mangaTitle, int $chapterNumber): string
|
|
||||||
{
|
|
||||||
return sprintf('%s/%s/%d', $this->projectDir . self::EXPORT_BASE_DIR, $mangaTitle, $chapterNumber) . '.cbz';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
namespace App\Service;
|
|
||||||
|
|
||||||
use App\Interface\ContentProviderInterface;
|
|
||||||
|
|
||||||
class MangaProviderFactory
|
|
||||||
{
|
|
||||||
public static function create($providerName): ContentProviderInterface
|
|
||||||
{
|
|
||||||
return match ($providerName) {
|
|
||||||
'LelScans' => new LelScansProviderService(),
|
|
||||||
'AutreManga' => new AutreMangaProviderService(),
|
|
||||||
default => throw new \Exception("Provider {$providerName} non supporté."),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -166,6 +166,21 @@ class MangaScraperService
|
|||||||
return json_decode(implode("", $output), true);
|
return json_decode(implode("", $output), true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @throws GuzzleException
|
||||||
|
*/
|
||||||
|
public function testScrapingHtml(string $mangaSlug, string $chapterNumber, ContentSource $contentSource): array
|
||||||
|
{
|
||||||
|
$chapterUrl = $contentSource->getChapterUrl($mangaSlug, $chapterNumber);
|
||||||
|
$html = $this->fetchHtml($chapterUrl);
|
||||||
|
|
||||||
|
if ($contentSource->getNextPageSelector() === null) {
|
||||||
|
return $this->scrapeVerticalReader($html, $contentSource);
|
||||||
|
} else {
|
||||||
|
return $this->scrapeHorizontalReader($chapterUrl, $contentSource);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @throws GuzzleException
|
* @throws GuzzleException
|
||||||
*/
|
*/
|
||||||
@@ -173,32 +188,32 @@ class MangaScraperService
|
|||||||
{
|
{
|
||||||
$chapterUrl = $mangaSource->getChapterUrl($manga->getSlug(), $chapter->getNumber());
|
$chapterUrl = $mangaSource->getChapterUrl($manga->getSlug(), $chapter->getNumber());
|
||||||
|
|
||||||
$pageData = [];
|
|
||||||
$currentPageUrl = $chapterUrl;
|
|
||||||
|
|
||||||
$tempDir = sys_get_temp_dir() . '/' . uniqid('manga_scraper_');
|
$tempDir = sys_get_temp_dir() . '/' . uniqid('manga_scraper_');
|
||||||
mkdir($tempDir);
|
mkdir($tempDir);
|
||||||
|
|
||||||
do {
|
$pageData = [];
|
||||||
$html = $this->fetchHtml($currentPageUrl);
|
|
||||||
$page = $this->extractMangaPageData($html, $mangaSource);
|
|
||||||
|
|
||||||
$imageName = sprintf('%03d.%s', count($pageData) + 1, pathinfo(parse_url($page['image_url'], PHP_URL_PATH), PATHINFO_EXTENSION));
|
if ($mangaSource->getNextPageSelector() === null) {
|
||||||
|
// Lecteur vertical
|
||||||
|
$html = $this->fetchHtml($chapterUrl);
|
||||||
|
$pageData = $this->scrapeVerticalReader($html, $mangaSource);
|
||||||
|
} else {
|
||||||
|
// Lecteur horizontal (paginé)
|
||||||
|
$pageData = $this->scrapeHorizontalReader($chapterUrl, $mangaSource);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Télécharger et sauvegarder les images
|
||||||
|
foreach ($pageData as $index => &$page) {
|
||||||
|
$imageName = sprintf('%03d.%s', $index + 1, pathinfo(parse_url($page['image_url'], PHP_URL_PATH), PATHINFO_EXTENSION));
|
||||||
$imagePath = $tempDir . '/' . $imageName;
|
$imagePath = $tempDir . '/' . $imageName;
|
||||||
|
|
||||||
$this->downloadAndSaveImage($page['image_url'], $imagePath);
|
$this->downloadAndSaveImage($page['image_url'], $imagePath);
|
||||||
|
|
||||||
$event = new PageScrappingProgressEvent($chapter->getId(), count($pageData) + 1, 0);
|
$event = new PageScrappingProgressEvent($chapter->getId(), $index + 1, count($pageData));
|
||||||
$this->eventDispatcher->dispatch($event, PageScrappingProgressEvent::NAME);
|
$this->eventDispatcher->dispatch($event, PageScrappingProgressEvent::NAME);
|
||||||
|
|
||||||
$pageData[] = [
|
$page['local_image_url'] = $imagePath;
|
||||||
'image_url' => $page['image_url'],
|
}
|
||||||
'local_image_url' => $imagePath,
|
|
||||||
'page_number' => count($pageData) + 1,
|
|
||||||
];
|
|
||||||
|
|
||||||
$currentPageUrl = $page['next_page_url'];
|
|
||||||
} while ($currentPageUrl);
|
|
||||||
|
|
||||||
$cbzFilePath = $this->generateCbzPath($manga, $chapter);
|
$cbzFilePath = $this->generateCbzPath($manga, $chapter);
|
||||||
$this->createCbzFile($tempDir, $pageData, $cbzFilePath);
|
$this->createCbzFile($tempDir, $pageData, $cbzFilePath);
|
||||||
@@ -210,7 +225,78 @@ class MangaScraperService
|
|||||||
// Nettoyage du répertoire temporaire
|
// Nettoyage du répertoire temporaire
|
||||||
$this->cleanupTempFiles($tempDir);
|
$this->cleanupTempFiles($tempDir);
|
||||||
|
|
||||||
return true;
|
return $pageData;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function scrapeVerticalReader(string $html, ContentSource $contentSource): array
|
||||||
|
{
|
||||||
|
$crawler = new Crawler($html);
|
||||||
|
$images = $crawler->filter($contentSource->getImageSelector());
|
||||||
|
|
||||||
|
$pageData = [];
|
||||||
|
foreach ($images as $index => $image) {
|
||||||
|
if($image->getAttribute('src') === ''){
|
||||||
|
$imgUrl = $image->getAttribute('data-src');
|
||||||
|
}else{
|
||||||
|
$imgUrl = $image->getAttribute('src');
|
||||||
|
}
|
||||||
|
$pageData[] = [
|
||||||
|
'image_url' => $this->cleanImageUrl($imgUrl),
|
||||||
|
'page_number' => $index + 1,
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
return $pageData;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @throws GuzzleException
|
||||||
|
*/
|
||||||
|
private function scrapeHorizontalReader(string $chapterUrl, ContentSource $contentSource): array
|
||||||
|
{
|
||||||
|
$pageData = [];
|
||||||
|
$currentPageUrl = $chapterUrl;
|
||||||
|
|
||||||
|
do {
|
||||||
|
$html = $this->fetchHtml($currentPageUrl);
|
||||||
|
$page = $this->extractMangaPageData($html, $contentSource);
|
||||||
|
|
||||||
|
$pageData[] = [
|
||||||
|
'image_url' => $this->cleanImageUrl($page['image_url']),
|
||||||
|
'page_number' => count($pageData) + 1,
|
||||||
|
];
|
||||||
|
|
||||||
|
$currentPageUrl = $page['next_page_url'];
|
||||||
|
} while ($currentPageUrl);
|
||||||
|
|
||||||
|
return $pageData;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Processes a single image
|
||||||
|
* @throws GuzzleException
|
||||||
|
*/
|
||||||
|
private function processImage(string $imgUrl, string $tempDir, array &$pageData, int $index, Chapter $chapter): void
|
||||||
|
{
|
||||||
|
$imgUrl = $this->cleanImageUrl($imgUrl);
|
||||||
|
$imageName = sprintf('%03d.%s', $index + 1, pathinfo(parse_url($imgUrl, PHP_URL_PATH), PATHINFO_EXTENSION));
|
||||||
|
$imagePath = $tempDir . '/' . $imageName;
|
||||||
|
|
||||||
|
$this->downloadAndSaveImage($imgUrl, $imagePath);
|
||||||
|
|
||||||
|
// $event = new PageScrappingProgressEvent($chapter->getId(), $index + 1, 0);
|
||||||
|
// $this->eventDispatcher->dispatch($event, PageScrappingProgressEvent::NAME);
|
||||||
|
|
||||||
|
$pageData[] = [
|
||||||
|
'image_url' => $imgUrl,
|
||||||
|
'local_image_url' => $imagePath,
|
||||||
|
'page_number' => $index + 1,
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
private function cleanImageUrl(string $url): string
|
||||||
|
{
|
||||||
|
return preg_replace('/[\x00-\x1F\x7F]/', '', trim($url));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -1,157 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
namespace App\Service;
|
|
||||||
|
|
||||||
use App\EventSubscriber\MangaScrapedEvent;
|
|
||||||
use GuzzleHttp\Client;
|
|
||||||
use GuzzleHttp\Exception\GuzzleException;
|
|
||||||
use Symfony\Component\DomCrawler\Crawler;
|
|
||||||
use Symfony\Component\Routing\Matcher\UrlMatcher;
|
|
||||||
use Symfony\Component\Routing\RequestContext;
|
|
||||||
use Symfony\Component\Routing\Route;
|
|
||||||
use Symfony\Component\Routing\RouteCollection;
|
|
||||||
use Symfony\Contracts\EventDispatcher\EventDispatcherInterface;
|
|
||||||
|
|
||||||
class MangaScraperServiceOld
|
|
||||||
{
|
|
||||||
const string IMG_BASE_DIR = '/public/manga-images';
|
|
||||||
private string $projectDir;
|
|
||||||
private EventDispatcherInterface $eventDispatcher;
|
|
||||||
|
|
||||||
public function __construct($projectDir, EventDispatcherInterface $eventDispatcher)
|
|
||||||
{
|
|
||||||
$this->projectDir = $projectDir;
|
|
||||||
$this->eventDispatcher = $eventDispatcher;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function extractMangaPageData(string $html): array
|
|
||||||
{
|
|
||||||
$baseUrl = 'https://lelscans.net';
|
|
||||||
//pour éviter à PhpStorm de gueuler...
|
|
||||||
$selector = 'img';
|
|
||||||
$crawler = new Crawler($html);
|
|
||||||
$imgUrl = $crawler->filter($selector)->attr('src');
|
|
||||||
$nextLink = $crawler->filter('a[title="Suivant"]');
|
|
||||||
|
|
||||||
if (!preg_match('/^https?:\/\//', $imgUrl)) {
|
|
||||||
$urlComponents = parse_url($baseUrl);
|
|
||||||
$scheme = $urlComponents['scheme'];
|
|
||||||
$host = $urlComponents['host'];
|
|
||||||
|
|
||||||
// Construit l'URL absolue de l'image
|
|
||||||
$imgUrl = $scheme . '://' . $host . '/' . ltrim($imgUrl, '/');
|
|
||||||
}
|
|
||||||
|
|
||||||
if($nextLink->count() > 0){
|
|
||||||
$nextUrl = $nextLink->attr('href');
|
|
||||||
}else{
|
|
||||||
$nextUrl = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return [
|
|
||||||
'image_url' => $imgUrl,
|
|
||||||
'next_page_url' => $nextUrl,
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @throws GuzzleException
|
|
||||||
*/
|
|
||||||
public function scrapeMangaChapter(string $chapterUrl, string $mangaTitle, float $chapterNumber): array|bool
|
|
||||||
{
|
|
||||||
if(!$this->isChapterAvailable($chapterUrl, $chapterNumber)){
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
$pageData = [];
|
|
||||||
$currentPageUrl = $chapterUrl;
|
|
||||||
|
|
||||||
$mangaDir = sprintf('%s/%s', $this->projectDir . self::IMG_BASE_DIR, $mangaTitle);
|
|
||||||
if (!is_dir($mangaDir)) {
|
|
||||||
mkdir($mangaDir, 0755, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Créez le dossier du chapitre s'il n'existe pas
|
|
||||||
$chapterDir = sprintf('%s/%s', $mangaDir, $chapterNumber);
|
|
||||||
if (!is_dir($chapterDir)) {
|
|
||||||
mkdir($chapterDir, 0755, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
do {
|
|
||||||
$html = $this->fetchHtml($currentPageUrl);
|
|
||||||
$page = $this->extractMangaPageData($html);
|
|
||||||
$pageData[] = $page;
|
|
||||||
$currentPageUrl = $page['next_page_url'];
|
|
||||||
|
|
||||||
// Construisez le nom de fichier de l'image
|
|
||||||
$imageName = sprintf('%03d.jpg', count($pageData));
|
|
||||||
|
|
||||||
// Construisez le chemin du fichier de l'image
|
|
||||||
$imagePath = sprintf('%s/%s', $chapterDir, $imageName);
|
|
||||||
|
|
||||||
// Téléchargez et enregistrez l'image
|
|
||||||
$this->downloadAndSaveImage($page['image_url'], $imagePath);
|
|
||||||
|
|
||||||
// Modifiez les données de la page pour inclure l'URL de l'image stockée localement
|
|
||||||
$pageData[count($pageData) - 1]['local_image_url'] = sprintf('/manga-images/%s/%s/%s', $mangaTitle, $chapterNumber, $imageName);
|
|
||||||
$pageData[count($pageData) - 1]['page_number'] = count($pageData);
|
|
||||||
|
|
||||||
} while ($currentPageUrl);
|
|
||||||
|
|
||||||
$event = new MangaScrapedEvent($mangaTitle, $chapterNumber, $pageData);
|
|
||||||
$this->eventDispatcher->dispatch($event, MangaScrapedEvent::NAME);
|
|
||||||
|
|
||||||
return $pageData;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @throws GuzzleException
|
|
||||||
*/
|
|
||||||
private function fetchHtml(string $url): string
|
|
||||||
{
|
|
||||||
$client = new Client();
|
|
||||||
$response = $client->get($url);
|
|
||||||
|
|
||||||
return (string) $response->getBody();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @throws GuzzleException
|
|
||||||
*/
|
|
||||||
private function downloadAndSaveImage(string $imageUrl, string $destinationPath): void
|
|
||||||
{
|
|
||||||
$client = new Client();
|
|
||||||
$response = $client->get($imageUrl);
|
|
||||||
|
|
||||||
file_put_contents($destinationPath, $response->getBody()->getContents());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @throws GuzzleException
|
|
||||||
*/
|
|
||||||
private function isChapterAvailable(string $chapterUrl, float $chapterNumber): bool
|
|
||||||
{
|
|
||||||
$html = $this->fetchHtml($chapterUrl);
|
|
||||||
$crawler = new Crawler($html);
|
|
||||||
$nextLink = $crawler->filter('a[title="Suivant"]');
|
|
||||||
|
|
||||||
if($nextLink->count() === 0){
|
|
||||||
return false;
|
|
||||||
}else{
|
|
||||||
$nextUrl = $nextLink->attr('href');
|
|
||||||
}
|
|
||||||
|
|
||||||
$routeCollection = new RouteCollection();
|
|
||||||
$routeCollection->add('manga_chapter', new Route('/scan-{manga}/{chapter}/{page}'));
|
|
||||||
$context = new RequestContext('/');
|
|
||||||
$matcher = new UrlMatcher($routeCollection, $context);
|
|
||||||
$path = parse_url($nextUrl, PHP_URL_PATH);
|
|
||||||
$parameters = $matcher->match($path);
|
|
||||||
|
|
||||||
if((float) $parameters['chapter'] !== $chapterNumber){
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -125,12 +125,19 @@ readonly class MangadexProvider implements MetadataProviderInterface
|
|||||||
|
|
||||||
private function getFeedWithPagination(string $externalId, int $page): array
|
private function getFeedWithPagination(string $externalId, int $page): array
|
||||||
{
|
{
|
||||||
return $this->client->get('/manga/' . $externalId . '/feed', [
|
try {
|
||||||
'limit' => 500,
|
$response = $this->client->get('/manga/' . $externalId . '/feed', [
|
||||||
'translatedLanguage' =>['en', 'fr'],
|
'limit' => 500,
|
||||||
'order' => ['chapter' => 'asc'],
|
'translatedLanguage' =>['en', 'fr'],
|
||||||
'offset' => $page * 500
|
'order' => ['chapter' => 'asc'],
|
||||||
]);
|
'offset' => $page * 500
|
||||||
|
]);
|
||||||
|
}catch(\Exception $e){
|
||||||
|
$this->notificationService->sendUpdate(['status' => 'error', 'message' => 'An error occurred while fetching data from Mangadex.']);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return $response;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function getMangaAggregate(Manga $manga): array
|
public function getMangaAggregate(Manga $manga): array
|
||||||
@@ -139,7 +146,12 @@ readonly class MangadexProvider implements MetadataProviderInterface
|
|||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
$response = $this->client->get('/manga/' . $manga->getExternalId() . '/aggregate');
|
try {
|
||||||
|
$response = $this->client->get('/manga/' . $manga->getExternalId() . '/aggregate');
|
||||||
|
}catch(\Exception $e){
|
||||||
|
// $this->notificationService->sendUpdate(['status' => 'error', 'message' => 'An error occurred while fetching data from Mangadex.']);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
$chapterEntities = [];
|
$chapterEntities = [];
|
||||||
if($response['result'] === 'ok'){
|
if($response['result'] === 'ok'){
|
||||||
|
|||||||
@@ -1,73 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
namespace App\Service;
|
|
||||||
|
|
||||||
use App\Entity\Manga;
|
|
||||||
use App\Interface\ContentProviderInterface;
|
|
||||||
use Symfony\Component\BrowserKit\HttpBrowser;
|
|
||||||
use Symfony\Component\BrowserKit\HttpBrowser as Client;
|
|
||||||
//use GuzzleHttp\Client;
|
|
||||||
use GuzzleHttp\Exception\GuzzleException;
|
|
||||||
use Symfony\Component\DomCrawler\Crawler;
|
|
||||||
use Symfony\Component\HttpClient\HttpClient;
|
|
||||||
|
|
||||||
class SushiScanProviderService
|
|
||||||
{
|
|
||||||
const PROVIDER_URL = 'https://sushiscan.net/catalogue/';
|
|
||||||
const MANGA_SLUG = '/{manga}/{chapter}/{page}';
|
|
||||||
|
|
||||||
const CONTENT_TYPE = ['volume', 'chapitre'];
|
|
||||||
private Client $client;
|
|
||||||
|
|
||||||
public function __construct()
|
|
||||||
{
|
|
||||||
$httpClient = HttpClient::create(['timeout' => 60]);
|
|
||||||
$this->client = new HttpBrowser($httpClient);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function getAvailableContent(Manga $manga)
|
|
||||||
{
|
|
||||||
$url = 'http://flaresolverr:8191/v1';
|
|
||||||
$jsonContent = json_encode([
|
|
||||||
'cmd' => 'request.get',
|
|
||||||
'url' => self::PROVIDER_URL . $manga->getSlug(),
|
|
||||||
'maxTimeout' => 90000,
|
|
||||||
]);
|
|
||||||
|
|
||||||
|
|
||||||
try{
|
|
||||||
$crawler = $this->client->request('POST', $url, [], [], [
|
|
||||||
'HTTP_CONTENT_TYPE' => 'application/json',
|
|
||||||
], $jsonContent);
|
|
||||||
|
|
||||||
}catch (\Exception $e) {
|
|
||||||
dd($e);
|
|
||||||
}
|
|
||||||
$contentList = [];
|
|
||||||
|
|
||||||
dd($crawler);
|
|
||||||
|
|
||||||
$crawler->filter('#chapterList ul > li')->each(function (Crawler $node) use (&$contentList) {
|
|
||||||
dump($node);
|
|
||||||
// $contentName = $node->text();
|
|
||||||
// $contentUrl = $node->attr('href');
|
|
||||||
// if ($contentName && $contentUrl) {
|
|
||||||
// $contentList[] = [
|
|
||||||
// 'name' => $contentName,
|
|
||||||
// 'url' => $contentUrl,
|
|
||||||
// ];
|
|
||||||
// }
|
|
||||||
});
|
|
||||||
|
|
||||||
return $contentList;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param string $mangaSlug
|
|
||||||
* @return array
|
|
||||||
*/
|
|
||||||
public function getChapterList(string $mangaSlug): array
|
|
||||||
{
|
|
||||||
// TODO: Implement getChapterList() method.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,111 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
namespace App\Twig\Components;
|
|
||||||
|
|
||||||
use App\Entity\Manga;
|
|
||||||
use App\Service\MangadexProvider;
|
|
||||||
use Doctrine\DBAL\Exception\UniqueConstraintViolationException;
|
|
||||||
use Doctrine\ORM\EntityManagerInterface;
|
|
||||||
use Symfony\Component\HttpFoundation\RedirectResponse;
|
|
||||||
use Symfony\Component\HttpFoundation\Response;
|
|
||||||
use Symfony\Component\Routing\Generator\UrlGeneratorInterface;
|
|
||||||
use Symfony\UX\LiveComponent\Attribute\AsLiveComponent;
|
|
||||||
use Symfony\UX\LiveComponent\Attribute\LiveAction;
|
|
||||||
use Symfony\UX\LiveComponent\Attribute\LiveProp;
|
|
||||||
use Symfony\UX\LiveComponent\ComponentToolsTrait;
|
|
||||||
use Symfony\UX\LiveComponent\DefaultActionTrait;
|
|
||||||
|
|
||||||
#[AsLiveComponent]
|
|
||||||
class NewMangaForm
|
|
||||||
{
|
|
||||||
use ComponentToolsTrait;
|
|
||||||
use DefaultActionTrait;
|
|
||||||
|
|
||||||
public ?Manga $manga = null;
|
|
||||||
#[LiveProp(writable: true)]
|
|
||||||
public array $mangaData = [];
|
|
||||||
|
|
||||||
#[LiveProp(writable: true)]
|
|
||||||
public ?int $index = 0;
|
|
||||||
|
|
||||||
public function __construct(private UrlGeneratorInterface $urlGenerator)
|
|
||||||
{
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
public function mount(Manga $manga): void
|
|
||||||
{
|
|
||||||
$this->manga = $manga;
|
|
||||||
$this->mangaData = [
|
|
||||||
'title' => $manga->getTitle(),
|
|
||||||
'slug' => $manga->getSlug(),
|
|
||||||
'description' => $manga->getDescription(),
|
|
||||||
'imageUrl' => $manga->getImageUrl(),
|
|
||||||
'status' => $manga->getStatus(),
|
|
||||||
'genres' => $manga->getGenres(),
|
|
||||||
'author' => $manga->getAuthor(),
|
|
||||||
'publicationYear' => $manga->getPublicationYear(),
|
|
||||||
'rating' => $manga->getRating(),
|
|
||||||
'externalId' => $manga->getExternalId(),
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
#[LiveAction]
|
|
||||||
public function saveManga(EntityManagerInterface $entityManager, MangadexProvider $mangadexProvider): Response
|
|
||||||
{
|
|
||||||
$manga = new Manga();
|
|
||||||
$manga->setTitle($this->mangaData['title'])
|
|
||||||
->setSlug($this->mangaData['slug'])
|
|
||||||
->setDescription($this->mangaData['description'])
|
|
||||||
->setImageUrl($this->mangaData['imageUrl'])
|
|
||||||
->setStatus($this->mangaData['status'])
|
|
||||||
->setGenres($this->mangaData['genres'])
|
|
||||||
->setAuthor($this->mangaData['author'])
|
|
||||||
->setPublicationYear($this->mangaData['publicationYear'])
|
|
||||||
->setRating($this->mangaData['rating'])
|
|
||||||
->setExternalId($this->mangaData['externalId']);
|
|
||||||
|
|
||||||
$mangaFeed = $mangadexProvider->getFeed($manga);
|
|
||||||
$mangaAggregate = $mangadexProvider->getMangaAggregate($manga);
|
|
||||||
|
|
||||||
$allChapters = array_merge($mangaFeed, $mangaAggregate);
|
|
||||||
|
|
||||||
$mergedChapters = [];
|
|
||||||
foreach ($allChapters as $chapter) {
|
|
||||||
$number = $chapter->getNumber();
|
|
||||||
|
|
||||||
if (isset($mergedChapters[$number])) {
|
|
||||||
$existingChapter = $mergedChapters[$number];
|
|
||||||
|
|
||||||
if (!empty($chapter->getExternalId()) ||
|
|
||||||
(empty($existingChapter->getExternalId()) && !strpos($chapter->getTitle(), 'Chapter ') == 0)) {
|
|
||||||
$mergedChapters[$number] = $chapter;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
$mergedChapters[$number] = $chapter;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach($mergedChapters as $chapter) {
|
|
||||||
$manga->addChapter($chapter);
|
|
||||||
}
|
|
||||||
|
|
||||||
$mangaChapterUrl = $this->urlGenerator->generate('app_manga_show', ['mangaSlug' => $manga->getSlug()]);
|
|
||||||
|
|
||||||
try {
|
|
||||||
foreach ($manga->getChapters() as $chapter) {
|
|
||||||
$entityManager->persist($chapter);
|
|
||||||
}
|
|
||||||
|
|
||||||
$entityManager->persist($manga);
|
|
||||||
$entityManager->flush();
|
|
||||||
} catch (\Exception $e) {
|
|
||||||
if ($e instanceof UniqueConstraintViolationException) {
|
|
||||||
return new RedirectResponse($mangaChapterUrl);
|
|
||||||
}
|
|
||||||
throw $e;
|
|
||||||
}
|
|
||||||
|
|
||||||
return new RedirectResponse($mangaChapterUrl);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
28
src/Twig/Extension/AppExtension.php
Normal file
28
src/Twig/Extension/AppExtension.php
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\Twig\Extension;
|
||||||
|
|
||||||
|
use Twig\Extension\AbstractExtension;
|
||||||
|
use Twig\TwigFunction;
|
||||||
|
|
||||||
|
class AppExtension extends AbstractExtension
|
||||||
|
{
|
||||||
|
public function getFunctions(): array
|
||||||
|
{
|
||||||
|
return [
|
||||||
|
new TwigFunction('get_placeholder', [$this, 'getPlaceholder']),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
public function getPlaceholder(string $fieldName): string
|
||||||
|
{
|
||||||
|
return match ($fieldName) {
|
||||||
|
'baseUrl' => 'https://example.com',
|
||||||
|
'imageSelector' => '.manga-image img',
|
||||||
|
'chapterUrlFormat' => 'https://example.com/manga/{slug}/chapter-{number}',
|
||||||
|
'nextPageSelector' => '.next-page',
|
||||||
|
'scrapingType' => 'Select scraping type',
|
||||||
|
default => '',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -38,7 +38,7 @@
|
|||||||
<ul class="ml-8 mt-2 space-y-4">
|
<ul class="ml-8 mt-2 space-y-4">
|
||||||
<li><a href="{{ path('app_settings_general') }}" class="hover:text-green-600">Général</a></li>
|
<li><a href="{{ path('app_settings_general') }}" class="hover:text-green-600">Général</a></li>
|
||||||
<li><a href="{{ path('app_settings_folders') }}" class="hover:text-green-600">Dossiers</a></li>
|
<li><a href="{{ path('app_settings_folders') }}" class="hover:text-green-600">Dossiers</a></li>
|
||||||
<li><a href="{{ path('app_settings_scrappers') }}" class="hover:text-green-600">Scrappers</a></li>
|
<li><a href="{{ path('app_settings_scrappers_list') }}" class="hover:text-green-600">Scrappers</a></li>
|
||||||
<li><a href="{{ path('app_settings_ui') }}" class="hover:text-green-600">UI</a></li>
|
<li><a href="{{ path('app_settings_ui') }}" class="hover:text-green-600">UI</a></li>
|
||||||
</ul>
|
</ul>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|||||||
46
templates/settings/scrapper_list.html.twig
Normal file
46
templates/settings/scrapper_list.html.twig
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
{% extends 'base.html.twig' %}
|
||||||
|
|
||||||
|
{% block title %}Scrapper Configurations{% endblock %}
|
||||||
|
|
||||||
|
{% block body %}
|
||||||
|
<div class="container mx-auto p-4">
|
||||||
|
<h1 class="text-2xl font-bold mb-6">Scrapper Configurations</h1>
|
||||||
|
|
||||||
|
<div class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
|
||||||
|
{% for contentSource in contentSources %}
|
||||||
|
<div class="relative flex flex-col justify-between bg-white rounded-sm border border-gray-200 shadow-md hover:shadow-lg transition-shadow duration-300 h-full group">
|
||||||
|
<div class="p-4">
|
||||||
|
<div class="flex flex-row items-center justify-between mb-2">
|
||||||
|
<h5 class="text-xl tracking-tight text-gray-900 truncate flex-grow">
|
||||||
|
{{ contentSource.baseUrl|replace({'http://': '', 'https://': ''})|trim('/', 'right') }}
|
||||||
|
</h5>
|
||||||
|
<a href="{{ contentSource.baseUrl }}" target="_blank" rel="noopener noreferrer" class="text-gray-400 hover:text-green-600 ml-2 z-10">
|
||||||
|
<i class="fas fa-external-link-alt"></i>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="p-4 bg-gray-50">
|
||||||
|
<div class="flex space-x-2">
|
||||||
|
<span class="px-2 py-1 text-xs font-semibold text-white bg-green-400 rounded-sm">
|
||||||
|
{{ contentSource.scrapingType }}
|
||||||
|
</span>
|
||||||
|
<span class="px-2 py-1 text-xs font-semibold text-white bg-green-400 rounded-sm">
|
||||||
|
{{ contentSource.nextPageSelector ? 'Horizontal' : 'Vertical' }}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<a href="{{ path('app_settings_scrappers', {'id': contentSource.id}) }}" class="absolute inset-0 z-0">
|
||||||
|
<span class="sr-only">Edit configuration</span>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
|
<a href="{{ path('app_settings_scrappers') }}" class="block p-6 bg-white rounded-sm border border-gray-200 shadow-md hover:bg-gray-100 flex items-center justify-center h-full">
|
||||||
|
<div class="text-center">
|
||||||
|
<i class="fas fa-plus text-4xl text-gray-400 mb-2"></i>
|
||||||
|
<p class="text-gray-600">Add New Configuration</p>
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endblock %}
|
||||||
66
templates/settings/scrappers.html.twig
Normal file
66
templates/settings/scrappers.html.twig
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
{% extends 'base.html.twig' %}
|
||||||
|
|
||||||
|
{% block title %}{{ isNew ? 'Create' : 'Edit' }} Scrapper Configuration{% endblock %}
|
||||||
|
|
||||||
|
{% block body %}
|
||||||
|
<div class="container mx-auto p-4">
|
||||||
|
<div class="bg-white shadow-lg rounded-sm overflow-hidden">
|
||||||
|
<div class="bg-gray-800 text-white p-4">
|
||||||
|
<h1 class="text-2xl font-bold">
|
||||||
|
<i class="fas fa-cog mr-2"></i>{{ isNew ? 'Create' : 'Edit' }} Scrapper Configuration
|
||||||
|
</h1>
|
||||||
|
</div>
|
||||||
|
<div class="p-6" data-controller="scrapper-configure">
|
||||||
|
{{ form_start(form, {'attr': {'class': 'space-y-6', 'data-scrapper-configure-target': 'form', 'data-action': 'submit->scrapper-configure#saveConfiguration'}}) }}
|
||||||
|
|
||||||
|
{% for field in form.children %}
|
||||||
|
<div class="mb-4">
|
||||||
|
{{ form_label(field, null, {'label_attr': {'class': 'block text-sm font-medium text-gray-700 mb-2'}}) }}
|
||||||
|
{{ form_widget(field, {'attr': {
|
||||||
|
'class': 'mt-1 block w-full border border-gray-300 rounded-md shadow-sm py-2 px-3 focus:outline-none focus:ring-green-500 focus:border-green-500 sm:text-sm',
|
||||||
|
'placeholder': get_placeholder(field.vars.name)
|
||||||
|
}}) }}
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
|
<div class="mt-6">
|
||||||
|
<button type="submit" class="w-full flex items-center justify-center py-2 px-4 border border-transparent rounded-md shadow-sm text-sm font-medium text-white bg-green-600 hover:bg-green-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-green-500">
|
||||||
|
<span class="mr-2">{{ isNew ? 'Save' : 'Update' }} Configuration</span>
|
||||||
|
<i class="fas fa-{{ isNew ? 'save' : 'edit' }}"></i>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{{ form_end(form) }}
|
||||||
|
|
||||||
|
<div class="mt-8 bg-gray-100 p-6 rounded-md">
|
||||||
|
<h2 class="text-xl font-semibold mb-4">
|
||||||
|
<i class="fas fa-vial mr-2"></i>Test Configuration
|
||||||
|
</h2>
|
||||||
|
<form
|
||||||
|
action="{{ path('app_settings_scrappers_test') }}"
|
||||||
|
method="POST"
|
||||||
|
data-scrapper-configure-target="testForm"
|
||||||
|
data-action="submit->scrapper-configure#testConfiguration"
|
||||||
|
class="space-y-4">
|
||||||
|
<div>
|
||||||
|
<label for="testMangaSlug" class="block text-sm font-medium text-gray-700 mb-2">Manga Slug</label>
|
||||||
|
<input type="text" class="mt-1 block w-full border border-gray-300 rounded-md shadow-sm py-2 px-3 focus:outline-none focus:ring-green-500 focus:border-green-500 sm:text-sm" id="testMangaSlug" name="mangaSlug" required>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label for="testChapterNumber" class="block text-sm font-medium text-gray-700 mb-2">Chapter Number</label>
|
||||||
|
<input type="text" class="mt-1 block w-full border border-gray-300 rounded-md shadow-sm py-2 px-3 focus:outline-none focus:ring-green-500 focus:border-green-500 sm:text-sm" id="testChapterNumber" name="chapterNumber" required>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<button type="submit" class="w-full flex items-center justify-center py-2 px-4 border border-transparent rounded-md shadow-sm text-sm font-medium text-white bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500">
|
||||||
|
<span class="mr-2">Test Configuration</span>
|
||||||
|
<i class="fas fa-play"></i>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div data-scrapper-configure-target="testResults" class="mt-8"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endblock %}
|
||||||
Reference in New Issue
Block a user