feat: ajout de la fonctionnalité de test de configuration de scraper, incluant la mise à jour de l'API pour tester les configurations en temps réel, la gestion des erreurs détaillées et l'intégration des tests unitaires pour valider le bon fonctionnement de cette nouvelle fonctionnalité.

This commit is contained in:
ext.jeremy.guillot@maxicoffee.domains
2025-07-06 17:01:04 +02:00
parent ee2a9b3750
commit cbb62989d4
11 changed files with 1751 additions and 101 deletions

View File

@@ -924,66 +924,36 @@
"post": {
"operationId": "api_mangachaptersfetch_post",
"tags": [
"Chapters"
"Mangadex"
],
"responses": {
"202": {
"description": "Chapters resource created",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Chapters"
}
},
"application/ld+json": {
"schema": {
"$ref": "#/components/schemas/Chapters.jsonld"
}
},
"text/html": {
"schema": {
"$ref": "#/components/schemas/Chapters"
}
},
"application/hal+json": {
"schema": {
"$ref": "#/components/schemas/Chapters.jsonhal"
}
}
},
"links": {}
},
"400": {
"description": "Invalid input"
"description": "Demande de r\u00e9cup\u00e9ration accept\u00e9e et mise en file d'attente"
},
"422": {
"description": "Unprocessable entity"
"description": "Donn\u00e9es de validation invalides"
}
},
"summary": "Creates a Chapters resource.",
"description": "Creates a Chapters resource.",
"summary": "R\u00e9cup\u00e9rer les chapitres d'un manga",
"description": "Lance le processus de r\u00e9cup\u00e9ration des chapitres depuis la source externe pour un manga donn\u00e9",
"parameters": [],
"requestBody": {
"description": "The new Chapters resource",
"description": "Donn\u00e9es requises pour r\u00e9cup\u00e9rer les chapitres",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Chapters"
}
},
"application/ld+json": {
"schema": {
"$ref": "#/components/schemas/Chapters.jsonld"
}
},
"text/html": {
"schema": {
"$ref": "#/components/schemas/Chapters"
}
},
"application/hal+json": {
"schema": {
"$ref": "#/components/schemas/Chapters.jsonhal"
"type": "object",
"properties": {
"mangaId": {
"type": "string",
"format": "uuid",
"description": "L'identifiant unique du manga",
"example": "123e4567-e89b-12d3-a456-426614174000"
}
},
"required": [
"mangaId"
]
}
}
},
@@ -1586,30 +1556,30 @@
"post": {
"operationId": "api_mangascreate-from-mangadex_post",
"tags": [
"Manga"
"Mangadex"
],
"responses": {
"201": {
"description": "Manga resource created",
"description": "Mangadex resource created",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Manga"
"$ref": "#/components/schemas/Mangadex"
}
},
"application/ld+json": {
"schema": {
"$ref": "#/components/schemas/Manga.jsonld"
"$ref": "#/components/schemas/Mangadex.jsonld"
}
},
"text/html": {
"schema": {
"$ref": "#/components/schemas/Manga"
"$ref": "#/components/schemas/Mangadex"
}
},
"application/hal+json": {
"schema": {
"$ref": "#/components/schemas/Manga.jsonhal"
"$ref": "#/components/schemas/Mangadex.jsonhal"
}
}
},
@@ -2534,6 +2504,299 @@
"deprecated": false
},
"parameters": []
},
"/api/scraping/test-configuration": {
"post": {
"operationId": "api_scrapingtest-configuration_post",
"tags": [
"Scraping",
"Configuration"
],
"responses": {
"200": {
"description": "Test de configuration r\u00e9ussi",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"success": {
"type": "boolean",
"description": "Indique si le test a r\u00e9ussi"
},
"imageUrls": {
"type": "array",
"items": {
"type": "string",
"format": "uri"
},
"description": "Liste des URLs d'images trouv\u00e9es"
},
"totalImages": {
"type": "integer",
"description": "Nombre total d'images trouv\u00e9es"
},
"testedUrl": {
"type": "string",
"format": "uri",
"description": "URL qui a \u00e9t\u00e9 test\u00e9e"
},
"scrapingType": {
"type": "string",
"description": "Type de scraping utilis\u00e9"
},
"errors": {
"type": "array",
"items": {
"type": "object",
"properties": {
"type": {
"type": "string",
"description": "Type d'erreur (selector_error, url_error, general_error)"
},
"field": {
"type": "string",
"description": "Champ concern\u00e9 par l'erreur"
},
"message": {
"type": "string",
"description": "Message d'erreur d\u00e9taill\u00e9"
},
"suggestion": {
"type": "string",
"description": "Suggestion pour corriger l'erreur"
}
}
},
"description": "Liste des erreurs d\u00e9taill\u00e9es (vide en cas de succ\u00e8s)"
}
}
},
"examples": {
"succes": {
"summary": "Test r\u00e9ussi",
"value": {
"success": true,
"imageUrls": [
"https://mangasite.example.com/images/chapter1/page1.jpg",
"https://mangasite.example.com/images/chapter1/page2.jpg",
"https://mangasite.example.com/images/chapter1/page3.jpg"
],
"totalImages": 3,
"testedUrl": "https://mangasite.example.com/manga/one-piece/chapter/1",
"scrapingType": "html",
"errors": []
}
},
"echec_selecteur": {
"summary": "\u00c9chec - S\u00e9lecteur invalide",
"value": {
"success": false,
"imageUrls": [],
"totalImages": 0,
"testedUrl": "https://mangasite.example.com/manga/one-piece/chapter/1",
"scrapingType": "html",
"errors": [
{
"type": "selector_error",
"field": "imageSelector",
"message": "Le s\u00e9lecteur d'image '.invalid-selector' ne trouve aucun \u00e9l\u00e9ment sur la page",
"suggestion": "V\u00e9rifiez que le s\u00e9lecteur CSS est correct et qu'il correspond aux \u00e9l\u00e9ments d'image sur la page"
}
]
}
},
"echec_url": {
"summary": "\u00c9chec - URL inaccessible",
"value": {
"success": false,
"imageUrls": [],
"totalImages": 0,
"testedUrl": "https://invalid-site.example.com/chapter/1",
"scrapingType": "html",
"errors": [
{
"type": "url_error",
"field": "testUrl",
"message": "Impossible d'acc\u00e9der \u00e0 l'URL de test: https://invalid-site.example.com/chapter/1",
"suggestion": "V\u00e9rifiez que l'URL est correcte et accessible, et que le chapitre existe"
}
]
}
}
}
}
}
},
"422": {
"description": "Erreur de validation des donn\u00e9es d'entr\u00e9e",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"type": {
"type": "string",
"example": "https://tools.ietf.org/html/rfc2616#section-10"
},
"title": {
"type": "string",
"example": "An error occurred"
},
"detail": {
"type": "string",
"example": "baseUrl: L'URL de base doit \u00eatre une URL valide"
},
"violations": {
"type": "array",
"items": {
"type": "object",
"properties": {
"propertyPath": {
"type": "string",
"description": "Champ contenant l'erreur"
},
"message": {
"type": "string",
"description": "Message d'erreur de validation"
}
}
}
}
}
},
"example": {
"type": "https://symfony.com/errors/validation",
"title": "Validation Failed",
"detail": "baseUrl: L'URL de base doit \u00eatre une URL valide",
"violations": [
{
"propertyPath": "baseUrl",
"message": "L'URL de base doit \u00eatre une URL valide"
}
]
}
}
}
},
"400": {
"description": "Invalid input"
}
},
"summary": "Tester une configuration de scraper",
"description": "Teste une configuration de scraper en temps r\u00e9el sans l'enregistrer dans la base de donn\u00e9es. Cette API permet de valider les param\u00e8tres de scraping et de voir imm\u00e9diatement les URLs d'images qui seraient extraites.",
"parameters": [],
"requestBody": {
"description": "Configuration du scraper \u00e0 tester",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"baseUrl": {
"type": "string",
"format": "uri",
"description": "URL de base du site web source",
"example": "https://mangasite.example.com"
},
"chapterUrlFormat": {
"type": "string",
"description": "Format d'URL pour acc\u00e9der aux chapitres avec placeholders {slug} et {chapter}",
"example": "https://mangasite.example.com/manga/{slug}/chapter/{chapter}"
},
"scrapingType": {
"type": "string",
"enum": [
"html",
"javascript"
],
"description": "Type de scraping \u00e0 utiliser (html pour les sites statiques, javascript pour les sites dynamiques)",
"example": "html"
},
"testUrl": {
"type": "string",
"format": "uri",
"description": "URL compl\u00e8te d'un chapitre existant \u00e0 utiliser pour le test",
"example": "https://mangasite.example.com/manga/one-piece/chapter/1"
},
"mangaSlug": {
"type": "string",
"description": "Slug du manga utilis\u00e9 dans les URLs (sera utilis\u00e9 pour construire les URLs futures)",
"example": "one-piece"
},
"chapterNumber": {
"type": "number",
"minimum": 0,
"description": "Num\u00e9ro du chapitre \u00e0 tester",
"example": 1
},
"imageSelector": {
"type": "string",
"nullable": true,
"description": "S\u00e9lecteur CSS pour identifier les images dans la page",
"example": "img.manga-page, .chapter-image img"
},
"nextPageSelector": {
"type": "string",
"nullable": true,
"description": "S\u00e9lecteur CSS pour le lien vers la page suivante (pour les lecteurs horizontaux)",
"example": "a.next-page, .navigation .next"
},
"chapterSelector": {
"type": "string",
"nullable": true,
"description": "S\u00e9lecteur CSS pour identifier la zone contenant le chapitre",
"example": ".chapter-content, #manga-reader"
}
},
"required": [
"baseUrl",
"chapterUrlFormat",
"scrapingType",
"testUrl",
"mangaSlug",
"chapterNumber"
]
},
"examples": {
"lecteur_vertical": {
"summary": "Configuration pour un lecteur vertical",
"description": "Exemple de configuration pour un site avec toutes les images sur une seule page",
"value": {
"baseUrl": "https://mangasite.example.com",
"chapterUrlFormat": "https://mangasite.example.com/manga/{slug}/chapter/{chapter}",
"scrapingType": "html",
"testUrl": "https://mangasite.example.com/manga/one-piece/chapter/1",
"mangaSlug": "one-piece",
"chapterNumber": 1,
"imageSelector": "img.manga-page",
"nextPageSelector": null,
"chapterSelector": ".chapter-content"
}
},
"lecteur_horizontal": {
"summary": "Configuration pour un lecteur horizontal",
"description": "Exemple de configuration pour un site avec navigation page par page",
"value": {
"baseUrl": "https://mangasite.example.com",
"chapterUrlFormat": "https://mangasite.example.com/read/{slug}/{chapter}/1",
"scrapingType": "html",
"testUrl": "https://mangasite.example.com/read/one-piece/1/1",
"mangaSlug": "one-piece",
"chapterNumber": 1,
"imageSelector": "#manga-image",
"nextPageSelector": "a.next-page",
"chapterSelector": ".reader-container"
}
}
}
}
},
"required": false
},
"deprecated": false
},
"parameters": []
}
},
"components": {
@@ -3458,6 +3721,12 @@
"slug": {
"type": "string"
},
"alternativeSlugs": {
"type": "array",
"items": {
"type": "string"
}
},
"description": {
"type": "string"
},
@@ -3530,6 +3799,12 @@
"slug": {
"type": "string"
},
"alternativeSlugs": {
"type": "array",
"items": {
"type": "string"
}
},
"description": {
"type": "string"
},
@@ -3623,6 +3898,12 @@
"slug": {
"type": "string"
},
"alternativeSlugs": {
"type": "array",
"items": {
"type": "string"
}
},
"description": {
"type": "string"
},
@@ -4348,6 +4629,19 @@
}
}
},
"Mangadex": {
"type": "object",
"description": "",
"deprecated": false,
"required": [
"externalId"
],
"properties": {
"externalId": {
"type": "string"
}
}
},
"Mangadex.MangaSearchCollection": {
"type": "object",
"description": "",
@@ -4436,6 +4730,81 @@
}
}
},
"Mangadex.jsonhal": {
"type": "object",
"description": "",
"deprecated": false,
"required": [
"externalId"
],
"properties": {
"_links": {
"type": "object",
"properties": {
"self": {
"type": "object",
"properties": {
"href": {
"type": "string",
"format": "iri-reference"
}
}
}
}
},
"externalId": {
"type": "string"
}
}
},
"Mangadex.jsonld": {
"type": "object",
"description": "",
"deprecated": false,
"required": [
"externalId"
],
"properties": {
"@context": {
"readOnly": true,
"oneOf": [
{
"type": "string"
},
{
"type": "object",
"properties": {
"@vocab": {
"type": "string"
},
"hydra": {
"type": "string",
"enum": [
"http://www.w3.org/ns/hydra/core#"
]
}
},
"required": [
"@vocab",
"hydra"
],
"additionalProperties": true
}
]
},
"@id": {
"readOnly": true,
"type": "string"
},
"@type": {
"readOnly": true,
"type": "string"
},
"externalId": {
"type": "string"
}
}
},
"Reader": {
"type": "object",
"description": "",
@@ -4624,6 +4993,256 @@
}
}
},
"Scraping.TestScraperConfigurationResource": {
"type": "object",
"description": "Teste une configuration de scraper sans l'enregistrer",
"deprecated": false,
"properties": {
"success": {
"type": "boolean"
},
"imageUrls": {
"type": "array",
"items": {
"type": "string",
"format": "uri"
}
},
"totalImages": {
"type": "integer",
"minimum": 0
},
"testedUrl": {
"type": "string",
"format": "uri"
},
"scrapingType": {
"type": "string",
"enum": [
"html",
"javascript"
]
},
"errors": {
"type": "array",
"items": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": [
"selector_error",
"url_error",
"general_error"
],
"description": "Type d'erreur rencontr\u00e9e"
},
"field": {
"type": "string",
"description": "Champ de configuration concern\u00e9 par l'erreur"
},
"message": {
"type": "string",
"description": "Message d'erreur d\u00e9taill\u00e9"
},
"suggestion": {
"type": "string",
"description": "Suggestion pour corriger l'erreur"
}
},
"required": [
"type",
"field",
"message",
"suggestion"
]
}
}
}
},
"Scraping.TestScraperConfigurationResource.jsonhal": {
"type": "object",
"description": "Teste une configuration de scraper sans l'enregistrer",
"deprecated": false,
"properties": {
"_links": {
"type": "object",
"properties": {
"self": {
"type": "object",
"properties": {
"href": {
"type": "string",
"format": "iri-reference"
}
}
}
}
},
"success": {
"type": "boolean"
},
"imageUrls": {
"type": "array",
"items": {
"type": "string",
"format": "uri"
}
},
"totalImages": {
"type": "integer",
"minimum": 0
},
"testedUrl": {
"type": "string",
"format": "uri"
},
"scrapingType": {
"type": "string",
"enum": [
"html",
"javascript"
]
},
"errors": {
"type": "array",
"items": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": [
"selector_error",
"url_error",
"general_error"
],
"description": "Type d'erreur rencontr\u00e9e"
},
"field": {
"type": "string",
"description": "Champ de configuration concern\u00e9 par l'erreur"
},
"message": {
"type": "string",
"description": "Message d'erreur d\u00e9taill\u00e9"
},
"suggestion": {
"type": "string",
"description": "Suggestion pour corriger l'erreur"
}
},
"required": [
"type",
"field",
"message",
"suggestion"
]
}
}
}
},
"Scraping.TestScraperConfigurationResource.jsonld": {
"type": "object",
"description": "Teste une configuration de scraper sans l'enregistrer",
"deprecated": false,
"properties": {
"@context": {
"readOnly": true,
"oneOf": [
{
"type": "string"
},
{
"type": "object",
"properties": {
"@vocab": {
"type": "string"
},
"hydra": {
"type": "string",
"enum": [
"http://www.w3.org/ns/hydra/core#"
]
}
},
"required": [
"@vocab",
"hydra"
],
"additionalProperties": true
}
]
},
"@id": {
"readOnly": true,
"type": "string"
},
"@type": {
"readOnly": true,
"type": "string"
},
"success": {
"type": "boolean"
},
"imageUrls": {
"type": "array",
"items": {
"type": "string",
"format": "uri"
}
},
"totalImages": {
"type": "integer",
"minimum": 0
},
"testedUrl": {
"type": "string",
"format": "uri"
},
"scrapingType": {
"type": "string",
"enum": [
"html",
"javascript"
]
},
"errors": {
"type": "array",
"items": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": [
"selector_error",
"url_error",
"general_error"
],
"description": "Type d'erreur rencontr\u00e9e"
},
"field": {
"type": "string",
"description": "Champ de configuration concern\u00e9 par l'erreur"
},
"message": {
"type": "string",
"description": "Message d'erreur d\u00e9taill\u00e9"
},
"suggestion": {
"type": "string",
"description": "Suggestion pour corriger l'erreur"
}
},
"required": [
"type",
"field",
"message",
"suggestion"
]
}
}
}
},
"Scraping.jsonhal": {
"type": "object",
"description": "",