feat: ajout d'une nouvelle infrastructure de scraping avec des scrapers pour HTML, HTML avancé et JavaScript, ainsi qu'une factory pour gérer leur création et leur sélection. Mise à jour des gestionnaires de commandes pour intégrer cette nouvelle architecture et améliorer la gestion des erreurs lors du scraping des chapitres.
This commit is contained in:
ext.jeremy.guillot@maxicoffee.domains
parent
cbb62989d4
commit
b456f9304d
520
public/puppeteer-scraper.js
Normal file
520
public/puppeteer-scraper.js
Normal file
@@ -0,0 +1,520 @@
|
||||
const puppeteer = require('puppeteer');
|
||||
|
||||
// Configuration par défaut
|
||||
const CONFIG = {
|
||||
// Timeout en millisecondes
|
||||
PAGE_TIMEOUT: 30000,
|
||||
NAVIGATION_TIMEOUT: 10000,
|
||||
SCROLL_DELAY: 100,
|
||||
SCROLL_DISTANCE: 100,
|
||||
// Timeout réduit pour la détection d'erreur
|
||||
ERROR_DETECTION_TIMEOUT: 5000,
|
||||
|
||||
// User agents pour contourner la détection
|
||||
USER_AGENTS: [
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
||||
],
|
||||
|
||||
// Arguments pour contourner la détection
|
||||
BROWSER_ARGS: [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-accelerated-2d-canvas',
|
||||
'--no-first-run',
|
||||
'--no-zygote',
|
||||
'--single-process',
|
||||
'--disable-gpu',
|
||||
'--disable-web-security',
|
||||
'--disable-features=VizDisplayCompositor',
|
||||
'--disable-blink-features=AutomationControlled'
|
||||
]
|
||||
};
|
||||
|
||||
class ChapterNotFoundError extends Error {
|
||||
constructor(chapterNumber) {
|
||||
super(`Chapter ${chapterNumber} not found`);
|
||||
this.name = 'ChapterNotFoundError';
|
||||
this.chapterNumber = chapterNumber;
|
||||
}
|
||||
}
|
||||
|
||||
class PuppeteerScraper {
|
||||
constructor(options = {}) {
|
||||
this.options = this.parseArguments(options);
|
||||
this.browser = null;
|
||||
this.page = null;
|
||||
this.imageUrls = new Set();
|
||||
this.lastResponseStatus = null;
|
||||
this.navigationError = null;
|
||||
}
|
||||
|
||||
parseArguments(options) {
|
||||
const args = process.argv.slice(2);
|
||||
const parsed = { ...options };
|
||||
|
||||
args.forEach(arg => {
|
||||
if (arg.startsWith('--')) {
|
||||
const [key, value] = arg.substring(2).split('=');
|
||||
parsed[key.replace(/-/g, '_')] = value === 'true' ? true : value === 'false' ? false : value;
|
||||
}
|
||||
});
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
async launch() {
|
||||
// Essayer de trouver un exécutable Chrome/Chromium disponible
|
||||
const possiblePaths = [
|
||||
process.env.CHROME_BIN,
|
||||
'/usr/bin/chromium',
|
||||
'/usr/bin/chromium-browser',
|
||||
'/usr/bin/google-chrome',
|
||||
'/usr/bin/google-chrome-stable',
|
||||
'/snap/bin/chromium'
|
||||
].filter(path => path); // Supprimer les valeurs nulles/undefined
|
||||
|
||||
let executablePath = null;
|
||||
|
||||
// Vérifier si on peut utiliser un des chemins
|
||||
for (const path of possiblePaths) {
|
||||
try {
|
||||
const fs = require('fs');
|
||||
if (fs.existsSync(path)) {
|
||||
executablePath = path;
|
||||
console.log(`Using Chrome at: ${path}`);
|
||||
break;
|
||||
}
|
||||
} catch (e) {
|
||||
// Continuer avec le chemin suivant
|
||||
}
|
||||
}
|
||||
|
||||
// Si aucun exécutable trouvé, laisser Puppeteer utiliser celui installé via npm
|
||||
this.browser = await puppeteer.launch({
|
||||
headless: 'new',
|
||||
executablePath: executablePath,
|
||||
args: CONFIG.BROWSER_ARGS
|
||||
});
|
||||
|
||||
this.page = await this.browser.newPage();
|
||||
|
||||
// Configuration anti-détection
|
||||
await this.setupAntiDetection();
|
||||
|
||||
console.log('Browser launched and configured');
|
||||
}
|
||||
|
||||
async setupAntiDetection() {
|
||||
// Rotation des User-Agent
|
||||
const userAgent = CONFIG.USER_AGENTS[Math.floor(Math.random() * CONFIG.USER_AGENTS.length)];
|
||||
await this.page.setUserAgent(userAgent);
|
||||
|
||||
// Écouter les réponses pour détecter rapidement les erreurs HTTP
|
||||
this.page.on('response', (response) => {
|
||||
// Ne surveiller que les réponses de navigation principales
|
||||
if (response.request().isNavigationRequest()) {
|
||||
this.lastResponseStatus = response.status();
|
||||
|
||||
if (response.status() >= 400) {
|
||||
this.navigationError = {
|
||||
status: response.status(),
|
||||
statusText: response.statusText(),
|
||||
url: response.url()
|
||||
};
|
||||
console.log(`❌ HTTP Error ${response.status()} detected for: ${response.url()}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Désactiver seulement les fonts et certains styles pour optimiser
|
||||
await this.page.setRequestInterception(true);
|
||||
this.page.on('request', (request) => {
|
||||
if (['font'].includes(request.resourceType())) {
|
||||
request.abort();
|
||||
} else {
|
||||
request.continue();
|
||||
}
|
||||
});
|
||||
|
||||
// Masquer les propriétés de détection de Puppeteer
|
||||
await this.page.evaluateOnNewDocument(() => {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
|
||||
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
|
||||
window.chrome = { runtime: {} };
|
||||
});
|
||||
|
||||
// Viewport aléatoire
|
||||
await this.page.setViewport({
|
||||
width: 1366 + Math.floor(Math.random() * 200),
|
||||
height: 768 + Math.floor(Math.random() * 200)
|
||||
});
|
||||
}
|
||||
|
||||
async navigateToPage(url, useReducedTimeout = false) {
|
||||
// Reset des variables de détection d'erreur
|
||||
this.lastResponseStatus = null;
|
||||
this.navigationError = null;
|
||||
|
||||
const timeout = useReducedTimeout ? CONFIG.ERROR_DETECTION_TIMEOUT : CONFIG.PAGE_TIMEOUT;
|
||||
|
||||
try {
|
||||
console.log(`🌐 Navigating to: ${url}`);
|
||||
|
||||
const response = await this.page.goto(url, {
|
||||
waitUntil: 'domcontentloaded', // Plus rapide que networkidle2 pour la détection d'erreur
|
||||
timeout: timeout
|
||||
});
|
||||
|
||||
// Vérifier immédiatement le code de statut de la réponse
|
||||
if (response && response.status() >= 400) {
|
||||
throw new Error(`HTTP ${response.status()}: ${response.statusText()} for URL: ${url}`);
|
||||
}
|
||||
|
||||
// Si pas d'erreur immédiate, attendre que le contenu se charge complètement
|
||||
if (!this.navigationError) {
|
||||
// Attendre un court délai pour permettre aux ressources de se charger
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
} else {
|
||||
throw new Error(`Navigation error: ${this.navigationError.status} ${this.navigationError.statusText}`);
|
||||
}
|
||||
|
||||
console.log(`✅ Successfully loaded: ${url}`);
|
||||
return response;
|
||||
|
||||
} catch (error) {
|
||||
// Distinguer les erreurs de timeout des erreurs HTTP
|
||||
if (error.message.includes('HTTP 4') || error.message.includes('HTTP 5')) {
|
||||
console.log(`🚫 HTTP Error detected quickly: ${error.message}`);
|
||||
throw error;
|
||||
} else if (error.message.includes('timeout')) {
|
||||
console.log(`⏱️ Navigation timeout for: ${url}`);
|
||||
throw new Error(`Navigation timeout after ${timeout}ms for URL: ${url}`);
|
||||
} else {
|
||||
console.log(`❌ Navigation error: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async navigateToPageWithFallback(url) {
|
||||
try {
|
||||
// Première tentative avec timeout réduit pour détection rapide d'erreur
|
||||
return await this.navigateToPage(url, true);
|
||||
} catch (error) {
|
||||
if (error.message.includes('HTTP 4') || error.message.includes('HTTP 5')) {
|
||||
// Erreur HTTP confirmée, ne pas réessayer
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Si c'est un timeout, réessayer avec timeout complet
|
||||
console.log(`🔄 Quick check failed, retrying with full timeout...`);
|
||||
return await this.navigateToPage(url, false);
|
||||
}
|
||||
}
|
||||
|
||||
async selectChapter(chapterSelector, chapterNumber) {
|
||||
try {
|
||||
console.log(`📚 Looking for chapter selector: ${chapterSelector}`);
|
||||
|
||||
// Attendre que le sélecteur soit présent
|
||||
await this.page.waitForSelector(chapterSelector, { timeout: CONFIG.NAVIGATION_TIMEOUT });
|
||||
|
||||
// Lister toutes les options disponibles
|
||||
const options = await this.page.$$eval(chapterSelector + ' option', opts =>
|
||||
opts.map(opt => ({
|
||||
value: opt.value,
|
||||
text: opt.textContent.trim(),
|
||||
selected: opt.selected
|
||||
}))
|
||||
);
|
||||
|
||||
console.log(`📖 Found ${options.length} chapter options`);
|
||||
|
||||
// Chercher l'option correspondant au chapitre demandé
|
||||
const targetOption = options.find(opt => {
|
||||
const text = opt.text.toLowerCase();
|
||||
const chapterStr = chapterNumber.toString();
|
||||
|
||||
return text.includes(chapterStr) ||
|
||||
text.includes(`chapitre ${chapterStr}`) ||
|
||||
text.includes(`chapter ${chapterStr}`) ||
|
||||
opt.value === chapterStr ||
|
||||
text.includes(`${chapterStr}.0`) ||
|
||||
text.includes(`${chapterStr} -`);
|
||||
});
|
||||
|
||||
if (targetOption) {
|
||||
console.log(`🎯 Found target chapter: ${targetOption.text} (value: ${targetOption.value})`);
|
||||
|
||||
// Sélectionner le chapitre
|
||||
await this.page.select(chapterSelector, targetOption.value);
|
||||
console.log('✅ Chapter selected, waiting for page reload...');
|
||||
|
||||
// Attendre que la page se recharge après la sélection
|
||||
try {
|
||||
await this.page.waitForNavigation({
|
||||
waitUntil: 'domcontentloaded',
|
||||
timeout: CONFIG.ERROR_DETECTION_TIMEOUT
|
||||
});
|
||||
console.log('🔄 Page reloaded after chapter selection');
|
||||
} catch (error) {
|
||||
console.log(`⚠️ Warning during chapter navigation: ${error.message}`);
|
||||
// Attendre un peu même si la navigation échoue
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
}
|
||||
|
||||
} else {
|
||||
// Lancer une exception spécifique pour le chapitre non trouvé
|
||||
throw new ChapterNotFoundError(chapterNumber);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
if (error instanceof ChapterNotFoundError) {
|
||||
// Re-lancer l'exception pour qu'elle soit gérée en amont
|
||||
throw error;
|
||||
}
|
||||
console.log(`⚠️ Error selecting chapter: ${error.message}`);
|
||||
// Continuer même si la sélection échoue pour les autres erreurs
|
||||
}
|
||||
}
|
||||
|
||||
async scrapeVertical() {
|
||||
const url = this.options.url;
|
||||
const imageSelector = this.options.image_selector;
|
||||
const waitForImages = this.options.wait_for_images === 'true';
|
||||
const shouldScroll = this.options.scroll === 'true';
|
||||
const chapterSelector = this.options.chapter_selector;
|
||||
const chapterNumber = this.options.chapter_number;
|
||||
|
||||
try {
|
||||
await this.navigateToPageWithFallback(url);
|
||||
} catch (error) {
|
||||
if (error.message.includes('HTTP 4') || error.message.includes('HTTP 5')) {
|
||||
console.log(`🚫 Cannot access page: ${error.message}`);
|
||||
return []; // Retourner un tableau vide pour les erreurs HTTP
|
||||
}
|
||||
throw error; // Re-lancer les autres erreurs
|
||||
}
|
||||
|
||||
// Gérer la sélection de chapitre si nécessaire
|
||||
if (chapterSelector && chapterNumber) {
|
||||
try {
|
||||
await this.selectChapter(chapterSelector, chapterNumber);
|
||||
} catch (error) {
|
||||
if (error instanceof ChapterNotFoundError) {
|
||||
console.log(`📚 MANGA_EXISTS_BUT_CHAPTER_NOT_FOUND: ${error.message}`);
|
||||
return {
|
||||
error: 'CHAPTER_NOT_FOUND',
|
||||
message: `Le manga existe mais le chapitre ${error.chapterNumber} n'est pas disponible.`,
|
||||
images: []
|
||||
};
|
||||
}
|
||||
throw error; // Re-lancer les autres erreurs
|
||||
}
|
||||
}
|
||||
|
||||
// Attendre le sélecteur d'image
|
||||
if (waitForImages) {
|
||||
await this.page.waitForSelector(imageSelector, { timeout: CONFIG.NAVIGATION_TIMEOUT });
|
||||
}
|
||||
|
||||
// Scroll pour charger toutes les images lazy-load
|
||||
if (shouldScroll) {
|
||||
await this.autoScroll();
|
||||
}
|
||||
|
||||
// Attendre un peu pour que les images se chargent (plus de temps pour lazy loading)
|
||||
await new Promise(resolve => setTimeout(resolve, 3000));
|
||||
|
||||
// Collecter les URLs d'images
|
||||
const imageUrls = await this.page.$$eval(imageSelector, imgs => {
|
||||
return imgs.map(img => {
|
||||
// Priorité au src, puis aux attributs data-*
|
||||
return img.src ||
|
||||
img.getAttribute('src') ||
|
||||
img.getAttribute('data-src') ||
|
||||
img.getAttribute('data-lazy-src') ||
|
||||
img.getAttribute('data-original');
|
||||
}).filter(url => url && url !== 'about:blank');
|
||||
});
|
||||
|
||||
console.log(`Found ${imageUrls.length} images`);
|
||||
return imageUrls;
|
||||
}
|
||||
|
||||
async scrapeHorizontal() {
|
||||
const url = this.options.url;
|
||||
const imageSelector = this.options.image_selector;
|
||||
const nextSelector = this.options.next_selector;
|
||||
const waitForImages = this.options.wait_for_images === 'true';
|
||||
const chapterSelector = this.options.chapter_selector;
|
||||
const chapterNumber = this.options.chapter_number;
|
||||
|
||||
let currentUrl = url;
|
||||
let pageCount = 0;
|
||||
const maxPages = 200; // Limite de sécurité
|
||||
|
||||
while (currentUrl && pageCount < maxPages) {
|
||||
console.log(`Scraping page ${pageCount + 1}: ${currentUrl}`);
|
||||
|
||||
try {
|
||||
await this.navigateToPageWithFallback(currentUrl);
|
||||
} catch (error) {
|
||||
if (error.message.includes('HTTP 4') || error.message.includes('HTTP 5')) {
|
||||
console.log(`🚫 Cannot access page ${pageCount + 1}: ${error.message}`);
|
||||
break; // Arrêter le scraping si on rencontre une 404
|
||||
}
|
||||
// Pour les autres erreurs, essayer de continuer
|
||||
console.log(`⚠️ Warning on page ${pageCount + 1}: ${error.message}, continuing...`);
|
||||
}
|
||||
|
||||
// Gérer la sélection de chapitre pour la première page seulement
|
||||
if (pageCount === 0 && chapterSelector && chapterNumber) {
|
||||
try {
|
||||
await this.selectChapter(chapterSelector, chapterNumber);
|
||||
} catch (error) {
|
||||
if (error instanceof ChapterNotFoundError) {
|
||||
console.log(`📚 MANGA_EXISTS_BUT_CHAPTER_NOT_FOUND: ${error.message}`);
|
||||
return {
|
||||
error: 'CHAPTER_NOT_FOUND',
|
||||
message: `Le manga existe mais le chapitre ${error.chapterNumber} n'est pas disponible.`,
|
||||
images: []
|
||||
};
|
||||
}
|
||||
throw error; // Re-lancer les autres erreurs
|
||||
}
|
||||
}
|
||||
|
||||
// Attendre le sélecteur d'image
|
||||
if (waitForImages) {
|
||||
try {
|
||||
await this.page.waitForSelector(imageSelector, { timeout: CONFIG.NAVIGATION_TIMEOUT });
|
||||
} catch (e) {
|
||||
console.log(`No image found on page ${pageCount + 1}, skipping`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Récupérer l'image de la page
|
||||
const imageUrl = await this.page.$eval(imageSelector, img => {
|
||||
return img.src ||
|
||||
img.getAttribute('src') ||
|
||||
img.getAttribute('data-src') ||
|
||||
img.getAttribute('data-lazy-src') ||
|
||||
img.getAttribute('data-original');
|
||||
}).catch(() => null);
|
||||
|
||||
if (imageUrl) {
|
||||
this.imageUrls.add(imageUrl);
|
||||
console.log(`Image found: ${imageUrl}`);
|
||||
}
|
||||
|
||||
// Chercher le bouton/lien suivant
|
||||
const nextElement = await this.page.$(nextSelector);
|
||||
if (!nextElement) {
|
||||
console.log('No next button found, ending scraping');
|
||||
break;
|
||||
}
|
||||
|
||||
// Récupérer l'URL suivante
|
||||
currentUrl = await nextElement.evaluate(el => {
|
||||
return el.href || el.getAttribute('href');
|
||||
});
|
||||
|
||||
if (!currentUrl) {
|
||||
console.log('No next URL found, ending scraping');
|
||||
break;
|
||||
}
|
||||
|
||||
pageCount++;
|
||||
await new Promise(resolve => setTimeout(resolve, 1000)); // Pause entre les pages
|
||||
}
|
||||
|
||||
return Array.from(this.imageUrls);
|
||||
}
|
||||
|
||||
async autoScroll() {
|
||||
await this.page.evaluate(async (config) => {
|
||||
await new Promise((resolve) => {
|
||||
let totalHeight = 0;
|
||||
let lastHeight = 0;
|
||||
|
||||
const timer = setInterval(() => {
|
||||
const scrollHeight = document.body.scrollHeight;
|
||||
|
||||
// Si la hauteur a changé, on continue
|
||||
if (scrollHeight !== lastHeight) {
|
||||
lastHeight = scrollHeight;
|
||||
totalHeight = 0; // Reset le counter car plus de contenu apparaît
|
||||
}
|
||||
|
||||
window.scrollBy(0, config.SCROLL_DISTANCE);
|
||||
totalHeight += config.SCROLL_DISTANCE;
|
||||
|
||||
// Arrêter si on a atteint le bas ET que rien de nouveau ne charge
|
||||
if (totalHeight >= scrollHeight) {
|
||||
clearInterval(timer);
|
||||
// Scroll final jusqu'à la vraie fin
|
||||
window.scrollTo(0, document.body.scrollHeight);
|
||||
resolve();
|
||||
}
|
||||
}, config.SCROLL_DELAY);
|
||||
});
|
||||
}, CONFIG);
|
||||
}
|
||||
|
||||
async close() {
|
||||
if (this.browser) {
|
||||
await this.browser.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const scraper = new PuppeteerScraper();
|
||||
|
||||
try {
|
||||
await scraper.launch();
|
||||
|
||||
let result = [];
|
||||
|
||||
if (scraper.options.mode === 'vertical') {
|
||||
result = await scraper.scrapeVertical();
|
||||
} else if (scraper.options.mode === 'horizontal') {
|
||||
result = await scraper.scrapeHorizontal();
|
||||
} else {
|
||||
throw new Error('Invalid mode. Use --mode=vertical or --mode=horizontal');
|
||||
}
|
||||
|
||||
// Vérifier si le résultat est un objet d'erreur ou un tableau d'URLs
|
||||
if (result && typeof result === 'object' && result.error === 'CHAPTER_NOT_FOUND') {
|
||||
// Cas où le chapitre n'est pas trouvé
|
||||
console.log(`CHAPTER_NOT_FOUND:${JSON.stringify(result)}`);
|
||||
} else {
|
||||
// Cas normal - nettoyer les URLs
|
||||
const imageUrls = Array.isArray(result) ? result : [];
|
||||
const cleanUrls = imageUrls.filter(url => url && typeof url === 'string');
|
||||
console.log(`RESULT:${JSON.stringify(cleanUrls)}`);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
if (error instanceof ChapterNotFoundError) {
|
||||
// Cette erreur est déjà gérée dans les fonctions de scraping
|
||||
// Mais au cas où elle remonterait ici
|
||||
console.log(`CHAPTER_NOT_FOUND:${JSON.stringify({
|
||||
error: 'CHAPTER_NOT_FOUND',
|
||||
message: `Le manga existe mais le chapitre ${error.chapterNumber} n'est pas disponible.`,
|
||||
images: []
|
||||
})}`);
|
||||
} else {
|
||||
console.error('Error:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
} finally {
|
||||
await scraper.close();
|
||||
}
|
||||
})();
|
||||
Reference in New Issue
Block a user