521 lines
20 KiB
JavaScript
521 lines
20 KiB
JavaScript
const puppeteer = require('puppeteer');
|
|
|
|
// Configuration par défaut
|
|
const CONFIG = {
|
|
// Timeout en millisecondes
|
|
PAGE_TIMEOUT: 30000,
|
|
NAVIGATION_TIMEOUT: 10000,
|
|
SCROLL_DELAY: 100,
|
|
SCROLL_DISTANCE: 100,
|
|
// Timeout réduit pour la détection d'erreur
|
|
ERROR_DETECTION_TIMEOUT: 5000,
|
|
|
|
// User agents pour contourner la détection
|
|
USER_AGENTS: [
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
],
|
|
|
|
// Arguments pour contourner la détection
|
|
BROWSER_ARGS: [
|
|
'--no-sandbox',
|
|
'--disable-setuid-sandbox',
|
|
'--disable-dev-shm-usage',
|
|
'--disable-accelerated-2d-canvas',
|
|
'--no-first-run',
|
|
'--no-zygote',
|
|
'--single-process',
|
|
'--disable-gpu',
|
|
'--disable-web-security',
|
|
'--disable-features=VizDisplayCompositor',
|
|
'--disable-blink-features=AutomationControlled'
|
|
]
|
|
};
|
|
|
|
class ChapterNotFoundError extends Error {
|
|
constructor(chapterNumber) {
|
|
super(`Chapter ${chapterNumber} not found`);
|
|
this.name = 'ChapterNotFoundError';
|
|
this.chapterNumber = chapterNumber;
|
|
}
|
|
}
|
|
|
|
class PuppeteerScraper {
|
|
constructor(options = {}) {
|
|
this.options = this.parseArguments(options);
|
|
this.browser = null;
|
|
this.page = null;
|
|
this.imageUrls = new Set();
|
|
this.lastResponseStatus = null;
|
|
this.navigationError = null;
|
|
}
|
|
|
|
parseArguments(options) {
|
|
const args = process.argv.slice(2);
|
|
const parsed = { ...options };
|
|
|
|
args.forEach(arg => {
|
|
if (arg.startsWith('--')) {
|
|
const [key, value] = arg.substring(2).split('=');
|
|
parsed[key.replace(/-/g, '_')] = value === 'true' ? true : value === 'false' ? false : value;
|
|
}
|
|
});
|
|
|
|
return parsed;
|
|
}
|
|
|
|
async launch() {
|
|
// Essayer de trouver un exécutable Chrome/Chromium disponible
|
|
const possiblePaths = [
|
|
process.env.CHROME_BIN,
|
|
'/usr/bin/chromium',
|
|
'/usr/bin/chromium-browser',
|
|
'/usr/bin/google-chrome',
|
|
'/usr/bin/google-chrome-stable',
|
|
'/snap/bin/chromium'
|
|
].filter(path => path); // Supprimer les valeurs nulles/undefined
|
|
|
|
let executablePath = null;
|
|
|
|
// Vérifier si on peut utiliser un des chemins
|
|
for (const path of possiblePaths) {
|
|
try {
|
|
const fs = require('fs');
|
|
if (fs.existsSync(path)) {
|
|
executablePath = path;
|
|
console.log(`Using Chrome at: ${path}`);
|
|
break;
|
|
}
|
|
} catch (e) {
|
|
// Continuer avec le chemin suivant
|
|
}
|
|
}
|
|
|
|
// Si aucun exécutable trouvé, laisser Puppeteer utiliser celui installé via npm
|
|
this.browser = await puppeteer.launch({
|
|
headless: 'new',
|
|
executablePath: executablePath,
|
|
args: CONFIG.BROWSER_ARGS
|
|
});
|
|
|
|
this.page = await this.browser.newPage();
|
|
|
|
// Configuration anti-détection
|
|
await this.setupAntiDetection();
|
|
|
|
console.log('Browser launched and configured');
|
|
}
|
|
|
|
async setupAntiDetection() {
|
|
// Rotation des User-Agent
|
|
const userAgent = CONFIG.USER_AGENTS[Math.floor(Math.random() * CONFIG.USER_AGENTS.length)];
|
|
await this.page.setUserAgent(userAgent);
|
|
|
|
// Écouter les réponses pour détecter rapidement les erreurs HTTP
|
|
this.page.on('response', (response) => {
|
|
// Ne surveiller que les réponses de navigation principales
|
|
if (response.request().isNavigationRequest()) {
|
|
this.lastResponseStatus = response.status();
|
|
|
|
if (response.status() >= 400) {
|
|
this.navigationError = {
|
|
status: response.status(),
|
|
statusText: response.statusText(),
|
|
url: response.url()
|
|
};
|
|
console.log(`❌ HTTP Error ${response.status()} detected for: ${response.url()}`);
|
|
}
|
|
}
|
|
});
|
|
|
|
// Désactiver seulement les fonts et certains styles pour optimiser
|
|
await this.page.setRequestInterception(true);
|
|
this.page.on('request', (request) => {
|
|
if (['font'].includes(request.resourceType())) {
|
|
request.abort();
|
|
} else {
|
|
request.continue();
|
|
}
|
|
});
|
|
|
|
// Masquer les propriétés de détection de Puppeteer
|
|
await this.page.evaluateOnNewDocument(() => {
|
|
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
|
Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
|
|
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
|
|
window.chrome = { runtime: {} };
|
|
});
|
|
|
|
// Viewport aléatoire
|
|
await this.page.setViewport({
|
|
width: 1366 + Math.floor(Math.random() * 200),
|
|
height: 768 + Math.floor(Math.random() * 200)
|
|
});
|
|
}
|
|
|
|
async navigateToPage(url, useReducedTimeout = false) {
|
|
// Reset des variables de détection d'erreur
|
|
this.lastResponseStatus = null;
|
|
this.navigationError = null;
|
|
|
|
const timeout = useReducedTimeout ? CONFIG.ERROR_DETECTION_TIMEOUT : CONFIG.PAGE_TIMEOUT;
|
|
|
|
try {
|
|
console.log(`🌐 Navigating to: ${url}`);
|
|
|
|
const response = await this.page.goto(url, {
|
|
waitUntil: 'domcontentloaded', // Plus rapide que networkidle2 pour la détection d'erreur
|
|
timeout: timeout
|
|
});
|
|
|
|
// Vérifier immédiatement le code de statut de la réponse
|
|
if (response && response.status() >= 400) {
|
|
throw new Error(`HTTP ${response.status()}: ${response.statusText()} for URL: ${url}`);
|
|
}
|
|
|
|
// Si pas d'erreur immédiate, attendre que le contenu se charge complètement
|
|
if (!this.navigationError) {
|
|
// Attendre un court délai pour permettre aux ressources de se charger
|
|
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
} else {
|
|
throw new Error(`Navigation error: ${this.navigationError.status} ${this.navigationError.statusText}`);
|
|
}
|
|
|
|
console.log(`✅ Successfully loaded: ${url}`);
|
|
return response;
|
|
|
|
} catch (error) {
|
|
// Distinguer les erreurs de timeout des erreurs HTTP
|
|
if (error.message.includes('HTTP 4') || error.message.includes('HTTP 5')) {
|
|
console.log(`🚫 HTTP Error detected quickly: ${error.message}`);
|
|
throw error;
|
|
} else if (error.message.includes('timeout')) {
|
|
console.log(`⏱️ Navigation timeout for: ${url}`);
|
|
throw new Error(`Navigation timeout after ${timeout}ms for URL: ${url}`);
|
|
} else {
|
|
console.log(`❌ Navigation error: ${error.message}`);
|
|
throw error;
|
|
}
|
|
}
|
|
}
|
|
|
|
async navigateToPageWithFallback(url) {
|
|
try {
|
|
// Première tentative avec timeout réduit pour détection rapide d'erreur
|
|
return await this.navigateToPage(url, true);
|
|
} catch (error) {
|
|
if (error.message.includes('HTTP 4') || error.message.includes('HTTP 5')) {
|
|
// Erreur HTTP confirmée, ne pas réessayer
|
|
throw error;
|
|
}
|
|
|
|
// Si c'est un timeout, réessayer avec timeout complet
|
|
console.log(`🔄 Quick check failed, retrying with full timeout...`);
|
|
return await this.navigateToPage(url, false);
|
|
}
|
|
}
|
|
|
|
async selectChapter(chapterSelector, chapterNumber) {
|
|
try {
|
|
console.log(`📚 Looking for chapter selector: ${chapterSelector}`);
|
|
|
|
// Attendre que le sélecteur soit présent
|
|
await this.page.waitForSelector(chapterSelector, { timeout: CONFIG.NAVIGATION_TIMEOUT });
|
|
|
|
// Lister toutes les options disponibles
|
|
const options = await this.page.$$eval(chapterSelector + ' option', opts =>
|
|
opts.map(opt => ({
|
|
value: opt.value,
|
|
text: opt.textContent.trim(),
|
|
selected: opt.selected
|
|
}))
|
|
);
|
|
|
|
console.log(`📖 Found ${options.length} chapter options`);
|
|
|
|
// Chercher l'option correspondant au chapitre demandé
|
|
const targetOption = options.find(opt => {
|
|
const text = opt.text.toLowerCase();
|
|
const chapterStr = chapterNumber.toString();
|
|
|
|
return text.includes(chapterStr) ||
|
|
text.includes(`chapitre ${chapterStr}`) ||
|
|
text.includes(`chapter ${chapterStr}`) ||
|
|
opt.value === chapterStr ||
|
|
text.includes(`${chapterStr}.0`) ||
|
|
text.includes(`${chapterStr} -`);
|
|
});
|
|
|
|
if (targetOption) {
|
|
console.log(`🎯 Found target chapter: ${targetOption.text} (value: ${targetOption.value})`);
|
|
|
|
// Sélectionner le chapitre
|
|
await this.page.select(chapterSelector, targetOption.value);
|
|
console.log('✅ Chapter selected, waiting for page reload...');
|
|
|
|
// Attendre que la page se recharge après la sélection
|
|
try {
|
|
await this.page.waitForNavigation({
|
|
waitUntil: 'domcontentloaded',
|
|
timeout: CONFIG.ERROR_DETECTION_TIMEOUT
|
|
});
|
|
console.log('🔄 Page reloaded after chapter selection');
|
|
} catch (error) {
|
|
console.log(`⚠️ Warning during chapter navigation: ${error.message}`);
|
|
// Attendre un peu même si la navigation échoue
|
|
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
}
|
|
|
|
} else {
|
|
// Lancer une exception spécifique pour le chapitre non trouvé
|
|
throw new ChapterNotFoundError(chapterNumber);
|
|
}
|
|
|
|
} catch (error) {
|
|
if (error instanceof ChapterNotFoundError) {
|
|
// Re-lancer l'exception pour qu'elle soit gérée en amont
|
|
throw error;
|
|
}
|
|
console.log(`⚠️ Error selecting chapter: ${error.message}`);
|
|
// Continuer même si la sélection échoue pour les autres erreurs
|
|
}
|
|
}
|
|
|
|
async scrapeVertical() {
|
|
const url = this.options.url;
|
|
const imageSelector = this.options.image_selector;
|
|
const waitForImages = this.options.wait_for_images === 'true';
|
|
const shouldScroll = this.options.scroll === 'true';
|
|
const chapterSelector = this.options.chapter_selector;
|
|
const chapterNumber = this.options.chapter_number;
|
|
|
|
try {
|
|
await this.navigateToPageWithFallback(url);
|
|
} catch (error) {
|
|
if (error.message.includes('HTTP 4') || error.message.includes('HTTP 5')) {
|
|
console.log(`🚫 Cannot access page: ${error.message}`);
|
|
return []; // Retourner un tableau vide pour les erreurs HTTP
|
|
}
|
|
throw error; // Re-lancer les autres erreurs
|
|
}
|
|
|
|
// Gérer la sélection de chapitre si nécessaire
|
|
if (chapterSelector && chapterNumber) {
|
|
try {
|
|
await this.selectChapter(chapterSelector, chapterNumber);
|
|
} catch (error) {
|
|
if (error instanceof ChapterNotFoundError) {
|
|
console.log(`📚 MANGA_EXISTS_BUT_CHAPTER_NOT_FOUND: ${error.message}`);
|
|
return {
|
|
error: 'CHAPTER_NOT_FOUND',
|
|
message: `Le manga existe mais le chapitre ${error.chapterNumber} n'est pas disponible.`,
|
|
images: []
|
|
};
|
|
}
|
|
throw error; // Re-lancer les autres erreurs
|
|
}
|
|
}
|
|
|
|
// Attendre le sélecteur d'image
|
|
if (waitForImages) {
|
|
await this.page.waitForSelector(imageSelector, { timeout: CONFIG.NAVIGATION_TIMEOUT });
|
|
}
|
|
|
|
// Scroll pour charger toutes les images lazy-load
|
|
if (shouldScroll) {
|
|
await this.autoScroll();
|
|
}
|
|
|
|
// Attendre un peu pour que les images se chargent (plus de temps pour lazy loading)
|
|
await new Promise(resolve => setTimeout(resolve, 3000));
|
|
|
|
// Collecter les URLs d'images
|
|
const imageUrls = await this.page.$$eval(imageSelector, imgs => {
|
|
return imgs.map(img => {
|
|
// Priorité au src, puis aux attributs data-*
|
|
return img.src ||
|
|
img.getAttribute('src') ||
|
|
img.getAttribute('data-src') ||
|
|
img.getAttribute('data-lazy-src') ||
|
|
img.getAttribute('data-original');
|
|
}).filter(url => url && url !== 'about:blank');
|
|
});
|
|
|
|
console.log(`Found ${imageUrls.length} images`);
|
|
return imageUrls;
|
|
}
|
|
|
|
async scrapeHorizontal() {
|
|
const url = this.options.url;
|
|
const imageSelector = this.options.image_selector;
|
|
const nextSelector = this.options.next_selector;
|
|
const waitForImages = this.options.wait_for_images === 'true';
|
|
const chapterSelector = this.options.chapter_selector;
|
|
const chapterNumber = this.options.chapter_number;
|
|
|
|
let currentUrl = url;
|
|
let pageCount = 0;
|
|
const maxPages = 200; // Limite de sécurité
|
|
|
|
while (currentUrl && pageCount < maxPages) {
|
|
console.log(`Scraping page ${pageCount + 1}: ${currentUrl}`);
|
|
|
|
try {
|
|
await this.navigateToPageWithFallback(currentUrl);
|
|
} catch (error) {
|
|
if (error.message.includes('HTTP 4') || error.message.includes('HTTP 5')) {
|
|
console.log(`🚫 Cannot access page ${pageCount + 1}: ${error.message}`);
|
|
break; // Arrêter le scraping si on rencontre une 404
|
|
}
|
|
// Pour les autres erreurs, essayer de continuer
|
|
console.log(`⚠️ Warning on page ${pageCount + 1}: ${error.message}, continuing...`);
|
|
}
|
|
|
|
// Gérer la sélection de chapitre pour la première page seulement
|
|
if (pageCount === 0 && chapterSelector && chapterNumber) {
|
|
try {
|
|
await this.selectChapter(chapterSelector, chapterNumber);
|
|
} catch (error) {
|
|
if (error instanceof ChapterNotFoundError) {
|
|
console.log(`📚 MANGA_EXISTS_BUT_CHAPTER_NOT_FOUND: ${error.message}`);
|
|
return {
|
|
error: 'CHAPTER_NOT_FOUND',
|
|
message: `Le manga existe mais le chapitre ${error.chapterNumber} n'est pas disponible.`,
|
|
images: []
|
|
};
|
|
}
|
|
throw error; // Re-lancer les autres erreurs
|
|
}
|
|
}
|
|
|
|
// Attendre le sélecteur d'image
|
|
if (waitForImages) {
|
|
try {
|
|
await this.page.waitForSelector(imageSelector, { timeout: CONFIG.NAVIGATION_TIMEOUT });
|
|
} catch (e) {
|
|
console.log(`No image found on page ${pageCount + 1}, skipping`);
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Récupérer l'image de la page
|
|
const imageUrl = await this.page.$eval(imageSelector, img => {
|
|
return img.src ||
|
|
img.getAttribute('src') ||
|
|
img.getAttribute('data-src') ||
|
|
img.getAttribute('data-lazy-src') ||
|
|
img.getAttribute('data-original');
|
|
}).catch(() => null);
|
|
|
|
if (imageUrl) {
|
|
this.imageUrls.add(imageUrl);
|
|
console.log(`Image found: ${imageUrl}`);
|
|
}
|
|
|
|
// Chercher le bouton/lien suivant
|
|
const nextElement = await this.page.$(nextSelector);
|
|
if (!nextElement) {
|
|
console.log('No next button found, ending scraping');
|
|
break;
|
|
}
|
|
|
|
// Récupérer l'URL suivante
|
|
currentUrl = await nextElement.evaluate(el => {
|
|
return el.href || el.getAttribute('href');
|
|
});
|
|
|
|
if (!currentUrl) {
|
|
console.log('No next URL found, ending scraping');
|
|
break;
|
|
}
|
|
|
|
pageCount++;
|
|
await new Promise(resolve => setTimeout(resolve, 1000)); // Pause entre les pages
|
|
}
|
|
|
|
return Array.from(this.imageUrls);
|
|
}
|
|
|
|
async autoScroll() {
|
|
await this.page.evaluate(async (config) => {
|
|
await new Promise((resolve) => {
|
|
let totalHeight = 0;
|
|
let lastHeight = 0;
|
|
|
|
const timer = setInterval(() => {
|
|
const scrollHeight = document.body.scrollHeight;
|
|
|
|
// Si la hauteur a changé, on continue
|
|
if (scrollHeight !== lastHeight) {
|
|
lastHeight = scrollHeight;
|
|
totalHeight = 0; // Reset le counter car plus de contenu apparaît
|
|
}
|
|
|
|
window.scrollBy(0, config.SCROLL_DISTANCE);
|
|
totalHeight += config.SCROLL_DISTANCE;
|
|
|
|
// Arrêter si on a atteint le bas ET que rien de nouveau ne charge
|
|
if (totalHeight >= scrollHeight) {
|
|
clearInterval(timer);
|
|
// Scroll final jusqu'à la vraie fin
|
|
window.scrollTo(0, document.body.scrollHeight);
|
|
resolve();
|
|
}
|
|
}, config.SCROLL_DELAY);
|
|
});
|
|
}, CONFIG);
|
|
}
|
|
|
|
async close() {
|
|
if (this.browser) {
|
|
await this.browser.close();
|
|
}
|
|
}
|
|
}
|
|
|
|
(async () => {
|
|
const scraper = new PuppeteerScraper();
|
|
|
|
try {
|
|
await scraper.launch();
|
|
|
|
let result = [];
|
|
|
|
if (scraper.options.mode === 'vertical') {
|
|
result = await scraper.scrapeVertical();
|
|
} else if (scraper.options.mode === 'horizontal') {
|
|
result = await scraper.scrapeHorizontal();
|
|
} else {
|
|
throw new Error('Invalid mode. Use --mode=vertical or --mode=horizontal');
|
|
}
|
|
|
|
// Vérifier si le résultat est un objet d'erreur ou un tableau d'URLs
|
|
if (result && typeof result === 'object' && result.error === 'CHAPTER_NOT_FOUND') {
|
|
// Cas où le chapitre n'est pas trouvé
|
|
console.log(`CHAPTER_NOT_FOUND:${JSON.stringify(result)}`);
|
|
} else {
|
|
// Cas normal - nettoyer les URLs
|
|
const imageUrls = Array.isArray(result) ? result : [];
|
|
const cleanUrls = imageUrls.filter(url => url && typeof url === 'string');
|
|
console.log(`RESULT:${JSON.stringify(cleanUrls)}`);
|
|
}
|
|
|
|
} catch (error) {
|
|
if (error instanceof ChapterNotFoundError) {
|
|
// Cette erreur est déjà gérée dans les fonctions de scraping
|
|
// Mais au cas où elle remonterait ici
|
|
console.log(`CHAPTER_NOT_FOUND:${JSON.stringify({
|
|
error: 'CHAPTER_NOT_FOUND',
|
|
message: `Le manga existe mais le chapitre ${error.chapterNumber} n'est pas disponible.`,
|
|
images: []
|
|
})}`);
|
|
} else {
|
|
console.error('Error:', error.message);
|
|
process.exit(1);
|
|
}
|
|
} finally {
|
|
await scraper.close();
|
|
}
|
|
})();
|