crawler.js

const { chromium } = require('playwright');
const fs = require('fs');
const path = require('path');
const archiver = require('archiver');
const PQueue = require('p-queue').default;

const OUTPUT_DIR = 'screenshots';
const VIEWPORTS = [
    { name: 'desktop', width: 1280, height: 800 },
    { name: 'tablet', width: 768, height: 1024 },
    { name: 'mobile', width: 375, height: 667 },
];

let shouldStop = false;

function sanitizeName(name) {
    return name.replace(/https?:\/\//, '').replace(/[^\w\-\/]/g, '_').replace(/__+/g, '_');
}

function extractLangFromURL(url) {
    const match = url.match(/\/(en|fr|es|de|ar|zh|pt|ru)\b/);
    return match ? match[1] : 'default';
}

function shouldCrawlLink(href) {
    const excludedExtensions = [
        '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.svg', '.webp',
        '.ico', '.pdf', '.doc', '.docx', '.xls', '.xlsx',
        '.zip', '.rar', '.mp4', '.mp3', '.avi', '.mov',
        '.wmv', '.flv', '.mkv', '.txt'
    ];

    const url = new URL(href, 'http://placeholder.com'); // Dummy base
    const path = url.pathname.toLowerCase();

    // Exclude common media/file extensions and WordPress uploads
    return (
        !excludedExtensions.some(ext => path.endsWith(ext)) &&
        !path.includes('/uploads/')
    );
}

async function getInternalLinks(page, baseDomain) {
    const links = await page.$$eval(
        'a[href]',
        (anchors, base) =>
            anchors.map(a => a.href).filter(href =>
                href.startsWith(base) &&
                !href.includes('#') &&
                !href.startsWith('mailto:') &&
                !href.startsWith('tel:')
            ),
        baseDomain
    );

    return [...new Set(links.filter(shouldCrawlLink))];
}

async function getLanguageLinks(page, baseDomain) {
    const langs = await page.$$eval(
        'a[href]',
        (anchors, base) =>
            anchors
                .map(a => a.href)
                .filter(href =>
                    href.startsWith(base) &&
                    /\/(en|fr|es|de|ar|zh|pt|ru)\b/.test(new URL(href).pathname)
                ),
        baseDomain
    );

    return [...new Set(langs.filter(shouldCrawlLink))];
}

async function waitForImagesToLoad(page) {
    await page.evaluate(() => {
        const selectors = Array.from(document.images);
        return Promise.all(
            selectors.map(img => {
                if (img.complete && img.naturalHeight !== 0) return Promise.resolve();
                return new Promise(resolve => {
                    img.onload = img.onerror = resolve;
                });
            })
        );
    });
}

async function scrollAndCapture(page, folderPath, viewportName, onProgress, url) {
    // Reset scroll position to top inside browser context
    await page.evaluate(() => {
        window.scrollTo(0, 0);
    });

    const scrollStep = 600;
    const scrollDelay = 300;

    let totalHeight = 0;
    let screenshotIndex = 0;

    while (true) {
        // Wait for all images loaded before each screenshot
        await waitForImagesToLoad(page);

        const filePath = path.join(folderPath, `${viewportName}_part_${screenshotIndex}.png`);
        await page.screenshot({ path: filePath });
        onProgress({ type: 'screenshot', url, viewport: viewportName, path: filePath, method: 'scrollPart', index: screenshotIndex });

        // Scroll down by scrollStep
        await page.evaluate((step) => {
            window.scrollBy(0, step);
        }, scrollStep);

        screenshotIndex++;
        await page.waitForTimeout(scrollDelay);

        // Get the scrollHeight after scrolling to check if we reached the bottom
        const scrollHeight = await page.evaluate(() => {
            return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
        });

        totalHeight += scrollStep;

        // window.innerHeight is inside browser context, so grab that as well
        const innerHeight = await page.evaluate(() => window.innerHeight);

        if (totalHeight >= scrollHeight - innerHeight) break;
    }
}

async function crawlPage(browser, url, depth, baseDomain, visited, queue, onProgress) {
    if (shouldStop || depth > 1 || visited.has(url)) return;
    visited.add(url);

    const context = await browser.newContext();
    const page = await context.newPage();

    try {
        await page.goto(url, { waitUntil: 'networkidle' });
        onProgress({ type: 'pageStart', url });

        const lang = extractLangFromURL(url);
        const folderPath = path.join(OUTPUT_DIR, lang, sanitizeName(url));
        if (!fs.existsSync(folderPath)) fs.mkdirSync(folderPath, { recursive: true });

        for (const vp of VIEWPORTS) {
            if (shouldStop) break;

            await page.setViewportSize({ width: vp.width, height: vp.height });

            // 1. Take classic full page screenshot (one big one)
            const fullPath = path.join(folderPath, `${vp.name}_full.png`);
            await page.screenshot({ path: fullPath, fullPage: true });
            onProgress({ type: 'screenshot', url, viewport: vp.name, path: fullPath, method: 'fullPage' });

            // 2. Then take scrolling slices screenshots
            await scrollAndCapture(page, folderPath, vp.name, onProgress, url);

            // After scrolling screenshots, reset scroll position for next viewport or future use
            await page.evaluate(() => window.scrollTo(0, 0));
        }

        const langLinks = await getLanguageLinks(page, baseDomain);
        for (const langLink of langLinks) {
            if (!visited.has(langLink)) {
                queue.add(() =>
                    crawlPage(browser, langLink, depth + 1, baseDomain, visited, queue, onProgress)
                );
            }
        }

        const links = await getInternalLinks(page, baseDomain);
        for (const link of links) {
            if (!visited.has(link)) {
                queue.add(() =>
                    crawlPage(browser, link, depth + 1, baseDomain, visited, queue, onProgress)
                );
            }
        }

    } catch (err) {
        onProgress({ type: 'error', url, message: err.message });
    } finally {
        await context.close();
    }
}

async function zipScreenshots() {
    const zipPath = 'screenshots.zip';
    const output = fs.createWriteStream(zipPath);
    const archive = archiver('zip', { zlib: { level: 9 } });

    return new Promise((resolve, reject) => {
        output.on('close', () => resolve(zipPath));
        archive.on('error', reject);
        archive.pipe(output);
        archive.directory(OUTPUT_DIR, false);
        archive.finalize();
    });
}

async function runCrawler(websites, onProgress) {
    shouldStop = false;

    if (fs.existsSync(OUTPUT_DIR)) fs.rmSync(OUTPUT_DIR, { recursive: true });
    fs.mkdirSync(OUTPUT_DIR);

    const browser = await chromium.launch({ headless: true });
    const queue = new PQueue({ concurrency: 5 });

    const visited = new Set();

    for (const site of websites) {
        if (shouldStop) break;

        const baseDomain = new URL(site).origin;

        onProgress({ type: 'websiteStart', url: site });

        // Don't await this — let PQueue manage it
        queue.add(() => crawlPage(browser, site, 0, baseDomain, visited, queue, onProgress));

        onProgress({ type: 'websiteQueued', url: site });
    }

    // ✅ Wait here until all queue tasks are complete
    await queue.onIdle();

    await browser.close();

    if (!shouldStop) {
        const zipPath = await zipScreenshots();
        onProgress({ type: 'done', zipPath });
    } else {
        onProgress({ type: 'stopped' });
    }
}

function stopCrawler() {
    shouldStop = true;
}

module.exports = { runCrawler, stopCrawler };