/**
 * Grupku.net Web Scraper in Node.js
 * Ekstraksi daftar grup WhatsApp & Telegram, URL undangan, kategori, dan deskripsi.
 */

const fs = require("fs");
const path = require("path");

const BASE_URL = "https://www.grupku.net/";
const SITEMAP_URL = "https://www.grupku.net/sitemap.xml";

// Helper HTML Entity Unescape
function unescapeHtml(text) {
  if (!text) return "";
  return text
    .replace(/&/g, "&")
    .replace(/&lt;/g, "<")
    .replace(/&gt;/g, ">")
    .replace(/&quot;/g, '"')
    .replace(/&#039;/g, "'")
    .replace(/&#8226;/g, "•")
    .replace(/&#9961;/g, "⛑")
    .replace(/&#65039;/g, "")
    .replace(/&#185;/g, "¹")
    .replace(/&#8902;/g, "⋆")
    .replace(/&#8881;/g, "⊵")
    .replace(/&#9733;/g, "★")
    .replace(/&#9829;/g, "♥")
    .replace(/&#([0-9]{1,6});/gi, (_, num) => String.fromCharCode(parseInt(num, 10)));
}

function cleanText(text) {
  if (!text) return "";
  const noTags = text.replace(/<[^>]+>/g, "");
  return unescapeHtml(noTags).replace(/\s+/g, " ").trim();
}

async function fetchWithRetry(url, retries = 2) {
  for (let i = 0; i <= retries; i++) {
    try {
      const res = await fetch(url, {
        headers: {
          "User-Agent":
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
        },
        signal: AbortSignal.timeout(12000),
      });
      if (res.ok) {
        return await res.text();
      }
    } catch (err) {
      if (i === retries) throw err;
      await new Promise((r) => setTimeout(r, 600));
    }
  }
  return "";
}

async function getAllGroupUrls() {
  const urls = new Set();

  // 1. Ambil dari sitemap.xml
  try {
    const sitemapXml = await fetchWithRetry(SITEMAP_URL);
    const matches = sitemapXml.match(/<loc>(https:\/\/www\.grupku\.net\/202[0-9]\/[0-9]{2}\/[^\s<]+\.html)<\/loc>/g) || [];
    for (const m of matches) {
      const cleanUrl = m.replace(/<\/?loc>/g, "").trim();
      urls.add(cleanUrl);
    }
  } catch (err) {
    console.error("[!] Error membaca sitemap:", err.message);
  }

  // 2. Ambil dari homepage
  try {
    const homeHtml = await fetchWithRetry(BASE_URL);
    const matches = homeHtml.match(/https:\/\/www\.grupku\.net\/202[0-9]\/[0-9]{2}\/[a-zA-Z0-9_\-]+\.html/g) || [];
    for (const u of matches) {
      urls.add(u.trim());
    }
  } catch (err) {
    console.error("[!] Error membaca homepage:", err.message);
  }

  return Array.from(urls);
}

function parseGroupPage(pageHtml, url) {
  // 1. Title
  let title = "Tanpa Judul";
  const h1Match = pageHtml.match(/<h1[^>]*class=['"]gs-single-title['"][^>]*>([\s\S]*?)<\/h1>/i);
  if (h1Match) {
    let raw = h1Match[1].replace(/<span[^>]*class=['"][^'"]*gs-tier-badge[^'"]*['"][^>]*>[\s\S]*?<\/span>/gi, "");
    title = cleanText(raw);
  } else {
    const ogTitle = pageHtml.match(/<meta[^>]+property=['"]og:title['"][^>]+content=['"]([^'"]+)['"]/i);
    if (ogTitle) title = cleanText(ogTitle[1]);
  }

  // 2. Direct Invite URL (WhatsApp / Telegram)
  let inviteUrl = "";
  const urlDiv = pageHtml.match(/<div[^>]*class=['"]group-url['"][^>]*>\s*(https?:\/\/[^\s<]+)\s*<\/div>/i);
  if (urlDiv) {
    inviteUrl = urlDiv[1].trim();
  } else {
    const direct = pageHtml.match(/https?:\/\/(?:chat\.whatsapp\.com|t\.me)\/[a-zA-Z0-9_\-\.\?&=]+/i);
    if (direct) {
      inviteUrl = direct[0].trim();
    }
  }

  // 3. Platform
  let platform = "Other";
  if (inviteUrl.includes("chat.whatsapp.com") || inviteUrl.includes("wa.me") || /whatsapp/i.test(pageHtml)) {
    platform = "WhatsApp";
  } else if (inviteUrl.includes("t.me") || /telegram/i.test(pageHtml)) {
    platform = "Telegram";
  }

  // 4. Description
  let description = "";
  const descMatch = pageHtml.match(/<div[^>]*class=['"]gs-single-desc['"][^>]*>[\s\S]*?<p>([\s\S]*?)<\/p>/i);
  if (descMatch) {
    description = cleanText(descMatch[1]);
  }

  // 5. Icon URL
  let iconUrl = "";
  const iconMatch = pageHtml.match(/<div[^>]*class=['"]gs-single-thumb['"][^>]*>[\s\S]*?<img[^>]+src=['"]([^'"]+)['"]/i) ||
                    pageHtml.match(/<meta[^>]+property=['"]og:image['"][^>]+content=['"]([^'"]+)['"]/i);
  if (iconMatch) {
    iconUrl = iconMatch[1];
  }

  // 6. Tags & Category
  const tagMatches = pageHtml.match(/<a[^>]*class=['"]gs-tag['"][^>]*>\s*([\s\S]*?)\s*<\/a>/gi) || [];
  const tags = tagMatches
    .map((t) => cleanText(t))
    .filter((t) => t && !t.includes("cat.term"));
  const uniqueTags = Array.from(new Set(tags));
  const category = uniqueTags[0] || "Umum";

  // 7. Date
  let date = "";
  const dateMatch = pageHtml.match(/<i class=['"]fas fa-calendar-alt['"]><\/i>\s*([^<]+)<\/span>/i);
  if (dateMatch) {
    date = cleanText(dateMatch[1]);
  }

  return {
    title,
    platform,
    category,
    tags: uniqueTags,
    invite_url: inviteUrl,
    description,
    date,
    icon_url: iconUrl,
    page_url: url,
  };
}

// Concurrency pool helper
async function pool(items, limit, fn) {
  const results = [];
  const executing = [];
  for (const item of items) {
    const p = Promise.resolve().then(() => fn(item));
    results.push(p);
    if (limit <= items.length) {
      const e = p.then(() => executing.splice(executing.indexOf(e), 1));
      executing.push(e);
      if (executing.length >= limit) {
        await Promise.race(executing);
      }
    }
  }
  return Promise.all(results);
}

async function main() {
  console.log("[*] Mengambil link grup dari grupku.net ...");
  const urls = await getAllGroupUrls();
  console.log(`[+] Ditemukan ${urls.length} postingan grup.`);

  if (urls.length === 0) {
    console.error("[-] Tidak ada URL yang ditemukan.");
    return;
  }

  console.log("[*] Mulai scraping data (concurrency 10)...");
  let processed = 0;
  const groups = [];

  await pool(urls, 10, async (url) => {
    try {
      const html = await fetchWithRetry(url);
      const data = parseGroupPage(html, url);
      if (data.invite_url) {
        groups.push(data);
      }
    } catch (err) {
      // ignore individual failure
    } finally {
      processed++;
      if (processed % 25 === 0 || processed === urls.length) {
        console.log(`    Progres: ${processed}/${urls.length} diproses (Grup valid: ${groups.length})`);
      }
    }
  });

  // Urutkan berdasarkan judul
  groups.sort((a, b) => a.title.localeCompare(b.title));

  // Simpan JSON
  const jsonPath = path.resolve(process.cwd(), "grupku_data.json");
  fs.writeFileSync(jsonPath, JSON.stringify(groups, null, 2), "utf-8");
  console.log(`[✓] Berhasil disimpan ke JSON: ${jsonPath}`);

  // Simpan CSV
  const csvPath = path.resolve(process.cwd(), "grupku_data.csv");
  const csvHeaders = ["Title", "Platform", "Category", "Invite URL", "Description", "Date", "Page URL"];
  const csvRows = [csvHeaders.join(",")];
  for (const g of groups) {
    const escapeCsv = (str) => `"${(str || "").replace(/"/g, '""')}"`;
    csvRows.push([
      escapeCsv(g.title),
      escapeCsv(g.platform),
      escapeCsv(g.category),
      escapeCsv(g.invite_url),
      escapeCsv(g.description),
      escapeCsv(g.date),
      escapeCsv(g.page_url),
    ].join(","));
  }
  fs.writeFileSync(csvPath, csvRows.join("\n"), "utf-8");
  console.log(`[✓] Berhasil disimpan ke CSV: ${csvPath}`);

  // Tampilkan Ringkasan
  console.log("\n========================================================");
  console.log(`HASIL SCRAPING NODE.JS (${groups.length} Grup Berhasil Diambil)`);
  console.log("========================================================");

  const stats = {};
  for (const g of groups) {
    stats[g.category] = (stats[g.category] || 0) + 1;
  }
  const topCategories = Object.entries(stats).sort((a, b) => b[1] - a[1]);
  console.log("Kategori Terbanyak:");
  topCategories.slice(0, 8).forEach(([cat, count], i) => {
    console.log(`  ${i + 1}. ${cat}: ${count} grup`);
  });
  console.log("========================================================\n");
}

main().catch(console.error);