Gistify
Back to Home
Gistify Snippet
kyiov
@kyiov
Sep 5·1 file(s)
View profile
Public

odcrawler.js — cari file di open directories via ODCrawler (150M+ link)

odcrawler.js — cari file di open directories via ODCrawler (150M+ link)

#github-gist#javascript#gist:87f95c3f1eb0825e17d390a593a8a7ae#gistsync:2026-09-04T02:46:57Z
2
Raw
odcrawler.jsjavascript
134 lines
1// odcrawler.js — cari file di open directories via ODCrawler.
2// Endpoint: search.odcrawler.xyz/elastic/links/_search. Pure JS, tanpa key.
3// Usage: bagian bawah file.
4
5const API = 'https://search.odcrawler.xyz/elastic/links/_search'
6const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36'
7
8async function search(query, opts = {}) {
9 const { from = 0, size = 20, filenameOnly = false, matchPhrase = false, extensions = [], extMode = 'off' } = opts
10 const field = filenameOnly ? 'filename' : 'url'
11 const body = {
12 size,
13 from,
14 highlight: { fields: { url: {}, filename: {} } },
15 query: {
16 bool: {
17 must: [matchPhrase ? { match_phrase: { [field]: query } } : { match: { [field]: query } }],
18 should: [matchPhrase ? { match_phrase: { filename: query } } : { match: { filename: query } }],
19 must_not: [],
20 },
21 },
22 }
23 if (extensions.length && extMode !== 'off') {
24 const list = [...new Set(extensions.flatMap((e) => [e, e.toLowerCase(), e.toUpperCase()]))]
25 const term = { terms: { extension: list } }
26 if (extMode === 'include') body.query.bool.must.push(term)
27 else body.query.bool.must_not.push(term)
28 }
29 const res = await fetch(API, {
30 method: 'POST',
31 headers: { 'User-Agent': UA, 'Content-Type': 'application/json' },
32 body: JSON.stringify(body),
33 signal: AbortSignal.timeout(60000),
34 })
35 if (!res.ok) throw new Error(`HTTP ${res.status}`)
36 const d = await res.json()
37 return {
38 total: d.hits?.total?.value ?? 0,
39 hits: (d.hits?.hits || []).map((h) => ({
40 id: h._id,
41 score: h._score,
42 url: h._source?.url || '',
43 filename: h._source?.filename || '',
44 size: h._source?.size ?? -1,
45 highlights: h.highlight
46 ? Object.fromEntries(Object.entries(h.highlight).map(([k, v]) => [k, v.join(' ')]))
47 : {},
48 })),
49 }
50}
51
52const fmtSize = (b) => {
53 if (b == null || b < 0) return ''
54 if (b === 0) return '0 B'
55 const u = ['B', 'KB', 'MB', 'GB', 'TB']
56 const i = Math.min(Math.floor(Math.log(b) / Math.log(1024)), u.length - 1)
57 return `${(b / 1024 ** i).toFixed(1)} ${u[i]}`
58}
59
60const isDirectRun = process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href
61
62if (isDirectRun) {
63 ;(async () => {
64 const args = process.argv.slice(2)
65 const asJson = args.includes('--json')
66 const filenameOnly = args.includes('--file')
67 const matchPhrase = args.includes('--phrase')
68 const outIdx = args.indexOf('--out')
69 const outPath = outIdx !== -1 ? (args[outIdx + 1] && !args[outIdx + 1].startsWith('--') ? args[outIdx + 1] : 'output/odcrawler.json') : null
70 const nIdx = args.indexOf('--n')
71 const size = nIdx !== -1 && args[nIdx + 1] ? parseInt(args[nIdx + 1], 10) : 20
72 const pageIdx = args.indexOf('--page')
73 const page = pageIdx !== -1 && args[pageIdx + 1] ? parseInt(args[pageIdx + 1], 10) : 0
74 const extIdx = args.indexOf('--ext')
75 let extensions = []
76 let extMode = 'off'
77 if (extIdx !== -1 && args[extIdx + 1]) {
78 extensions = args[extIdx + 1].split(',').map((e) => e.trim().replace(/^\./, '')).filter(Boolean)
79 extMode = args.includes('--exclude') ? 'exclude' : 'include'
80 }
81 const query = args.find((a) => !a.startsWith('--') && !/^\d+$/.test(a))
82
83 if (!query) {
84 console.error('ERROR: butuh kata kunci. Contoh: node odcrawler.js "ubuntu iso"')
85 process.exit(1)
86 }
87
88 if (!asJson) {
89 console.log(`== odcrawler scraper ==`)
90 console.log(`query : "${query}"${filenameOnly ? ' (filename-only)' : ''}${extensions.length ? ` ext=${extensions.join(',')} (${extMode})` : ''}`)
91 }
92
93 try {
94 const r = await search(query, { from: page * size, size, filenameOnly, matchPhrase, extensions, extMode })
95 const data = { query, total: r.total, returned: r.hits.length, hits: r.hits }
96
97 if (outPath) {
98 const { mkdir, writeFile } = await import('node:fs/promises')
99 const { dirname } = await import('node:path')
100 await mkdir(dirname(outPath), { recursive: true })
101 await writeFile(outPath, JSON.stringify(data, null, 2))
102 console.log(`saved : ${outPath}`)
103 }
104
105 if (asJson) {
106 console.log(JSON.stringify(data, null, 2))
107 } else {
108 console.log(`\ntotal : ${r.total}+ hasil (hal ${page + 1}, ${size}/hal)`)
109 r.hits.forEach((h, i) => {
110 const name = h.filename || h.url.split('/').pop() || h.url
111 console.log(`\n${page * size + i + 1}. ${name}${h.size >= 0 ? ` [${fmtSize(h.size)}]` : ''}`)
112 console.log(` ${h.url}`)
113 })
114 console.log('\ndone')
115 }
116 } catch (e) {
117 console.error(`ERROR: ${e.message || e}`)
118 process.exit(1)
119 }
120 })()
121}
122
123/*
124 * USAGE
125 * node odcrawler.js "kata kunci" cari file di open directories
126 * node odcrawler.js <q> --n 50 50 hasil
127 * node odcrawler.js <q> --page 2 halaman 2
128 * node odcrawler.js <q> --file cari di nama file saja
129 * node odcrawler.js <q> --phrase match frasa persis
130 * node odcrawler.js <q> --ext iso hanya ekstensi tertentu (koma: iso,zip)
131 * node odcrawler.js <q> --ext mp4 --exclude kecualikan ekstensi
132 * node odcrawler.js <q> --json json lengkap
133 * node odcrawler.js <q> --out [f] simpan ke file
134 */
Discussion & Comments0

No comments yet — be the first to leave feedback or start a code discussion.

Sign in with your GitHub account to write comments and join the discussion.