Gistify
Back to Home
Gistify Snippet
kyiov
@kyiov
Sep 5·1 file(s)
View profile
Public

fso.js — scrape fullstackopen.com: part + chapter + konten kursus

#github-gist#javascript#scraper#gist:6b87681bdecc6d9e5e8715ca44d7bdae
10
Raw
fso.jsjavascript
210 lines
1#!/usr/bin/env node
2// fso.js — scrape fullstackopen.com: daftar part + isi chapter
3
4const BASE = 'https://fullstackopen.com'
5const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36'
6
7async function get(url) {
8 const r = await fetch(url, { redirect: 'follow', headers: { 'User-Agent': UA, 'Accept': 'text/html,*/*' } })
9 if (r.status !== 200) throw new Error(`HTTP ${r.status}`)
10 return await r.text()
11}
12
13function decode(s) {
14 return s
15 .replace(/'/g, "'").replace(/'/g, "'").replace(/"/g, '"')
16 .replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&amp;/g, '&')
17 .replace(/&nbsp;/g, ' ').replace(/&hellip;/g, '...').replace(/&mdash;/g, '—')
18}
19
20function clean(s) {
21 return decode(s).replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim()
22}
23
24// daftar part + chapter dari homepage
25function index(html) {
26 const parts = []
27 const seen = new Set()
28 const re = /href="(\/en\/part\d+(?:\/[a-z0-9_]+)?\/?)"/g
29 let m
30 while ((m = re.exec(html)) !== null) {
31 if (seen.has(m[1])) continue
32 seen.add(m[1])
33 const path = m[1].replace(/\/$/, '')
34 const seg = path.split('/')
35 const part = seg[2]
36 let p = parts.find(x => x.id === part)
37 if (!p) {
38 p = { id: part, title: part, chapters: [] }
39 parts.push(p)
40 }
41 if (seg[3]) p.chapters.push({ slug: seg[3], url: `${BASE}${path}/` })
42 }
43 return parts
44}
45
46// title part + daftar chapter dari halaman part
47async function partInfo(partId) {
48 const html = await get(`${BASE}/en/${partId}/`)
49 const titleM = /<h1[^>]*>([\s\S]*?)<\/h1>/.exec(html)
50 let title = titleM ? clean(titleM[1]) : partId
51 // beberapa halaman part pakai sub-header di intro
52 const subM = /<div class="part-intro[^"]*">[\s\S]*?<p[^>]*>([\s\S]*?)<\/p>/.exec(html)
53 if (subM) {
54 const t = clean(subM[1])
55 if (t && t.length < 120 && !t.includes('moved')) title = t
56 }
57 const chapters = []
58 const re = new RegExp(`href="(\/en\/${partId}\/[a-z0-9_]+)\/?"`, 'g')
59 let m
60 while ((m = re.exec(html)) !== null) {
61 const slug = m[1].split('/').pop()
62 if (!chapters.find(c => c.slug === slug)) chapters.push({ slug, url: `${BASE}${m[1]}/` })
63 }
64 return { id: partId, title, chapters }
65}
66
67// parse konten chapter -> sections
68function chapter(html, url) {
69 const titleM = /<h1[^>]*>([\s\S]*?)<\/h1>/.exec(html)
70 const title = titleM ? clean(titleM[1]) : url
71 let main = /<main id="main-content">([\s\S]*?)<\/main>/.exec(html)
72 if (!main) main = [null, html]
73 const body = main[1]
74
75 const sections = []
76 // pecah per heading h2/h3 (skip h1 - judul)
77 const parts = body.split(/<h([23])[^>]*>/)
78 // parts[0] = sebelum heading pertama
79 let cur = null
80 const push = () => {
81 if (cur && (cur.body.trim() || cur.heading)) sections.push(cur)
82 }
83 for (let i = 1; i < parts.length; i += 2) {
84 push()
85 const level = parts[i]
86 const hEnd = parts[i + 1].indexOf('</h')
87 const heading = clean(parts[i + 1].slice(0, hEnd))
88 const rest = parts[i + 1].slice(hEnd + 5) // setelah </hN>
89 cur = { level: Number(level), heading, body: '' }
90 // sisa hingga heading berikutnya
91 const idx = rest.search(/<h[23][^>]*>/)
92 const content = idx >= 0 ? rest.slice(0, idx) : rest
93 cur.body = bodyText(content)
94 if (idx >= 0) {
95 // masukkan sisa kembali
96 cur._rest = rest.slice(idx)
97 }
98 }
99 push()
100
101 // gabungkan _rest bila ada (kasus heading tidak split sempurna)
102 const out = []
103 for (const s of sections) {
104 if (s._rest) {
105 const m2 = s._rest.split(/<h([23])[^>]*>/)
106 let cur2 = s
107 for (let j = 1; j < m2.length; j += 2) {
108 if (cur2.body.trim()) out.push({ level: cur2.level, heading: cur2.heading, body: cur2.body.trim() })
109 const hEnd = m2[j + 1].indexOf('</h')
110 cur2 = { level: Number(m2[j]), heading: clean(m2[j + 1].slice(0, hEnd)), body: '' }
111 const rest2 = m2[j + 1].slice(hEnd + 5)
112 cur2.body = bodyText(rest2)
113 }
114 if (cur2.body.trim()) out.push({ level: cur2.level, heading: cur2.heading, body: cur2.body.trim() })
115 } else {
116 out.push({ level: s.level, heading: s.heading, body: s.body.trim() })
117 }
118 }
119 return { title, url, sections: out.filter(s => s.heading || s.body) }
120}
121
122// HTML -> teks section (p, li, pre/code jadi blok)
123function bodyText(html) {
124 const out = []
125 // code blocks dulu
126 const cleaned = html.replace(/<pre[^>]*>[\s\S]*?<\/pre>|<code[^>]*>[\s\S]*?<\/code>/g, (code) => {
127 out.push('```\n' + decode(code.replace(/<[^>]+>/g, '')).replace(/^\s+|\s+$/g, '') + '\n```')
128 return ' '
129 })
130 // list items
131 const parts = cleaned.split(/(<li[\s\S]*?<\/li>)/g)
132 for (const part of parts) {
133 if (/^<li/.test(part)) {
134 out.push('- ' + clean(part))
135 } else {
136 const t = clean(part)
137 if (t) out.push(t)
138 }
139 }
140 return out.join('\n')
141}
142
143function dumpSections(sections) {
144 sections.forEach(s => {
145 if (s.heading) console.log(`\n## ${s.heading}`)
146 if (s.body) console.log(s.body)
147 })
148}
149
150async function run() {
151 const args = process.argv.slice(2)
152 const asJson = args.includes('--json')
153 const outIdx = args.indexOf('--out')
154 const outFile = outIdx >= 0 ? (args[outIdx + 1] && !args[outIdx + 1].startsWith('--') ? args[outIdx + 1] : 'output/fso.json') : null
155 const target = args.find(a => !a.startsWith('--'))
156
157 let out
158 if (!target) {
159 // daftar semua part + chapter (fetch tiap halaman part)
160 const html = await get(`${BASE}/en/`)
161 const parts = index(html)
162 const list = []
163 for (const p of parts) {
164 const info = await partInfo(p.id)
165 list.push({ part: info.id, title: info.title, chapters: info.chapters.length })
166 }
167 out = { type: 'index', parts: list }
168 } else if (/^part\d+\/[a-z0-9_]+$/i.test(target)) {
169 // 1 chapter
170 const [partId, slug] = target.split('/')
171 const url = `${BASE}/en/${partId}/${slug}/`
172 const html = await get(url)
173 out = { type: 'chapter', ...chapter(html, url), part: partId }
174 } else if (/^part\d+$/i.test(target)) {
175 // semua chapter part
176 const info = await partInfo(target)
177 const chapters = []
178 for (const c of info.chapters) {
179 const html = await get(c.url)
180 chapters.push(chapter(html, c.url))
181 }
182 out = { type: 'part', part: info.id, title: info.title, chapters }
183 } else {
184 console.error('usage: node fso.js daftar part\n node fso.js part3 semua chapter part3\n node fso.js part3/part3b 1 chapter\n node fso.js <x> --json/--out')
185 process.exit(1)
186 }
187
188 if (asJson) {
189 console.log(JSON.stringify(out, null, 2))
190 } else if (out.type === 'index') {
191 out.parts.forEach(p => console.log(`${p.part} — ${p.chapters} chapter`))
192 } else if (out.type === 'part') {
193 for (const c of out.chapters) {
194 console.log(`\n===== ${c.title} (${c.sections.length} section) =====`)
195 dumpSections(c.sections)
196 }
197 } else {
198 console.log(`${out.part}: ${out.title}`)
199 dumpSections(out.sections)
200 }
201 if (outFile) {
202 const { mkdir, writeFile } = await import('node:fs/promises')
203 const { dirname } = await import('node:path')
204 await mkdir(dirname(outFile), { recursive: true })
205 await writeFile(outFile, JSON.stringify(out, null, 2))
206 console.error(`\nsaved: ${outFile}`)
207 }
208}
209
210run().catch(e => { console.error('error:', e.message); process.exit(1) })
Discussion & Comments0

No comments yet — be the first to leave feedback or start a code discussion.

Sign in with your GitHub account to write comments and join the discussion.