Scraper cheat sheet - realpython.com (anti-bot)

#github-gist#javascript#gist:5f2371ed7c859c4f291df7818aa1cb6f#gistsync:2026-09-02T10:46:07Z

0 / kyiov

Raw

realpython.jsjavascript

136 lines

1	`import { exec } from 'child_process'`
2	`import { promisify } from 'util'`
3	`import fs from 'fs'`
4
5	`const execAsync = promisify(exec)`
6
7	const PYTHON_BRIDGE = `
8	`import scrapling`
9	`from scrapling.fetchers import StealthyFetcher`
10	`from bs4 import BeautifulSoup`
11	`import json`
12	`import re`
13
14	`def clean_code_block(text):`
15	`raw_lines = text.split('\\n')`
16	`cleaned_lines = []`
17	`current_line = []`
18
19	`for l in raw_lines:`
20	`s = l.strip()`
21	`if not s:`
22	`if current_line:`
23	`cleaned_lines.append(" ".join(current_line))`
24	`current_line = []`
25	`continue`
26
27	`if s.startswith('#') or s.startswith('$') or s.startswith('>>>'):`
28	`if current_line:`
29	`cleaned_lines.append(" ".join(current_line))`
30	`current_line = []`
31	`cleaned_lines.append(s)`
32	`else:`
33	`current_line.append(s)`
34
35	`if current_line:`
36	`cleaned_lines.append(" ".join(current_line))`
37
38	`res = "\\n".join(cleaned_lines)`
39	`res = re.sub(r'\\s+([,.:;\\]\\)\\}])', r'\\1', res)`
40	`res = re.sub(r'([\\[\\(\\{])\\s+', r'\\1', res)`
41	`res = re.sub(r'\\s=\\s', ' = ', res)`
42	`res = re.sub(r'\\s\\+\\=\\s', ' += ', res)`
43	`res = re.sub(r'\\s\\-\\=\\s', ' -= ', res)`
44	`return res.strip()`
45
46	`def fetch_cheatsheet():`
47	`try:`
48	`fetcher = StealthyFetcher()`
49	`page = fetcher.fetch('https://realpython.com/cheatsheets/python/', network_idle=True)`
50	`if page.status != 200:`
51	`print(json.dumps({"status": False, "error": f"HTTP {page.status}"}))`
52	`return`
53
54	`soup = BeautifulSoup(page.html_content, 'html.parser')`
55	`title = "Python 3 Cheat Sheet"`
56
57	`sections = []`
58
59	`# Blacklist of promotional keywords`
60	`PROMO_WORDS = [`
61	`'real python', 'show/hide', 'continue your learning', 'bonus',`
62	`'download this', 'newsletter', 'quizzes', 'podcast', 'get a python cheat',`
63	`'learning paths', 'video courses', 'written tutorials', 'interactive quizzes',`
64	`'do you want to go deeper'`
65	`]`
66
67	`for sec_tag in soup.find_all('section', class_='section2'):`
68	`h2 = sec_tag.find('h2')`
69	`if not h2:`
70	`continue`
71	`sec_title = h2.get_text().strip()`
72
73	`# Skip promotional sections`
74	`if any(pw in sec_title.lower() for pw in ['related tutorial', 'bonus', 'resources', 'curriculum']):`
75	`continue`
76
77	`items = []`
78	`for child in sec_tag.find_all(['pre', 'p']):`
79	`if child.name == 'pre':`
80	`code_clean = clean_code_block(child.get_text())`
81	`if code_clean:`
82	`items.append({'type': 'code', 'language': 'python', 'content': code_clean})`
83	`elif child.name == 'p':`
84	`t = child.get_text().strip()`
85	`if t and len(t) > 3 and not any(pw in t.lower() for pw in PROMO_WORDS):`
86	`items.append({'type': 'text', 'content': t})`
87
88	`if any(it['type'] == 'code' for it in items):`
89	`sections.append({`
90	`'title': sec_title,`
91	`'items': items`
92	`})`
93
94	`print(json.dumps({`
95	`"status": True,`
96	`"title": title,`
97	`"total_sections": len(sections),`
98	`"sections": sections`
99	`}))`
100	`except Exception as e:`
101	`print(json.dumps({"status": False, "error": str(e)}))`
102
103	`if __name__ == "__main__":`
104	`fetch_cheatsheet()`
105	`
106
107	`let cacheCheatsheet = null`
108	`let lastFetch = 0`
109
110	`export async function getPythonCheatSheet(forceRefresh = false) {`
111	`const now = Date.now()`
112	`if (!forceRefresh && cacheCheatsheet && (now - lastFetch < 3600000)) {`
113	`return cacheCheatsheet`
114	`}`
115
116	`try {`
117	`const tmpScript = '/tmp/rp_cheatsheet_clean.py'`
118	`await fs.promises.writeFile(tmpScript, PYTHON_BRIDGE, 'utf-8')`
119
120	const { stdout } = await execAsync(`${process.env.SCRAPLING_PYTHON \|\| "/tmp/scrapping-venv/bin/python"} ${tmpScript}`, { maxBuffer: 15 * 1024 * 1024 })
121	`const json = JSON.parse(stdout.trim().split('\n').pop())`
122
123	`if (!json.status) throw new Error(json.error \|\| 'Gagal mengekstrak cheatsheet.')`
124
125	`cacheCheatsheet = json`
126	`lastFetch = now`
127	`return json`
128	`} catch (e) {`
129	`console.error('[RealPython CheatSheet Error]:', e.message)`
130	`throw e`
131	`}`
132	`}`
133
134	`export default {`
135	`getPythonCheatSheet`
136	`}`

Discussion & Comments0

No comments yet — be the first to leave feedback or start a code discussion.