-def fetch(url: str, timeout: int) -> None:
- with requests.session() as s:
- with s.get(url, timeout=timeout) as r:
- r.raise_for_status()
+
+def parse(content: bytes) -> BeautifulSoup:
+ return BeautifulSoup(content, 'html.parser')
+
+
+def clean(html: BeautifulSoup) -> BeautifulSoup:
+ for eb in html.find_all("div", class_="post-edit-box"):
+ eb.decompose()
+ for footer in html.find_all("div", class_="post-footer"):
+ footer.decompose()
+ return html
+
+
+def replies(html: BeautifulSoup) -> Iterable[Tag]:
+ def text() -> Tag:
+ body = html.body
+ assert body
+ text = body.find_next("div", class_="post-post")
+ assert isinstance(text, Tag)
+ return text
+
+ def the_replies() -> Iterable[Tag]:
+ rs = html.find_all("div", class_="post-reply")
+ assert all(isinstance(r, Tag) for r in rs)
+ return rs
+
+ return itertools.chain([text()], the_replies())
+
+
+def process(spec: Spec) -> None:
+ spec.texout.write(b'\\documentclass{article}\n\\begin{document}\n')
+ html = clean(parse(spec.fetcher.fetch(spec.url)))
+ for r in replies(html):
+ spec.texout.write(spec.texifier.texify(r))
+ spec.texout.write(b'\\end{document}\n')