+def fetch(url: str, session: requests.Session, timeout: int) -> BeautifulSoup:
+ with session.get(url, timeout=timeout) as r:
+ r.raise_for_status()
+ return BeautifulSoup(r.text, 'html.parser')
+
+
+def clean(html: BeautifulSoup) -> BeautifulSoup:
+ for eb in html.find_all("div", class_="post-edit-box"):
+ eb.decompose()
+ for footer in html.find_all("div", class_="post-footer"):
+ footer.decompose()
+ return html
+
+
+def replies(html: BeautifulSoup) -> Iterable[Tag]:
+ def text() -> Tag:
+ body = html.body
+ assert body
+ text = body.find_next("div", class_="post-post")
+ assert isinstance(text, Tag)
+ return text
+
+ def the_replies() -> Iterable[Tag]:
+ rs = html.find_all("div", class_="post-reply")
+ assert all(isinstance(r, Tag) for r in rs)
+ return rs
+
+ return itertools.chain([text()], the_replies())
+
+
+def html_to_tex(pandoc: str, tag: Tag) -> bytes:
+ return subprocess.run([pandoc, '--from=html', '--to=latex'],
+ input=tag.encode(),
+ stdout=subprocess.PIPE,
+ check=True).stdout
+
+
+def process(
+ url: str,
+ session: requests.Session,
+ timeout: int,
+ texout: IO[bytes],
+ pandoc: str,
+) -> None:
+ html = clean(fetch(url, session, timeout))
+ for r in replies(html):
+ texout.write(html_to_tex(pandoc, r))
+
+