from dataclasses import dataclass
import itertools
-from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
+from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
from typing import Iterable
from bs4.element import Tag
from images import ImageStore
+from spec import Spec
from texify import Texifier
class Thread:
- def __init__(self, html: BeautifulSoup) -> None:
- self._html = html
+ def __init__(self, spec: Spec) -> None:
+ def find_next_thread(dom: BeautifulSoup) -> str | None:
+ for c in dom.findChildren('div', class_='post-navheader'):
+ for a in c.findChildren('a'):
+ if 'Next Post' in a.text and 'href' in a.attrs and isinstance(
+ a.attrs['href'], str):
+ return urljoin(spec.url, a.attrs['href'])
+ return None
+
+ spec.log('Fetching HTML...\r')
+ html = spec.fetcher.fetch(spec.url)
+ flat_html = spec.fetcher.fetch(flatURL(spec.url))
+ spec.log('Parsing HTML...\r')
+ self._next_thread = find_next_thread(
+ BeautifulSoup(spec.htmlfilter(html), 'html.parser'))
+ self._dom = BeautifulSoup(spec.htmlfilter(flat_html), 'html.parser')
+
+ def title(self) -> str | None:
+ span = self._dom.findChild("span", id="post-title")
+ if not isinstance(span, Tag):
+ return None
+ return span.text.strip()
+
+ def next_thread(self) -> str | None:
+ return self._next_thread
def chunkDOMs(self) -> Iterable[Tag]:
def text() -> Tag:
- body = self._html.body
+ body = self._dom.body
assert body
text = body.find_next("div", class_="post-post")
assert isinstance(text, Tag)
return text
def the_replies() -> Iterable[Tag]:
- rs = self._html.find_all("div", class_="post-reply")
+ rs = self._dom.find_all("div", class_="post-reply")
assert all(isinstance(r, Tag) for r in rs)
return rs