X-Git-Url: http://git.scottworley.com/paperdoorknob/blobdiff_plain/62043b2bafbcb57beae7559235bd6984b6275b55..refs/heads/main:/glowfic.py diff --git a/glowfic.py b/glowfic.py index 8561cd7..275c163 100644 --- a/glowfic.py +++ b/glowfic.py @@ -5,20 +5,24 @@ # Free Software Foundation, version 3. -from abc import ABC, abstractmethod -from dataclasses import dataclass +import dataclasses import itertools -from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse +from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse -from typing import Iterable +from typing import Any, Iterable from bs4 import BeautifulSoup from bs4.element import Tag from images import ImageStore +from spec import Spec from texify import Texifier +def ilen(it: Iterable[Any]) -> int: + return sum(1 for _ in it) + + def _removeViewFromURL(url: str) -> str: u = urlparse(url) old_qs = parse_qsl(u.query) @@ -36,7 +40,7 @@ def flatURL(url: str) -> str: return urlunparse(u._replace(query=urlencode(qs))) -@dataclass(frozen=True) +@dataclasses.dataclass(frozen=True) class Chunk: icon: str | None character: Tag | None @@ -55,20 +59,66 @@ class Chunk: # * Humans in the community tend to use "posts" to mean chunks. -def chunkDOMs(html: BeautifulSoup) -> Iterable[Tag]: - def text() -> Tag: - body = html.body - assert body - text = body.find_next("div", class_="post-post") - assert isinstance(text, Tag) - return text +class Thread: - def the_replies() -> Iterable[Tag]: - rs = html.find_all("div", class_="post-reply") - assert all(isinstance(r, Tag) for r in rs) - return rs + def __init__(self, spec: Spec) -> None: + def find_next_thread(dom: BeautifulSoup) -> str | None: + for c in dom.findChildren('div', class_='post-navheader'): + for a in c.findChildren('a'): + if 'Next Post' in a.text and 'href' in a.attrs and isinstance( + a.attrs['href'], str): + return urljoin(spec.url, a.attrs['href']) + return None - return itertools.chain([text()], the_replies()) + spec.log('Fetching HTML...\r') + html = spec.fetcher.fetch(spec.url) + flat_html = spec.fetcher.fetch(flatURL(spec.url)) + spec.log('Parsing HTML...\r') + self._next_thread = find_next_thread( + BeautifulSoup(spec.htmlfilter(html), 'html.parser')) + self._dom = BeautifulSoup(spec.htmlfilter(flat_html), 'html.parser') + self._spec = spec + + def title(self) -> str | None: + span = self._dom.findChild("span", id="post-title") + if not isinstance(span, Tag): + return None + return span.text.strip() + + def next_thread(self) -> str | None: + return self._next_thread + + def chunkDOMs(self) -> Iterable[Tag]: + def text() -> Tag: + body = self._dom.body + assert body + text = body.find_next("div", class_="post-post") + assert isinstance(text, Tag) + return text + + def the_replies() -> Iterable[Tag]: + rs = self._dom.find_all("div", class_="post-reply") + assert all(isinstance(r, Tag) for r in rs) + return rs + + return itertools.chain([text()], the_replies()) + + def emit(self) -> None: + self._spec.log('Counting chunks...\r') + num_chunks = ilen(self.chunkDOMs()) + title = self.title() or "chunk" + for i, r in enumerate(self.chunkDOMs()): + percent = 100.0 * (i + 1) / num_chunks + self._spec.log( + f'Processing {title} {i+1} of {num_chunks} ({percent:.1f}%)\r') + self._spec.domfilter(r) + chunk = makeChunk(r, self._spec.images) + self._spec.texout.write( + self._spec.texfilter(renderChunk(self._spec.texifier, chunk))) + self._spec.log('') + next_url = self.next_thread() + if next_url is not None: + Thread(dataclasses.replace(self._spec, url=next_url)).emit() def makeChunk(chunk_dom: Tag, image_store: ImageStore) -> Chunk: @@ -111,98 +161,68 @@ def makeChunk(chunk_dom: Tag, image_store: ImageStore) -> Chunk: content) -def renderIcon(icon_path: str | None, image_size: float) -> bytes | None: - if icon_path is None: - return None - return b'\\includegraphics[width=%fmm,height=%fmm,keepaspectratio]{%s}' % ( - image_size, image_size, icon_path.encode('UTF-8')) - - -class Layout(ABC): - - @abstractmethod - def renderChunk(self, chunk: Chunk) -> bytes: - raise NotImplementedError() - - -class ContentOnlyLayout(Layout): - - def __init__(self, texifier: Texifier) -> None: - self._texifier = texifier +def renderChunk(texifier: Texifier, chunk: Chunk) -> bytes: + return b''.join([ + br'\glowhead{', + br'\glowicon{%s}' % chunk.icon.encode('UTF-8') if chunk.icon else b'', + b'}{', + texifier.texify(chunk.character) if chunk.character else b'', + b'}{', + texifier.texify(chunk.screen_name) if chunk.screen_name else b'', + b'}{', + texifier.texify(chunk.author) if chunk.author else b'', + b'}', + texifier.texify(chunk.content)]) + + +ContentOnlyLayout = br''' +\newcommand{\glowhead}[4]{} +''' + + +BelowIconLayout = br''' +\newcommand{\glowhead}[4]{\wrapstuffclear +\begin{wrapstuff}[l] +\fbox{ +\begin{varwidth}{0.5\textwidth} + \smash{\parbox[t][0pt]{0pt}{ + \setlength{\fboxrule}{0.2pt} + \setlength{\fboxsep}{0pt} + \vspace{-3.4pt} + \fbox{\hspace{107mm}} + }\\*} + \vspace{-1em} +\begin{center} +#1\ifnotempty +{#1}{\\*}#2\ifnotempty +{#2}{\\*}#3\ifnotempty +{#3}{\\*}#4 +\end{center} +\end{varwidth} +} +\end{wrapstuff} - def renderChunk(self, chunk: Chunk) -> bytes: - return self._texifier.texify(chunk.content) + b'\n' +\strut +\noindent}''' -class BelowIconLayout(Layout): - def __init__(self, texifier: Texifier, image_size: float) -> None: - self._texifier = texifier - self._image_size = image_size +# Why is \textwidth not the width of the text? +# Why is the width of the text .765\textwidth? +BesideIconLayout = br''' +\newcommand{\glowhead}[4]{ - def renderChunk(self, chunk: Chunk) -> bytes: - icon = renderIcon(chunk.icon, self._image_size) - meta = [icon] if icon else [] - meta += [self._texifier.texify(x) - for x in [chunk.character, chunk.screen_name, chunk.author] - if x is not None] +\strut - return b'''\\wrapstuffclear -\\begin{wrapstuff}[l] -\\fbox{ -\\begin{varwidth}{0.5\\textwidth} - \\smash{\\parbox[t][0pt]{0pt}{ - \\setlength{\\fboxrule}{0.2pt} - \\setlength{\\fboxsep}{0pt} - \\vspace{-3.4pt} - \\fbox{\\hspace{107mm}} - }\\\\*} - \\vspace{-1em} -\\begin{center} -%s -\\end{center} -\\end{varwidth} +\noindent\fbox{ +#1 +\parbox[b]{.765\textwidth}{ +\begin{center} +#2\ifnotempty +{#2}{\\*}#3\ifnotempty +{#3}{\\*}#4 +\end{center} } -\\end{wrapstuff} - -\\strut - -\\noindent %s -''' % ( - b'\\\\*'.join(meta), - self._texifier.texify(chunk.content)) - - -class BesideIconLayout(Layout): - - def __init__(self, texifier: Texifier, image_size: float) -> None: - self._texifier = texifier - self._image_size = image_size - - def renderChunk(self, chunk: Chunk) -> bytes: - icon = renderIcon(chunk.icon, self._image_size) - meta = [ - chunk.character, - chunk.screen_name, - chunk.author, - ] - - # Why is \textwidth not the width of the text? - # Why is the width of the text .765\textwidth? - return b'''\\noindent\\fbox{ -%s -\\parbox[b]{.765\\textwidth}{ -\\begin{center} -%s -\\end{center} -} -}\\\\* -\\vspace{-0.75em}\\\\* -\\noindent %s - -\\strut - -''' % ( - icon if icon else b'', - b'\\\\*'.join(self._texifier.texify(x) for x in meta if x is not None), - self._texifier.texify(chunk.content)) +}\\* +\vspace{-0.75em}\\* +\noindent}'''