From: Scott Worley Date: Fri, 12 Jan 2024 12:05:14 +0000 (-0800) Subject: Fetch the non-flat view to get the next-thread link X-Git-Url: http://git.scottworley.com/paperdoorknob/commitdiff_plain/53f396b459889b46e6feea48e8fa8e89b2452e9f Fetch the non-flat view to get the next-thread link --- diff --git a/glowfic.py b/glowfic.py index 5ffaac3..81a5342 100644 --- a/glowfic.py +++ b/glowfic.py @@ -58,10 +58,21 @@ class Chunk: class Thread: def __init__(self, spec: Spec) -> None: + def find_next_thread(dom: BeautifulSoup) -> str | None: + for c in dom.findChildren('div', class_='post-navheader'): + for a in c.findChildren('a'): + if 'Next Post' in a.text and 'href' in a.attrs and isinstance( + a.attrs['href'], str): + return a.attrs['href'] + return None + spec.log('Fetching HTML...\r') - html = spec.fetcher.fetch(flatURL(spec.url)) + html = spec.fetcher.fetch(spec.url) + flat_html = spec.fetcher.fetch(flatURL(spec.url)) spec.log('Parsing HTML...\r') - self._dom = BeautifulSoup(spec.htmlfilter(html), 'html.parser') + self._next_thread = find_next_thread( + BeautifulSoup(spec.htmlfilter(html), 'html.parser')) + self._dom = BeautifulSoup(spec.htmlfilter(flat_html), 'html.parser') def title(self) -> str | None: span = self._dom.findChild("span", id="post-title") @@ -69,6 +80,9 @@ class Thread: return None return span.text.strip() + def next_thread(self) -> str | None: + return self._next_thread + def chunkDOMs(self) -> Iterable[Tag]: def text() -> Tag: body = self._dom.body diff --git a/glowfic_test.py b/glowfic_test.py index 52d6f86..68debf9 100644 --- a/glowfic_test.py +++ b/glowfic_test.py @@ -17,7 +17,7 @@ from texify import PandocTexifier def spec_for_testing(html: bytes) -> Spec: return Spec('test', - FakeFetcher({'test?view=flat': html}), + FakeFetcher({'test': html, 'test?view=flat': html}), FakeImageStore(), lambda x: x, lambda x: None, @@ -102,6 +102,18 @@ class TestThread(unittest.TestCase): ''')) self.assertEqual(t.title(), 'Teh Story!') + def testNextThread(self) -> None: + t = Thread(spec_for_testing(b''' + +
+
Next Post »
+
« Previous Post
+ +
+
The "post"
+ ''')) + self.assertEqual(t.next_thread(), '/posts/4567') + if __name__ == '__main__': unittest.main() diff --git a/paperdoorknob_test.py b/paperdoorknob_test.py index f2cd5f1..571dda6 100644 --- a/paperdoorknob_test.py +++ b/paperdoorknob_test.py @@ -121,7 +121,8 @@ class TestProcessFromFakeFetcher(BaseTestProcess, unittest.TestCase): def fetcher(self) -> Fetcher: with open('testdata/this-is-glowfic.html', 'rb') as f: - return FakeFetcher({'fic?view=flat': f.read(9999)}) + html = f.read(9999) + return FakeFetcher({'fic': html, 'fic?view=flat': html}) if __name__ == '__main__':