X-Git-Url: http://git.scottworley.com/paperdoorknob/blobdiff_plain/75c5665e089279f9de1a38b31e03f9736ecdc17e..e3b5fdf02164b45e85da8898196612aa51060760:/fetch.py?ds=sidebyside diff --git a/fetch.py b/fetch.py index 1d20701..e998394 100644 --- a/fetch.py +++ b/fetch.py @@ -13,6 +13,12 @@ from typing import IO, Iterator import requests import requests_cache +from version import paperdoorknob_version + + +_headers = {'User-Agent': f'paperdoorknob/{paperdoorknob_version} ' + + '(https://git.scottworley.com/paperdoorknob/)'} + class Fetcher(ABC): @abstractmethod @@ -27,7 +33,7 @@ class _SessionFetcher(Fetcher): self._timeout = timeout def fetch(self, url: str) -> bytes: - with self._session.get(url, timeout=self._timeout) as r: + with self._session.get(url, timeout=self._timeout, headers=_headers) as r: r.raise_for_status() return r.content @@ -44,7 +50,7 @@ class _CachingFetcher(Fetcher): self._cache_hit_count = 0 def fetch(self, url: str) -> bytes: - with self._session.get(url, timeout=self._timeout) as r: + with self._session.get(url, timeout=self._timeout, headers=_headers) as r: r.raise_for_status() self._request_count += 1 self._cache_hit_count += int(r.from_cache) @@ -89,7 +95,7 @@ class FakeFetcher(Fetcher): def fetch(self, url: str) -> bytes: self._fetch_count += 1 if url not in self._resources: - raise requests.HTTPError("URL not found") + raise requests.HTTPError("URL not found", url) return self._resources[url] def request_count(self) -> int: