mirror of
https://github.com/skoelle/buildbroken-blog-archive.git
synced 2026-09-17 18:30:25 +00:00
289 lines
11 KiB
Python
289 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Laedt die Wayback-Snapshots des "build broken" Blogs (aztec-project.org/blog/) herunter.
|
|
|
|
Verfahren:
|
|
1. CDX-API: fuer jede gewuenschte URL den letzten Snapshot mit Status 200 ermitteln.
|
|
2. HTML ueber den id_-Modifier laden: liefert das rohe Original-HTML ohne
|
|
Wayback-Toolbar und ohne Link-Rewriting.
|
|
3. Alle <img src> aus dem HTML extrahieren, per CDX nach einem Snapshot suchen und
|
|
ueber den im_-Modifier als rohe Bilddaten laden.
|
|
Alle Dateien landen flach in archive/wayback-html/ (Basisname der Original-URL).
|
|
|
|
Zusaetzlich wird archive/wayback-html/.captures.json geschrieben, das fuer jede
|
|
HTML-Datei den Capture-Timestamp und die heruntergeladenen Bilder dokumentiert
|
|
(grundlage fuer archive_url im Frontmatter und den Parsing-Report).
|
|
|
|
Nutzung:
|
|
python download_wayback_archive.py --urls ../URLS.md --out ../archive/wayback-html
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import sys
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
from urllib.parse import urljoin, urlparse
|
|
|
|
import requests
|
|
|
|
CDX_ENDPOINT = "https://web.archive.org/cdx/search/cdx"
|
|
WAYBACK_BASE = "https://web.archive.org/web/{ts}{modifier}/{url}"
|
|
ORIGIN = "https://aztec-project.org/blog/"
|
|
|
|
# Fehlerhaft geschriebene Dateinamen im Live-Blog: fuer die angegebene URL gibt es
|
|
# keinen Wayback-Capture, aber fuer den korrigierten Namen. Der Downloader faellt
|
|
# nach der Hauptschleife auf den korrigierten Namen zurueck.
|
|
URL_ALIASES = {
|
|
"async.html": "aync.html",
|
|
}
|
|
|
|
|
|
def read_urls_file(urls_path: Path):
|
|
"""Liest die Dateinamen aus der ersten Spalte der URLS.md-Tabelle."""
|
|
names = []
|
|
for line in urls_path.read_text(encoding="utf-8").splitlines():
|
|
m = re.search(r"^\s*\|\s*`([^`]+\.html)`\s*\|", line)
|
|
if m:
|
|
names.append(m.group(1))
|
|
return names
|
|
|
|
|
|
def cdx_latest(url: str, cache: dict = None):
|
|
if cache is not None and url in cache:
|
|
return cache[url]
|
|
params = {
|
|
"url": url,
|
|
"output": "json",
|
|
"filter": "statuscode:200",
|
|
"limit": "-1",
|
|
}
|
|
result = None
|
|
for attempt in range(4):
|
|
try:
|
|
r = requests.get(CDX_ENDPOINT, params=params, timeout=90)
|
|
r.raise_for_status()
|
|
rows = r.json()
|
|
if rows and len(rows) >= 2:
|
|
result = rows[-1][1]
|
|
break
|
|
except requests.RequestException as exc:
|
|
if attempt == 3:
|
|
print(f" ! CDX-Fehler fuer {url}: {exc}")
|
|
break
|
|
time.sleep(3 * (attempt + 1))
|
|
if cache is not None:
|
|
cache[url] = result
|
|
return result
|
|
|
|
|
|
def fetch_raw(url: str, ts: str, modifier: str):
|
|
wb_url = WAYBACK_BASE.format(ts=ts, modifier=modifier, url=url)
|
|
for attempt in range(4):
|
|
try:
|
|
r = requests.get(wb_url, timeout=180, allow_redirects=True)
|
|
r.raise_for_status()
|
|
return r.content
|
|
except requests.RequestException as exc:
|
|
if attempt == 3:
|
|
print(f" ! Download-Fehler {modifier} fuer {url}: {exc}")
|
|
return None
|
|
time.sleep(3 * (attempt + 1))
|
|
return None
|
|
|
|
|
|
def extract_image_srcs(html_bytes: bytes):
|
|
"""Extrahiert alle <img src=...> als originale (unrewrittene) URLs."""
|
|
srcs = set()
|
|
for m in re.finditer(r"<img\b[^>]*?\bsrc\s*=\s*([\"'])(.*?)\1", html_bytes.decode("utf-8", "ignore"), re.IGNORECASE | re.DOTALL):
|
|
srcs.add(m.group(2))
|
|
return srcs
|
|
|
|
|
|
def cdx_lookup_many(image_urls, cache, workers=4):
|
|
"""Fuehrt CDX-Lookups parallel aus; liefert dict url -> timestamp."""
|
|
found = {}
|
|
pending = [u for u in image_urls if u not in cache]
|
|
with ThreadPoolExecutor(max_workers=workers) as ex:
|
|
futs = {}
|
|
for i, u in enumerate(pending):
|
|
time.sleep(0.2)
|
|
futs[ex.submit(cdx_latest, u, cache)] = u
|
|
for fut in as_completed(futs):
|
|
u = futs[fut]
|
|
ts = fut.result()
|
|
if ts:
|
|
found[u] = ts
|
|
for u in pending:
|
|
if u in cache and cache[u]:
|
|
found[u] = cache[u]
|
|
return found
|
|
|
|
|
|
WAYBACK_REWRITE = re.compile(r"^https?://web\.archive\.org/web/\d+(?:id_|im_)?/")
|
|
|
|
|
|
def unwrap_wayback_url(url: str) -> str:
|
|
"""Entfernt einen web.archive.org-Rewrite-Praefix, falls vorhanden."""
|
|
return WAYBACK_REWRITE.sub("", url)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--urls", required=True, help="Pfad zur URLS.md")
|
|
parser.add_argument("--out", required=True, help="Zielordner (archive/wayback-html)")
|
|
parser.add_argument("--pause", type=float, default=1.0, help="Pause zwischen Requests (Sekunden)")
|
|
args = parser.parse_args()
|
|
|
|
out_dir = Path(args.out)
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
names = read_urls_file(Path(args.urls))
|
|
if not names:
|
|
print("Keine Dateinamen aus URLS.md gelesen.", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
print(f"Gelesen: {len(names)} Eintraege aus {args.urls}\n")
|
|
|
|
cap_path = out_dir / ".captures.json"
|
|
captures = {}
|
|
if cap_path.exists():
|
|
captures = json.loads(cap_path.read_text(encoding="utf-8"))
|
|
print(f"Resume: {len(captures)} Eintraege bereits vorhanden\n")
|
|
|
|
cdx_cache_path = out_dir / ".cdx_cache.json"
|
|
cdx_cache = {}
|
|
if cdx_cache_path.exists():
|
|
cdx_cache = json.loads(cdx_cache_path.read_text(encoding="utf-8"))
|
|
print(f"CDX-Cache: {len(cdx_cache)} URLs bekannt\n")
|
|
|
|
for name in names:
|
|
out_path = out_dir / name
|
|
if name in captures and captures[name].get("html_ts") and out_path.exists() and out_path.stat().st_size > 0:
|
|
print(f"== {name} (bereits verarbeitet, uebersprungen)")
|
|
continue
|
|
page_url = ORIGIN + name
|
|
print(f"== {name}")
|
|
ts = cdx_latest(page_url, cdx_cache)
|
|
if not ts:
|
|
print(f" ! KEIN Snapshot (Status 200) gefunden: {page_url}")
|
|
captures[name] = {"html_ts": None, "images": {}}
|
|
cdx_cache_path.write_text(json.dumps(cdx_cache), encoding="utf-8")
|
|
continue
|
|
|
|
out_path = out_dir / name
|
|
if out_path.exists() and out_path.stat().st_size > 0:
|
|
html = out_path.read_bytes()
|
|
print(f" HTML vorhanden ({len(html)} Bytes, ts={ts})")
|
|
else:
|
|
html = fetch_raw(page_url, ts, "id_")
|
|
if html is None:
|
|
captures[name] = {"html_ts": ts, "images": {}}
|
|
continue
|
|
out_path.write_bytes(html)
|
|
print(f" HTML ok ({len(html)} Bytes, ts={ts})")
|
|
|
|
entry = {"html_ts": ts, "images": {}}
|
|
if name == "blog.html":
|
|
print(" (Indexseite: Bilder werden uebersprungen)")
|
|
else:
|
|
img_srcs = []
|
|
for raw_src in sorted(extract_image_srcs(html)):
|
|
if raw_src.startswith("data:"):
|
|
continue
|
|
full = urljoin(page_url, raw_src)
|
|
full = unwrap_wayback_url(full)
|
|
if "aztec-project.org" not in full:
|
|
continue
|
|
fname = Path(urlparse(full).path).name
|
|
if not fname:
|
|
continue
|
|
img_srcs.append((full, fname))
|
|
if img_srcs:
|
|
found = cdx_lookup_many([u for u, _ in img_srcs], cdx_cache)
|
|
else:
|
|
found = {}
|
|
for full, fname in img_srcs:
|
|
img_ts = found.get(full)
|
|
if not img_ts:
|
|
print(f" ! Bild ohne Snapshot: {full}")
|
|
continue
|
|
target = out_dir / fname
|
|
if not target.exists() or target.stat().st_size == 0:
|
|
data = fetch_raw(full, img_ts, "im_")
|
|
if data is None:
|
|
continue
|
|
target.write_bytes(data)
|
|
entry["images"][fname] = {"url": full, "ts": img_ts}
|
|
print(f" Bild {fname} ok (ts={img_ts})")
|
|
time.sleep(args.pause)
|
|
captures[name] = entry
|
|
cap_path.write_text(json.dumps(captures, indent=2), encoding="utf-8")
|
|
cdx_cache_path.write_text(json.dumps(cdx_cache), encoding="utf-8")
|
|
time.sleep(args.pause)
|
|
|
|
# Alias-Pass: fehlerhaft geschriebene Dateinamen, fuer die es im Archiv nur
|
|
# unter dem korrigierten Namen einen Capture gibt (z.B. async -> aync).
|
|
for alias_name, real_name in URL_ALIASES.items():
|
|
real_path = out_dir / real_name
|
|
if real_path.exists() and real_path.stat().st_size > 0 and real_name in captures:
|
|
print(f"== {alias_name} -> {real_name} (bereits vorhanden)")
|
|
continue
|
|
print(f"== {alias_name} -> {real_name} (Alias)")
|
|
real_url = ORIGIN + real_name
|
|
ts = cdx_latest(real_url, cdx_cache)
|
|
if not ts:
|
|
print(f" ! KEIN Snapshot fuer Alias {real_name}: {real_url}")
|
|
captures[real_name] = {"html_ts": None, "images": {}}
|
|
continue
|
|
html = fetch_raw(real_url, ts, "id_")
|
|
if html is None:
|
|
captures[real_name] = {"html_ts": ts, "images": {}}
|
|
continue
|
|
real_path.write_bytes(html)
|
|
print(f" HTML ok ({len(html)} Bytes, ts={ts})")
|
|
entry = {"html_ts": ts, "images": {}}
|
|
img_srcs = []
|
|
for raw_src in sorted(extract_image_srcs(html)):
|
|
if raw_src.startswith("data:"):
|
|
continue
|
|
full = urljoin(real_url, raw_src)
|
|
full = unwrap_wayback_url(full)
|
|
if "aztec-project.org" not in full:
|
|
continue
|
|
fname = Path(urlparse(full).path).name
|
|
if not fname:
|
|
continue
|
|
img_srcs.append((full, fname))
|
|
if img_srcs:
|
|
found = cdx_lookup_many([u for u, _ in img_srcs], cdx_cache)
|
|
else:
|
|
found = {}
|
|
for full, fname in img_srcs:
|
|
img_ts = found.get(full)
|
|
if not img_ts:
|
|
print(f" ! Bild ohne Snapshot: {full}")
|
|
continue
|
|
target = out_dir / fname
|
|
if not target.exists() or target.stat().st_size == 0:
|
|
data = fetch_raw(full, img_ts, "im_")
|
|
if data is None:
|
|
continue
|
|
target.write_bytes(data)
|
|
entry["images"][fname] = {"url": full, "ts": img_ts}
|
|
print(f" Bild {fname} ok (ts={img_ts})")
|
|
time.sleep(args.pause)
|
|
captures[real_name] = entry
|
|
cap_path.write_text(json.dumps(captures, indent=2), encoding="utf-8")
|
|
cdx_cache_path.write_text(json.dumps(cdx_cache), encoding="utf-8")
|
|
|
|
cap_path.write_text(json.dumps(captures, indent=2), encoding="utf-8")
|
|
cdx_cache_path.write_text(json.dumps(cdx_cache), encoding="utf-8")
|
|
print(f"\nMetadaten geschrieben: {cap_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|