initial commit

scripts/download_wayback_archive.py

@@ -0,0 +1,288 @@
+#!/usr/bin/env python3
+"""
+Laedt die Wayback-Snapshots des "build broken" Blogs (aztec-project.org/blog/) herunter.
+
+Verfahren:
+  1. CDX-API: fuer jede gewuenschte URL den letzten Snapshot mit Status 200 ermitteln.
+  2. HTML ueber den id_-Modifier laden: liefert das rohe Original-HTML ohne
+     Wayback-Toolbar und ohne Link-Rewriting.
+  3. Alle <img src> aus dem HTML extrahieren, per CDX nach einem Snapshot suchen und
+     ueber den im_-Modifier als rohe Bilddaten laden.
+Alle Dateien landen flach in archive/wayback-html/ (Basisname der Original-URL).
+
+Zusaetzlich wird archive/wayback-html/.captures.json geschrieben, das fuer jede
+HTML-Datei den Capture-Timestamp und die heruntergeladenen Bilder dokumentiert
+(grundlage fuer archive_url im Frontmatter und den Parsing-Report).
+
+Nutzung:
+    python download_wayback_archive.py --urls ../URLS.md --out ../archive/wayback-html
+"""
+
+import argparse
+import json
+import re
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from urllib.parse import urljoin, urlparse
+
+import requests
+
+CDX_ENDPOINT = "https://web.archive.org/cdx/search/cdx"
+WAYBACK_BASE = "https://web.archive.org/web/{ts}{modifier}/{url}"
+ORIGIN = "https://aztec-project.org/blog/"
+
+# Fehlerhaft geschriebene Dateinamen im Live-Blog: fuer die angegebene URL gibt es
+# keinen Wayback-Capture, aber fuer den korrigierten Namen. Der Downloader faellt
+# nach der Hauptschleife auf den korrigierten Namen zurueck.
+URL_ALIASES = {
+    "async.html": "aync.html",
+}
+
+
+def read_urls_file(urls_path: Path):
+    """Liest die Dateinamen aus der ersten Spalte der URLS.md-Tabelle."""
+    names = []
+    for line in urls_path.read_text(encoding="utf-8").splitlines():
+        m = re.search(r"^\s*\|\s*`([^`]+\.html)`\s*\|", line)
+        if m:
+            names.append(m.group(1))
+    return names
+
+
+def cdx_latest(url: str, cache: dict = None):
+    if cache is not None and url in cache:
+        return cache[url]
+    params = {
+        "url": url,
+        "output": "json",
+        "filter": "statuscode:200",
+        "limit": "-1",
+    }
+    result = None
+    for attempt in range(4):
+        try:
+            r = requests.get(CDX_ENDPOINT, params=params, timeout=90)
+            r.raise_for_status()
+            rows = r.json()
+            if rows and len(rows) >= 2:
+                result = rows[-1][1]
+            break
+        except requests.RequestException as exc:
+            if attempt == 3:
+                print(f"  ! CDX-Fehler fuer {url}: {exc}")
+                break
+            time.sleep(3 * (attempt + 1))
+    if cache is not None:
+        cache[url] = result
+    return result
+
+
+def fetch_raw(url: str, ts: str, modifier: str):
+    wb_url = WAYBACK_BASE.format(ts=ts, modifier=modifier, url=url)
+    for attempt in range(4):
+        try:
+            r = requests.get(wb_url, timeout=180, allow_redirects=True)
+            r.raise_for_status()
+            return r.content
+        except requests.RequestException as exc:
+            if attempt == 3:
+                print(f"  ! Download-Fehler {modifier} fuer {url}: {exc}")
+                return None
+            time.sleep(3 * (attempt + 1))
+    return None
+
+
+def extract_image_srcs(html_bytes: bytes):
+    """Extrahiert alle <img src=...> als originale (unrewrittene) URLs."""
+    srcs = set()
+    for m in re.finditer(r"<img\b[^>]*?\bsrc\s*=\s*([\"'])(.*?)\1", html_bytes.decode("utf-8", "ignore"), re.IGNORECASE | re.DOTALL):
+        srcs.add(m.group(2))
+    return srcs
+
+
+def cdx_lookup_many(image_urls, cache, workers=4):
+    """Fuehrt CDX-Lookups parallel aus; liefert dict url -> timestamp."""
+    found = {}
+    pending = [u for u in image_urls if u not in cache]
+    with ThreadPoolExecutor(max_workers=workers) as ex:
+        futs = {}
+        for i, u in enumerate(pending):
+            time.sleep(0.2)
+            futs[ex.submit(cdx_latest, u, cache)] = u
+        for fut in as_completed(futs):
+            u = futs[fut]
+            ts = fut.result()
+            if ts:
+                found[u] = ts
+    for u in pending:
+        if u in cache and cache[u]:
+            found[u] = cache[u]
+    return found
+
+
+WAYBACK_REWRITE = re.compile(r"^https?://web\.archive\.org/web/\d+(?:id_|im_)?/")
+
+
+def unwrap_wayback_url(url: str) -> str:
+    """Entfernt einen web.archive.org-Rewrite-Praefix, falls vorhanden."""
+    return WAYBACK_REWRITE.sub("", url)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--urls", required=True, help="Pfad zur URLS.md")
+    parser.add_argument("--out", required=True, help="Zielordner (archive/wayback-html)")
+    parser.add_argument("--pause", type=float, default=1.0, help="Pause zwischen Requests (Sekunden)")
+    args = parser.parse_args()
+
+    out_dir = Path(args.out)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    names = read_urls_file(Path(args.urls))
+    if not names:
+        print("Keine Dateinamen aus URLS.md gelesen.", file=sys.stderr)
+        sys.exit(1)
+
+    print(f"Gelesen: {len(names)} Eintraege aus {args.urls}\n")
+
+    cap_path = out_dir / ".captures.json"
+    captures = {}
+    if cap_path.exists():
+        captures = json.loads(cap_path.read_text(encoding="utf-8"))
+        print(f"Resume: {len(captures)} Eintraege bereits vorhanden\n")
+
+    cdx_cache_path = out_dir / ".cdx_cache.json"
+    cdx_cache = {}
+    if cdx_cache_path.exists():
+        cdx_cache = json.loads(cdx_cache_path.read_text(encoding="utf-8"))
+        print(f"CDX-Cache: {len(cdx_cache)} URLs bekannt\n")
+
+    for name in names:
+        out_path = out_dir / name
+        if name in captures and captures[name].get("html_ts") and out_path.exists() and out_path.stat().st_size > 0:
+            print(f"== {name} (bereits verarbeitet, uebersprungen)")
+            continue
+        page_url = ORIGIN + name
+        print(f"== {name}")
+        ts = cdx_latest(page_url, cdx_cache)
+        if not ts:
+            print(f"  ! KEIN Snapshot (Status 200) gefunden: {page_url}")
+            captures[name] = {"html_ts": None, "images": {}}
+            cdx_cache_path.write_text(json.dumps(cdx_cache), encoding="utf-8")
+            continue
+
+        out_path = out_dir / name
+        if out_path.exists() and out_path.stat().st_size > 0:
+            html = out_path.read_bytes()
+            print(f"  HTML vorhanden ({len(html)} Bytes, ts={ts})")
+        else:
+            html = fetch_raw(page_url, ts, "id_")
+            if html is None:
+                captures[name] = {"html_ts": ts, "images": {}}
+                continue
+            out_path.write_bytes(html)
+            print(f"  HTML ok ({len(html)} Bytes, ts={ts})")
+
+        entry = {"html_ts": ts, "images": {}}
+        if name == "blog.html":
+            print("  (Indexseite: Bilder werden uebersprungen)")
+        else:
+            img_srcs = []
+            for raw_src in sorted(extract_image_srcs(html)):
+                if raw_src.startswith("data:"):
+                    continue
+                full = urljoin(page_url, raw_src)
+                full = unwrap_wayback_url(full)
+                if "aztec-project.org" not in full:
+                    continue
+                fname = Path(urlparse(full).path).name
+                if not fname:
+                    continue
+                img_srcs.append((full, fname))
+            if img_srcs:
+                found = cdx_lookup_many([u for u, _ in img_srcs], cdx_cache)
+            else:
+                found = {}
+            for full, fname in img_srcs:
+                img_ts = found.get(full)
+                if not img_ts:
+                    print(f"  ! Bild ohne Snapshot: {full}")
+                    continue
+                target = out_dir / fname
+                if not target.exists() or target.stat().st_size == 0:
+                    data = fetch_raw(full, img_ts, "im_")
+                    if data is None:
+                        continue
+                    target.write_bytes(data)
+                entry["images"][fname] = {"url": full, "ts": img_ts}
+                print(f"  Bild {fname} ok (ts={img_ts})")
+                time.sleep(args.pause)
+        captures[name] = entry
+        cap_path.write_text(json.dumps(captures, indent=2), encoding="utf-8")
+        cdx_cache_path.write_text(json.dumps(cdx_cache), encoding="utf-8")
+        time.sleep(args.pause)
+
+    # Alias-Pass: fehlerhaft geschriebene Dateinamen, fuer die es im Archiv nur
+    # unter dem korrigierten Namen einen Capture gibt (z.B. async -> aync).
+    for alias_name, real_name in URL_ALIASES.items():
+        real_path = out_dir / real_name
+        if real_path.exists() and real_path.stat().st_size > 0 and real_name in captures:
+            print(f"== {alias_name} -> {real_name} (bereits vorhanden)")
+            continue
+        print(f"== {alias_name} -> {real_name} (Alias)")
+        real_url = ORIGIN + real_name
+        ts = cdx_latest(real_url, cdx_cache)
+        if not ts:
+            print(f"  ! KEIN Snapshot fuer Alias {real_name}: {real_url}")
+            captures[real_name] = {"html_ts": None, "images": {}}
+            continue
+        html = fetch_raw(real_url, ts, "id_")
+        if html is None:
+            captures[real_name] = {"html_ts": ts, "images": {}}
+            continue
+        real_path.write_bytes(html)
+        print(f"  HTML ok ({len(html)} Bytes, ts={ts})")
+        entry = {"html_ts": ts, "images": {}}
+        img_srcs = []
+        for raw_src in sorted(extract_image_srcs(html)):
+            if raw_src.startswith("data:"):
+                continue
+            full = urljoin(real_url, raw_src)
+            full = unwrap_wayback_url(full)
+            if "aztec-project.org" not in full:
+                continue
+            fname = Path(urlparse(full).path).name
+            if not fname:
+                continue
+            img_srcs.append((full, fname))
+        if img_srcs:
+            found = cdx_lookup_many([u for u, _ in img_srcs], cdx_cache)
+        else:
+            found = {}
+        for full, fname in img_srcs:
+            img_ts = found.get(full)
+            if not img_ts:
+                print(f"  ! Bild ohne Snapshot: {full}")
+                continue
+            target = out_dir / fname
+            if not target.exists() or target.stat().st_size == 0:
+                data = fetch_raw(full, img_ts, "im_")
+                if data is None:
+                    continue
+                target.write_bytes(data)
+            entry["images"][fname] = {"url": full, "ts": img_ts}
+            print(f"  Bild {fname} ok (ts={img_ts})")
+            time.sleep(args.pause)
+        captures[real_name] = entry
+        cap_path.write_text(json.dumps(captures, indent=2), encoding="utf-8")
+        cdx_cache_path.write_text(json.dumps(cdx_cache), encoding="utf-8")
+
+    cap_path.write_text(json.dumps(captures, indent=2), encoding="utf-8")
+    cdx_cache_path.write_text(json.dumps(cdx_cache), encoding="utf-8")
+    print(f"\nMetadaten geschrieben: {cap_path}")
+
+
+if __name__ == "__main__":
+    main()

scripts/parse_buildbroken_archive.py

@@ -0,0 +1,595 @@
+#!/usr/bin/env python3
+"""
+Parser fuer das "build broken" Blog-Archiv (Wayback-Downloads) -> Hugo-Content.
+
+Ziel:
+  - Liest die heruntergeladenen web.archive.org HTML-Dateien aus einem Quellordner
+  - Entfernt Wayback-Toolbar/Wrapper-Markup
+  - Extrahiert Titel, Datum, Autor, Kategorie, Tags, Content-HTML und optional Kommentare
+  - Laedt referenzierte Bilder herunter (falls online erreichbar) bzw. kopiert sie aus
+    einem lokalen Assets-Ordner und schreibt sie nach static/images/<slug>/
+  - Konvertiert den Content nach Markdown
+  - Schreibt content/posts/<slug>.md mit YAML-Frontmatter fuer Hugo
+
+Voraussetzungen:
+    pip install beautifulsoup4 markdownify requests python-slugify lxml
+
+Nutzung:
+    python parse_buildbroken_archive.py \
+        --source ../archive/wayback-html \
+        --output .. \
+        --download-images
+"""
+
+import argparse
+import json
+import re
+import sys
+import shutil
+from pathlib import Path
+from urllib.parse import urljoin, urlparse
+
+from bs4 import BeautifulSoup
+from markdownify import MarkdownConverter
+from slugify import slugify
+
+try:
+    import requests
+except ImportError:
+    requests = None
+
+
+class HugoConverter(MarkdownConverter):
+    """Markdownify-Konverter, der <img>-Tags als HTML erhaelt (statt sie zu
+    ![](...) zu konvertieren), damit width/height und align-Klassen aus dem
+    Original-Design nicht verloren gehen."""
+
+    def convert_img(self, el, text, parent_tags):
+        alt = el.attrs.get("alt", "") or ""
+        src = el.attrs.get("src", "") or ""
+        title = el.attrs.get("title", "") or ""
+        w = el.attrs.get("width")
+        h = el.attrs.get("height")
+        cls = " ".join(el.attrs.get("class") or [])
+        attrs = []
+        if alt:
+            attrs.append(f'alt="{alt}"')
+        if title:
+            attrs.append(f'title="{title}"')
+        if w:
+            attrs.append(f'width="{w}"')
+        if h:
+            attrs.append(f'height="{h}"')
+        if cls:
+            attrs.append(f'class="{cls}"')
+        attr_str = " " + " ".join(attrs) if attrs else ""
+        return f'<img src="{src}"{attr_str}/>'
+
+
+def to_md(html):
+    return HugoConverter(heading_style="ATX").convert(html)
+
+
+WAYBACK_WRAPPER_IDS = ["wm-ipp-base", "wm-ipp", "donato"]
+WAYBACK_WRAPPER_CLASSES = ["wb-autocomplete-suggestions"]
+
+CONTENT_SELECTORS = [
+    ("div", {"class": "entry"}),
+    ("div", {"class": "post"}),
+    ("div", {"class": "entry-content"}),
+    ("article", {}),
+    ("div", {"id": "content"}),
+]
+
+TITLE_SELECTORS = [
+    ("h2", {"class": "posttitle"}),
+    ("h2", {"class": "entry-title"}),
+    ("h1", {"class": "entry-title"}),
+    ("h2", {}),
+    ("title", {}),
+]
+
+DATE_BADGE_SELECTORS = [
+    ("div", {"class": "datestamp"}),
+    ("div", {"class": "date"}),
+    ("span", {"class": "date"}),
+]
+
+META_LINE_PATTERN = re.compile(
+    r"(?:Geschrieben von|Posted by)\s+(?P<author>.+?)\s+in\s+"
+    r"(?P<category>.*?)(?:,\s*tags:\s*(?P<tags>.*))?$",
+    re.IGNORECASE,
+)
+
+FOOTER_LINE_PATTERN = re.compile(
+    r"Geschrieben am\s+(?P<date>.+?)\s+um\s+(?P<time>[\d:apm\s]+)\s+und ist zu finden in",
+    re.IGNORECASE,
+)
+
+COMMENT_SELECTORS = [
+    ("div", {"class": "commentlist"}),
+    ("ol", {"class": "commentlist"}),
+    ("div", {"id": "comments"}),
+]
+
+# Slug-Overrides: Der Dateiname im Wayback-Archiv weicht vom kanonischen Slug ab
+# (Tippfehler im Live-Blog, kein Parser-/Downloader-Fehler).
+# aync.html: Blogbetreiber hat "async" falsch geschrieben, kein Capture fuer async.html.
+SLUG_OVERRIDES = {
+    "aync.html": "async",
+}
+
+MONTHS_DE = {
+    "Jan": "01", "Feb": "02", "Mär": "03", "Mar": "03", "Apr": "04",
+    "Mai": "05", "May": "05", "Jun": "06", "Jul": "07", "Aug": "08",
+    "Sep": "09", "Okt": "10", "Oct": "10", "Nov": "11", "Dez": "12", "Dec": "12",
+}
+
+
+def strip_wayback_chrome(soup: BeautifulSoup) -> None:
+    for wid in WAYBACK_WRAPPER_IDS:
+        el = soup.find(id=wid)
+        if el:
+            el.decompose()
+    for wcls in WAYBACK_WRAPPER_CLASSES:
+        for el in soup.find_all(class_=wcls):
+            el.decompose()
+    for comment in soup.find_all(string=lambda s: isinstance(s, str) and "BEGIN WAYBACK TOOLBAR INSERT" in s):
+        comment.extract()
+    for tag in soup(["script", "style", "noscript", "iframe"]):
+        tag.decompose()
+
+
+def find_first(soup, selectors):
+    for name, attrs in selectors:
+        el = soup.find(name, attrs=attrs) if attrs else soup.find(name)
+        if el:
+            return el
+    return None
+
+
+def extract_date_from_badge(date_el):
+    if not date_el:
+        return None
+    text = date_el.get_text(" ", strip=True)
+    m = re.search(r"([A-Za-zäöü]{3})\s*(\d{1,2})\s*(\d{4})", text)
+    if m:
+        mon, day, year = m.groups()
+        mon_num = MONTHS_DE.get(mon[:3].capitalize())
+        if mon_num:
+            return f"{year}-{mon_num}-{int(day):02d}"
+    return None
+
+
+def extract_date_from_footer(container):
+    text = container.get_text(" ", strip=True)
+    m = FOOTER_LINE_PATTERN.search(text)
+    if not m:
+        return None
+    raw = m.group("date")
+    m2 = re.search(r"(\d{1,2})\.?\s*([A-Za-zäöü]+)\s*(\d{4})", raw)
+    if not m2:
+        return None
+    day, mon_name, year = m2.groups()
+    mon_map = {
+        "januar": "01", "februar": "02", "märz": "03", "april": "04",
+        "mai": "05", "juni": "06", "juli": "07", "august": "08",
+        "september": "09", "oktober": "10", "november": "11", "dezember": "12",
+    }
+    mon_num = mon_map.get(mon_name.lower())
+    if not mon_num:
+        return None
+    return f"{year}-{mon_num}-{int(day):02d}"
+
+
+def extract_meta_line(soup):
+    # Nur im .postinfo <small> suchen, nicht im gesamten Dokument
+    small = soup.find("small")
+    if not small:
+        return None, None, []
+    text = small.get_text(" ", strip=True)
+    m = META_LINE_PATTERN.search(text)
+    if not m:
+        return None, None, []
+    author = m.group("author").strip()
+    categories = [c.strip() for c in m.group("category").split(",") if c.strip()]
+    tags_raw = m.group("tags") or ""
+    tags = [t.strip() for t in tags_raw.split(",") if t.strip()]
+    return author, categories, tags
+
+
+def download_or_copy_image(src, page_url, source_dir, assets_out_dir, download):
+    assets_out_dir.mkdir(parents=True, exist_ok=True)
+    filename = Path(urlparse(src).path).name
+    if not filename:
+        return None
+    target = assets_out_dir / filename
+
+    if not target.exists():
+        local_candidate = source_dir / filename
+        if local_candidate.exists():
+            shutil.copy2(local_candidate, target)
+        elif download and requests is not None and src.startswith("http"):
+            try:
+                resp = requests.get(src, timeout=15)
+                resp.raise_for_status()
+                target.write_bytes(resp.content)
+            except Exception as exc:
+                print(f"  ! Bild konnte nicht geladen werden: {src} ({exc})")
+                return None
+        else:
+            print(f"  ! Bild nicht gefunden (weder lokal noch Download aktiviert): {src}")
+            return None
+    return filename
+
+
+def process_images(content_el, page_url, source_dir, static_dir, slug, download):
+    out_dir = static_dir / "images" / slug
+    for img in content_el.find_all("img"):
+        src = img.get("src")
+        if not src:
+            continue
+        full_src = urljoin(page_url or "", src)
+        new_name = download_or_copy_image(full_src, page_url, source_dir, out_dir, download)
+        if new_name:
+            img["src"] = f"/images/{slug}/{new_name}"
+        else:
+            # Bild nicht auffindbar (weder lokal noch im Archiv): kaputten Verweis entfernen
+            img.decompose()
+
+
+def remove_placeholder_headings(content_el):
+    """Entfernt reine Trenn-Headings wie <h2>.</h2> aus dem Content."""
+    for h in content_el.find_all(["h1", "h2", "h3", "h4"]):
+        text = h.get_text(strip=True)
+        if text and not any(ch.isalnum() for ch in text):
+            h.decompose()
+
+
+def remove_postmetadata(content_el):
+    """Entfernt die WordPress-Artikel-Fusszeile (<p class="postmetadata">),
+    die Meta-Zeile und RSS/Antwort/trackback-Links enthaelt (redundant zum
+    Frontmatter und auf einer statischen Seite ohne Funktion)."""
+    for p in content_el.find_all("p", class_="postmetadata"):
+        p.decompose()
+
+
+def remove_share_buttons(content_el):
+    """Entfernt AddToAny-Share-Buttons (soziales Teilen, auf einem statischen
+    Archiv ohne Funktion)."""
+    for a in content_el.find_all("a", href=True):
+        cls = " ".join(a.get("class") or [])
+        if "addtoany" in cls or "share_save" in cls:
+            a.decompose()
+
+
+CODE_LINE_STYLE = re.compile(r"margin\s*:\s*0px")
+
+
+def _code_line_text(el):
+    return el.get_text(" ", strip=False).strip().replace("\xa0", " ").replace("\u2003", " ")
+
+
+def merge_sequential_pres(content_el):
+    """Fasst aufeinanderfolgende <pre>-Geschwister (Word-Export mit Zeilennummern)
+    innerhalb desselben Parent zu einem zusammengehoerenden Codeblock zusammen."""
+    for parent in content_el.find_all(True):
+        pres = parent.find_all("pre", recursive=False)
+        if len(pres) < 2:
+            continue
+        idx = 0
+        while idx < len(pres):
+            pre = pres[idx]
+            if not CODE_LINE_STYLE.search(pre.get("style", "").replace(" ", "")):
+                idx += 1
+                continue
+            block = [pre]
+            idx += 1
+            while idx < len(pres) and CODE_LINE_STYLE.search(pres[idx].get("style", "").replace(" ", "")):
+                block.append(pres[idx])
+                idx += 1
+            if len(block) > 1:
+                lines = [_code_line_text(el) for el in block]
+                code_text = "\n".join(lines)
+                code_text = re.sub(r"\n{3,}", "\n\n", code_text)
+                merged = content_el.new_tag("pre")
+                code = content_el.new_tag("code")
+                code.string = code_text
+                merged.append(code)
+                block[0].insert_before(merged)
+                for el in block:
+                    el.decompose()
+
+
+def strip_code_line_numbers(content_el):
+    """Entfernt fuehrende Zeilennummern (Word-Export) aus <pre><code> Inhalt."""
+    for pre in content_el.find_all("pre"):
+        code = pre.find("code")
+        if not code or not code.string:
+            continue
+        lines = [re.sub(r"^\s*\d+\s+", "", ln) for ln in code.string.split("\n")]
+        code.string = "\n".join(lines)
+
+
+def convert_code_blocks(content_el):
+    """Alle Word-Export-Code-Markierungen zu sauberen <pre><code>-Bloecken."""
+    merge_sequential_pres(content_el)
+    convert_code_paragraphs(content_el)
+    strip_code_line_numbers(content_el)
+
+
+CODE_LINE_HINT = re.compile(r"[{};=<>()\[\]//]|^\s*$")
+
+
+def _looks_like_code(el):
+    """margin:0px-Absatz ist Code, wenn er Syntax-Highlighting-Spans oder
+    typische Code-Zeichen enthaelt (sonst z.B. Fliesstext im Word-Export)."""
+    if el.find("span", style=lambda s: s and "color:" in s):
+        return True
+    return bool(CODE_LINE_HINT.search(el.get_text(" ", strip=True)))
+
+
+def convert_code_paragraphs(content_el):
+    """Fasst aufeinanderfolgende Word-Export-Absaetze (margin: 0px) mit Code-
+    Formatierung zu <pre><code>-Bloecken zusammen, damit C#-Snippets erhalten
+    bleiben statt als Fliesstext-Escaping zu enden. Nur Geschwister innerhalb
+    desselben Parent werden zusammengefasst (nicht ueber div-Grenzen hinweg)."""
+    for parent in content_el.find_all(True):
+        paragraphs = parent.find_all("p", recursive=False)
+        idx = 0
+        while idx < len(paragraphs):
+            p = paragraphs[idx]
+            is_code = p.get("style", "") and CODE_LINE_STYLE.search(p.get("style", "").replace(" ", ""))
+            if not is_code or not _looks_like_code(p):
+                idx += 1
+                continue
+            block = [p]
+            idx += 1
+            while idx < len(paragraphs) and CODE_LINE_STYLE.search(paragraphs[idx].get("style", "").replace(" ", "")):
+                if not _looks_like_code(paragraphs[idx]):
+                    break
+                block.append(paragraphs[idx])
+                idx += 1
+            if block:
+                lines = [_code_line_text(el) for el in block]
+                code_text = "\n".join(lines)
+                code_text = re.sub(r"\n{3,}", "\n\n", code_text)
+                pre = content_el.new_tag("pre")
+                code = content_el.new_tag("code")
+                code.string = code_text
+                pre.append(code)
+                block[0].insert_before(pre)
+                for el in block:
+                    el.decompose()
+
+
+WAYBACK_REWRITE = re.compile(r"^https?://web\.archive\.org/web/\d+(?:id_|im_)?/")
+
+
+def unwrap_wayback_url(url: str) -> str:
+    """Entfernt einen web.archive.org-Rewrite-Praefix, falls vorhanden."""
+    return WAYBACK_REWRITE.sub("", url)
+
+
+def process_links(content_el, page_url, permalink_map):
+    for a in content_el.find_all("a", href=True):
+        href = unwrap_wayback_url(a["href"])
+
+        # KickIt-Button (dotnet-kicks): Bild behalten, Link entfernen
+        cls = " ".join(a.get("class") or [])
+        if "kickit" in cls.lower() or "kick-it" in cls.lower() or "dotnet-kicks" in a["href"]:
+            a.unwrap()
+            continue
+
+        # Links auf alte .html-Seiten (aztec-project.org/blog/xyz.html)
+        if "aztec-project.org/blog/" in href:
+            filename = Path(urlparse(href).path).name
+            if filename.endswith(".html"):
+                target_slug = SLUG_OVERRIDES.get(filename, slugify(filename[:-5]))
+                a["href"] = f"/posts/{target_slug}/"
+            continue
+        # WordPress-Permalinks (blog.aztec-project.org/2009/10/23/<wp-slug>/)
+        m = re.search(r"blog\.aztec-project\.org/\d{4}/\d{2}/\d{2}/([^/?#]+)", href)
+        if m:
+            wp_slug = m.group(1)
+            target = permalink_map.get(wp_slug)
+            if target:
+                a["href"] = f"/posts/{target}/"
+            else:
+                a["href"] = href
+            continue
+        # Alle uebrigen Links: Wayback-Rewrite-Praefix entfernen
+        a["href"] = unwrap_wayback_url(a["href"])
+
+
+def convert_comments(soup, page_url, source_dir, static_dir, slug, download, permalink_map):
+    comments_el = find_first(soup, COMMENT_SELECTORS)
+    if not comments_el:
+        return ""
+    process_images(comments_el, page_url, source_dir, static_dir, slug, download)
+    process_links(comments_el, page_url, permalink_map)
+
+    # WP-UI-Elemente (Thread-Toggle) und leere Boilerplate entfernen
+    to_remove = []
+    for tag in comments_el.descendants:
+        if not getattr(tag, "name", None):
+            continue
+        cls = tag.get("class") or []
+        if "switch-post" in cls or "says" in cls:
+            to_remove.append(tag)
+    for tag in to_remove:
+        tag.decompose()
+
+    # Struktur: .comment-author vcard -> strong (Name), .commentmetadata -> Zeit in eigener Zeile
+    for li in comments_el.find_all("li"):
+        author = li.find(class_="comment-author") or li.find("cite")
+        meta = li.find(class_="commentmetadata")
+        if author:
+            cite = author.find("cite") or author
+            cite.name = "strong"
+            cite["class"] = ["comment-author-name"]
+        if meta:
+            meta.name = "div"
+            meta["class"] = ["comment-metadata"]
+
+    return to_md(str(comments_el)).strip()
+
+
+def load_captures(source_dir):
+    cap_path = source_dir / ".captures.json"
+    if cap_path.exists():
+        return json.loads(cap_path.read_text(encoding="utf-8"))
+    return {}
+
+
+def collect_permalink_map(source_dir):
+    """Mappt die WordPress-Permalink-Slugs (blog.aztec-project.org/.../<wp-slug>/) auf
+    die lokalen Hugo-Slugs. Der permalink-Slug steht im h2.posttitle-Link jeder Seite."""
+    mapping = {}
+    for p in source_dir.glob("*.html"):
+        if p.name in {"blog.html", "about.html"}:
+            continue
+        soup = BeautifulSoup(p.read_text(encoding="utf-8", errors="ignore"), "lxml")
+        pt = soup.select_one("h2.posttitle a")
+        if not pt or not pt.get("href"):
+            continue
+        m = re.search(r"blog\.aztec-project\.org/\d{4}/\d{2}/\d{2}/([^/?#]+)", pt["href"])
+        if not m:
+            continue
+        wp_slug = m.group(1)
+        our_slug = SLUG_OVERRIDES.get(p.name, slugify(p.stem))
+        mapping[wp_slug] = our_slug
+    return mapping
+
+
+def archive_url_for(name, captures):
+    entry = captures.get(name)
+    if not entry or not entry.get("html_ts"):
+        return None
+    ts = entry["html_ts"]
+    return f"https://web.archive.org/web/{ts}/https://aztec-project.org/blog/{name}"
+
+
+def parse_post(html_path, source_dir, static_dir, content_dir, download_images, page_url_base, captures=None, permalink_map=None):
+    raw = html_path.read_text(encoding="utf-8", errors="ignore")
+    soup = BeautifulSoup(raw, "lxml")
+    strip_wayback_chrome(soup)
+
+    title_el = find_first(soup, TITLE_SELECTORS)
+    title = title_el.get_text(strip=True) if title_el else html_path.stem
+    title = re.sub(r"\s*\|\s*build broken.*$", "", title, flags=re.IGNORECASE)
+
+    content_el = find_first(soup, CONTENT_SELECTORS)
+    if not content_el:
+        print(f"  ! Kein Content-Block gefunden fuer {html_path.name} - ueberspringe")
+        return None
+
+    # Meta-Zeile nur im .postinfo suchen (nicht im ganzen Dokument)
+    postinfo = soup.find("div", class_="postinfo")
+    meta_source = postinfo if postinfo else soup
+
+    date_badge = find_first(soup, DATE_BADGE_SELECTORS)
+    date = extract_date_from_badge(date_badge) or extract_date_from_footer(content_el)
+    date_uncertain = False
+    if not date:
+        m = re.search(r"(19|20)\d{2}", html_path.stem)
+        if m:
+            date = f"{m.group(0)}-01-01"
+        else:
+            date = "1970-01-01"
+        date_uncertain = True
+
+    author, category, tags = extract_meta_line(meta_source)
+
+    slug = SLUG_OVERRIDES.get(html_path.name, slugify(html_path.stem))
+    page_url = urljoin(page_url_base, html_path.name)
+
+    process_images(content_el, page_url, source_dir, static_dir, slug, download_images)
+    remove_placeholder_headings(content_el)
+    remove_postmetadata(content_el)
+    remove_share_buttons(content_el)
+    convert_code_blocks(content_el)
+    process_links(content_el, page_url, permalink_map)
+
+    body_md = to_md(str(content_el)).strip()
+    body_md = re.sub(r"\n{3,}", "\n\n", body_md)
+
+    comments_md = convert_comments(soup, page_url, source_dir, static_dir, slug, download_images, permalink_map)
+
+    frontmatter_lines = [
+        "---",
+        f'title: "{title.replace(chr(34), chr(39))}"',
+        f"date: {date}",
+        f"slug: {slug}",
+        f'original_url: "https://aztec-project.org/blog/{html_path.name}"',
+    ]
+    if date_uncertain:
+        frontmatter_lines.append("date_uncertain: true")
+    archive_url = archive_url_for(html_path.name, captures or {})
+    if archive_url:
+        frontmatter_lines.append(f'archive_url: "{archive_url}"')
+    if author:
+        frontmatter_lines.append(f'author: "{author}"')
+    if category:
+        cat_list = ", ".join(f'"{c}"' for c in category)
+        frontmatter_lines.append(f"categories: [{cat_list}]")
+    if tags:
+        tag_list = ", ".join(f'"{t}"' for t in tags)
+        frontmatter_lines.append(f"tags: [{tag_list}]")
+    frontmatter_lines.append("---")
+    frontmatter = "\n".join(frontmatter_lines)
+
+    parts = [frontmatter, "", body_md]
+    if comments_md:
+        parts += ["", "## Kommentare (Archiv)", "", comments_md]
+
+    out_path = content_dir / f"{slug}.md"
+    out_path.write_text("\n".join(parts) + "\n", encoding="utf-8")
+    print(f"  OK -> {out_path}")
+    return out_path
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--source", required=True, help="Ordner mit den heruntergeladenen HTML-Dateien")
+    parser.add_argument("--output", required=True, help="Zielordner (Hugo-Projektwurzel)")
+    parser.add_argument("--download-images", action="store_true")
+    parser.add_argument("--page-url-base", default="https://aztec-project.org/blog/")
+    args = parser.parse_args()
+
+    source_dir = Path(args.source)
+    output_dir = Path(args.output)
+    content_dir = output_dir / "content" / "posts"
+    static_dir = output_dir / "static"
+    content_dir.mkdir(parents=True, exist_ok=True)
+    static_dir.mkdir(parents=True, exist_ok=True)
+
+    if not source_dir.exists():
+        print(f"Quellordner {source_dir} existiert nicht.", file=sys.stderr)
+        sys.exit(1)
+
+    captures = load_captures(source_dir)
+
+    html_files = sorted(source_dir.glob("*.html"))
+    if not html_files:
+        print("Keine .html Dateien im Quellordner gefunden.", file=sys.stderr)
+        sys.exit(1)
+
+    skip_as_post = {"blog.html", "about.html"}
+    permalink_map = collect_permalink_map(source_dir)
+
+    print(f"Gefunden: {len(html_files)} HTML-Dateien\n")
+
+    for html_path in html_files:
+        print(f"Verarbeite {html_path.name} ...")
+        if html_path.name in skip_as_post:
+            print("  - Startseite/Index wird ignoriert (wird von Hugo generiert)")
+        else:
+            parse_post(html_path, source_dir, static_dir, content_dir,
+                       args.download_images, args.page_url_base, captures, permalink_map)
+
+    print("\nFertig.")
+
+
+if __name__ == "__main__":
+    main()

scripts/requirements.txt

@@ -0,0 +1,5 @@
+beautifulsoup4
+markdownify
+requests
+python-slugify
+lxml