From 12e36b2f17e87614ed85fbbc046372d7dc8340bd Mon Sep 17 00:00:00 2001 From: Stefan Koelle Date: Wed, 5 Aug 2026 15:10:55 +0200 Subject: [PATCH] fix db sync bugs --- src/carddav_client.py | 30 +++++++++++++++++++------- src/db.py | 4 +++- src/sync.py | 50 +++++++++++++++++++++++++++++-------------- src/vcard_parser.py | 4 ++-- 4 files changed, 61 insertions(+), 27 deletions(-) diff --git a/src/carddav_client.py b/src/carddav_client.py index f552dd6..76c733a 100644 --- a/src/carddav_client.py +++ b/src/carddav_client.py @@ -89,8 +89,10 @@ class CardDAVClient: def sync_collection(self, collection_url: str, sync_token: str | None): """ - Führt REPORT sync-collection aus. Gibt (changed_or_new_vcards, deleted_hrefs, new_sync_token) zurück. - changed_or_new_vcards: list[str] roher vCard-Text + Führt REPORT sync-collection aus. + Gibt (changed_vcards, etags, deleted_hrefs, new_sync_token) zurück. + changed_vcards: list[str] roher vCard-Text + etags: list[str|None] ETag pro vCard (None wenn nicht vorhanden) deleted_hrefs: list[str] Hrefs von gelöschten Kontakten (status 404) """ token_element = f"{sync_token}" if sync_token else "" @@ -102,7 +104,7 @@ class CardDAVClient: """ root = self._request("REPORT", collection_url, body, depth="1") - vcards, deleted_hrefs = [], [] + vcards, etags, deleted_hrefs = [], [], [] for response in root.findall("d:response", NS): status_el = response.find(".//d:status", NS) status_text = status_el.text if status_el is not None else "" @@ -114,28 +116,40 @@ class CardDAVClient: deleted_hrefs.append(href) continue + if not any(s in status_text for s in ("200", "207")): + logger.warning("Unerwarteter Status %s für %s, überspringe", status_text, href) + continue + + etag_el = response.find("d:getetag", NS) + etag = etag_el.text if etag_el is not None else None + data = response.find(".//card:address-data", NS) if data is not None and data.text: vcards.append(data.text) + etags.append(etag) new_token_el = root.find("d:sync-token", NS) new_token = new_token_el.text if new_token_el is not None else None - return vcards, deleted_hrefs, new_token + return vcards, etags, deleted_hrefs, new_token - def fetch_all_vcards(self, collection_url: str) -> list[str]: - """Fallback für den allerersten, vollen Abruf über addressbook-query.""" + def fetch_all_vcards(self, collection_url: str) -> tuple[list[str], list[str | None]]: + """Fallback für den allerersten, vollen Abruf über addressbook-query. + Gibt (vcards, etags) zurück.""" body = """ """ root = self._request("REPORT", collection_url, body, depth="1") - vcards = [] + vcards, etags = [], [] for response in root.findall("d:response", NS): + etag_el = response.find("d:getetag", NS) + etag = etag_el.text if etag_el is not None else None data = response.find(".//card:address-data", NS) if data is not None and data.text: vcards.append(data.text) - return vcards + etags.append(etag) + return vcards, etags def discover_collection(self) -> str: principal = self.discover_principal() diff --git a/src/db.py b/src/db.py index 5620c2d..e0751ba 100644 --- a/src/db.py +++ b/src/db.py @@ -5,7 +5,7 @@ import logging import os import uuid from contextlib import contextmanager -from datetime import date +from datetime import date, datetime import pymysql from pymysql.cursors import DictCursor @@ -95,9 +95,11 @@ def finish_sync_run(conn, run_id: str, status: str, upserted: int = None, delete def upsert_contacts(conn, contacts: list[dict], run_id: str): if not contacts: return + now = datetime.now() with conn.cursor() as cur: for c in contacts: c["sync_run_id"] = run_id + c["last_synced_at"] = now c = _sanitize_contact(c) cols = list(c.keys()) placeholders = ", ".join(["%s"] * len(cols)) diff --git a/src/sync.py b/src/sync.py index 675f6d4..ced55ae 100644 --- a/src/sync.py +++ b/src/sync.py @@ -9,10 +9,10 @@ vollständiger Re-Sync.""" import logging import sys -from config import Config -from carddav_client import CardDAVClient, ICLOUD_BASE_URL, SyncTokenInvalid -from vcard_parser import parse_vcard import db +from carddav_client import ICLOUD_BASE_URL, CardDAVClient, SyncTokenInvalid +from config import Config +from vcard_parser import parse_vcard logging.basicConfig(level=Config.LOG_LEVEL, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("sync") @@ -29,35 +29,53 @@ def sync_account(conn, account, href_to_uid_cache: dict): try: if not stored_token: logger.info("[%s] Kein sync-token vorhanden, führe initialen Full-Sync aus", account.name) - raw_vcards = client.fetch_all_vcards(collection_url) - contacts = [c for c in (parse_vcard(v, account.name) for v in raw_vcards) if c] + raw_vcards, raw_etags = client.fetch_all_vcards(collection_url) + contacts = [] + for v, etag in zip(raw_vcards, raw_etags): + c = parse_vcard(v, account.name, etag=etag) + if c: + contacts.append(c) + else: + logger.warning("[%s] vCard konnte nicht geparst werden, überspringe", account.name) db.replace_all_contacts_for_account(conn, account.name, contacts, run_id) - _, _, new_token = client.sync_collection(collection_url, None) - if new_token: - db.save_sync_token(conn, account.name, new_token) db.finish_sync_run(conn, run_id, "success", upserted=len(contacts), deleted=0) logger.info("[%s] Initialer Sync abgeschlossen: %d Kontakte", account.name, len(contacts)) return try: - changed_vcards, deleted_hrefs, new_token = client.sync_collection(collection_url, stored_token) + changed_vcards, etags, deleted_hrefs, new_token = client.sync_collection(collection_url, stored_token) except SyncTokenInvalid: logger.warning("[%s] sync-token vom Server abgelehnt, führe vollen Re-Sync aus", account.name) db.clear_sync_token(conn, account.name) - raw_vcards = client.fetch_all_vcards(collection_url) - contacts = [c for c in (parse_vcard(v, account.name) for v in raw_vcards) if c] + raw_vcards, raw_etags = client.fetch_all_vcards(collection_url) + contacts = [] + for v, etag in zip(raw_vcards, raw_etags): + c = parse_vcard(v, account.name, etag=etag) + if c: + contacts.append(c) + else: + logger.warning("[%s] vCard konnte nicht geparst werden, überspringe", account.name) db.replace_all_contacts_for_account(conn, account.name, contacts, run_id) - _, _, new_token = client.sync_collection(collection_url, None) - if new_token: - db.save_sync_token(conn, account.name, new_token) db.finish_sync_run(conn, run_id, "success", upserted=len(contacts), deleted=0) logger.info("[%s] Re-Sync abgeschlossen: %d Kontakte", account.name, len(contacts)) return - contacts = [c for c in (parse_vcard(v, account.name) for v in changed_vcards) if c] + contacts = [] + for v, etag in zip(changed_vcards, etags): + c = parse_vcard(v, account.name, etag=etag) + if c: + contacts.append(c) + else: + logger.warning("[%s] vCard konnte nicht geparst werden, überspringe", account.name) db.upsert_contacts(conn, contacts, run_id) - deleted_uids = [href.rstrip("/").rsplit("/", 1)[-1].replace(".vcf", "") for href in deleted_hrefs] + deleted_uids = [] + for href in deleted_hrefs: + uid = href.rstrip("/").rsplit("/", 1)[-1].replace(".vcf", "") + if uid: + deleted_uids.append(uid) + else: + logger.warning("[%s] Konnte UID nicht aus href extrahieren: %s", account.name, href) db.delete_contacts_by_href_uids(conn, account.name, deleted_uids) if new_token: diff --git a/src/vcard_parser.py b/src/vcard_parser.py index ca823f4..ddc6e86 100644 --- a/src/vcard_parser.py +++ b/src/vcard_parser.py @@ -33,7 +33,7 @@ def _type_str(obj) -> str: return "other" -def parse_vcard(raw_text: str, account: str) -> dict | None: +def parse_vcard(raw_text: str, account: str, etag: str | None = None) -> dict | None: try: vcard = vobject.readOne(raw_text) except Exception as exc: @@ -81,7 +81,7 @@ def parse_vcard(raw_text: str, account: str) -> dict | None: return { "account": account, "uid": uid, - "etag": None, + "etag": etag, "full_name": _scalar(_get(vcard, "fn")), "given_name": given_name, "family_name": family_name,