Fix outage data loss with per-target fetch cursors; audit cleanups; 1.8.1

eufy_sync/__init__.py

@@ -1,6 +1,6 @@
 """Sync Eufy smart scale body composition data to Garmin Connect and Strava."""
 
-__version__ = "1.8.0"
+__version__ = "1.8.1"
 
 # Public API for programmatic use
 from eufy_sync.garmin_auth import GarminAuth

eufy_sync/cli/app.py

@@ -1,12 +1,29 @@
 """The eufy-sync command line entry point and sync driver."""
 from __future__ import annotations
 
+import logging
 import sys
 from pathlib import Path
 
 from eufy_sync import platform_support
 from eufy_sync.cli import doctor, maintenance, profiles, setup, shared, status, updater
 
+logger = logging.getLogger("eufy_sync")
+
+
+def _target_label(total_counts: dict[str, int]) -> str:
+    """Human label for the targets that received data, e.g. "Garmin and Strava"."""
+    return " and ".join(n.capitalize() for n in total_counts if total_counts[n] > 0)
+
+
+def _tally_run(user, counts: dict[str, int], errors: dict[str, str], total_counts: dict[str, int], failures: list) -> None:
+    """Fold one sync_user result into the run totals."""
+    for target_name, count in counts.items():
+        total_counts[target_name] = total_counts.get(target_name, 0) + count
+    for target_name, err in errors.items():
+        failures.append((f"{user.name}/{target_name}", err))
+    logger.info("User %s: synced %s", user.name, counts)
+
 
 def main() -> None:
     try:
@@ -91,8 +108,13 @@ def _main() -> None:
         print("Credentials moved into the system keychain.")
         sys.exit(0)
 
-    # Handle Launch Agent install/uninstall
+    # Handle Launch Agent install/uninstall. Installing needs a config first:
+    # a scheduled agent on an unconfigured machine would just fail (and
+    # notify) every few hours.
     if args.install_agent:
+        if not config_path.exists():
+            print("No config found. Run eufy-sync first to set up.")
+            sys.exit(1)
         maintenance._install_launch_agent()
         return
 
@@ -189,13 +211,10 @@ def _main() -> None:
         return
 
     # Run sync
-    import logging
     from eufy_sync.sync import sync_user
     from eufy_sync.state import SyncState
     from eufy_sync.eufy_client import AmbiguousProfileError
 
-    logger = logging.getLogger("eufy_sync")
-
     updater._check_for_updates()
 
     backfill = args.backfill_days
@@ -216,11 +235,7 @@ def _main() -> None:
         for user in config.users:
             try:
                 counts, errors = sync_user(user, state, backfill_days=backfill, headless=args.headless, dry_run=args.dry_run)
-                for target_name, count in counts.items():
-                    total_counts[target_name] = total_counts.get(target_name, 0) + count
-                for target_name, err in errors.items():
-                    failures.append((f"{user.name}/{target_name}", err))
-                logger.info("User %s: synced %s", user.name, counts)
+                _tally_run(user, counts, errors, total_counts, failures)
             except AmbiguousProfileError as e:
                 interactive = not args.headless and sys.stdin.isatty()
                 if interactive:
@@ -231,11 +246,7 @@ def _main() -> None:
                     print("Saved. Syncing your profile now...")
                     try:
                         counts, errors = sync_user(user, state, backfill_days=backfill, headless=args.headless, dry_run=args.dry_run)
-                        for target_name, count in counts.items():
-                            total_counts[target_name] = total_counts.get(target_name, 0) + count
-                        for target_name, err in errors.items():
-                            failures.append((f"{user.name}/{target_name}", err))
-                        logger.info("User %s: synced %s", user.name, counts)
+                        _tally_run(user, counts, errors, total_counts, failures)
                     except Exception as retry_error:
                         logger.exception("Failed to sync user %s after profile selection", user.name)
                         failures.append((user.name, str(retry_error)))
@@ -291,15 +302,15 @@ def _main() -> None:
             failure_notify.clear_network_failures()
 
         if args.dry_run:
-            target_label = " and ".join(n.capitalize() for n in total_counts if total_counts[n] > 0)
+            target_label = _target_label(total_counts)
             if total > 0:
                 print(f"[DRY RUN] Would sync {total} measurement{'s' if total != 1 else ''} to {target_label}.")
             else:
                 print("[DRY RUN] Would sync 0 measurements. Nothing new to sync.")
             sys.exit(1 if failures else 0)
 
         if total > 0:
-            target_label = " and ".join(n.capitalize() for n in total_counts if total_counts[n] > 0)
+            target_label = _target_label(total_counts)
             platform_support.notify("eufy-sync", f"Synced {total} measurement{'s' if total != 1 else ''} to {target_label}")
 
         if first_run:
@@ -308,7 +319,7 @@ def _main() -> None:
                 print("First sync failed. Fix the issue above, then run eufy-sync again.")
             else:
                 if total > 0:
-                    target_label = " and ".join(n.capitalize() for n in total_counts if total_counts[n] > 0)
+                    target_label = _target_label(total_counts)
                     print("")
                     print(f"Synced {total} measurements to {target_label}.")
                 maintenance._offer_launch_agent()

eufy_sync/cli/maintenance.py

@@ -222,18 +222,20 @@ def _uninstall(data_dir: Path, config_path: Path | None = None, db_path: Path |
         except Exception:
             print("Note: could not clear keychain entries (the keychain may be locked).")
 
-    # Remove data directory (preserving DB if requested)
+    # Remove data directory. A kept DB at a custom --db path lives outside
+    # data_dir, so only the default location needs the selective sweep.
+    preserve_default_db = keep_db and db_path == default_db_path and db_path.exists()
     if data_dir.exists():
-        if keep_db and db_path.exists() and db_path == default_db_path:
-            # Remove everything except state.db
-            for item in data_dir.iterdir():
-                if item.name != "state.db":
-                    if item.is_dir():
-                        shutil.rmtree(item)
-                    else:
-                        item.unlink()
-        else:
+        if not preserve_default_db:
             shutil.rmtree(data_dir)
+        else:
+            for item in data_dir.iterdir():
+                if item.name == "state.db":
+                    continue
+                if item.is_dir():
+                    shutil.rmtree(item)
+                else:
+                    item.unlink()
 
     # A custom --config/--db path lives outside data_dir, so it survives the
     # rmtree above and must be removed explicitly.

eufy_sync/cli/status.py

@@ -107,7 +107,7 @@ def _show_status(state, users: list) -> None:
             if status["state"] == "valid":
                 print("Garmin auth: valid (auto-refreshes; re-login only if it expires)")
             else:
-                print("Garmin auth: not connected - run: eufy-sync --reauth")
+                print("Garmin auth: not connected - run: eufy-sync --reauth garmin")
 
         # Strava token health
         if user.strava:

eufy_sync/state.py

@@ -1,9 +1,22 @@
 from __future__ import annotations
 
 import sqlite3
-from datetime import date, datetime
+from datetime import date, datetime, timedelta
 from pathlib import Path
 
+# Response recorded when an upload is skipped because Garmin already holds an
+# entry for that date from another source. Such rows dedup the measurement
+# (is_synced) but do not count as "we uploaded on this date".
+SKIPPED_IN_GARMIN_RESPONSE = '{"skipped": "already_in_garmin"}'
+
+
+def _date_window(local_date: date) -> tuple[str, str]:
+    """ISO-string bounds that contain every timestamp whose local date could
+    be local_date. Offsets are under a day, so the string's date part differs
+    from the local date by at most one; ISO strings compare lexicographically.
+    """
+    return (local_date - timedelta(days=1)).isoformat(), (local_date + timedelta(days=2)).isoformat()
+
 
 class SyncState:
     def __init__(self, db_path: Path):
@@ -83,14 +96,6 @@ def _migrate_weight_only_column(self) -> None:
                 "ALTER TABLE sync_log ADD COLUMN weight_only INTEGER NOT NULL DEFAULT 0"
             )
 
-    def has_any_syncs(self, user_name: str, target: str) -> bool:
-        """Check if a target has ever been synced to."""
-        cursor = self._conn.execute(
-            "SELECT 1 FROM sync_log WHERE user_name = ? AND target = ? LIMIT 1",
-            (user_name, target),
-        )
-        return cursor.fetchone() is not None
-
     def is_synced(self, user_name: str, measurement_id: str, target: str) -> bool:
         cursor = self._conn.execute(
             "SELECT 1 FROM sync_log WHERE user_name = ? AND eufy_measurement_id = ? AND target = ?",
@@ -99,16 +104,22 @@ def is_synced(self, user_name: str, measurement_id: str, target: str) -> bool:
         return cursor.fetchone() is not None
 
     def has_synced_on_date(self, user_name: str, target: str, local_date: date) -> bool:
-        """Whether WE have already synced something to target for this local
+        """Whether WE have already uploaded something to target for this local
         calendar date. Used to tell "our own earlier upload today" apart from
         "another source already has this date in Garmin" (see the same-date
-        guard in sync.py). The table is small, so the date comparison is done
-        in Python rather than as date math against mixed-tz ISO strings in
-        SQL.
+        guard in sync.py). Skipped-because-already-in-Garmin rows do not
+        count: treating them as uploads would disable the guard for a second
+        measurement on the same day and create the very duplicate the guard
+        exists to prevent. SQL narrows to a two-day window; the exact check
+        against mixed-tz ISO strings stays in Python.
         """
+        lo, hi = _date_window(local_date)
         cursor = self._conn.execute(
-            "SELECT measurement_timestamp FROM sync_log WHERE user_name = ? AND target = ?",
-            (user_name, target),
+            "SELECT measurement_timestamp FROM sync_log"
+            " WHERE user_name = ? AND target = ?"
+            " AND measurement_timestamp >= ? AND measurement_timestamp < ?"
+            " AND (response IS NULL OR response != ?)",
+            (user_name, target, lo, hi, SKIPPED_IN_GARMIN_RESPONSE),
         )
         for (ts,) in cursor.fetchall():
             if datetime.fromisoformat(ts).astimezone().date() == local_date:
@@ -137,13 +148,15 @@ def record_sync(
     def weight_only_syncs_on_date(self, user_name: str, target: str, local_date: date) -> list[dict]:
         """Weight-only (raw Wi-Fi) syncs for a local calendar date that have
         not been upgraded to a full record yet. Each dict carries
-        measurement_id, measurement_timestamp, and weight_kg. Date comparison
-        happens in Python for the same reason as has_synced_on_date."""
+        measurement_id, measurement_timestamp, and weight_kg. Windowed in SQL,
+        exact date check in Python, as in has_synced_on_date."""
+        lo, hi = _date_window(local_date)
         cursor = self._conn.execute(
             """SELECT eufy_measurement_id, measurement_timestamp, weight_kg
                FROM sync_log
-               WHERE user_name = ? AND target = ? AND weight_only = 1""",
-            (user_name, target),
+               WHERE user_name = ? AND target = ? AND weight_only = 1
+                 AND measurement_timestamp >= ? AND measurement_timestamp < ?""",
+            (user_name, target, lo, hi),
         )
         return [
             {"measurement_id": mid, "measurement_timestamp": ts, "weight_kg": kg}
@@ -162,17 +175,20 @@ def mark_upgraded(self, user_name: str, measurement_id: str, target: str) -> Non
         )
         self._conn.commit()
 
-    def get_latest_sync_timestamp(self, user_name: str) -> int | None:
-        """Return the unix timestamp of the most recent synced measurement, or None."""
-        cursor = self._conn.execute(
-            "SELECT MAX(measurement_timestamp) FROM sync_log WHERE user_name = ?",
-            (user_name,),
-        )
-        row = cursor.fetchone()
+    def get_latest_sync_timestamp(self, user_name: str, target: str | None = None) -> int | None:
+        """Return the unix timestamp of the most recent synced measurement,
+        or None. With target, only that target's rows count - each target
+        keeps its own fetch cursor so one target's progress cannot hide
+        measurements another target missed while it was down.
+        """
+        query = "SELECT MAX(measurement_timestamp) FROM sync_log WHERE user_name = ?"
+        params: tuple = (user_name,)
+        if target is not None:
+            query += " AND target = ?"
+            params = (user_name, target)
+        row = self._conn.execute(query, params).fetchone()
         if row and row[0]:
-            from datetime import datetime
-            dt = datetime.fromisoformat(row[0])
-            return int(dt.timestamp())
+            return int(datetime.fromisoformat(row[0]).timestamp())
         return None
 
     def get_history(self, user_name: str, limit: int = 14) -> list[dict]:

eufy_sync/sync.py

@@ -11,6 +11,7 @@
 
 from eufy_sync.config import UserConfig
 from eufy_sync.eufy_client import AmbiguousProfileError, EufyClient
+from eufy_sync import state as state_module
 from eufy_sync.state import SyncState
 from eufy_sync.transform import transform
 
@@ -101,25 +102,24 @@ def sync_user(user: UserConfig, state: SyncState, backfill_days: int | None = No
             # GarminConnectTooManyRequestsError) still work.
             raise first_exception
 
-        # Determine how far back to fetch
-        after_timestamp: int | None = None
+        # Determine how far back to fetch: from the OLDEST per-target cursor,
+        # not a shared one. If one target was down (auth failing) while
+        # another kept syncing, a shared cursor would advance past the outage
+        # window and the recovered target would silently never receive those
+        # measurements. Re-fetched ones are cheap: the up-to-date target
+        # skips them via is_synced. A target with no syncs yet (first run or
+        # newly added) backfills 7 days.
         if backfill_days:
             after_timestamp = int(time.time()) - (backfill_days * 86400)
         else:
-            # Check if any target is newly added (no syncs yet)
-            new_target = any(
-                not state.has_any_syncs(user.name, name) for name, _ in targets
-            )
-            if new_target:
-                # New target added - backfill 7 days so existing measurements sync
-                after_timestamp = int(time.time()) - (7 * 86400)
-                logger.info("New sync target detected for %s, backfilling 7 days", user.name)
-            else:
-                after_timestamp = state.get_latest_sync_timestamp(user.name)
-                if after_timestamp is None:
-                    # First run - default to last 7 days
-                    after_timestamp = int(time.time()) - (7 * 86400)
-                    logger.info("First run for %s, defaulting to 7-day backfill", user.name)
+            default_cursor = int(time.time()) - (7 * 86400)
+            cursors = []
+            for name, _ in targets:
+                ts = state.get_latest_sync_timestamp(user.name, name)
+                if ts is None:
+                    logger.info("No prior syncs to %s for %s, backfilling 7 days", name, user.name)
+                cursors.append(ts if ts is not None else default_cursor)
+            after_timestamp = min(cursors)
 
         measurements = _retry(
             lambda: eufy.fetch_measurements(after_timestamp=after_timestamp),
@@ -192,7 +192,7 @@ def sync_user(user: UserConfig, state: SyncState, backfill_days: int | None = No
                         weight_kg=m.weight_kg,
                         synced_at=datetime.now(timezone.utc).isoformat(),
                         target="garmin",
-                        response='{"skipped": "already_in_garmin"}',
+                        response=state_module.SKIPPED_IN_GARMIN_RESPONSE,
                     )
                     continue
 
@@ -210,13 +210,12 @@ def sync_user(user: UserConfig, state: SyncState, backfill_days: int | None = No
                         lambda: client.upload_body_composition(body_comp),
                         f"Garmin upload ({m.measurement_id})",
                     )
-                    response_str = json.dumps(result) if result else None
                 else:
                     result = _retry(
                         lambda: client.update_weight(m.weight_kg),
                         f"Strava upload ({m.measurement_id})",
                     )
-                    response_str = json.dumps(result) if result else None
+                response_str = json.dumps(result) if result else None
 
                 if not synced_already:
                     state.record_sync(

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "eufy-sync"
-version = "1.8.0"
+version = "1.8.1"
 description = "Sync Eufy smart scale body composition data to Garmin Connect and Strava"
 readme = "README.md"
 license = "MIT"
@@ -43,8 +43,10 @@ dependencies = [
 eufy-sync = "eufy_sync.cli:main"
 
 [project.urls]
+Homepage = "https://github.com/sturimcode/eufy-sync"
 Repository = "https://github.com/sturimcode/eufy-sync"
 Issues = "https://github.com/sturimcode/eufy-sync/issues"
+Funding = "https://ko-fi.com/sturim"
 
 [tool.setuptools.packages.find]
 include = ["eufy_sync*"]