Add the cloud sync engine

Walks the local store and maps it onto the cloud object model per the
design doc: captures, generations (+ version wavs), profiles (+ samples,
avatar), and the settings singletons. Records carry the full row (paths
storage-relative so restores re-anchor under the destination data dir);
audio travels as per-asset ciphertext blobs.

Envelopes are randomized, so a new cloud_sync_state table remembers both
the plaintext fingerprint (did content actually change?) and the
ciphertext hash the server holds (re-declare unchanged blobs without
re-encrypting or re-uploading). updated_at is excluded from fingerprints
- its onupdate trigger fires as a side effect of applying pulls and
would bounce synced objects otherwise. Push runs before pull; conflicts
are last-writer-wins per object. Own pushes echoing back through the
feed are recognized by ciphertext hash and skipped.

Tests simulate two machines (own DB, data dir, keychain) syncing through
an in-process fake cloud: full backup/restore with byte-identical audio,
incremental edit (only the record blob re-uploads), delete propagation,
LWW convergence, and the blindness check that server storage only ever
holds VBX1 envelopes.

backend/tests/fake_cloud.py

@@ -0,0 +1,188 @@
+"""An in-process fake of the voicebox-cloud API for tests.
+
+Implements the surface the desktop sync client uses — devices, the account-key
+escrow, the encrypted object store, the sync feed, and blob storage — behind an
+httpx.MockTransport, mirroring apps/api in the voicebox-cloud repo. Every
+request body is kept in ``seen_bodies`` so tests can assert what the server was
+shown (never key material, never plaintext).
+"""
+
+import json
+
+import httpx
+
+STORAGE_HOST = "http://cloud.test/__storage/"
+
+
+class FakeCloud:
+    def __init__(self):
+        self.devices: dict[str, dict] = {}
+        self.account_key: dict | None = None
+        self.objects: dict[str, dict] = {}  # objectId -> row (incl. assets dict)
+        self.storage: dict[str, bytes] = {}  # key -> ciphertext
+        self.seen_bodies: list[bytes] = []
+        self._next_device = 0
+        self._next_object = 0
+        self._seq = 0
+
+    def transport(self) -> httpx.MockTransport:
+        return httpx.MockTransport(self.handle)
+
+    # -- helpers --------------------------------------------------------------
+
+    @staticmethod
+    def _ok(data, status=200):
+        return httpx.Response(status, json={"ok": True, "data": data})
+
+    def _seq_next(self) -> int:
+        self._seq += 1
+        return self._seq
+
+    def _find_object(self, kind: str, client_id: str) -> dict | None:
+        return next((o for o in self.objects.values() if o["kind"] == kind and o["clientId"] == client_id), None)
+
+    # -- request routing --------------------------------------------------------
+
+    def handle(self, request: httpx.Request) -> httpx.Response:
+        if request.content:
+            self.seen_bodies.append(request.content)
+        path, method = request.url.path, request.method
+
+        if path.startswith("/__storage/"):
+            key = path[len("/__storage/") :]
+            if method == "PUT":
+                self.storage[key] = request.content
+                return httpx.Response(200)
+            data = self.storage.get(key)
+            return httpx.Response(200, content=data) if data is not None else httpx.Response(404)
+
+        if path == "/v1/devices" and method == "POST":
+            return self._register_device(json.loads(request.content))
+        if path == "/v1/devices" and method == "GET":
+            return self._ok(list(self.devices.values()))
+        if path == "/v1/devices/account-key" and method == "PUT":
+            self.account_key = json.loads(request.content)
+            return self._ok(None)
+        if path == "/v1/devices/account-key" and method == "GET":
+            return self._ok(self.account_key)
+        if path.startswith("/v1/devices/") and path.endswith("/wrapped-key"):
+            device_id = path.split("/")[3]
+            if method == "POST":
+                self.devices[device_id]["wrappedMasterKey"] = json.loads(request.content)["wrappedMasterKey"]
+                return self._ok(None)
+            return self._ok({"wrappedMasterKey": self.devices[device_id]["wrappedMasterKey"]})
+
+        if path == "/v1/objects" and method == "POST":
+            return self._upsert_object(json.loads(request.content))
+        if path.startswith("/v1/objects/") and path.endswith("/commit"):
+            return self._commit(path.split("/")[3])
+        if path.startswith("/v1/objects/") and method == "DELETE":
+            obj = self.objects.get(path.split("/")[3])
+            if obj:
+                obj["deleted"] = True
+                obj["seq"] = self._seq_next()
+            return self._ok(None)
+
+        if path == "/v1/sync/changes" and method == "GET":
+            return self._changes(request.url.params)
+
+        return httpx.Response(404, json={"ok": False, "error": {"message": f"unhandled {method} {path}"}})
+
+    # -- endpoint implementations ----------------------------------------------
+
+    def _register_device(self, body: dict) -> httpx.Response:
+        self._next_device += 1
+        device_id = f"dev-{self._next_device}"
+        self.devices[device_id] = {
+            "id": device_id,
+            "name": body["name"],
+            "publicKey": body["publicKey"],
+            "wrappedMasterKey": None,
+            "revokedAt": None,
+        }
+        return self._ok({"deviceId": device_id, "accountHasKey": self.account_key is not None}, 201)
+
+    def _upsert_object(self, body: dict) -> httpx.Response:
+        obj = self._find_object(body["kind"], body["clientId"])
+        if obj is None:
+            self._next_object += 1
+            obj = {
+                "id": f"obj-{self._next_object}",
+                "kind": body["kind"],
+                "clientId": body["clientId"],
+                "version": 0,
+                "deleted": False,
+                "record": None,
+                "assets": {},
+            }
+            self.objects[obj["id"]] = obj
+
+        obj["version"] = max(obj["version"], body["version"])
+        obj["seq"] = self._seq_next()
+        obj["deleted"] = False
+
+        uploads = []
+        record = body.get("record")
+        if record and (obj["record"] is None or obj["record"]["hash"] != record["hash"]):
+            key = f"o/{obj['id']}/record"
+            obj["record"] = {**record, "key": key}
+            uploads.append({"for": "record", "key": key, "url": STORAGE_HOST + key})
+        for asset in body.get("assets", []):
+            caid = asset["clientAssetId"]
+            existing = obj["assets"].get(caid)
+            if existing is None or existing["hash"] != asset["hash"]:
+                key = f"o/{obj['id']}/a/{caid}"
+                obj["assets"][caid] = {**asset, "key": key}
+                uploads.append({"for": f"asset:{caid}", "key": key, "url": STORAGE_HOST + key})
+        return self._ok({"objectId": obj["id"], "seq": obj["seq"], "uploads": uploads}, 201)
+
+    def _commit(self, object_id: str) -> httpx.Response:
+        obj = self.objects.get(object_id)
+        if obj is None:
+            return httpx.Response(404, json={"ok": False, "error": {"message": "object not found"}})
+        missing = []
+        if obj["record"] and obj["record"]["key"] not in self.storage:
+            missing.append("record")
+        for caid, asset in obj["assets"].items():
+            if asset["key"] not in self.storage:
+                missing.append(f"asset:{caid}")
+        if missing:
+            return httpx.Response(409, json={"ok": False, "error": {"message": f"uploads missing: {missing}"}})
+        return self._ok({"objectId": object_id, "committed": True})
+
+    def _changes(self, params) -> httpx.Response:
+        since = int(params.get("since", 0))
+        limit = int(params.get("limit", 200))
+        rows = sorted((o for o in self.objects.values() if o["seq"] > since), key=lambda o: o["seq"])[:limit]
+        changes = [
+            {
+                "id": o["id"],
+                "kind": o["kind"],
+                "clientId": o["clientId"],
+                "version": o["version"],
+                "seq": o["seq"],
+                "deleted": o["deleted"],
+                "record": (
+                    {
+                        "hash": o["record"]["hash"],
+                        "size": o["record"]["size"],
+                        "url": STORAGE_HOST + o["record"]["key"],
+                    }
+                    if o["record"]
+                    else None
+                ),
+                "assets": [
+                    {
+                        "clientAssetId": caid,
+                        "role": a["role"],
+                        "hash": a["hash"],
+                        "size": a["size"],
+                        "url": STORAGE_HOST + a["key"] if a["key"] in self.storage else None,
+                    }
+                    for caid, a in o["assets"].items()
+                ],
+            }
+            for o in rows
+        ]
+        cursor = rows[-1]["seq"] if rows else since
+        return self._ok({"changes": changes, "cursor": cursor, "hasMore": len(rows) == limit})

backend/tests/test_cloud_account.py

@@ -6,10 +6,8 @@ the master key and recovery phrase never appear in anything sent to the server.
 """
 
 import base64
-import json
 from datetime import datetime
 
-import httpx
 import keyring
 import keyring.backend
 import pytest
@@ -20,6 +18,7 @@ from backend.database.models import Base, CloudSettings
 from backend.services import cloud_account, cloud_crypto, cloud_keys
 from backend.services.cloud_account import CloudAccountError
 from backend.services.cloud_api import CloudApiClient
+from backend.tests.fake_cloud import FakeCloud
 
 USER_A = "user-a"
 
@@ -41,57 +40,6 @@ class InMemoryKeyring(keyring.backend.KeyringBackend):
         self.store.pop((service, username), None)
 
 
-class FakeCloud:
-    """Just enough of the cloud API for the identity flows, plus a transcript
-    of every request body so tests can assert what the server was shown."""
-
-    def __init__(self):
-        self.devices: dict[str, dict] = {}
-        self.account_key: dict | None = None
-        self.seen_bodies: list[bytes] = []
-        self._next_id = 0
-
-    def transport(self) -> httpx.MockTransport:
-        return httpx.MockTransport(self.handle)
-
-    def handle(self, request: httpx.Request) -> httpx.Response:
-        if request.content:
-            self.seen_bodies.append(request.content)
-        path, method = request.url.path, request.method
-
-        if path == "/v1/devices" and method == "POST":
-            body = json.loads(request.content)
-            self._next_id += 1
-            device_id = f"dev-{self._next_id}"
-            self.devices[device_id] = {
-                "id": device_id,
-                "name": body["name"],
-                "publicKey": body["publicKey"],
-                "wrappedMasterKey": None,
-                "revokedAt": None,
-            }
-            return self._ok({"deviceId": device_id, "accountHasKey": self.account_key is not None}, 201)
-        if path == "/v1/devices" and method == "GET":
-            return self._ok(list(self.devices.values()))
-        if path == "/v1/devices/account-key" and method == "PUT":
-            self.account_key = json.loads(request.content)
-            return self._ok(None)
-        if path == "/v1/devices/account-key" and method == "GET":
-            return self._ok(self.account_key)
-        if path.endswith("/wrapped-key") and method == "POST":
-            device_id = path.split("/")[3]
-            self.devices[device_id]["wrappedMasterKey"] = json.loads(request.content)["wrappedMasterKey"]
-            return self._ok(None)
-        if path.endswith("/wrapped-key") and method == "GET":
-            device_id = path.split("/")[3]
-            return self._ok({"wrappedMasterKey": self.devices[device_id]["wrappedMasterKey"]})
-        return httpx.Response(404, json={"ok": False, "error": {"message": f"unhandled {method} {path}"}})
-
-    @staticmethod
-    def _ok(data, status=200):
-        return httpx.Response(status, json={"ok": True, "data": data})
-
-
 @pytest.fixture
 def fake_keyring(monkeypatch):
     backend = InMemoryKeyring()

backend/tests/test_cloud_sync.py

@@ -0,0 +1,278 @@
+"""Tests for the cloud sync engine (services/cloud_sync.py).
+
+Simulates two installs ("machines") of the desktop app — each with its own
+SQLite database, data directory, and keychain — syncing through the in-process
+FakeCloud. Covers the full backup → restore path, incremental pushes, deletes,
+and the blindness invariant (nothing plaintext ever lands in server storage).
+"""
+
+from datetime import datetime
+
+import keyring
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from backend import config
+from backend.database.models import (
+    Base,
+    Capture,
+    CaptureSettings,
+    CloudSettings,
+    Generation,
+    GenerationVersion,
+    ProfileSample,
+    VoiceProfile,
+)
+from backend.services import cloud_account, cloud_sync
+from backend.services.cloud_api import CloudApiClient
+from backend.tests.fake_cloud import FakeCloud
+
+AUDIO_A = b"RIFFfake-capture-audio" + b"\x11" * 4000
+AUDIO_B = b"RIFFfake-generation-audio" + b"\x22" * 4000
+AUDIO_C = b"RIFFfake-version-audio" + b"\x33" * 4000
+AUDIO_D = b"RIFFfake-sample-audio" + b"\x44" * 4000
+AVATAR = b"\x89PNGfake-avatar" + b"\x55" * 500
+
+
+class Install:
+    """One simulated machine: its own DB, data dir, and keychain store."""
+
+    def __init__(self, root, name: str):
+        self.data_dir = root / name
+        self.data_dir.mkdir()
+        self.keychain: dict[tuple[str, str], str] = {}
+        engine = create_engine("sqlite://")
+        Base.metadata.create_all(engine)
+        self.db = sessionmaker(bind=engine)()
+        self.db.add(
+            CloudSettings(
+                id=1,
+                api_key=f"voicebox_{name}",
+                device_name=name,
+                account_user_id="user-1",
+                connected_at=datetime(2026, 7, 1),
+            )
+        )
+        self.db.commit()
+
+    def activate(self, monkeypatch):
+        """Point global config + keychain at this machine."""
+        config.set_data_dir(self.data_dir)
+        store = self.keychain
+        monkeypatch.setattr(keyring, "get_password", lambda s, u: store.get((s, u)))
+        monkeypatch.setattr(keyring, "set_password", lambda s, u, p: store.__setitem__((s, u), p))
+        monkeypatch.setattr(keyring, "delete_password", lambda s, u: store.pop((s, u), None))
+
+    def write_file(self, relative: str, data: bytes) -> str:
+        path = self.data_dir / relative
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_bytes(data)
+        return str(path)
+
+
+@pytest.fixture
+def cloud(monkeypatch):
+    fake = FakeCloud()
+    monkeypatch.setattr(
+        cloud_sync,
+        "CloudApiClient",
+        lambda url, key: CloudApiClient(url, key, transport=fake.transport()),
+    )
+    monkeypatch.setattr(
+        cloud_account,
+        "_client",
+        lambda row: CloudApiClient("http://cloud.test", row.api_key, transport=fake.transport()),
+    )
+    return fake
+
+
+def seed_content(install: Install) -> None:
+    db = install.db
+    profile = VoiceProfile(
+        id="prof-1",
+        name="Morgan",
+        description="test voice",
+        language="en",
+        avatar_path=install.write_file("profiles/prof-1/avatar.png", AVATAR),
+        personality="dry wit",
+    )
+    db.add(profile)
+    db.add(
+        ProfileSample(
+            id="samp-1",
+            profile_id="prof-1",
+            audio_path=install.write_file("profiles/prof-1/samples/samp-1.wav", AUDIO_D),
+            reference_text="hello there",
+        )
+    )
+    db.add(
+        Capture(
+            id="cap-1",
+            audio_path=install.write_file("captures/cap-1.wav", AUDIO_A),
+            source="dictation",
+            language="en",
+            transcript_raw="the raw transcript",
+            transcript_refined="the refined transcript",
+        )
+    )
+    db.add(
+        Generation(
+            id="gen-1",
+            profile_id="prof-1",
+            text="hello world",
+            audio_path=install.write_file("generations/gen-1.wav", AUDIO_B),
+            status="completed",
+            source="manual",
+        )
+    )
+    db.add(
+        GenerationVersion(
+            id="ver-1",
+            generation_id="gen-1",
+            label="Take 2",
+            audio_path=install.write_file("generations/ver-1.wav", AUDIO_C),
+        )
+    )
+    db.add(CaptureSettings(id=1, stt_model="turbo", language="auto"))
+    db.commit()
+
+
+async def connect(install: Install, monkeypatch, phrase: str | None = None) -> str | None:
+    install.activate(monkeypatch)
+    result = await cloud_account.setup_device(install.db)
+    if phrase is not None:
+        await cloud_account.restore_with_phrase(install.db, phrase)
+    return result
+
+
+class TestSyncEngine:
+    async def test_backup_then_restore_on_second_machine(self, cloud, tmp_path, monkeypatch):
+        a = Install(tmp_path, "machine-a")
+        phrase = await connect(a, monkeypatch)
+        seed_content(a)
+
+        report = await cloud_sync.run_sync(a.db)
+        assert report.pushed == 4  # capture, generation, profile, capture_settings
+        assert report.pulled == 0  # own echoes are recognized by ciphertext hash
+
+        # Server blindness: every stored blob is a VBX1 envelope, no plaintext.
+        assert cloud.storage
+        for blob in cloud.storage.values():
+            assert blob[:4] == b"VBX1"
+            assert b"transcript" not in blob
+            assert AUDIO_A not in blob
+        for body in cloud.seen_bodies:
+            assert b"the raw transcript" not in body
+            assert b"Morgan" not in body
+
+        # Fresh machine: restore identity from phrase, then pull everything.
+        b = Install(tmp_path, "machine-b")
+        await connect(b, monkeypatch, phrase=phrase)
+        report_b = await cloud_sync.run_sync(b.db)
+        assert report_b.pulled == report.pushed
+
+        cap = b.db.query(Capture).one()
+        assert cap.id == "cap-1"
+        assert cap.transcript_raw == "the raw transcript"
+        assert (b.data_dir / "captures/cap-1.wav").read_bytes() == AUDIO_A
+
+        prof = b.db.query(VoiceProfile).one()
+        assert prof.name == "Morgan"
+        assert prof.personality == "dry wit"
+        assert (b.data_dir / "profiles/prof-1/avatar.png").read_bytes() == AVATAR
+        samp = b.db.query(ProfileSample).one()
+        assert samp.reference_text == "hello there"
+        assert (b.data_dir / "profiles/prof-1/samples/samp-1.wav").read_bytes() == AUDIO_D
+
+        gen = b.db.query(Generation).one()
+        assert gen.text == "hello world"
+        assert (b.data_dir / "generations/gen-1.wav").read_bytes() == AUDIO_B
+        ver = b.db.query(GenerationVersion).one()
+        assert ver.label == "Take 2"
+        assert (b.data_dir / "generations/ver-1.wav").read_bytes() == AUDIO_C
+
+        settings = b.db.query(CaptureSettings).one()
+        assert settings.stt_model == "turbo"
+
+        # Second sync on B is a no-op in both directions.
+        report_b2 = await cloud_sync.run_sync(b.db)
+        assert (report_b2.pushed, report_b2.pulled) == (0, 0)
+
+    async def test_incremental_edit_propagates(self, cloud, tmp_path, monkeypatch):
+        a = Install(tmp_path, "machine-a")
+        phrase = await connect(a, monkeypatch)
+        seed_content(a)
+        await cloud_sync.run_sync(a.db)
+
+        b = Install(tmp_path, "machine-b")
+        await connect(b, monkeypatch, phrase=phrase)
+        await cloud_sync.run_sync(b.db)
+
+        # Edit on A: only the capture should push, and only its record blob
+        # should re-upload (the audio is unchanged).
+        a.activate(monkeypatch)
+        blobs_before = dict(cloud.storage)
+        cap = a.db.query(Capture).one()
+        cap.transcript_refined = "edited on machine A"
+        a.db.commit()
+        report_a = await cloud_sync.run_sync(a.db)
+        assert report_a.pushed == 1
+        changed_keys = [k for k, v in cloud.storage.items() if blobs_before.get(k) != v]
+        assert changed_keys == [k for k in changed_keys if k.endswith("/record")]
+
+        b.activate(monkeypatch)
+        report_b = await cloud_sync.run_sync(b.db)
+        assert report_b.pulled == 1
+        assert b.db.query(Capture).one().transcript_refined == "edited on machine A"
+
+    async def test_delete_propagates(self, cloud, tmp_path, monkeypatch):
+        a = Install(tmp_path, "machine-a")
+        phrase = await connect(a, monkeypatch)
+        seed_content(a)
+        await cloud_sync.run_sync(a.db)
+
+        b = Install(tmp_path, "machine-b")
+        await connect(b, monkeypatch, phrase=phrase)
+        await cloud_sync.run_sync(b.db)
+
+        a.activate(monkeypatch)
+        cap = a.db.query(Capture).one()
+        a.db.delete(cap)
+        a.db.commit()
+        report_a = await cloud_sync.run_sync(a.db)
+        assert report_a.pushed_deletes == 1
+
+        b.activate(monkeypatch)
+        report_b = await cloud_sync.run_sync(b.db)
+        assert report_b.pulled_deletes == 1
+        assert b.db.query(Capture).count() == 0
+
+    async def test_last_writer_wins_on_conflict(self, cloud, tmp_path, monkeypatch):
+        a = Install(tmp_path, "machine-a")
+        phrase = await connect(a, monkeypatch)
+        seed_content(a)
+        await cloud_sync.run_sync(a.db)
+
+        b = Install(tmp_path, "machine-b")
+        await connect(b, monkeypatch, phrase=phrase)
+        await cloud_sync.run_sync(b.db)
+
+        # Concurrent edits to the same capture on both machines.
+        a.activate(monkeypatch)
+        a.db.query(Capture).one().transcript_refined = "A's edit"
+        a.db.commit()
+        await cloud_sync.run_sync(a.db)
+
+        b.activate(monkeypatch)
+        b.db.query(Capture).one().transcript_refined = "B's edit"
+        b.db.commit()
+        await cloud_sync.run_sync(b.db)  # B pushes after A: B is the last writer
+
+        a.activate(monkeypatch)
+        await cloud_sync.run_sync(a.db)
+        assert a.db.query(Capture).one().transcript_refined == "B's edit"
+
+        b.activate(monkeypatch)
+        await cloud_sync.run_sync(b.db)
+        assert b.db.query(Capture).one().transcript_refined == "B's edit"