Fetch: carry attestation signers' identities; skip an unverifiable bookmark d36ba2f79f95

    /// oxygenhub#266: a bookmark whose latest attestation a delegate made,

    /// signing no change, is fetched by a third party. The fetch bundle

    /// carries the delegate's identity even when it asks for no change,

    /// so a client as it was (import the bundle, then check each

    /// bookmark's attestation) verifies the move instead of failing the

    /// whole fetch with "unknown key".

    #[test]

    fn a_bookmark_attested_by_a_key_that_signed_no_change_is_fetched() {

        let dir = tempfile::tempdir().unwrap();

        let (host, mut user, _, _, _) = setup(dir.path(), true, false);

        let agent_home = dir.path().join("agent-home");

        let mut agent_repo = Repo::init(&dir.path().join("agent-work"))

            .unwrap()

            .with_home(agent_home.clone());

        let (agent, agent_id) = agent_repo

            .create_identity("agent", "agent@example", IdentityScope::User)

            .unwrap();

        user.delegate_add(agent_id, Scope::CHANGES | Scope::REFS, None)

            .unwrap();

        assert!(user.push("origin", false).unwrap().iter().all(|x| x.ok));

        let mut as_agent = Repo::open(&dir.path().join("work"))

            .unwrap()

            .with_home(agent_home);

        as_agent

            .set_user_config("author.signer", &agent.to_hex())

            .unwrap();

        let heads = as_agent.bookmark_heads().unwrap();

        as_agent.bookmark_create("feature", Some(&heads)).unwrap();

        let r = as_agent.push_bookmark("origin", "feature", false).unwrap();

        assert!(r.iter().all(|x| x.ok), "{r:?}");

        host.refresh().unwrap();

        let (_, sa) = host.latest_ref_attest("feature").unwrap().unwrap();

        assert_eq!(sa.sigs[0].key, agent, "the agent attested the move");

        // What a fetch that names no change brings: the agent's identity.

        let remote = crate::protocol::LocalRemote::open(&dir.path().join("host")).unwrap();

        let carries = |b: &Bundle| {

            b.objects

                .iter()

                .any(|(t, body)| *t == ObjectType::Identity && hash_object(*t, body) == agent)

        };

        assert!(carries(

            &crate::protocol::Remote::fetch(&remote, &[], None).unwrap()

        ));

        assert!(carries(&host.bundle_for_filtered(&[], None).unwrap()));

        // A third party's clone takes both bookmarks, verified.

        let theirs = Repo::clone(

            dir.path().join("host").to_str().unwrap(),

            &dir.path().join("theirs"),

        )

        .unwrap();

        let tracked = theirs.view().unwrap().remotes["origin"].clone();

        assert_eq!(

            tracked.get("feature"),

            host.view().unwrap().bookmarks.get("feature")

        );

        assert!(theirs.store().get_identity(&agent).is_ok());

        let report = theirs.fetch("origin").unwrap();

        assert!(report.skipped.is_empty(), "{:?}", report.skipped);

    }

    /// Make `bookmark`'s latest attestation on `host` one signed as `key`

    /// with `sk`: by a key the host never saw (as a host that did not send

    /// the signer's identity looked), or a bad signature.

    fn reattest(host: &Repo, bookmark: &str, key: KeyId, sk: &SecretKey) {

        let state = host.view().unwrap().bookmarks[bookmark];

        let (_, last) = host.latest_ref_attest(bookmark).unwrap().unwrap();

        let AttestBody::Ref {

            old_state, serial, ..

        } = last.attest.body

        else {

            panic!("not a ref attestation")

        };

        let attest = Attest {

            repo: host.id().unwrap(),

            hlc: Hlc::now(),

            body: AttestBody::Ref {

                name: bookmark.into(),

                old_state,

                new_state: state,

                serial: serial + 1,

            },

        };

        let r = host.store().put_attest(&attest).unwrap();

        host.store()

            .put_signature(&r, &sk.sign(Domain::Attest, &key, &attest.hash()))

            .unwrap();

        host.set_ref_attest(bookmark, &r).unwrap();

    }

    /// oxygenhub#266: a bookmark whose attestation a client cannot verify

    /// (its signer unknown, or its signature bad) is left as it was, named

    /// with why, and the rest are fetched. The current bookmark is not

    /// skipped: a clone, fetch or pull that would take it fails.

    #[test]

    fn an_unverifiable_bookmark_is_skipped_but_never_the_current_one() {

        let dir = tempfile::tempdir().unwrap();

        let (host, user, _, _, _) = setup(dir.path(), false, false);

        let author = user.config.author_key.unwrap();

        let heads = user.bookmark_heads().unwrap();

        for b in ["unknown", "forged", "fine"] {

            user.bookmark_create(b, Some(&heads)).unwrap();

            let r = user.push_bookmark("origin", b, false).unwrap();

            assert!(r.iter().all(|x| x.ok), "{r:?}");

        }

        host.refresh().unwrap();

        let stranger_sk = SecretKey::generate();

        let stranger = stranger_sk.identity("s", "s@example", 1).hash();

        reattest(&host, "unknown", stranger, &stranger_sk);

        reattest(&host, "forged", author, &stranger_sk);

        let url = dir.path().join("host").to_string_lossy().to_string();

        let (theirs, skipped) =

            Repo::clone_reporting(&url, &dir.path().join("theirs"), None, &[], None).unwrap();

        let names: Vec<&str> = skipped.iter().map(|(n, _)| n.as_str()).collect();

        assert_eq!(names, ["forged", "unknown"]);

        assert!(skipped[1].1.contains("unknown key"), "{skipped:?}");

        assert!(skipped[1].1.contains("origin/unknown"), "{skipped:?}");

        let tracked = theirs.view().unwrap().remotes["origin"].clone();

        assert!(tracked.contains_key("main") && tracked.contains_key("fine"));

        assert!(!tracked.contains_key("unknown") && !tracked.contains_key("forged"));

        assert_eq!(theirs.config.bookmark, "main");

        assert_eq!(theirs.fetch("origin").unwrap().skipped.len(), 2);

        assert_eq!(theirs.pull("origin").unwrap().skipped.len(), 2);

        // The current bookmark's attestation unverifiable: a fetch and a

        // pull fail, and so does a fresh clone.

        reattest(&host, "main", stranger, &stranger_sk);

        for err in [

            theirs.fetch("origin").unwrap_err(),

            theirs.pull("origin").unwrap_err(),

            Repo::clone(&url, &dir.path().join("again"))

                .err()

                .expect("the clone fails"),

        ] {

            let err = err.to_string();

            assert!(

                err.contains("attestation for origin/main: unknown key"),

                "{err}"

            );

        }

    }

    /// Bookmarks left as they were because their attestation could not be

    /// verified (its signer unknown here, or its signature bad), each with

    /// why (oxygenhub#266). The current bookmark is never skipped: its

    /// fetch fails instead.

    pub skipped: Vec<(String, String)>,

    /// What the fetch left as it was ([`FetchReport::skipped`]).

    pub skipped: Vec<(String, String)>,

        self.bundle_of(changes, None, false, false)

    ///

    /// This is what a server answers a fetch with, so it also carries what

    /// verifies the attestations it serves beside it (see

    /// [`Repo::served_attest_signers`]), also when `changes` is empty.

        self.bundle_of(changes, max_blob, false, true)

        self.bundle_of(changes, max_blob, true, true)

    /// The keys that signed the attestations this repository serves with

    /// its references: each bookmark's latest and each tag's. A fetch

    /// bundle carries their identities and certificates (spec §16.2), so a

    /// client can check an attestation whose signer signed none of the

    /// changes it receives, a delegate or a short-lived key that only

    /// moved a bookmark (oxygenhub#266). One pointer and one signature

    /// list per reference; nothing per change.

    pub fn served_attest_signers(&self) -> Result<BTreeSet<KeyId>> {

        let view = self.view()?;

        let pointers = view

            .bookmarks

            .keys()

            .map(|b| self.ref_attest_path(b))

            .chain(view.tags.keys().map(|t| self.tag_attest_path(t)));

        let mut keys = BTreeSet::new();

        for p in pointers {

            let Ok(text) = fs::read_to_string(&p) else {

                continue;

            };

            let Some(r) = ObjectRef::from_hex(text.trim()) else {

                continue;

            };

            keys.extend(self.store.signatures(&r)?.into_iter().map(|s| s.key));

        }

        Ok(keys)

    }

    /// `serving`: a fetch's answer, which carries the identities and

    /// certificates of [`Repo::served_attest_signers`] too.

        serving: bool,

        let signers: Vec<KeyId> = if serving {

            let signers = self.served_attest_signers()?;

            for k in &signers {

                add_object(k, &mut objects);

            }

            signers.into_iter().collect()

        } else {

            Vec::new()

        };

        self.attach_certificates_for(&mut bundle, &signers)?;

        &self,

        sa: &SignedAttest,

        name: &str,

        old_state: &ObjectRef,

        new_state: &ObjectRef,

        last_serial: Option<u64>,

        allow_equal: bool,

    ) -> std::result::Result<u64, String> {

        let serial =

            self.check_ref_attest_body(sa, name, old_state, new_state, last_serial, allow_equal)?;

        self.verify_attest_signature(sa)?;

        Ok(serial)

    }

    /// What [`Repo::check_ref_attest`] checks short of the signature: the

    /// attestation describes this update of this repository with a serial

    /// past the last seen.

    fn check_ref_attest_body(

        Ok(*serial)

    }

    /// An attestation's first signature verifies with its signer's

    /// identity, which must be stored here.

    fn verify_attest_signature(&self, sa: &SignedAttest) -> std::result::Result<(), String> {

        verify_signature(&id, sig, Domain::Attest, &r).map_err(|e| e.to_string())

        let mut skipped = Vec::new();

                    .check_ref_attest_body(

                // A signature this client cannot verify (its signer unknown

                // here, as from a host that did not send the identity, or

                // bad) fails that bookmark alone: it stays as it was and

                // the rest are fetched (oxygenhub#266). The current

                // bookmark, which a pull would take, still fails the fetch.

                // A policy's requirements were checked above and still fail.

                if let Err(e) = self.verify_attest_signature(sa) {

                    let why = format!("attestation for {remote_name}/{}: {e}", r.name);

                    if r.name == self.config.bookmark {

                        return fail(why);

                    }

                    skipped.push((r.name, why));

                    continue;

                }

            skipped,

        let skipped = self.fetch(remote_name)?.skipped;

            skipped,

        Ok(Self::clone_reporting(url, dest, max_blob, sparse, depth)?.0)

    }

    /// Clone as [`Repo::clone_with`] (or [`Repo::clone_shallow`] with a

    /// `depth`), also returning the bookmarks the fetch left out because

    /// their attestation could not be verified ([`FetchReport::skipped`]),

    /// for the caller to warn about. The bookmark checked out is never one

    /// of them: that fails the clone.

    pub fn clone_reporting(

        url: &str,

        dest: &Path,

        max_blob: Option<u64>,

        sparse: &[String],

        depth: Option<u32>,

    ) -> Result<(Repo, Vec<(String, String)>)> {

        Ok(Self::clone_using(url, dest, None, &[], None, &options)?.0)

    ) -> Result<(Repo, Vec<(String, String)>)> {

        let skipped = if let Some(depth) = depth {

            Vec::new()

            repo.fetch("origin")?.skipped

        };

        // The remote's default bookmark, else ours (main), else its only

        // one; a bookmark the fetch skipped counts, so the clone fails

        // rather than check out another (oxygenhub#266).

        let mut theirs = op.view.remotes.get("origin").cloned().unwrap_or_default();

        for (name, _) in &skipped {

            theirs.insert(name.clone(), ObjectRef::ZERO);

        }

        if let Some((_, why)) = skipped.iter().find(|(name, _)| *name == bookmark) {

            return fail(format!("cannot check out {bookmark}: {why}"));

        }

        Ok((repo, skipped))

    pub(crate) fn tag_attest_path(&self, name: &str) -> PathBuf {

/// oxygenhub#266: a short-lived key its issuer certified moves a bookmark

/// and signs no change (a keyless agent naming a bookmark). A third

/// party's clone gets the key's identity, its certificate and the

/// issuer's identity with the fetch, so the policy's subject grant

/// authorizes the move and the signature verifies.

#[test]

fn a_bookmark_moved_by_a_certified_key_alone_is_fetched() {

    use crate::cert::{Certificate, SignedCertificate};

    use crate::identity::Scope;

    hermetic();

    let dir = tempfile::tempdir().unwrap();

    let root = dir.path();

    let _origin = Repo::init(&root.join("origin")).unwrap();

    let url = root.join("origin").to_string_lossy().to_string();

    let mut a = Repo::clone(&url, &root.join("a"))

        .unwrap()

        .with_home(root.join("home-a"));

    a.create_identity("Ada", "[email protected]", IdentityScope::User)

        .unwrap();

    write(&root.join("a"), "a.txt", b"a\n");

    a.commit("a", &[]).unwrap();

    a.policy_init().unwrap();

    assert!(a.push("origin", false).unwrap().iter().all(|r| r.ok));

    let hub_sk = SecretKey::generate();

    let hub = hub_sk.identity("hub", "[email protected]", 1);

    a.policy_grant_subject(

        "agent:claude@ada",

        hub.clone(),

        Scope::CHANGES | Scope::REFS,

        None,

    )

    .unwrap();

    assert!(a.push("origin", false).unwrap().iter().all(|r| r.ok));

    let now = Hlc::now().wall_ms;

    let mut agent = Repo::clone(&url, &root.join("agent"))

        .unwrap()

        .with_home(root.join("home-agent"));

    let (key, id) = agent

        .create_identity("claude", "[email protected]", IdentityScope::User)

        .unwrap();

    let cert = SignedCertificate::sign(

        Certificate {

            key: id,

            issuer: hub.hash(),

            subject: "agent:claude@ada".into(),

            claims: vec![],

            scope: Scope::CHANGES | Scope::REFS,

            not_before: now - 60_000,

            expires: now + 600_000,

            grant: None,

        },

        &hub,

        &hub_sk,

    );

    agent.store().put_identity(&hub).unwrap();

    agent.store().put_certificate(&cert).unwrap();

    let heads = agent.bookmark_heads().unwrap();

    agent.bookmark_create("agent-work", Some(&heads)).unwrap();

    let r = agent.push_bookmark("origin", "agent-work", false).unwrap();

    assert!(r.iter().all(|x| x.ok), "{r:?}");

    let host = Repo::open(&root.join("origin")).unwrap();

    let (_, sa) = host.latest_ref_attest("agent-work").unwrap().unwrap();

    assert_eq!(sa.sigs[0].key, key, "the certified key alone moved it");

    // A fetch naming no change brings the certificate and both identities.

    let bundle = host.bundle_raw_for(&[], None).unwrap();

    assert!(bundle.changes.is_empty() && bundle.raw_changes.is_empty());

    assert_eq!(bundle.certificates.len(), 1);

    for k in [key, hub.hash()] {

        assert!(bundle

            .objects

            .iter()

            .any(|(t, b)| *t == ObjectType::Identity && hash_object(*t, b) == k));

    }

    let c = Repo::clone(&url, &root.join("c")).unwrap();

    let tracked = c.view().unwrap().remotes["origin"].clone();

    assert_eq!(

        tracked.get("agent-work"),

        host.view().unwrap().bookmarks.get("agent-work")

    );

    assert!(c.store().certificate_of(&key).unwrap().is_some());

}

Every published bookmark update is accompanied by an attestation (§14.4) of kind `ref` that binds the repository ID, the reference name, the previous state, the new state, and a per-reference monotonically increasing `serial`. Servers MUST store the latest attestation per reference and serve it with the reference. Clients MUST verify the attestation against policy before updating a `remotes/` reference and MUST reject an attestation whose `serial` is lower than one they have already recorded for that reference (rollback protection). Clients SHOULD warn when the newest attestation's `hlc.wall_ms` is older than `remote.max-age` (default 30 days) and the remote claims no update. A fetched bookmark whose attestation cannot be verified because its signer's identity is unknown to the client or its signature does not verify is skipped, not fatal, when it is not the current bookmark: its `remotes/` reference stays as it was, the client warns naming it and why, and the other bookmarks are fetched (oxygenhub#266). For the current bookmark (the one a `pull` takes, or a `clone` checks out) the fetch, pull or clone still fails, and an attestation that is missing under a policy, unauthorized, rolled back or not describing the update fails the fetch as before.

| `fetch` request | `{0: [hashes], 1: max blob size or null}`; response a `Bundle` without blobs above the size; also when it names no change, the bundle carries the identities of the keys that signed the attestations served with `refs` (each bookmark's latest and each tag's) and, for a short-lived key, its certificate and its issuer's identity (§14.7), each once, so a client can check an attestation whose signer signed none of the changes it receives, such as a delegate's or a keyless agent's bookmark move (oxygenhub#266) |

    let (repo, skipped) = Repo::clone_reporting(url, &dest, max_blob, sparse, depth)?;

    warn_skipped(&skipped);

                ("skipped", skipped_json(&report.skipped)),

    warn_skipped(&report.skipped);

}

/// Bookmarks a fetch left as they were, their attestation unverifiable

/// (oxygenhub#266): one warning each, naming it and why.

fn warn_skipped(skipped: &[(String, String)]) {

    for (name, why) in skipped {

        eprintln!(

            "{} {} {}",

            ui::caution(),

            ui::warn(format!("{name} not updated:")),

            ui::dim(why)

        );

    }

}

fn skipped_json(skipped: &[(String, String)]) -> String {

    json::arr(

        skipped

            .iter()

            .map(|(name, why)| json::obj([("name", json::s(name)), ("reason", json::s(why))])),

    )

                ("others", json::arr(r.others.iter().map(|o| json::s(o)))),

                ("skipped", skipped_json(&r.skipped))

        warn_skipped(&r.skipped);

    warn_skipped(&r.skipped);