OXYGENHUB
# e71583501b1d oxygenhub / crates / hub / src / diffusion.rs
12.0 KiB · Rust · last changed Diffusion: turning it on 24 days ago source blame history raw
f677d627 Andy McCall ↶ 1
//! Diffusion (#148, docs/diffusion.md): an upstream's changes reach the
2
//! repositories that pin it as a subrepo.
3
//!
4
//! D1 starts with the reverse pin index (§4.1): when a push moves a
5
//! repository's default bookmark, or the bookmark its work lands on, each
6
//! Oxygen subrepo pin in the new state is recorded. That answers "who
7
//! consumes this repository, at which state, and how far behind", which
8
//! an upstream's page shows as Used by.
9
10
use crate::auth::roles;
11
use crate::db::{self, accounts::Account, repos::RepoRow, Pool};
12
use crate::repos::RepoName;
13
use crate::Shared;
14
use oxy_core::hash::ChangeHash;
15
use serde::Serialize;
16
17
/// How far back landed proposals teach where a repository's work lands.
18
pub const LEARN_DAYS: i64 = 90;
19
20
/// The bookmark a repository's work lands on (§4.3): the most common
21
/// target of its proposals landed in the last `LEARN_DAYS`, else its
22
/// default bookmark.
23
pub async fn target(pool: &Pool, row: &RepoRow) -> String {
24
    let since = db::now_ms() - LEARN_DAYS * 24 * 3600 * 1000;
25
    let learned: Option<String> = sqlx::query_scalar(
26
        "SELECT target FROM proposals WHERE repo_id = ? AND state = 'landed' AND landed_ms >= ?
27
         GROUP BY target ORDER BY COUNT(*) DESC, MAX(landed_ms) DESC LIMIT 1",
28
    )
29
    .bind(row.id)
30
    .bind(since)
31
    .fetch_optional(pool)
32
    .await
33
    .ok()
34
    .flatten();
35
    learned.filter(|t| !t.is_empty()).unwrap_or_else(|| row.default_bookmark.clone())
36
}
37
38
/// The bookmarks whose pins are indexed: the default and the learned
39
/// target (one when they are the same).
40
pub async fn followed_bookmarks(pool: &Pool, row: &RepoRow) -> Vec<String> {
41
    let mut out = vec![row.default_bookmark.clone()];
42
    let t = target(pool, row).await;
43
    if !out.contains(&t) {
44
        out.push(t);
45
    }
46
    out
47
}
48
49
/// Record `row`'s pins on `bookmark` at `heads`, replacing what was there.
50
pub async fn record(app: &Shared, row: &RepoRow, bookmark: &str, heads: &[ChangeHash]) {
51
    let Some(name) = RepoName::new(&row.owner, &row.name) else { return };
52
    let pins = if heads.is_empty() { Vec::new() } else { crate::flow::pins_in(app, name, heads.to_vec()).await };
53
    let now = db::now_ms();
54
    let Ok(mut tx) = app.db.begin().await else { return };
55
    let mut ok = sqlx::query("DELETE FROM diffusion_pins WHERE consumer_id = ? AND bookmark = ?")
56
        .bind(row.id)
57
        .bind(bookmark)
58
        .execute(&mut *tx)
59
        .await
60
        .is_ok();
61
    for (path, id, pinned) in &pins {
62
        let heads: Vec<String> = pinned.iter().map(|h| h.to_hex()).collect();
63
        ok &=
64
            sqlx::query("INSERT INTO diffusion_pins (consumer_id, bookmark, path, upstream, heads, updated_ms) VALUES (?, ?, ?, ?, ?, ?)")
65
                .bind(row.id)
66
                .bind(bookmark)
67
                .bind(path)
68
                .bind(hex::encode(id))
69
                .bind(heads.join("+"))
70
                .bind(now)
71
                .execute(&mut *tx)
72
                .await
73
                .is_ok();
74
    }
75
    if ok {
76
        tx.commit().await.ok();
77
    }
78
}
79
80
/// After a push: index the pins of each moved bookmark that is followed.
81
pub async fn after_push(app: &Shared, row: &RepoRow, moved: &[(String, String, Vec<ChangeHash>)]) {
82
    let followed = followed_bookmarks(&app.db, row).await;
83
    for (bookmark, _, heads) in moved {
84
        if followed.contains(bookmark) {
85
            record(app, row, bookmark, heads).await;
86
        }
87
    }
88
}
89
90
/// Index every repository's followed bookmarks, for a hub that had no
91
/// index before (run once at start, in the background).
92
pub async fn backfill(app: &Shared) {
93
    let Ok(rows) = db::repos::list(&app.db).await else { return };
94
    for row in rows {
95
        let Some(name) = RepoName::new(&row.owner, &row.name) else { continue };
96
        for bookmark in followed_bookmarks(&app.db, &row).await {
97
            let b = bookmark.clone();
98
            let heads = app.repos.read(&name, move |r, _| heads_of(r, &b)).await;
99
            if let Ok(heads) = heads {
100
                record(app, &row, &bookmark, &heads).await;
101
            }
102
        }
103
    }
104
}
105
106
/// A bookmark's heads as the repository has them now (not a cached
107
/// list, which a push refreshes a moment later); empty when it has none.
108
fn heads_of(r: &oxy_core::repo::Repo, bookmark: &str) -> Result<Vec<ChangeHash>, crate::repos::RepoError> {
109
    Ok(r.bookmarks()?.into_iter().find(|b| b.name == bookmark).map(|b| b.heads).unwrap_or_default())
110
}
111
112
/// One consumer of an upstream, as an upstream's page and API show it.
113
#[derive(Clone, Debug, Serialize)]
114
pub struct Consumer {
115
    pub owner: String,
116
    pub name: String,
117
    pub bookmark: String,
118
    pub path: String,
119
    /// The upstream heads it pins, hex.
120
    pub pinned: Vec<String>,
121
    /// Upstream changes on its default bookmark the pin lacks; `None` when
122
    /// the hub cannot tell (the pinned changes are not in the upstream).
123
    pub behind: Option<usize>,
124
}
125
126
/// The consumers of `row` that `account` may read, with how far behind
127
/// the upstream's default bookmark each pin is.
128
pub async fn consumers(app: &Shared, row: &RepoRow, account: Option<&Account>) -> Result<Vec<Consumer>, crate::repos::RepoError> {
129
    let Some(name) = RepoName::new(&row.owner, &row.name) else { return Ok(Vec::new()) };
130
    let id = app.repos.read(&name, |r, _| Ok(r.id()?)).await?;
131
    let found: Vec<(i64, String, String, String)> = sqlx::query_as(
132
        "SELECT consumer_id, bookmark, path, heads FROM diffusion_pins WHERE upstream = ? ORDER BY consumer_id, bookmark, path",
133
    )
134
    .bind(hex::encode(id))
135
    .fetch_all(&app.db)
136
    .await
137
    .map_err(|e| crate::repos::RepoError::Io(std::io::Error::other(e)))?;
138
    let mut out = Vec::new();
139
    let mut pinned_sets = Vec::new();
140
    for (consumer_id, bookmark, path, heads) in found {
141
        let Ok(Some(c)) = db::repos::get_by_id(&app.db, consumer_id).await else { continue };
142
        if !roles::allows(roles::role_for(&app.db, account, &c).await.ok().flatten(), crate::auth::Access::Read) {
143
            continue;
144
        }
145
        let pinned: Vec<String> = heads.split('+').filter(|h| !h.is_empty()).map(String::from).collect();
146
        pinned_sets.push(pinned.iter().filter_map(|h| ChangeHash::from_hex(h)).collect::<Vec<_>>());
147
        out.push(Consumer { owner: c.owner, name: c.name, bookmark, path, pinned, behind: None });
148
    }
149
    if out.is_empty() {
150
        return Ok(out);
151
    }
152
    let default = row.default_bookmark.clone();
153
    let behind = app
154
        .repos
155
        .read(&name, move |r, cache| {
156
            let tip = heads_of(r, &default)?;
157
            let tip = cache.summary(r, &tip)?;
158
            Ok(pinned_sets
159
                .iter()
160
                .map(|p| cache.summary(r, p).ok().map(|s| tip.changes.iter().filter(|c| !s.changes.contains(*c)).count()))
161
                .collect::<Vec<_>>())
162
        })
163
        .await?;
164
    for (c, b) in out.iter_mut().zip(behind) {
165
        c.behind = b;
166
    }
167
    Ok(out)
168
}
e7158350 Andy McCall ↶ 169
170
/// When a consumer takes an upstream's moves (§4.2).
171
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
172
#[serde(rename_all = "lowercase")]
173
pub enum Follow {
174
    Off,
175
    /// When the upstream state's checks pass.
176
    Green,
177
    /// On every move.
178
    Any,
179
}
180
181
impl Follow {
182
    pub fn parse(s: &str) -> Option<Follow> {
183
        match s.trim() {
184
            "off" => Some(Follow::Off),
185
            "green" => Some(Follow::Green),
186
            "any" => Some(Follow::Any),
187
            _ => None,
188
        }
189
    }
190
}
191
192
/// How one consumer follows one subrepo, from its `.oxymodules` entry, its
193
/// repository setting and the site's default, in that order.
194
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
195
pub struct Rule {
196
    pub follow: Follow,
197
    /// Land a green bump itself (D5); D1 only proposes.
198
    pub land_auto: bool,
199
    /// The bookmark to bump, when `.oxymodules` names one (else learned).
200
    pub target: Option<String>,
201
    /// Paths the consumer watches: upstream changes touching them are
202
    /// flagged in a bump's changelog.
203
    pub watch: Vec<String>,
204
    /// Where `follow` came from: `.oxymodules`, `setting` or `site`.
205
    pub from: &'static str,
206
}
207
208
/// The Diffusion keys of `path`'s entry in `.oxymodules` text: `follow`,
209
/// `land`, `target` and `watch` (comma-separated, or a TOML list).
210
pub fn module_keys(text: &str, path: &str) -> (Option<Follow>, Option<bool>, Option<String>, Vec<String>) {
211
    let (mut follow, mut land, mut target, mut watch) = (None, None, None, Vec::new());
212
    let mut here = false;
213
    for line in text.lines() {
214
        let line = line.trim();
215
        if line.is_empty() || line.starts_with('#') {
216
            continue;
217
        }
218
        if let Some(inner) = line.strip_prefix('[').and_then(|l| l.strip_suffix(']')) {
219
            here = inner.trim().trim_matches('"').trim_end_matches('/') == path.trim_end_matches('/');
220
            continue;
221
        }
222
        let (true, Some((k, v))) = (here, line.split_once('=')) else { continue };
223
        let v = v.trim();
224
        let unquoted = v.trim_matches('"').trim();
225
        match k.trim() {
226
            "follow" => follow = Follow::parse(unquoted),
227
            "land" => land = Some(unquoted == "auto"),
228
            "target" if !unquoted.is_empty() => target = Some(unquoted.to_string()),
229
            "watch" => {
230
                watch = v
231
                    .trim_start_matches('[')
232
                    .trim_end_matches(']')
233
                    .split(',')
234
                    .map(|w| w.trim().trim_matches('"').trim().to_string())
235
                    .filter(|w| !w.is_empty())
236
                    .collect()
237
            }
238
            _ => {}
239
        }
240
    }
241
    (follow, land, target, watch)
242
}
243
244
/// How `row` follows its subrepo at `path`, given its `.oxymodules` text.
245
pub fn rule(site_default: &str, row: &RepoRow, oxymodules: Option<&str>, path: &str) -> Rule {
246
    let (follow, land, target, watch) = oxymodules.map(|t| module_keys(t, path)).unwrap_or_default();
247
    let (follow, from) = match follow {
248
        Some(f) => (f, ".oxymodules"),
249
        None => match Follow::parse(&row.diffusion) {
250
            Some(f) => (f, "setting"),
251
            None => (Follow::parse(site_default).unwrap_or(Follow::Off), "site"),
252
        },
253
    };
254
    Rule { follow, land_auto: land.unwrap_or(false), target, watch, from }
255
}
256
257
#[cfg(test)]
258
mod tests {
259
    use super::*;
260
261
    const MODULES: &str = r#"
262
[rstar-api]
263
url = https://oxygenhub.io/doomsdayone/rstar-api
264
follow = green
265
land = "propose"
266
target = develop
267
watch = ["src/x16/irq*", "src/x16/frame*"]
268
269
[other]
270
url = https://example/other
271
follow = any
272
"#;
273
274
    fn row(diffusion: &str) -> RepoRow {
275
        RepoRow {
276
            id: 1,
277
            owner: "o".into(),
278
            name: "n".into(),
279
            description: String::new(),
280
            default_bookmark: "main".into(),
281
            visibility: "public".into(),
282
            created_ms: 0,
283
            updated_ms: 0,
284
            git_mirror: false,
285
            git_mirror_state: String::new(),
286
            forked_from: None,
287
            topics: String::new(),
288
            flow: String::new(),
289
            diffusion: diffusion.into(),
290
            ..Default::default()
291
        }
292
    }
293
294
    #[test]
295
    fn oxymodules_keys_are_read_per_entry() {
296
        let (f, land, target, watch) = module_keys(MODULES, "rstar-api");
297
        assert_eq!((f, land, target.as_deref()), (Some(Follow::Green), Some(false), Some("develop")));
298
        assert_eq!(watch, vec!["src/x16/irq*", "src/x16/frame*"]);
299
        assert_eq!(module_keys(MODULES, "other").0, Some(Follow::Any));
300
        assert_eq!(module_keys(MODULES, "missing"), (None, None, None, vec![]));
301
        assert_eq!(module_keys("[a]\nwatch = x, y\n", "a").3, vec!["x", "y"]);
302
    }
303
304
    #[test]
305
    fn oxymodules_wins_then_the_setting_then_the_site() {
306
        let r = rule("off", &row("any"), Some(MODULES), "rstar-api");
307
        assert_eq!((r.follow, r.from), (Follow::Green, ".oxymodules"));
308
        let r = rule("off", &row("any"), Some("[rstar-api]\nurl = x\n"), "rstar-api");
309
        assert_eq!((r.follow, r.from), (Follow::Any, "setting"));
310
        let r = rule("green", &row(""), None, "rstar-api");
311
        assert_eq!((r.follow, r.from), (Follow::Green, "site"));
312
        assert_eq!(rule("nonsense", &row(""), None, "x").follow, Follow::Off);
313
    }
314
}