Assistants: what to do when a proposal is too much for the model c08cec372c81

    fn a_refusal_for_length_is_told_from_others() {

        for e in [

            "400 This model's maximum context length is 8192 tokens. Please reduce the length of the messages.",

            "400 prompt is too long: 210000 tokens > 200000 maximum",

            "413 Request too large",

            "400 input length exceeds the context window",

        ] {

            assert!(super::too_long(e), "{e}");

        }

        for e in ["401 invalid api key", "500 too long a queue", "no answer within 90 seconds", "400 model not found"] {

            assert!(!super::too_long(e), "{e}");

        }

    }

    #[test]

    // Every file with its size first, so a model told only part of the

    // diff still knows the whole (oxygenhub#248).

    text.push_str(&format!("\n## Files ({})\n\n", diffs.len()));

    let mut parts: Vec<(String, String)> = Vec::with_capacity(diffs.len());

        let (added, removed) = f.hunks.iter().fold((0, 0), |(a, r), h| (a + h.added.len(), r + h.removed.len()));

        text.push_str(&format!("- {} ({}, +{added} -{removed})\n", f.path, f.kind));

        let mut diff = format!("### {} ({})\n", f.path, f.kind);

            parts.push((f.path.clone(), diff));

        parts.push((f.path.clone(), diff));

    text.push_str("\n## Diff\n\n");

    // Whole files, in order, while they fit. One that doesn't is cut where

    // the room ends if there is a fair amount left, else left out (a

    // smaller one after it may still go in); either way it is named.

    let head = text.chars().count();

    let mut room = budget.saturating_sub(head + 200);

    let mut left_out = Vec::new();

    for (path, diff) in parts {

        let n = diff.chars().count();

        if n <= room {

            text.push_str(&diff);

            room -= n;

        } else if room >= 1_500 {

            let keep = room - 80;

            text.extend(diff.chars().take(keep));

            text.push_str("\n(the rest of this file was cut to fit)\n```\n\n");

            left_out.push(format!("{path} (cut)"));

            room = 0;

        } else {

            left_out.push(path);

        }

    }

    if !left_out.is_empty() {

        let names: String = left_out.join(", ").chars().take(room.max(400)).collect();

        text.push_str(&format!("(left out to fit the context budget, {} file(s): {names})\n", left_out.len()));

    ask_within(app, a, &system, budget, BACKGROUND_TIMEOUT, |room| review_context(row, p, &loaded, room)).await

}

/// Ask with the context cut to fit `budget` characters, system prompt

/// included (oxygenhub#248). A provider that says the prompt is too long

/// for its model is asked once more with a third of the room; whatever

/// fails is said plainly, with what to do about it.

async fn ask_within(

    app: &Shared,

    a: &Assistant,

    system: &str,

    budget: usize,

    wait: Duration,

    context: impl Fn(usize) -> String,

) -> Result<String, String> {

    let room = |b: usize| b.saturating_sub(system.chars().count());

    match ask(app, a, system, &context(room(budget)), wait).await {

        Err(e) if too_long(&e) => {

            let smaller = (budget / 3).max(1_000);

            tracing::info!("assistant {}: too long at {budget} characters ({e}); asking again at {smaller}", a.name);

            ask(app, a, system, &context(room(smaller)), wait).await.map_err(|e2| {

                format!(

                    "the proposal was too large for {}'s model at {budget} characters and again at {smaller} ({e2}): lower this assistant's context budget",

                    a.name

                )

            })

        }

        Err(e) if e.starts_with("no answer within") => Err(format!(

            "{e}: a large proposal on a slow or local model can take longer; lower this assistant's context budget, or use a faster model"

        )),

        other => other,

    }

}

/// Whether a provider's refusal says the prompt is too long for its model.

fn too_long(e: &str) -> bool {

    let l = e.to_lowercase();

    (l.starts_with("400") || l.starts_with("413") || l.starts_with("422"))

        && [

            "too long",

            "too large",

            "context length",

            "context window",

            "maximum context",

            "too many tokens",

            "exceeds",

            "reduce the length",

            "input length",

        ]

        .iter()

        .any(|k| l.contains(k))

    ask_within(app, a, &system, budget, BACKGROUND_TIMEOUT, |room| review_context(row, p, &loaded, room)).await

            ask_within(app, a, &system, budget, CALL_TIMEOUT, |room| draft_context(row, title, target, changes, diffs, room)).await

/// A stand-in that refuses a prompt longer than `limit` bytes as a provider

/// does when it overflows the model, and keeps each call's length.

async fn mock_limited(limit: usize) -> (String, std::sync::Arc<std::sync::Mutex<Vec<usize>>>) {

    use std::sync::{Arc, Mutex};

    let seen: Arc<Mutex<Vec<usize>>> = Arc::new(Mutex::new(vec![]));

    let s2 = seen.clone();

    let router = axum::Router::new().route(

        "/v1/chat/completions",

        axum::routing::post(move |body: String| {

            let s2 = s2.clone();

            async move {

                s2.lock().unwrap().push(body.len());

                if body.len() > limit {

                    let msg = format!("This model's maximum context length is {limit} tokens. Please reduce the length of the messages.");

                    return (StatusCode::BAD_REQUEST, axum::Json(serde_json::json!({ "error": { "message": msg } })));

                }

                (

                    StatusCode::OK,

                    axum::Json(serde_json::json!({ "choices": [{ "message": { "role": "assistant", "content": "Small draft." } }] })),

                )

            }

        }),

    );

    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();

    let port = listener.local_addr().unwrap().port();

    tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });

    (format!("http://127.0.0.1:{port}/v1/chat/completions"), seen)

}

/// A proposal too large for the model (oxygenhub#248): the provider refuses

/// it, the hub asks again with a third of the room, and the answer comes.

#[tokio::test(flavor = "multi_thread")]

async fn a_proposal_too_large_for_the_model_is_asked_again_smaller() {

    let h = hub().await;

    let app = &h.app;

    let andy = register(app, "andy").await;

    let repo = RepoName::parse("andy/demo").unwrap();

    app.repos.create(&repo, None).unwrap();

    db::repos::create(&app.db, "andy", "demo", "", hub_api::Visibility::Public).await.unwrap();

    let work = tempfile::tempdir().unwrap();

    let mut client = Repo::init(work.path()).unwrap();

    client.create_identity("Andy McCall", "[email protected]", oxy_core::repo::IdentityScope::User).unwrap();

    client.config.set("remote.origin.url", &format!("{}/andy/demo", h.base)).unwrap();

    client.config.set("remote.origin.token", "site").unwrap();

    client.save_config().unwrap();

    write(work.path(), "a.txt", b"one\n");

    client.snapshot("t").unwrap();

    let base = client.commit("base", &[]).unwrap();

    assert!(client.push("origin", false).unwrap().iter().all(|r| r.ok));

    client.bookmark_create("feature", Some(&[base])).unwrap();

    client.bookmark_switch("feature", false).unwrap();

    // A big change: 200 lines of 60 characters, about 12 KB of diff.

    let big: String = (0..200).map(|i| format!("{i:04} {}\n", "x".repeat(55))).collect();

    write(work.path(), "big.txt", big.as_bytes());

    client.snapshot("t").unwrap();

    client.commit("a big file", &[]).unwrap();

    assert!(client.push_bookmark("origin", "feature", false).unwrap().iter().all(|r| r.ok));

    let (endpoint, seen) = mock_limited(6_000).await;

    let admin = get(app, "/admin", Some(&andy)).await.body;

    let ep = endpoint.replace(':', "%3A").replace('/', "%2F");

    let r = post(

        app,

        "/admin/assistants",

        Some(&andy),

        &format!(

            "_csrf={}&action=add&name=Small&endpoint={ep}&api=chat-completions&model=small-1&token=sk-test&enabled=1&budget=12000",

            csrf_of(&admin)

        ),

    )

    .await;

    assert!(r.body.contains("Small added and answered"), "{}", r.body);

    seen.lock().unwrap().clear();

    let page = get(app, "/andy/demo/proposals/new?source=feature&target=main", Some(&andy)).await.body;

    let r = post(

        app,

        "/andy/demo/proposals/new/draft",

        Some(&andy),

        &format!("_csrf={}&title=Big&body=&target=main&source=feature&assistant=Small", csrf_of(&page)),

    )

    .await;

    assert_eq!(r.status, StatusCode::OK, "{}", r.body);

    assert!(r.body.contains("Small draft."), "{}", r.body);

    let calls = seen.lock().unwrap().clone();

    assert_eq!(calls.len(), 2, "refused once, then asked smaller: {calls:?}");

    assert!(calls[0] > 6_000 && calls[1] <= 6_000, "{calls:?}");

}

- **Too much for the model** (oxygenhub#248). The context lists every changed file with its +/− counts before the diff, then gives whole files in order while they fit: one that doesn't is cut where the room ends (marked) when a fair amount is left, else left out, and every file cut or left out is named, so the model knows what it did not see. A provider that refuses the prompt as too long for its model (`400`/`413`/`422` saying so) is asked once more with a third of the budget; if that fails too, the event says the proposal was too large at both sizes and to lower the assistant's budget. A timeout says the same about slow or local models. A local Ollama cuts a prompt longer than its context window without saying so: set its context (`OLLAMA_CONTEXT_LENGTH`, or the model's `num_ctx`) to fit the assistant's budget, about four characters a token.