Asking for help — an agent that consults a more capable model (Agents kit)

Agent.Consults = "<model name>" · Agent.StuckAfter = 2 · Agent.ConfidenceFloor · AgentHelp { From, Why, AskedModel, Question, Context, Answer, Status, CostMicros } · policy AgentsMayConsult

An agent can name one of the app's models to consult. The model doing the work may then ask it one focused question; work that keeps failing asks once before it gives up; and a decision the agent is not sure enough of is asked again. The consulted model gets the question and the context given with it — no conversation, no tools — and the call is charged to the same piece of work. Each request is an `AgentHelp` row and a line of the record.

Summary#

[Actor(Roles = [ShopRole.Helper])]
public class Estimator : Agent {
  public override string Consults => "Senior";            // one of the app's models, by name
  public override int StuckAfter => 2;                    // two failed attempts in a row: ask once, then fail
  public override decimal ConfidenceFloor => 0.8m;        // a decision under this is asked again of Senior
}

Signature#

class Agent { virtual string Consults; virtual int StuckAfter; virtual decimal ConfidenceFloor; }

entity AgentHelp {
  string Agent; AgentTask? Task;
  HelpFrom From;            // Model, Person
  HelpReason Why;           // Chosen (the model asked), Stuck, LowConfidence
  string AskedModel; string Question; string Context; string? Answer;
  HelpStatus Status;        // Waiting, Answered, Refused, Unavailable
  long CostMicros;
}

policy AgentsMayConsult => true;       // an app's answer to "may an agent consult on this caller's account?"
[Event] AgentAskedForHelp

Description#

When does an agent consult?#

Three ways, all of them only when Consults names a model. It chooses to: consult is then one of the working model's tools, taking a question and the context it needs answered against. It is stuck: after StuckAfter failed attempts in a row at one job, the kit asks once on its behalf and hands the advice back; if it fails again the work ends as Failed. It is not sure: a Decide<T> whose confidence is under ConfidenceFloor is asked again of the consulted model, and that answer is the one used.

What is the consulted model given?#

The question and the context that came with it. Not the conversation, not the tools, and no rights: it advises, and the working model decides what to do with the advice.

What does a consult cost, and who pays?#

It is an ordinary metered call: charged to the same piece of work, held to the same limits, on the meter with the purpose consult. It is never stepped down to a cheaper model — a consult that does not fit the budget is refused and the working model is told so. A model that cannot be reached gives no help, and the work goes on without it. Leave room for it: a call reserves what it could cost before it leaves, and a stronger model's reservation can be larger than a small job's whole limit.

How do I switch it off, or see that it happened?#

Leave Consults unset and nothing of an agent's work is ever sent to another model. An app that names one can still answer AgentsMayConsult per caller. Every request — answered, refused or unavailable — is an AgentHelp row and an AskedForHelp line on the record, and raises AgentAskedForHelp.

Examples#

app Workshop {
  use Osysharp.Agents;
  use Osysharp.Agents.Testing;
  use Osysharp.Llm;
  use Osysharp.Identity;
  use Osysharp.Observability;
  model "model/**/*.osy";
}

using Osysharp.Agents;
using Osysharp.Agents.Testing;
using Osysharp.Identity;
using Osysharp.Llm;

app.Llm = new LlmSetup {
  Models = [
    new LlmModel { Name = "Everyday", Client = new ScriptedModel { Name = "Everyday" }, Model = "scripted-everyday",
                   Price = new LlmPrice { Input = 1, Output = 5 }, MaxOutputTokens = 1000 },
    new LlmModel { Name = "Senior",   Client = new ScriptedModel { Name = "Senior" },   Model = "scripted-senior",
                   Price = new LlmPrice { Input = 10, Output = 50 }, MaxOutputTokens = 1000 },
  ],
  Default = "Everyday",
};
app.Agents = new AgentsSetup { Agents = [new Estimator()], TaskBudget = 0.05m };

[Role] enum ShopRole { Authenticator, Mechanic, Manager, Helper }
partial entity RoleGrant { security { allow read when IsAuthenticated; } }
policy IsMechanic => RoleGrant.Any(g => g.Member == user && g.Role == ShopRole.Mechanic);
policy IsManager  => RoleGrant.Any(g => g.Member == user && g.Role == ShopRole.Manager);
policy IsHelper   => RoleGrant.Any(g => g.Member == user && g.Role == ShopRole.Helper);

// The kit's questions, answered by this app.
policy UsesAgents       => IsMechanic || IsManager;   // gives agents work, reads the work they asked for
policy SupervisesAgents => IsManager;                 // names sponsors, pauses and stops, reads that work happened
policy AuditsAgents     => IsManager;                 // reads everything, content included
policy ReadsModelUsage   => IsManager;
policy ManagesModelSpend => IsManager;

entity Person : Principal {
  security { allow read when IsAuthenticated || IsAuthenticator; allow create when IsAuthenticator; }
}

/// One bike in for repair.
entity Repair {
  [Required, Unique, MaxLength(40)] string Ticket;
  [Required, MaxLength(200)] string Fault;
  [MaxLength(400)] string Note = "";
  /// What the customer is charged. Only a mechanic or the manager sets it.
  decimal Price;
  security {
    allow read, update when IsMechanic || IsManager || IsHelper;
    allow create when IsMechanic || IsManager;
    deny update Price when IsHelper;
  }
}

/// What an estimate says.
public class Estimate {
  public decimal Hours;
  public string Summary = "";
  public bool NeedsParts;
}

/// The parts a fault needs.
[Action("Look up the parts a fault needs", Query)]
public List<string> PartsFor(string fault) { return fault.Contains("brake") ? ["brake pads", "cable"] : []; }

/// Write a note on the repair's ticket.
[Action("Write a note on the repair")]
public string NoteRepair(Repair repair, string note) { repair.Note = note; return $"Noted on {repair.Ticket}."; }

/// Set what the customer is charged.
[Action("Set the price")]
public string SetPrice(Repair repair, decimal price) { repair.Price = price; return $"{repair.Ticket} costs {price}."; }

/// Works out how long one repair takes.
public class EstimateRepair : Skill<Repair, Estimate> {
  public override string Description => "Estimate one repair.";
  public override string Instructions => "Look up the parts first. Write the estimate as a note on the repair.";
  public override List<ActionInfo> Tools => [Actions.Of(PartsFor), Actions.Of(NoteRepair), Actions.Of(SetPrice)];
  public override string Brief(Repair r) => $"Ticket {r.Ticket}: {r.Fault}. Its id is {r.Id}.";
  public override string? Check(Estimate e) => e.Hours <= 0 ? "An estimate takes some time." : null;
}

/// Estimates the repairs that come in. It holds the Helper role and nothing more.
[Actor(Roles = [ShopRole.Helper])]
public class Estimator : Agent {
  public static Person Principal => new Person { Email = "[email protected]", DisplayName = "Estimator" };
  public override string Purpose => "You estimate bicycle repairs for a small workshop.";
  public override List<SkillBase> Skills => [new EstimateRepair()];
  public override Permits May => Permits.ChangesAlone();        // it notes and prices by itself
  public override string Consults => "Senior";
  // Room for the advice: a call to Senior reserves up to $0.05 before it leaves.
  public override AgentLimits Limits => new AgentLimits { PerTask = 0.20m };
}

[TestFixture]
void Shop() {
  var mia = new Person { Email = "[email protected]", DisplayName = "Mia" };
  var noor = new Person { Email = "[email protected]", DisplayName = "Noor" };
  new RoleGrant { Member = mia, Role = ShopRole.Mechanic };
  new RoleGrant { Member = noor, Role = ShopRole.Manager };
  new Repair { Ticket = "T-7", Fault = "rear brake does not bite" };
  new AgentSeat { Agent = "Estimator", Sponsor = noor };          // the manager answers for the agent
}
principal Mia  => Person.Single(p => p.Email == "[email protected]");
principal Noor => Person.Single(p => p.Email == "[email protected]");

string RepairArgs(Repair r, string rest) { return "{\"repair\":\"" + r.Id.ToString() + "\"" + rest + "}"; }

[Test(Shop)]
[runas(Mia)]
void The_working_model_asks_a_focused_question_and_is_given_the_advice() {
  var repair = Repair.Single(r => r.Ticket == "T-7");
  var script = AgentScript.Of(
      Says.Consult("Does a brake that does not bite need new pads or a new cable?", "Rear rim brake, eight years old."),
      Says.HandBack(new Estimate { Hours = 1, Summary = "Pads first; check the cable.", NeedsParts = true }))
    .Advises("Senior", "Pads first. Replace the cable only if it is frayed.");
  ScriptedAnswer.Stub((model, request) => script.Answer(model, request));

  var outcome = await Agents.Of<Estimator>().Do(new EstimateRepair(), repair);

  Assert.True(outcome.Done);
  Assert.Equal(["Everyday", "Senior", "Everyday"], Trajectory.Models(outcome.Task));
  var help = AgentHelp.Single();
  Assert.Equal(HelpReason.Chosen, help.Why);
  Assert.Equal("Senior", help.AskedModel);
  Assert.Equal(HelpStatus.Answered, help.Status);
  Assert.Equal("Pads first. Replace the cable only if it is frayed.", help.Answer);
  // The advice is part of what this piece of work cost.
  Assert.Equal(outcome.CostMicros, AgentTask.Single().SpentMicros);
  Assert.True(help.CostMicros > 0 && help.CostMicros < outcome.CostMicros);
}

[Test(Shop)]
[runas(Mia)]
void Work_that_is_stuck_consults_once_and_goes_on() {
  var repair = Repair.Single(r => r.Ticket == "T-7");
  var script = AgentScript.Of(
      Says.HandBack(new Estimate { Hours = 0 }),
      Says.HandBack(new Estimate { Hours = -1 }),
      Says.HandBack(new Estimate { Hours = 1, Summary = "One hour." }))
    .Advises("Senior", "An estimate is a positive number of hours. One hour suits a brake.");
  ScriptedAnswer.Stub((model, request) => script.Answer(model, request));

  var outcome = await Agents.Of<Estimator>().Do(new EstimateRepair(), repair);

  Assert.True(outcome.Done);
  Assert.Equal(1, outcome.Result.Hours);
  Assert.Equal(1, Trajectory.Helps(outcome.Task));
  Assert.Equal(HelpReason.Stuck, AgentHelp.Single().Why);
}