projectatlas_core/
toon.rs

1//! Purpose: Render `ProjectAtlas` responses with the TOON standard encoder.
2
3use crate::health::{HealthFinding, Severity};
4use crate::outline::FileOutline;
5use crate::symbols::{CodeSymbol, SymbolRelation};
6use crate::telemetry::{TokenOverview, TokenTrendReport};
7use crate::{IndexedNode, Overview, RankedNode};
8use serde::Serialize;
9use serde_json::{Value, json};
10
11/// Render a repository overview as standard TOON.
12#[must_use]
13pub fn render_overview(overview: &Overview) -> String {
14    encode_agent_payload(&json!({ "overview": overview }))
15}
16
17/// Build the agent-facing folder/file row projection used by TOON and JSON.
18#[must_use]
19pub fn render_node_rows(label: &str, nodes: &[IndexedNode]) -> Vec<serde_json::Value> {
20    nodes
21        .iter()
22        .map(|node| {
23            let purpose = node.purpose.purpose.as_deref().unwrap_or("");
24            let content_summary = node.summary.as_deref().unwrap_or("");
25            match label {
26                "folders" => json!({
27                    "path": node.node.path,
28                    "kind": node.node.kind.to_string(),
29                    "folder_purpose": purpose,
30                    "content_summary": content_summary,
31                    "status": node.purpose.status.to_string(),
32                    "purpose_source": node.purpose.source,
33                    "purpose_agent_reviewed": node.purpose.agent_reviewed(),
34                }),
35                "files" => json!({
36                    "path": node.node.path,
37                    "kind": node.node.kind.to_string(),
38                    "language": node.node.language.as_deref().unwrap_or(""),
39                    "file_purpose": purpose,
40                    "content_summary": content_summary,
41                    "status": node.purpose.status.to_string(),
42                    "purpose_source": node.purpose.source,
43                    "purpose_agent_reviewed": node.purpose.agent_reviewed(),
44                }),
45                _ => json!({
46                    "path": node.node.path,
47                    "kind": node.node.kind.to_string(),
48                    "purpose": purpose,
49                    "content_summary": content_summary,
50                    "status": node.purpose.status.to_string(),
51                    "purpose_source": node.purpose.source,
52                    "purpose_agent_reviewed": node.purpose.agent_reviewed(),
53                }),
54            }
55        })
56        .collect()
57}
58
59/// Build the agent-facing ranked row projection with bounded reasons.
60#[must_use]
61pub fn render_ranked_node_rows(label: &str, nodes: &[RankedNode]) -> Vec<serde_json::Value> {
62    nodes
63        .iter()
64        .map(|ranked| {
65            let mut row = render_node_rows(label, std::slice::from_ref(&ranked.node))
66                .into_iter()
67                .next()
68                .unwrap_or_else(|| json!({}));
69            if let Some(object) = row.as_object_mut() {
70                object.insert("reasons".to_string(), json!(ranked.reasons));
71                object.insert("reason_codes".to_string(), json!(ranked.reason_codes));
72                object.insert(
73                    "connection_counts".to_string(),
74                    json!(ranked.connection_counts),
75                );
76                object.insert("connections".to_string(), json!(ranked.connections));
77                object.insert(
78                    "connections_truncated".to_string(),
79                    json!(ranked.connections_truncated),
80                );
81                object.insert("next_call".to_string(), json!(ranked.next_call));
82            }
83            row
84        })
85        .collect()
86}
87
88/// Render indexed nodes as standard TOON.
89#[must_use]
90pub fn render_nodes(label: &str, nodes: &[IndexedNode]) -> String {
91    let rows = render_node_rows(label, nodes);
92    encode_agent_payload(&json!({ label: rows }))
93}
94
95/// Render ranked indexed nodes as standard TOON.
96#[must_use]
97pub fn render_ranked_nodes(label: &str, nodes: &[RankedNode]) -> String {
98    let rows = render_ranked_node_rows(label, nodes);
99    encode_agent_payload(&json!({ label: rows }))
100}
101
102/// Render an outline as standard TOON.
103#[must_use]
104pub fn render_outline(outline: &FileOutline) -> String {
105    encode_agent_payload(&json!({ "outline": outline }))
106}
107
108/// Render health findings as standard TOON.
109#[must_use]
110pub fn render_health(findings: &[HealthFinding]) -> String {
111    let rows = findings
112        .iter()
113        .map(|finding| {
114            json!({
115                "severity": render_severity(finding.severity),
116                "id": finding.id,
117                "category": finding.category,
118                "path": finding.path,
119                "related_path": finding.related_path.as_deref().unwrap_or(""),
120                "message": finding.message,
121                "recommendation": finding.recommendation,
122            })
123        })
124        .collect::<Vec<_>>();
125    encode_agent_payload(&json!({ "health_findings": rows }))
126}
127
128/// Render token savings overview as standard TOON.
129#[must_use]
130pub fn render_token_overview(overview: &TokenOverview) -> String {
131    let savings_rate = percentage_label(overview.savings_rate);
132    let buckets = overview
133        .buckets
134        .iter()
135        .map(|bucket| {
136            json!({
137                "token_savings_bucket": bucket.token_savings_bucket,
138                "provider": bucket.provider,
139                "model": bucket.model,
140                "tokenizer_backend": bucket.tokenizer_backend,
141                "accuracy": bucket.accuracy,
142                "baseline_kind": bucket.baseline_kind,
143                "confidence": bucket.confidence,
144                "accounting_layer": bucket.accounting_layer,
145                "estimate_method": bucket.estimate_method,
146                "denominator_kind": bucket.denominator_kind,
147                "dedupe_scope": bucket.dedupe_scope,
148                "calls": bucket.calls,
149                "baseline_tokens": bucket.estimated_without_projectatlas,
150                "emitted_tokens": bucket.estimated_with_projectatlas,
151                "saved_tokens": bucket.estimated_saved,
152                "savings_rate": percentage_label(bucket.savings_rate),
153            })
154        })
155        .collect::<Vec<_>>();
156    encode_agent_payload(&json!({
157        "token_savings": {
158            "estimate_kind": overview.estimate_kind,
159            "estimator": overview.estimator,
160            "estimate_scope": overview.estimate_scope,
161            "detail_availability": overview.detail_availability,
162            "calls": overview.calls,
163            "estimated_without_projectatlas": overview.estimated_without_projectatlas,
164            "estimated_with_projectatlas": overview.estimated_with_projectatlas,
165            "estimated_saved": overview.estimated_saved,
166            "legacy_gross_estimated_saved": overview.legacy_gross_estimated_saved,
167            "measured_tokens_saved": overview.measured_tokens_saved,
168            "gross_modeled_tokens_avoided": overview.gross_modeled_tokens_avoided,
169            "deduped_modeled_tokens_avoided": overview.deduped_modeled_tokens_avoided,
170            "average_modeled_tokens_avoided": overview.average_modeled_tokens_avoided,
171            "average_tokens_avoided": overview.average_tokens_avoided,
172            "maximum_tokens_avoided": overview.maximum_tokens_avoided,
173            "tokens_avoided": overview.tokens_avoided,
174            "average_policy": {
175                "directory_walk_baseline_percent": overview.average_policy.directory_walk_baseline_percent,
176                "atlas_payload_percent": overview.average_policy.atlas_payload_percent,
177                "evidence": overview.average_policy.evidence.as_str(),
178            },
179            "repeated_baselines_deduped": overview.repeated_baselines_deduped,
180            "likely_file_reads_avoided": overview.likely_file_reads_avoided,
181            "read_avoidance": {
182                "likely_file_reads_avoided": overview.likely_file_reads_avoided,
183                "observed_file_read_replacements": overview.observed_file_read_replacements,
184                "modeled_file_reads_avoided": overview.modeled_file_reads_avoided,
185                "scope": overview.read_avoidance_scope,
186                "confidence": overview.read_avoidance_confidence,
187                "plain_language": "ProjectAtlas summaries, search results, and slices were used instead of opening likely whole files.",
188            },
189            "agent_efficiency": overview.agent_efficiency,
190            "calibration": overview.calibration,
191            "savings_rate": savings_rate,
192            "totals": {
193                "baseline_tokens": overview.estimated_without_projectatlas,
194                "emitted_tokens": overview.estimated_with_projectatlas,
195                "saved_tokens": overview.estimated_saved,
196                "legacy_gross_saved_tokens": overview.legacy_gross_estimated_saved,
197                "measured_saved_tokens": overview.measured_tokens_saved,
198                "gross_modeled_avoided_tokens": overview.gross_modeled_tokens_avoided,
199                "deduped_modeled_avoided_tokens": overview.deduped_modeled_tokens_avoided,
200                "average_modeled_avoided_tokens": overview.average_modeled_tokens_avoided,
201                "average_tokens_avoided": overview.average_tokens_avoided,
202                "maximum_tokens_avoided": overview.maximum_tokens_avoided,
203                "tokens_avoided": overview.tokens_avoided,
204                "likely_file_reads_avoided": overview.likely_file_reads_avoided,
205                "savings_rate": savings_rate,
206            },
207            "buckets": buckets,
208        }
209    }))
210}
211
212/// Render token savings trends as standard TOON.
213#[must_use]
214pub fn render_token_trends(report: &TokenTrendReport) -> String {
215    let periods = report
216        .periods
217        .iter()
218        .map(|period| {
219            let buckets = period
220                .buckets
221                .iter()
222                .map(|bucket| {
223                    json!({
224                        "token_savings_bucket": bucket.token_savings_bucket,
225                        "provider": bucket.provider,
226                        "model": bucket.model,
227                        "tokenizer_backend": bucket.tokenizer_backend,
228                        "accuracy": bucket.accuracy,
229                        "baseline_kind": bucket.baseline_kind,
230                        "confidence": bucket.confidence,
231                        "accounting_layer": bucket.accounting_layer,
232                        "estimate_method": bucket.estimate_method,
233                        "denominator_kind": bucket.denominator_kind,
234                        "dedupe_scope": bucket.dedupe_scope,
235                        "calls": bucket.calls,
236                        "baseline_tokens": bucket.estimated_without_projectatlas,
237                        "emitted_tokens": bucket.estimated_with_projectatlas,
238                        "saved_tokens": bucket.estimated_saved,
239                        "savings_rate": percentage_label(bucket.savings_rate),
240                    })
241                })
242                .collect::<Vec<_>>();
243            json!({
244                "period": period.period,
245                "calls": period.calls,
246                "baseline_tokens": period.estimated_without_projectatlas,
247                "emitted_tokens": period.estimated_with_projectatlas,
248                "saved_tokens": period.estimated_saved,
249                "savings_rate": percentage_label(period.savings_rate),
250                "buckets": buckets,
251            })
252        })
253        .collect::<Vec<_>>();
254    encode_agent_payload(&json!({
255        "token_trends": {
256            "estimate_kind": report.estimate_kind,
257            "estimator": report.estimator,
258            "estimate_scope": report.estimate_scope,
259            "session": report.session.as_deref().unwrap_or("all sessions"),
260            "window": report.window,
261            "detail_availability": report.detail_availability,
262            "periods": periods,
263        }
264    }))
265}
266
267/// Render symbols as standard TOON.
268#[must_use]
269pub fn render_symbols(symbols: &[CodeSymbol]) -> String {
270    encode_agent_payload(&json!({ "symbols": render_symbol_rows(symbols) }))
271}
272
273/// Project symbols into the stable agent-facing row shape.
274#[must_use]
275pub fn render_symbol_rows(symbols: &[CodeSymbol]) -> Vec<Value> {
276    symbols
277        .iter()
278        .map(|symbol| {
279            let mut row = json!({
280                "path": symbol.path,
281                "kind": symbol.kind.to_string(),
282                "name": symbol.name,
283                "start": symbol.line_start,
284                "end": symbol.line_end,
285                "parent": symbol.parent.as_deref().unwrap_or(""),
286                "parser": symbol.parser.to_string(),
287                "signature": symbol.signature,
288                "exported": symbol.exported,
289                "documentation": symbol.documentation.as_deref().unwrap_or(""),
290            });
291            if let (Some(object), Some(selector)) = (row.as_object_mut(), symbol.source_selector) {
292                object.insert("source_selector".to_string(), json!(selector));
293            }
294            row
295        })
296        .collect()
297}
298
299/// Render symbol relations as standard TOON.
300#[must_use]
301pub fn render_symbol_relations(relations: &[SymbolRelation]) -> String {
302    let rows = relations
303        .iter()
304        .map(|relation| {
305            json!({
306                "path": relation.path,
307                "kind": relation.kind.to_string(),
308                "source": relation.source_name,
309                "target": relation.target_name,
310                "line": relation.line,
311                "parser": relation.parser.to_string(),
312                "context": relation.context,
313            })
314        })
315        .collect::<Vec<_>>();
316    encode_agent_payload(&json!({ "symbol_relations": rows }))
317}
318
319/// Encode a serializable payload through the standard TOON Rust implementation.
320#[must_use]
321pub fn encode_agent_payload<T>(payload: &T) -> String
322where
323    T: Serialize,
324{
325    match toon_format::encode_default(payload) {
326        Ok(mut encoded) => {
327            encoded.push('\n');
328            encoded
329        }
330        Err(error) => format!("toon_error: {}\n", encode_error_text(&error.to_string())),
331    }
332}
333
334/// Encode one string using TOON by wrapping it in an object and extracting text.
335#[must_use]
336pub fn encode_error_text(value: &str) -> String {
337    match toon_format::encode_default(&json!({ "value": value })) {
338        Ok(encoded) => encoded
339            .strip_prefix("value: ")
340            .map_or_else(|| quoted_fallback(value), ToString::to_string),
341        Err(_) => quoted_fallback(value),
342    }
343}
344
345/// Render a severity enum as a stable TOON value.
346fn render_severity(severity: Severity) -> &'static str {
347    severity.as_str()
348}
349
350/// Format an optional savings rate as a stable display label.
351fn percentage_label(rate: Option<f64>) -> String {
352    rate.map_or_else(
353        || "unknown".to_string(),
354        |value| format!("{:.1}%", value * 100.0),
355    )
356}
357
358/// Return a conservative quoted fallback for rare encoder failures.
359fn quoted_fallback(value: &str) -> String {
360    let escaped = value
361        .replace('\\', "\\\\")
362        .replace('"', "\\\"")
363        .replace('\n', "\\n")
364        .replace('\r', "\\r")
365        .replace('\t', "\\t");
366    format!("\"{escaped}\"")
367}
368
369#[cfg(test)]
370mod tests {
371    use super::{encode_agent_payload, render_symbols, render_token_overview, render_token_trends};
372    use crate::symbols::{CodeSymbol, ParserKind, SymbolKind};
373    use crate::telemetry::{
374        TOKEN_BASELINE_DIRECTORY_WALK, TokenOverview, TokenTrendReport, TokenTrendWindow,
375        UsageDetailAvailability, usage_from_estimates, usage_from_text,
376    };
377    use serde_json::{Value, json};
378
379    #[test]
380    fn renders_round_trippable_toon_with_standard_decoder() -> Result<(), Box<dyn std::error::Error>>
381    {
382        let toon = encode_agent_payload(&serde_json::json!({
383            "items": [
384                {"path": "src/lib.rs", "text": "alpha,beta"},
385                {"path": "src/main.rs", "text": "line\nbreak"}
386            ]
387        }));
388        let decoded: Value = toon_format::decode_default(&toon)?;
389        if decoded["items"][0]["text"] != "alpha,beta" {
390            return Err("first decoded item did not round-trip".into());
391        }
392        if decoded["items"][1]["text"] != "line\nbreak" {
393            return Err("second decoded item did not round-trip".into());
394        }
395        Ok(())
396    }
397
398    #[test]
399    fn renders_symbols_as_tabular_toon() {
400        let toon = render_symbols(&[CodeSymbol {
401            path: "src/lib.rs".to_string(),
402            language: Some("rust".to_string()),
403            name: "scan".to_string(),
404            kind: SymbolKind::Function,
405            signature: "fn scan()".to_string(),
406            exported: false,
407            documentation: None,
408            line_start: 1,
409            line_end: 3,
410            source_selector: None,
411            parent: None,
412            parser: ParserKind::TreeSitter,
413            detail: Some("function_item".to_string()),
414        }]);
415        assert!(toon.contains(
416            "symbols[1]{path,kind,name,start,end,parent,parser,signature,exported,documentation}:"
417        ));
418    }
419
420    #[test]
421    fn renders_token_overview_with_read_avoidance_section() -> Result<(), Box<dyn std::error::Error>>
422    {
423        let mut folder = usage_from_estimates("s", "folders", None, None, 101, 20);
424        folder.denominator_kind = TOKEN_BASELINE_DIRECTORY_WALK.to_string();
425        let overview = TokenOverview::from_events(&[
426            usage_from_text("s", "summary", None, None, "abcdefghijkl", "abcd"),
427            usage_from_estimates("s", "search", None, None, 100, 20),
428            folder,
429        ]);
430        let toon = render_token_overview(&overview);
431        let decoded: Value = toon_format::decode_default(&toon)?;
432        let token_savings = &decoded["token_savings"];
433
434        require_json_eq(
435            &token_savings["average_tokens_avoided"],
436            &json!(overview.average_tokens_avoided),
437            "average tokens avoided",
438        )?;
439        require_json_eq(
440            &token_savings["maximum_tokens_avoided"],
441            &json!(overview.maximum_tokens_avoided),
442            "maximum tokens avoided",
443        )?;
444        require_json_eq(
445            &token_savings["tokens_avoided"],
446            &token_savings["average_tokens_avoided"],
447            "primary average compatibility alias",
448        )?;
449        require_json_eq(
450            &token_savings["average_policy"]["directory_walk_baseline_percent"],
451            &json!(50),
452            "average directory-walk policy",
453        )?;
454
455        require_json_eq(
456            &token_savings["likely_file_reads_avoided"],
457            &json!(2),
458            "top-level read avoidance",
459        )?;
460        require_json_eq(
461            &token_savings["read_avoidance"]["likely_file_reads_avoided"],
462            &json!(2),
463            "section read avoidance",
464        )?;
465        require_json_eq(
466            &token_savings["read_avoidance"]["observed_file_read_replacements"],
467            &json!(1),
468            "observed read replacements",
469        )?;
470        require_json_eq(
471            &token_savings["read_avoidance"]["modeled_file_reads_avoided"],
472            &json!(1),
473            "modeled read avoidance",
474        )?;
475        require_json_eq(
476            &token_savings["totals"]["likely_file_reads_avoided"],
477            &json!(2),
478            "total read avoidance",
479        )?;
480        require_json_eq(
481            &token_savings["read_avoidance"]["plain_language"],
482            &json!(
483                "ProjectAtlas summaries, search results, and slices were used instead of opening likely whole files."
484            ),
485            "plain language read avoidance",
486        )?;
487        Ok(())
488    }
489
490    #[test]
491    fn token_toon_reports_retained_detail_truth_for_overview_and_trends()
492    -> Result<(), Box<dyn std::error::Error>> {
493        let mut overview = TokenOverview::from_events(&[]);
494        overview.set_detail_availability(UsageDetailAvailability::Partial);
495        let overview_toon = render_token_overview(&overview);
496        let decoded_overview: Value = toon_format::decode_default(&overview_toon)?;
497        require_json_eq(
498            &decoded_overview["token_savings"]["detail_availability"],
499            &json!("partial"),
500            "overview detail availability",
501        )?;
502
503        let mut trends = TokenTrendReport::new(None, TokenTrendWindow::Month, Vec::new());
504        trends.set_detail_availability(UsageDetailAvailability::Expired);
505        let trends_toon = render_token_trends(&trends);
506        let decoded_trends: Value = toon_format::decode_default(&trends_toon)?;
507        require_json_eq(
508            &decoded_trends["token_trends"]["detail_availability"],
509            &json!("expired"),
510            "trend detail availability",
511        )?;
512        Ok(())
513    }
514
515    fn require_json_eq(
516        actual: &Value,
517        expected: &Value,
518        label: &str,
519    ) -> Result<(), Box<dyn std::error::Error>> {
520        if actual == expected {
521            Ok(())
522        } else {
523            Err(format!("{label}: expected {expected}, got {actual}").into())
524        }
525    }
526}
projectatlas_core/toon.rs

projectatlas_core/
toon.rs
