projectatlas_db/
content_classification.rs

1//! Persist and batch file content classifications inside index publication.
2
3use super::{AtlasStore, DbError, DbResult, IndexPublicationGuard, numbered_placeholders};
4use projectatlas_core::Node;
5use projectatlas_core::language::ContentClassification;
6use rusqlite::types::Value;
7use rusqlite::{Connection, OptionalExtension, params, params_from_iter};
8use std::collections::{BTreeMap, BTreeSet};
9
10/// Maximum exact paths admitted to one classification batch.
11pub const MAX_FILE_CONTENT_CLASSIFICATION_PATHS: usize = 256;
12/// Maximum rows returned by one classification/path page.
13pub const MAX_FILE_CONTENT_CLASSIFICATION_PAGE_ROWS: u32 = 1_000;
14/// Paths bound per statement, below supported `SQLite` variable ceilings.
15const FILE_CONTENT_CLASSIFICATION_BIND_PATHS: usize = 48;
16
17/// One persisted classification for an exact admitted file path.
18#[derive(Clone, Debug, Eq, PartialEq)]
19pub struct FileContentClassification {
20    /// Repository-relative file path using forward slashes.
21    pub path: String,
22    /// Registry-owned closed content role.
23    pub classification: ContentClassification,
24}
25
26/// One bounded classification/path page.
27#[derive(Clone, Debug, Eq, PartialEq)]
28pub struct FileContentClassificationPage {
29    /// Rows in stable repository-path order.
30    pub rows: Vec<FileContentClassification>,
31    /// Whether at least one additional row exists.
32    pub truncated: bool,
33}
34
35impl IndexPublicationGuard<'_> {
36    /// Upsert one bounded classification batch inside the parent publication.
37    ///
38    /// # Errors
39    ///
40    /// Returns an error before mutation for duplicate, blank, oversized, absent,
41    /// inactive, or non-file paths, or when `SQLite` rejects the batch.
42    pub fn upsert_file_content_classification_batch(
43        &mut self,
44        rows: &[FileContentClassification],
45    ) -> DbResult<()> {
46        validate_classification_batch(&self.store.connection, rows)?;
47        if rows.is_empty() {
48            return Ok(());
49        }
50        let values = (0..rows.len())
51            .map(|index| format!("(?{}, ?{})", index * 2 + 1, index * 2 + 2))
52            .collect::<Vec<_>>()
53            .join(", ");
54        let sql = format!(
55            "INSERT INTO file_content_classifications(path, classification)
56             VALUES {values}
57             ON CONFLICT(path) DO UPDATE SET classification = excluded.classification"
58        );
59        let parameters = rows
60            .iter()
61            .flat_map(|row| {
62                [
63                    Value::Text(row.path.clone()),
64                    Value::Text(row.classification.as_str().to_string()),
65                ]
66            })
67            .collect::<Vec<_>>();
68        let savepoint = self.store.validated_savepoint()?;
69        savepoint
70            .prepare_cached(&sql)?
71            .execute(params_from_iter(parameters))?;
72        savepoint.commit()?;
73        Ok(())
74    }
75}
76
77impl AtlasStore {
78    /// Load classifications for exact file paths in one bounded set of statements.
79    ///
80    /// # Errors
81    ///
82    /// Returns an error for an oversized request, missing classification, invalid
83    /// persisted value, cancellation, or `SQLite` failure. No partial result is
84    /// returned.
85    pub fn file_content_classifications_for_paths(
86        &self,
87        paths: &[String],
88    ) -> DbResult<Vec<FileContentClassification>> {
89        let paths = unique_paths(paths)?;
90        let mut by_path = BTreeMap::new();
91        for chunk in paths.chunks(FILE_CONTENT_CLASSIFICATION_BIND_PATHS) {
92            let placeholders = numbered_placeholders(1, chunk.len());
93            let sql = format!(
94                "SELECT path, classification
95                   FROM file_content_classifications
96                  WHERE path IN ({placeholders})
97                  ORDER BY path"
98            );
99            let mut statement = self.connection.prepare_cached(&sql)?;
100            let rows = statement.query_map(params_from_iter(chunk.iter()), |row| {
101                Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
102            })?;
103            for row in rows {
104                let (path, value) = row?;
105                let classification = parse_classification(value)?;
106                by_path.insert(path, classification);
107            }
108        }
109        paths
110            .into_iter()
111            .map(|path| {
112                let classification = by_path.remove(&path).ok_or_else(|| {
113                    DbError::FileContentClassificationMissing { path: path.clone() }
114                })?;
115                Ok(FileContentClassification {
116                    path,
117                    classification,
118                })
119            })
120            .collect()
121    }
122
123    /// Load one stable path page through the classification/path index.
124    ///
125    /// # Errors
126    ///
127    /// Returns an error for an invalid limit, corrupt persisted value, or
128    /// `SQLite` failure.
129    pub fn file_content_classification_page(
130        &self,
131        classification: ContentClassification,
132        after_path: Option<&str>,
133        limit: u32,
134    ) -> DbResult<FileContentClassificationPage> {
135        if limit == 0 || limit > MAX_FILE_CONTENT_CLASSIFICATION_PAGE_ROWS {
136            return Err(DbError::FileContentClassificationLimit {
137                requested: limit,
138                maximum: MAX_FILE_CONTENT_CLASSIFICATION_PAGE_ROWS,
139            });
140        }
141        let fetch = i64::from(limit) + 1;
142        let mut statement = self.connection.prepare_cached(
143            "SELECT path, classification
144               FROM file_content_classifications
145              WHERE classification = ?1 AND path > ?2
146              ORDER BY classification, path
147              LIMIT ?3",
148        )?;
149        let rows = statement.query_map(
150            params![classification.as_str(), after_path.unwrap_or(""), fetch],
151            |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)),
152        )?;
153        let mut rows = rows
154            .map(|row| {
155                let (path, value) = row?;
156                Ok(FileContentClassification {
157                    path,
158                    classification: parse_classification(value)?,
159                })
160            })
161            .collect::<DbResult<Vec<_>>>()?;
162        let truncated = rows.len() > limit as usize;
163        if truncated {
164            rows.pop();
165        }
166        Ok(FileContentClassificationPage { rows, truncated })
167    }
168}
169
170/// Remove rows whose owning file node is no longer current.
171pub(crate) fn delete_absent_file_content_classifications(connection: &Connection) -> DbResult<()> {
172    connection.execute(
173        "DELETE FROM file_content_classifications
174          WHERE path IN (SELECT path FROM nodes WHERE exists_now = 0)",
175        [],
176    )?;
177    Ok(())
178}
179
180/// Remove classifications before a node changes away from file ownership.
181pub(crate) fn delete_non_file_content_classifications(
182    connection: &Connection,
183    nodes: &[Node],
184) -> DbResult<()> {
185    let mut statement =
186        connection.prepare_cached("DELETE FROM file_content_classifications WHERE path = ?1")?;
187    for node in nodes
188        .iter()
189        .filter(|node| node.kind != projectatlas_core::NodeKind::File)
190    {
191        statement.execute([&node.path])?;
192    }
193    Ok(())
194}
195
196/// Require exactly one classification for every current admitted file.
197pub(crate) fn validate_complete_file_content_classifications(
198    connection: &Connection,
199) -> DbResult<()> {
200    let missing = connection
201        .query_row(
202            "SELECT nodes.path
203               FROM nodes
204               LEFT JOIN file_content_classifications AS classification
205                 ON classification.path = nodes.path
206              WHERE nodes.kind = 'file'
207                AND nodes.exists_now = 1
208                AND classification.path IS NULL
209              ORDER BY nodes.path
210              LIMIT 1",
211            [],
212            |row| row.get::<_, String>(0),
213        )
214        .optional()?;
215    if let Some(path) = missing {
216        return Err(DbError::FileContentClassificationMissing { path });
217    }
218    let stale = connection
219        .query_row(
220            "SELECT classification.path
221               FROM file_content_classifications AS classification
222               JOIN nodes ON nodes.path = classification.path
223              WHERE nodes.kind <> 'file' OR nodes.exists_now <> 1
224              ORDER BY classification.path
225              LIMIT 1",
226            [],
227            |row| row.get::<_, String>(0),
228        )
229        .optional()?;
230    if let Some(path) = stale {
231        return Err(DbError::FileContentClassificationNotCurrent { path });
232    }
233    Ok(())
234}
235
236/// Reject malformed input before any batch mutation.
237fn validate_classification_batch(
238    connection: &Connection,
239    rows: &[FileContentClassification],
240) -> DbResult<()> {
241    if rows.len() > MAX_FILE_CONTENT_CLASSIFICATION_PATHS {
242        return Err(DbError::FileContentClassificationBatchTooLarge {
243            requested: rows.len(),
244            maximum: MAX_FILE_CONTENT_CLASSIFICATION_PATHS,
245        });
246    }
247    let mut paths = BTreeSet::new();
248    for row in rows {
249        if row.path.is_empty() {
250            return Err(DbError::PathNotIndexed {
251                path: row.path.clone(),
252            });
253        }
254        if !paths.insert(row.path.as_str()) {
255            return Err(DbError::FileContentClassificationDuplicatePath {
256                path: row.path.clone(),
257            });
258        }
259    }
260    for chunk in rows.chunks(FILE_CONTENT_CLASSIFICATION_BIND_PATHS) {
261        let values = (0..chunk.len())
262            .map(|index| format!("(?{})", index + 1))
263            .collect::<Vec<_>>()
264            .join(", ");
265        let sql = format!(
266            "WITH requested(path) AS (VALUES {values})
267             SELECT requested.path
268               FROM requested
269               LEFT JOIN nodes
270                 ON nodes.path = requested.path
271                AND nodes.kind = 'file'
272                AND nodes.exists_now = 1
273              WHERE nodes.path IS NULL
274              ORDER BY requested.path
275              LIMIT 1"
276        );
277        let invalid = connection
278            .query_row(
279                &sql,
280                params_from_iter(chunk.iter().map(|row| &row.path)),
281                |row| row.get::<_, String>(0),
282            )
283            .optional()?;
284        if let Some(path) = invalid {
285            return Err(DbError::PathNotIndexed { path });
286        }
287    }
288    Ok(())
289}
290
291/// Normalize a bounded exact-path request without duplicate round trips.
292fn unique_paths(paths: &[String]) -> DbResult<Vec<String>> {
293    let paths = paths.iter().cloned().collect::<BTreeSet<_>>();
294    if paths.len() > MAX_FILE_CONTENT_CLASSIFICATION_PATHS {
295        return Err(DbError::FileContentClassificationBatchTooLarge {
296            requested: paths.len(),
297            maximum: MAX_FILE_CONTENT_CLASSIFICATION_PATHS,
298        });
299    }
300    Ok(paths.into_iter().collect())
301}
302
303/// Parse one closed persisted classification without fallback.
304pub(crate) fn parse_classification(value: String) -> DbResult<ContentClassification> {
305    ContentClassification::from_db(&value).ok_or(DbError::InvalidEnum {
306        field: "file_content_classifications.classification",
307        value,
308    })
309}
310
311#[cfg(test)]
312mod tests {
313    use super::*;
314    use projectatlas_core::{Node, NodeKind};
315    use std::error::Error;
316    use std::fs;
317    use std::io;
318
319    #[test]
320    fn classifications_batch_page_reopen_and_follow_file_ownership() -> Result<(), Box<dyn Error>> {
321        let temp = tempfile::tempdir()?;
322        let root = temp.path().join("repository");
323        fs::create_dir(&root)?;
324        let database = temp.path().join("projectatlas.db");
325        let mut store = AtlasStore::open_for_project(&database, &root)?;
326        store.replace_scan(&[
327            file_node("docs/a.md"),
328            file_node("docs/b.md"),
329            file_node("src/lib.rs"),
330            folder_node("target"),
331        ])?;
332
333        let rows = vec![
334            classified("docs/a.md", ContentClassification::Documentation),
335            classified("docs/b.md", ContentClassification::Documentation),
336            classified("src/lib.rs", ContentClassification::Source),
337        ];
338        let mut publication = store.begin_index_publication("classification-round-trip")?;
339        publication.upsert_file_content_classification_batch(&rows)?;
340        publication.complete()?;
341
342        let exact = store.file_content_classifications_for_paths(&[
343            "src/lib.rs".to_string(),
344            "docs/b.md".to_string(),
345            "docs/a.md".to_string(),
346        ])?;
347        require_eq(&exact, &rows, "stable exact-path classifications")?;
348        let first = store.file_content_classification_page(
349            ContentClassification::Documentation,
350            None,
351            1,
352        )?;
353        require(
354            first.truncated && first.rows == vec![rows[0].clone()],
355            "classification LIMIT + 1 page",
356        )?;
357        let second = store.file_content_classification_page(
358            ContentClassification::Documentation,
359            Some("docs/a.md"),
360            10,
361        )?;
362        require(
363            !second.truncated && second.rows == vec![rows[1].clone()],
364            "classification keyset continuation",
365        )?;
366        let plan = store
367            .connection
368            .prepare(
369                "EXPLAIN QUERY PLAN
370                 SELECT path, classification
371                   FROM file_content_classifications
372                  WHERE classification = 'documentation' AND path > ''
373                  ORDER BY classification, path
374                  LIMIT 11",
375            )?
376            .query_map([], |row| row.get::<_, String>(3))?
377            .collect::<Result<Vec<_>, _>>()?;
378        require(
379            plan.iter().any(|detail| {
380                detail.contains("idx_file_content_classifications_classification_path")
381            }) && plan
382                .iter()
383                .all(|detail| !detail.contains("USE TEMP B-TREE")),
384            &format!("classification page did not use its covering index: {plan:?}"),
385        )?;
386        drop(store);
387
388        let mut reopened = AtlasStore::open_for_project(&database, &root)?;
389        require_eq(
390            &reopened.file_content_classifications_for_paths(&["src/lib.rs".to_string()])?,
391            &vec![rows[2].clone()],
392            "reopened classification",
393        )?;
394        reopened.replace_scan(&[
395            file_node("docs/a.md"),
396            file_node("docs/b.md"),
397            folder_node("src/lib.rs"),
398        ])?;
399        let missing = require_error(
400            reopened.file_content_classifications_for_paths(&["src/lib.rs".to_string()]),
401            "file-to-folder transition retained its classification",
402        )?;
403        require(
404            matches!(missing, DbError::FileContentClassificationMissing { .. }),
405            "file-to-folder cleanup returned the wrong error",
406        )?;
407        Ok(())
408    }
409
410    #[test]
411    fn classifications_reject_invalid_ownership_values_and_batches_atomically()
412    -> Result<(), Box<dyn Error>> {
413        let mut store = AtlasStore::in_memory()?;
414        store.replace_scan(&[
415            file_node("src/a.rs"),
416            file_node("src/b.rs"),
417            folder_node("src"),
418        ])?;
419
420        let folder_error = store.connection.execute(
421            "INSERT INTO file_content_classifications(path, classification)
422             VALUES('src', 'source')",
423            [],
424        );
425        require(
426            folder_error.is_err(),
427            "folder accepted a file classification",
428        )?;
429        let value_error = store.connection.execute(
430            "UPDATE file_content_classifications
431                SET classification = 'executable'
432              WHERE path = 'src/a.rs'",
433            [],
434        );
435        require(
436            value_error.is_err(),
437            "open classification value was accepted",
438        )?;
439
440        let duplicate = classified("src/a.rs", ContentClassification::Source);
441        let mut publication = store.begin_index_publication("classification-atomicity")?;
442        let duplicate_error = require_error(
443            publication.upsert_file_content_classification_batch(&[duplicate.clone(), duplicate]),
444            "duplicate classification path was accepted",
445        )?;
446        require(
447            matches!(
448                duplicate_error,
449                DbError::FileContentClassificationDuplicatePath { .. }
450            ),
451            "duplicate classification returned the wrong error",
452        )?;
453        let missing_error = require_error(
454            publication.upsert_file_content_classification_batch(&[classified(
455                "missing.rs",
456                ContentClassification::Source,
457            )]),
458            "missing path accepted a classification",
459        )?;
460        require(
461            matches!(missing_error, DbError::PathNotIndexed { .. }),
462            "missing classification path returned the wrong error",
463        )?;
464        publication.connection.execute_batch(
465            "CREATE TEMP TRIGGER abort_second_classification
466             BEFORE UPDATE OF classification ON file_content_classifications
467             WHEN NEW.path = 'src/b.rs'
468             BEGIN SELECT RAISE(ABORT, 'injected classification failure'); END;",
469        )?;
470        let injected = require_error(
471            publication.upsert_file_content_classification_batch(&[
472                classified("src/a.rs", ContentClassification::Source),
473                classified("src/b.rs", ContentClassification::Documentation),
474            ]),
475            "injected classification failure committed",
476        )?;
477        require(
478            matches!(injected, DbError::Sqlite(_)),
479            "injected classification failure returned the wrong error",
480        )?;
481        let unchanged = publication.file_content_classifications_for_paths(&[
482            "src/a.rs".to_string(),
483            "src/b.rs".to_string(),
484        ])?;
485        require(
486            unchanged
487                .iter()
488                .all(|row| row.classification == ContentClassification::Opaque),
489            "failed classification statement exposed partial mutation",
490        )?;
491        publication.complete()?;
492        Ok(())
493    }
494
495    #[test]
496    fn classification_reads_fail_closed_on_corrupt_closed_value() -> Result<(), Box<dyn Error>> {
497        let mut store = AtlasStore::in_memory()?;
498        store.replace_scan(&[file_node("src/lib.rs")])?;
499        store
500            .connection
501            .execute_batch("PRAGMA ignore_check_constraints = ON")?;
502        store.connection.execute(
503            "UPDATE file_content_classifications
504                SET classification = 'corrupt'
505              WHERE path = 'src/lib.rs'",
506            [],
507        )?;
508        store
509            .connection
510            .execute_batch("PRAGMA ignore_check_constraints = OFF")?;
511        let error = require_error(
512            store.file_content_classifications_for_paths(&["src/lib.rs".to_string()]),
513            "corrupt classification was coerced",
514        )?;
515        require(
516            matches!(error, DbError::InvalidEnum { .. }),
517            "corrupt classification returned the wrong error",
518        )?;
519        Ok(())
520    }
521
522    fn classified(path: &str, classification: ContentClassification) -> FileContentClassification {
523        FileContentClassification {
524            path: path.to_string(),
525            classification,
526        }
527    }
528
529    fn file_node(path: &str) -> Node {
530        Node {
531            path: path.to_string(),
532            kind: NodeKind::File,
533            parent_path: path.rsplit_once('/').map(|(parent, _)| parent.to_string()),
534            extension: None,
535            language: None,
536            size_bytes: Some(1),
537            mtime_ns: Some(1),
538            content_hash: Some(format!("hash-{path}")),
539        }
540    }
541
542    fn folder_node(path: &str) -> Node {
543        Node {
544            path: path.to_string(),
545            kind: NodeKind::Folder,
546            parent_path: path.rsplit_once('/').map(|(parent, _)| parent.to_string()),
547            extension: None,
548            language: None,
549            size_bytes: None,
550            mtime_ns: None,
551            content_hash: None,
552        }
553    }
554
555    fn require(condition: bool, message: &str) -> Result<(), Box<dyn Error>> {
556        if condition {
557            Ok(())
558        } else {
559            Err(io::Error::other(message).into())
560        }
561    }
562
563    fn require_error<T>(result: DbResult<T>, message: &str) -> Result<DbError, Box<dyn Error>> {
564        match result {
565            Ok(_) => Err(io::Error::other(message).into()),
566            Err(error) => Ok(error),
567        }
568    }
569
570    fn require_eq<T: std::fmt::Debug + PartialEq>(
571        actual: &T,
572        expected: &T,
573        label: &str,
574    ) -> Result<(), Box<dyn Error>> {
575        if actual == expected {
576            Ok(())
577        } else {
578            Err(io::Error::other(format!(
579                "{label} mismatch: expected {expected:?}, got {actual:?}"
580            ))
581            .into())
582        }
583    }
584}
projectatlas_db/content_classification.rs

projectatlas_db/
content_classification.rs
