From 9d8f1e0bbafc2e3e55052b3e60badb93fd7053e4 Mon Sep 17 00:00:00 2001 From: loki5512344 Date: Wed, 9 Sep 2026 22:02:09 +0200 Subject: [PATCH] feat(parser): tree-sitter TS/JS import extraction + module specifier resolver - 6 import forms: static/type/side-effect export-from, require, dynamic import() - specifier-only rewrites: byte spans exclude quotes, statements never reformatted - resolver: exact path > extension guessing (.ts/.tsx/.js/.jsx/.mjs/.cjs) > .d.ts > index.* - 22 unit tests (11 parser + 11 resolver) --- src/parser/resolve.rs | 172 ++++++++++++++++++++++++++++++++++++++- src/parser/ts.rs | 142 +++++++++++++++++++++++++++++++- src/parser/ts_extract.rs | 91 +++++++++++++++++++++ 3 files changed, 399 insertions(+), 6 deletions(-) create mode 100644 src/parser/ts_extract.rs diff --git a/src/parser/resolve.rs b/src/parser/resolve.rs index dc91e8e..b636895 100644 --- a/src/parser/resolve.rs +++ b/src/parser/resolve.rs @@ -5,9 +5,27 @@ //! package specifiers (not starting with `.`) are out of project scope and //! resolve to `None`. +use std::ffi::OsString; use std::path::{Path, PathBuf}; use crate::core::index::FileSet; +use crate::core::normalize_rel_path; + +/// Supported extensions, in Node/TS resolution priority order. +const EXTENSIONS: [&str; 6] = ["ts", "tsx", "js", "jsx", "mjs", "cjs"]; + +/// Ambient declaration files: only consulted after every real module +/// candidate missed (last resort). +const DECLARATION_EXT: &str = "d.ts"; + +/// Append `.{ext}` to a path without touching its existing extension +/// (specifiers are joined verbatim, matching Node's lookup). +fn with_ext(base: &Path, ext: &str) -> PathBuf { + let mut joined = OsString::from(base.as_os_str()); + joined.push("."); + joined.push(ext); + PathBuf::from(joined) +} /// Resolve `specifier` (e.g. `"../utils/fmt"`) written in the file at /// `importer` (project-relative), against the indexed `files`. @@ -21,6 +39,156 @@ use crate::core::index::FileSet; /// Returns `None` for bare specifiers or unresolvable paths. #[must_use] pub fn resolve_module(importer: &Path, specifier: &str, files: &FileSet) -> Option { - let _ = (importer, specifier, files); - todo!("parser agent: implement resolver") + if !specifier.starts_with('.') { + return None; // bare package specifier: outside project scope + } + // `importer` is project-relative, so `..` segments that walk past the + // root collapse to `None` here instead of escaping the index. + let base = normalize_rel_path(&importer.parent()?.join(specifier))?; + if files.contains(&base) { + return Some(base); + } + for ext in EXTENSIONS { + let candidate = with_ext(&base, ext); + if files.contains(&candidate) { + return Some(candidate); + } + } + let declaration = with_ext(&base, DECLARATION_EXT); + if files.contains(&declaration) { + return Some(declaration); + } + let index_dir = base.join("index"); + EXTENSIONS + .iter() + .map(|ext| with_ext(&index_dir, ext)) + .find(|candidate| files.contains(candidate)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn files(paths: &[&str]) -> FileSet { + let mut set = FileSet::default(); + for path in paths { + set.add(PathBuf::from(path)); + } + set + } + + #[track_caller] + fn resolve(importer: &str, specifier: &str, set: &FileSet) -> Option { + resolve_module(Path::new(importer), specifier, set) + } + + #[test] + fn exact_specifier_with_extension() { + let set = files(&["src/a.ts"]); + assert_eq!( + resolve("src/b.ts", "./a.ts", &set), + Some(PathBuf::from("src/a.ts")) + ); + } + + #[test] + fn guesses_extensions_in_priority_order() { + let set = files(&["src/a.ts", "src/a.js"]); + assert_eq!( + resolve("src/b.ts", "./a", &set), + Some(PathBuf::from("src/a.ts")) + ); + let only_js = files(&["src/a.js"]); + assert_eq!( + resolve("src/b.ts", "./a", &only_js), + Some(PathBuf::from("src/a.js")) + ); + } + + #[test] + fn jsx_and_component_specifiers() { + let set = files(&["src/Comp.tsx"]); + assert_eq!( + resolve("src/app.tsx", "./Comp", &set), + Some(PathBuf::from("src/Comp.tsx")) + ); + } + + #[test] + fn declaration_file_is_last_resort() { + let set = files(&["types/globals.d.ts"]); + assert_eq!( + resolve("src/a.ts", "../types/globals", &set), + Some(PathBuf::from("types/globals.d.ts")) + ); + // a real module wins over the declaration file + let both = files(&["types/globals.d.ts", "types/globals.ts"]); + assert_eq!( + resolve("src/a.ts", "../types/globals", &both), + Some(PathBuf::from("types/globals.ts")) + ); + } + + #[test] + fn directory_index_resolution() { + let set = files(&["pkg/index.ts", "other/index.js", "other/util.ts"]); + assert_eq!( + resolve("src/a.ts", "../pkg", &set), + Some(PathBuf::from("pkg/index.ts")) + ); + assert_eq!(resolve("src/a.ts", "./", &set), None); + assert_eq!( + resolve("src/a.ts", "../other", &set), + Some(PathBuf::from("other/index.js")) + ); + } + + #[test] + fn bare_specifiers_are_not_project_paths() { + let set = files(&["react.ts", "lodash.ts"]); + assert_eq!(resolve("src/a.ts", "react", &set), None); + assert_eq!(resolve("src/a.ts", "@scope/pkg", &set), None); + assert_eq!(resolve("src/a.ts", "/abs/path", &set), None); + } + + #[test] + fn escaping_the_project_root_resolves_to_none() { + let set = files(&["evil/x.ts", "outside.ts"]); + assert_eq!(resolve("src/a.ts", "../../evil/x", &set), None); + assert_eq!(resolve("a.ts", "../outside", &set), None); + } + + #[test] + fn nested_parent_walks_land_inside_the_project() { + let set = files(&["lib/x.ts", "src/app/main.ts"]); + assert_eq!( + resolve("src/app/main.ts", "../../lib/x", &set), + Some(PathBuf::from("lib/x.ts")) + ); + } + + #[test] + fn forward_slash_specifiers_and_redundant_dots() { + let set = files(&["src/nested/dir/deep.ts"]); + assert_eq!( + resolve("src/app.ts", "./nested/./dir/deep", &set), + Some(PathBuf::from("src/nested/dir/deep.ts")) + ); + } + + #[test] + fn unresolvable_relative_specifier_is_none() { + let set = files(&["src/a.ts"]); + assert_eq!(resolve("src/a.ts", "./missing", &set), None); + assert_eq!(resolve("src/a.ts", "./styles.css", &set), None); + } + + #[test] + fn root_level_importer_resolves_plain_sibling() { + let set = files(&["index.ts", "b.ts"]); + assert_eq!( + resolve("index.ts", "./b", &set), + Some(PathBuf::from("b.ts")) + ); + } } diff --git a/src/parser/ts.rs b/src/parser/ts.rs index 2659ce6..b1be26b 100644 --- a/src/parser/ts.rs +++ b/src/parser/ts.rs @@ -1,7 +1,10 @@ //! Tree-sitter based frontend for TypeScript/JavaScript. //! -//! CONTRACT: see [`crate::parser`]. Implement `extract_imports` using -//! `tree-sitter-typescript` grammars. +//! CONTRACT: see [`crate::parser`]. Traversal lives in [`ts_extract`]. + +// Inline module so the folder stays at `mod.rs` + 3 files. +#[path = "ts_extract.rs"] +mod ts_extract; use super::{ImportRecord, Language, SourceLanguage}; @@ -23,7 +26,138 @@ impl Language for TreeSitterTs { self.lang } - fn extract_imports(&self, _source: &str) -> Vec { - todo!("parser agent: implement tree-sitter extraction") + fn extract_imports(&self, source: &str) -> Vec { + ts_extract::extract(self.lang, source) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(lang: SourceLanguage, source: &str) -> Vec { + TreeSitterTs::new(lang).extract_imports(source) + } + + /// Assert `records[i]` is specifier `spec` over `source` with quotes + /// `q`, at the byte span implied by the occurrence of `q.spec.q`. + #[track_caller] + fn assert_import(source: &str, records: &[ImportRecord], index: usize, spec: &str) { + let rec = &records[index]; + assert_eq!(rec.specifier, spec); + assert_eq!(&source[rec.span.clone()], spec); + let before = source[..rec.span.start].bytes().last().unwrap_or(b'\n'); + let after = source.as_bytes()[rec.span.end]; + assert!( + (before == b'"' || before == b'\'') && before == after, + "span must sit exactly between matching quotes, got {before:?} / {after:?}" + ); + } + + #[test] + fn import_statements_with_default_and_named_bindings() { + let src = "import X from './a'\nimport { A, B } from \"../utils/b\";\n"; + let recs = parse(SourceLanguage::TypeScript, src); + assert_eq!(recs.len(), 2); + assert_import(src, &recs, 0, "./a"); + assert_import(src, &recs, 1, "../utils/b"); + assert!(recs.iter().all(|r| !r.is_dynamic)); + } + + #[test] + fn bare_side_effect_import() { + let src = "import './styles.css';\n"; + let recs = parse(SourceLanguage::TypeScript, src); + assert_eq!(recs.len(), 1); + assert_import(src, &recs, 0, "./styles.css"); + assert!(!recs[0].is_dynamic); + } + + #[test] + fn type_only_import_is_static() { + let src = "import type { A } from './types';\n"; + let recs = parse(SourceLanguage::TypeScript, src); + assert_eq!(recs.len(), 1); + assert_import(src, &recs, 0, "./types"); + assert!(!recs[0].is_dynamic); + } + + #[test] + fn export_from_reexport() { + let src = "export { A } from './a';\nexport * from './b';\n"; + let recs = parse(SourceLanguage::TypeScript, src); + assert_eq!(recs.len(), 2); + assert_import(src, &recs, 0, "./a"); + assert_import(src, &recs, 1, "./b"); + } + + #[test] + fn require_call_is_dynamic() { + let src = "const x = require('./a');\n"; + let recs = parse(SourceLanguage::TypeScript, src); + assert_eq!(recs.len(), 1); + assert_import(src, &recs, 0, "./a"); + assert!(recs[0].is_dynamic); + } + + #[test] + fn dynamic_import_await_and_chained() { + let src = "const a = await import('./a');\nimport('./b').then((m) => m);\n"; + let recs = parse(SourceLanguage::TypeScript, src); + assert_eq!(recs.len(), 2); + assert_import(src, &recs, 0, "./a"); + assert_import(src, &recs, 1, "./b"); + assert!(recs.iter().all(|r| r.is_dynamic)); + } + + #[test] + fn declarations_and_comments_are_not_captured() { + let src = "export const x = 1;\n// import y from './nope'\n/* require('./nah') */\n"; + assert!(parse(SourceLanguage::TypeScript, src).is_empty()); + } + + #[test] + fn nested_imports_inside_functions_and_blocks() { + let src = "function f() { if (true) { const m = require('./deep'); } }\n"; + let recs = parse(SourceLanguage::JavaScript, src); + assert_eq!(recs.len(), 1); + assert_import(src, &recs, 0, "./deep"); + } + + #[test] + fn jsx_file_uses_tsx_grammar() { + // The `require` sits inside a JSX attribute body: only the TSX + // grammar parses that expression tree (TS reads ` = recs.iter().map(|r| r.span.start).collect(); + assert_eq!(spans, { + let mut s = spans.clone(); + s.sort(); + s + }); + assert_eq!(recs.len(), 3); + } + + #[test] + fn plain_javascript_parses_with_the_ts_grammar() { + let src = "const x = require('./a');\nimport y from './b';\n"; + let recs = parse(SourceLanguage::JavaScript, src); + assert_eq!(recs.len(), 2); + assert_import(src, &recs, 1, "./b"); } } diff --git a/src/parser/ts_extract.rs b/src/parser/ts_extract.rs new file mode 100644 index 0000000..6378bba --- /dev/null +++ b/src/parser/ts_extract.rs @@ -0,0 +1,91 @@ +//! Tree-sitter traversal backing [`crate::parser::ts`]. +//! +//! Finds every module reference: `import`/`export … from` statements (the +//! grammar exposes the specifier via the `source` field), `require("…")` +//! calls and dynamic `import("…")` calls. Nesting depth is arbitrary, so +//! the walk is fully recursive. + +use tree_sitter::{Language, Node, Parser}; + +use super::{ImportRecord, SourceLanguage}; + +/// Record for a `string` node used as a specifier: the span covers the +/// text between the quote characters (the grammar only emits ASCII +/// `"`/`'` quotes for import sources), never the quotes themselves. +fn record(string: Node, source: &str, is_dynamic: bool) -> Option { + let (start, end) = (string.start_byte(), string.end_byte()); + let text = &source[start..end]; + let quote = *text.as_bytes().first()?; + if quote != b'"' && quote != b'\'' || text.as_bytes().last() != Some("e) { + return None; + } + Some(ImportRecord { + specifier: text[1..text.len() - 1].to_string(), + span: start + 1..end - 1, + is_dynamic, + }) +} + +/// Record for a `require("…")` / `import("…")` call whose first argument +/// is a string literal. Both are flagged `is_dynamic` per the contract in +/// [`crate::parser`]. +fn call_record(call: Node, source: &str) -> Option { + let func = call.child_by_field_name("function")?; + let is_require = func.kind() == "identifier" + && source.get(func.start_byte()..func.end_byte()) == Some("require"); + if func.kind() != "import" && !is_require { + return None; + } + let args = call.child_by_field_name("arguments")?; + let first = args.named_child(0).filter(|n| n.kind() == "string")?; + record(first, source, true) +} + +/// Pre-order walk pushing every module reference under `node`. +fn visit(node: Node, source: &str, out: &mut Vec) { + match node.kind() { + // `import "./a.css"` and `export * from "./a"` also carry a + // `source` field; `export const x = …` has none and is skipped. + "import_statement" | "export_statement" => out.extend( + node.child_by_field_name("source") + .and_then(|string| record(string, source, false)), + ), + "call_expression" => out.extend(call_record(node, source)), + _ => {} + } + let mut cursor = node.walk(); + for child in node.named_children(&mut cursor) { + visit(child, source, out); + } +} + +/// Grammar for a language variant. The TypeScript grammar also parses +/// plain JavaScript; only JSX needs the TSX variant. +fn grammar(lang: SourceLanguage) -> Language { + match lang { + SourceLanguage::Tsx => tree_sitter_typescript::LANGUAGE_TSX.into(), + SourceLanguage::TypeScript | SourceLanguage::JavaScript => { + tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into() + } + } +} + +/// Parse `source` and collect its imports in byte-offset order. +/// A parse failure yields whatever the partial tree understood (possibly +/// nothing); it never panics. +pub(super) fn extract(lang: SourceLanguage, source: &str) -> Vec { + // tree-sitter's Parser holds raw pointers; build one per call instead + // of storing it in the (Sync) frontend struct. + let mut parser = Parser::new(); + let grammar = grammar(lang); + if parser.set_language(&grammar).is_err() { + return Vec::new(); + } + let Some(tree) = parser.parse(source, None) else { + return Vec::new(); + }; + let mut out = Vec::new(); + visit(tree.root_node(), source, &mut out); + out.sort_by_key(|rec| rec.span.start); + out +}