feat(parser): tree-sitter TS/JS import extraction + module specifier resolver

- 6 import forms: static/type/side-effect export-from, require, dynamic import()
- specifier-only rewrites: byte spans exclude quotes, statements never reformatted
- resolver: exact path > extension guessing (.ts/.tsx/.js/.jsx/.mjs/.cjs) > .d.ts > index.*
- 22 unit tests (11 parser + 11 resolver)
This commit is contained in:
loki5512344 2026-09-09 22:02:09 +02:00
parent a33ed2eb30
commit 9d8f1e0bba
Signed by: boba
GPG key ID: 253067914055423B
3 changed files with 399 additions and 6 deletions

View file

@ -5,9 +5,27 @@
//! package specifiers (not starting with `.`) are out of project scope and //! package specifiers (not starting with `.`) are out of project scope and
//! resolve to `None`. //! resolve to `None`.
use std::ffi::OsString;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use crate::core::index::FileSet; use crate::core::index::FileSet;
use crate::core::normalize_rel_path;
/// Supported extensions, in Node/TS resolution priority order.
const EXTENSIONS: [&str; 6] = ["ts", "tsx", "js", "jsx", "mjs", "cjs"];
/// Ambient declaration files: only consulted after every real module
/// candidate missed (last resort).
const DECLARATION_EXT: &str = "d.ts";
/// Append `.{ext}` to a path without touching its existing extension
/// (specifiers are joined verbatim, matching Node's lookup).
fn with_ext(base: &Path, ext: &str) -> PathBuf {
let mut joined = OsString::from(base.as_os_str());
joined.push(".");
joined.push(ext);
PathBuf::from(joined)
}
/// Resolve `specifier` (e.g. `"../utils/fmt"`) written in the file at /// Resolve `specifier` (e.g. `"../utils/fmt"`) written in the file at
/// `importer` (project-relative), against the indexed `files`. /// `importer` (project-relative), against the indexed `files`.
@ -21,6 +39,156 @@ use crate::core::index::FileSet;
/// Returns `None` for bare specifiers or unresolvable paths. /// Returns `None` for bare specifiers or unresolvable paths.
#[must_use] #[must_use]
pub fn resolve_module(importer: &Path, specifier: &str, files: &FileSet) -> Option<PathBuf> { pub fn resolve_module(importer: &Path, specifier: &str, files: &FileSet) -> Option<PathBuf> {
let _ = (importer, specifier, files); if !specifier.starts_with('.') {
todo!("parser agent: implement resolver") return None; // bare package specifier: outside project scope
}
// `importer` is project-relative, so `..` segments that walk past the
// root collapse to `None` here instead of escaping the index.
let base = normalize_rel_path(&importer.parent()?.join(specifier))?;
if files.contains(&base) {
return Some(base);
}
for ext in EXTENSIONS {
let candidate = with_ext(&base, ext);
if files.contains(&candidate) {
return Some(candidate);
}
}
let declaration = with_ext(&base, DECLARATION_EXT);
if files.contains(&declaration) {
return Some(declaration);
}
let index_dir = base.join("index");
EXTENSIONS
.iter()
.map(|ext| with_ext(&index_dir, ext))
.find(|candidate| files.contains(candidate))
}
#[cfg(test)]
mod tests {
use super::*;
fn files(paths: &[&str]) -> FileSet {
let mut set = FileSet::default();
for path in paths {
set.add(PathBuf::from(path));
}
set
}
#[track_caller]
fn resolve(importer: &str, specifier: &str, set: &FileSet) -> Option<PathBuf> {
resolve_module(Path::new(importer), specifier, set)
}
#[test]
fn exact_specifier_with_extension() {
let set = files(&["src/a.ts"]);
assert_eq!(
resolve("src/b.ts", "./a.ts", &set),
Some(PathBuf::from("src/a.ts"))
);
}
#[test]
fn guesses_extensions_in_priority_order() {
let set = files(&["src/a.ts", "src/a.js"]);
assert_eq!(
resolve("src/b.ts", "./a", &set),
Some(PathBuf::from("src/a.ts"))
);
let only_js = files(&["src/a.js"]);
assert_eq!(
resolve("src/b.ts", "./a", &only_js),
Some(PathBuf::from("src/a.js"))
);
}
#[test]
fn jsx_and_component_specifiers() {
let set = files(&["src/Comp.tsx"]);
assert_eq!(
resolve("src/app.tsx", "./Comp", &set),
Some(PathBuf::from("src/Comp.tsx"))
);
}
#[test]
fn declaration_file_is_last_resort() {
let set = files(&["types/globals.d.ts"]);
assert_eq!(
resolve("src/a.ts", "../types/globals", &set),
Some(PathBuf::from("types/globals.d.ts"))
);
// a real module wins over the declaration file
let both = files(&["types/globals.d.ts", "types/globals.ts"]);
assert_eq!(
resolve("src/a.ts", "../types/globals", &both),
Some(PathBuf::from("types/globals.ts"))
);
}
#[test]
fn directory_index_resolution() {
let set = files(&["pkg/index.ts", "other/index.js", "other/util.ts"]);
assert_eq!(
resolve("src/a.ts", "../pkg", &set),
Some(PathBuf::from("pkg/index.ts"))
);
assert_eq!(resolve("src/a.ts", "./", &set), None);
assert_eq!(
resolve("src/a.ts", "../other", &set),
Some(PathBuf::from("other/index.js"))
);
}
#[test]
fn bare_specifiers_are_not_project_paths() {
let set = files(&["react.ts", "lodash.ts"]);
assert_eq!(resolve("src/a.ts", "react", &set), None);
assert_eq!(resolve("src/a.ts", "@scope/pkg", &set), None);
assert_eq!(resolve("src/a.ts", "/abs/path", &set), None);
}
#[test]
fn escaping_the_project_root_resolves_to_none() {
let set = files(&["evil/x.ts", "outside.ts"]);
assert_eq!(resolve("src/a.ts", "../../evil/x", &set), None);
assert_eq!(resolve("a.ts", "../outside", &set), None);
}
#[test]
fn nested_parent_walks_land_inside_the_project() {
let set = files(&["lib/x.ts", "src/app/main.ts"]);
assert_eq!(
resolve("src/app/main.ts", "../../lib/x", &set),
Some(PathBuf::from("lib/x.ts"))
);
}
#[test]
fn forward_slash_specifiers_and_redundant_dots() {
let set = files(&["src/nested/dir/deep.ts"]);
assert_eq!(
resolve("src/app.ts", "./nested/./dir/deep", &set),
Some(PathBuf::from("src/nested/dir/deep.ts"))
);
}
#[test]
fn unresolvable_relative_specifier_is_none() {
let set = files(&["src/a.ts"]);
assert_eq!(resolve("src/a.ts", "./missing", &set), None);
assert_eq!(resolve("src/a.ts", "./styles.css", &set), None);
}
#[test]
fn root_level_importer_resolves_plain_sibling() {
let set = files(&["index.ts", "b.ts"]);
assert_eq!(
resolve("index.ts", "./b", &set),
Some(PathBuf::from("b.ts"))
);
}
} }

View file

@ -1,7 +1,10 @@
//! Tree-sitter based frontend for TypeScript/JavaScript. //! Tree-sitter based frontend for TypeScript/JavaScript.
//! //!
//! CONTRACT: see [`crate::parser`]. Implement `extract_imports` using //! CONTRACT: see [`crate::parser`]. Traversal lives in [`ts_extract`].
//! `tree-sitter-typescript` grammars.
// Inline module so the folder stays at `mod.rs` + 3 files.
#[path = "ts_extract.rs"]
mod ts_extract;
use super::{ImportRecord, Language, SourceLanguage}; use super::{ImportRecord, Language, SourceLanguage};
@ -23,7 +26,138 @@ impl Language for TreeSitterTs {
self.lang self.lang
} }
fn extract_imports(&self, _source: &str) -> Vec<ImportRecord> { fn extract_imports(&self, source: &str) -> Vec<ImportRecord> {
todo!("parser agent: implement tree-sitter extraction") ts_extract::extract(self.lang, source)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn parse(lang: SourceLanguage, source: &str) -> Vec<ImportRecord> {
TreeSitterTs::new(lang).extract_imports(source)
}
/// Assert `records[i]` is specifier `spec` over `source` with quotes
/// `q`, at the byte span implied by the occurrence of `q.spec.q`.
#[track_caller]
fn assert_import(source: &str, records: &[ImportRecord], index: usize, spec: &str) {
let rec = &records[index];
assert_eq!(rec.specifier, spec);
assert_eq!(&source[rec.span.clone()], spec);
let before = source[..rec.span.start].bytes().last().unwrap_or(b'\n');
let after = source.as_bytes()[rec.span.end];
assert!(
(before == b'"' || before == b'\'') && before == after,
"span must sit exactly between matching quotes, got {before:?} / {after:?}"
);
}
#[test]
fn import_statements_with_default_and_named_bindings() {
let src = "import X from './a'\nimport { A, B } from \"../utils/b\";\n";
let recs = parse(SourceLanguage::TypeScript, src);
assert_eq!(recs.len(), 2);
assert_import(src, &recs, 0, "./a");
assert_import(src, &recs, 1, "../utils/b");
assert!(recs.iter().all(|r| !r.is_dynamic));
}
#[test]
fn bare_side_effect_import() {
let src = "import './styles.css';\n";
let recs = parse(SourceLanguage::TypeScript, src);
assert_eq!(recs.len(), 1);
assert_import(src, &recs, 0, "./styles.css");
assert!(!recs[0].is_dynamic);
}
#[test]
fn type_only_import_is_static() {
let src = "import type { A } from './types';\n";
let recs = parse(SourceLanguage::TypeScript, src);
assert_eq!(recs.len(), 1);
assert_import(src, &recs, 0, "./types");
assert!(!recs[0].is_dynamic);
}
#[test]
fn export_from_reexport() {
let src = "export { A } from './a';\nexport * from './b';\n";
let recs = parse(SourceLanguage::TypeScript, src);
assert_eq!(recs.len(), 2);
assert_import(src, &recs, 0, "./a");
assert_import(src, &recs, 1, "./b");
}
#[test]
fn require_call_is_dynamic() {
let src = "const x = require('./a');\n";
let recs = parse(SourceLanguage::TypeScript, src);
assert_eq!(recs.len(), 1);
assert_import(src, &recs, 0, "./a");
assert!(recs[0].is_dynamic);
}
#[test]
fn dynamic_import_await_and_chained() {
let src = "const a = await import('./a');\nimport('./b').then((m) => m);\n";
let recs = parse(SourceLanguage::TypeScript, src);
assert_eq!(recs.len(), 2);
assert_import(src, &recs, 0, "./a");
assert_import(src, &recs, 1, "./b");
assert!(recs.iter().all(|r| r.is_dynamic));
}
#[test]
fn declarations_and_comments_are_not_captured() {
let src = "export const x = 1;\n// import y from './nope'\n/* require('./nah') */\n";
assert!(parse(SourceLanguage::TypeScript, src).is_empty());
}
#[test]
fn nested_imports_inside_functions_and_blocks() {
let src = "function f() { if (true) { const m = require('./deep'); } }\n";
let recs = parse(SourceLanguage::JavaScript, src);
assert_eq!(recs.len(), 1);
assert_import(src, &recs, 0, "./deep");
}
#[test]
fn jsx_file_uses_tsx_grammar() {
// The `require` sits inside a JSX attribute body: only the TSX
// grammar parses that expression tree (TS reads `<Button` as a
// type assertion and loses the nested call).
let src = "export const C = () => <Button onClick={() => require('./a')} />;\n";
let recs = parse(SourceLanguage::Tsx, src);
assert_eq!(recs.len(), 1);
assert_import(src, &recs, 0, "./a");
assert!(recs[0].is_dynamic);
assert!(
parse(SourceLanguage::TypeScript, src).is_empty(),
"JSX must need the TSX grammar to parse fully"
);
}
#[test]
fn records_are_in_byte_order_with_distinct_spans() {
let src = "import a from './a';\nexport * from './b';\nconst c = require('./c');\n";
let recs = parse(SourceLanguage::TypeScript, src);
let spans: Vec<_> = recs.iter().map(|r| r.span.start).collect();
assert_eq!(spans, {
let mut s = spans.clone();
s.sort();
s
});
assert_eq!(recs.len(), 3);
}
#[test]
fn plain_javascript_parses_with_the_ts_grammar() {
let src = "const x = require('./a');\nimport y from './b';\n";
let recs = parse(SourceLanguage::JavaScript, src);
assert_eq!(recs.len(), 2);
assert_import(src, &recs, 1, "./b");
} }
} }

91
src/parser/ts_extract.rs Normal file
View file

@ -0,0 +1,91 @@
//! Tree-sitter traversal backing [`crate::parser::ts`].
//!
//! Finds every module reference: `import`/`export … from` statements (the
//! grammar exposes the specifier via the `source` field), `require("…")`
//! calls and dynamic `import("…")` calls. Nesting depth is arbitrary, so
//! the walk is fully recursive.
use tree_sitter::{Language, Node, Parser};
use super::{ImportRecord, SourceLanguage};
/// Record for a `string` node used as a specifier: the span covers the
/// text between the quote characters (the grammar only emits ASCII
/// `"`/`'` quotes for import sources), never the quotes themselves.
fn record(string: Node, source: &str, is_dynamic: bool) -> Option<ImportRecord> {
let (start, end) = (string.start_byte(), string.end_byte());
let text = &source[start..end];
let quote = *text.as_bytes().first()?;
if quote != b'"' && quote != b'\'' || text.as_bytes().last() != Some(&quote) {
return None;
}
Some(ImportRecord {
specifier: text[1..text.len() - 1].to_string(),
span: start + 1..end - 1,
is_dynamic,
})
}
/// Record for a `require("…")` / `import("…")` call whose first argument
/// is a string literal. Both are flagged `is_dynamic` per the contract in
/// [`crate::parser`].
fn call_record(call: Node, source: &str) -> Option<ImportRecord> {
let func = call.child_by_field_name("function")?;
let is_require = func.kind() == "identifier"
&& source.get(func.start_byte()..func.end_byte()) == Some("require");
if func.kind() != "import" && !is_require {
return None;
}
let args = call.child_by_field_name("arguments")?;
let first = args.named_child(0).filter(|n| n.kind() == "string")?;
record(first, source, true)
}
/// Pre-order walk pushing every module reference under `node`.
fn visit(node: Node, source: &str, out: &mut Vec<ImportRecord>) {
match node.kind() {
// `import "./a.css"` and `export * from "./a"` also carry a
// `source` field; `export const x = …` has none and is skipped.
"import_statement" | "export_statement" => out.extend(
node.child_by_field_name("source")
.and_then(|string| record(string, source, false)),
),
"call_expression" => out.extend(call_record(node, source)),
_ => {}
}
let mut cursor = node.walk();
for child in node.named_children(&mut cursor) {
visit(child, source, out);
}
}
/// Grammar for a language variant. The TypeScript grammar also parses
/// plain JavaScript; only JSX needs the TSX variant.
fn grammar(lang: SourceLanguage) -> Language {
match lang {
SourceLanguage::Tsx => tree_sitter_typescript::LANGUAGE_TSX.into(),
SourceLanguage::TypeScript | SourceLanguage::JavaScript => {
tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into()
}
}
}
/// Parse `source` and collect its imports in byte-offset order.
/// A parse failure yields whatever the partial tree understood (possibly
/// nothing); it never panics.
pub(super) fn extract(lang: SourceLanguage, source: &str) -> Vec<ImportRecord> {
// tree-sitter's Parser holds raw pointers; build one per call instead
// of storing it in the (Sync) frontend struct.
let mut parser = Parser::new();
let grammar = grammar(lang);
if parser.set_language(&grammar).is_err() {
return Vec::new();
}
let Some(tree) = parser.parse(source, None) else {
return Vec::new();
};
let mut out = Vec::new();
visit(tree.root_node(), source, &mut out);
out.sort_by_key(|rec| rec.span.start);
out
}