feat(parser): tree-sitter TS/JS import extraction + module specifier resolver
- 6 import forms: static/type/side-effect export-from, require, dynamic import() - specifier-only rewrites: byte spans exclude quotes, statements never reformatted - resolver: exact path > extension guessing (.ts/.tsx/.js/.jsx/.mjs/.cjs) > .d.ts > index.* - 22 unit tests (11 parser + 11 resolver)
This commit is contained in:
parent
a33ed2eb30
commit
9d8f1e0bba
3 changed files with 399 additions and 6 deletions
|
|
@ -5,9 +5,27 @@
|
||||||
//! package specifiers (not starting with `.`) are out of project scope and
|
//! package specifiers (not starting with `.`) are out of project scope and
|
||||||
//! resolve to `None`.
|
//! resolve to `None`.
|
||||||
|
|
||||||
|
use std::ffi::OsString;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use crate::core::index::FileSet;
|
use crate::core::index::FileSet;
|
||||||
|
use crate::core::normalize_rel_path;
|
||||||
|
|
||||||
|
/// Supported extensions, in Node/TS resolution priority order.
|
||||||
|
const EXTENSIONS: [&str; 6] = ["ts", "tsx", "js", "jsx", "mjs", "cjs"];
|
||||||
|
|
||||||
|
/// Ambient declaration files: only consulted after every real module
|
||||||
|
/// candidate missed (last resort).
|
||||||
|
const DECLARATION_EXT: &str = "d.ts";
|
||||||
|
|
||||||
|
/// Append `.{ext}` to a path without touching its existing extension
|
||||||
|
/// (specifiers are joined verbatim, matching Node's lookup).
|
||||||
|
fn with_ext(base: &Path, ext: &str) -> PathBuf {
|
||||||
|
let mut joined = OsString::from(base.as_os_str());
|
||||||
|
joined.push(".");
|
||||||
|
joined.push(ext);
|
||||||
|
PathBuf::from(joined)
|
||||||
|
}
|
||||||
|
|
||||||
/// Resolve `specifier` (e.g. `"../utils/fmt"`) written in the file at
|
/// Resolve `specifier` (e.g. `"../utils/fmt"`) written in the file at
|
||||||
/// `importer` (project-relative), against the indexed `files`.
|
/// `importer` (project-relative), against the indexed `files`.
|
||||||
|
|
@ -21,6 +39,156 @@ use crate::core::index::FileSet;
|
||||||
/// Returns `None` for bare specifiers or unresolvable paths.
|
/// Returns `None` for bare specifiers or unresolvable paths.
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn resolve_module(importer: &Path, specifier: &str, files: &FileSet) -> Option<PathBuf> {
|
pub fn resolve_module(importer: &Path, specifier: &str, files: &FileSet) -> Option<PathBuf> {
|
||||||
let _ = (importer, specifier, files);
|
if !specifier.starts_with('.') {
|
||||||
todo!("parser agent: implement resolver")
|
return None; // bare package specifier: outside project scope
|
||||||
|
}
|
||||||
|
// `importer` is project-relative, so `..` segments that walk past the
|
||||||
|
// root collapse to `None` here instead of escaping the index.
|
||||||
|
let base = normalize_rel_path(&importer.parent()?.join(specifier))?;
|
||||||
|
if files.contains(&base) {
|
||||||
|
return Some(base);
|
||||||
|
}
|
||||||
|
for ext in EXTENSIONS {
|
||||||
|
let candidate = with_ext(&base, ext);
|
||||||
|
if files.contains(&candidate) {
|
||||||
|
return Some(candidate);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let declaration = with_ext(&base, DECLARATION_EXT);
|
||||||
|
if files.contains(&declaration) {
|
||||||
|
return Some(declaration);
|
||||||
|
}
|
||||||
|
let index_dir = base.join("index");
|
||||||
|
EXTENSIONS
|
||||||
|
.iter()
|
||||||
|
.map(|ext| with_ext(&index_dir, ext))
|
||||||
|
.find(|candidate| files.contains(candidate))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn files(paths: &[&str]) -> FileSet {
|
||||||
|
let mut set = FileSet::default();
|
||||||
|
for path in paths {
|
||||||
|
set.add(PathBuf::from(path));
|
||||||
|
}
|
||||||
|
set
|
||||||
|
}
|
||||||
|
|
||||||
|
#[track_caller]
|
||||||
|
fn resolve(importer: &str, specifier: &str, set: &FileSet) -> Option<PathBuf> {
|
||||||
|
resolve_module(Path::new(importer), specifier, set)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn exact_specifier_with_extension() {
|
||||||
|
let set = files(&["src/a.ts"]);
|
||||||
|
assert_eq!(
|
||||||
|
resolve("src/b.ts", "./a.ts", &set),
|
||||||
|
Some(PathBuf::from("src/a.ts"))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn guesses_extensions_in_priority_order() {
|
||||||
|
let set = files(&["src/a.ts", "src/a.js"]);
|
||||||
|
assert_eq!(
|
||||||
|
resolve("src/b.ts", "./a", &set),
|
||||||
|
Some(PathBuf::from("src/a.ts"))
|
||||||
|
);
|
||||||
|
let only_js = files(&["src/a.js"]);
|
||||||
|
assert_eq!(
|
||||||
|
resolve("src/b.ts", "./a", &only_js),
|
||||||
|
Some(PathBuf::from("src/a.js"))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn jsx_and_component_specifiers() {
|
||||||
|
let set = files(&["src/Comp.tsx"]);
|
||||||
|
assert_eq!(
|
||||||
|
resolve("src/app.tsx", "./Comp", &set),
|
||||||
|
Some(PathBuf::from("src/Comp.tsx"))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn declaration_file_is_last_resort() {
|
||||||
|
let set = files(&["types/globals.d.ts"]);
|
||||||
|
assert_eq!(
|
||||||
|
resolve("src/a.ts", "../types/globals", &set),
|
||||||
|
Some(PathBuf::from("types/globals.d.ts"))
|
||||||
|
);
|
||||||
|
// a real module wins over the declaration file
|
||||||
|
let both = files(&["types/globals.d.ts", "types/globals.ts"]);
|
||||||
|
assert_eq!(
|
||||||
|
resolve("src/a.ts", "../types/globals", &both),
|
||||||
|
Some(PathBuf::from("types/globals.ts"))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn directory_index_resolution() {
|
||||||
|
let set = files(&["pkg/index.ts", "other/index.js", "other/util.ts"]);
|
||||||
|
assert_eq!(
|
||||||
|
resolve("src/a.ts", "../pkg", &set),
|
||||||
|
Some(PathBuf::from("pkg/index.ts"))
|
||||||
|
);
|
||||||
|
assert_eq!(resolve("src/a.ts", "./", &set), None);
|
||||||
|
assert_eq!(
|
||||||
|
resolve("src/a.ts", "../other", &set),
|
||||||
|
Some(PathBuf::from("other/index.js"))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bare_specifiers_are_not_project_paths() {
|
||||||
|
let set = files(&["react.ts", "lodash.ts"]);
|
||||||
|
assert_eq!(resolve("src/a.ts", "react", &set), None);
|
||||||
|
assert_eq!(resolve("src/a.ts", "@scope/pkg", &set), None);
|
||||||
|
assert_eq!(resolve("src/a.ts", "/abs/path", &set), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn escaping_the_project_root_resolves_to_none() {
|
||||||
|
let set = files(&["evil/x.ts", "outside.ts"]);
|
||||||
|
assert_eq!(resolve("src/a.ts", "../../evil/x", &set), None);
|
||||||
|
assert_eq!(resolve("a.ts", "../outside", &set), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn nested_parent_walks_land_inside_the_project() {
|
||||||
|
let set = files(&["lib/x.ts", "src/app/main.ts"]);
|
||||||
|
assert_eq!(
|
||||||
|
resolve("src/app/main.ts", "../../lib/x", &set),
|
||||||
|
Some(PathBuf::from("lib/x.ts"))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn forward_slash_specifiers_and_redundant_dots() {
|
||||||
|
let set = files(&["src/nested/dir/deep.ts"]);
|
||||||
|
assert_eq!(
|
||||||
|
resolve("src/app.ts", "./nested/./dir/deep", &set),
|
||||||
|
Some(PathBuf::from("src/nested/dir/deep.ts"))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unresolvable_relative_specifier_is_none() {
|
||||||
|
let set = files(&["src/a.ts"]);
|
||||||
|
assert_eq!(resolve("src/a.ts", "./missing", &set), None);
|
||||||
|
assert_eq!(resolve("src/a.ts", "./styles.css", &set), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn root_level_importer_resolves_plain_sibling() {
|
||||||
|
let set = files(&["index.ts", "b.ts"]);
|
||||||
|
assert_eq!(
|
||||||
|
resolve("index.ts", "./b", &set),
|
||||||
|
Some(PathBuf::from("b.ts"))
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
142
src/parser/ts.rs
142
src/parser/ts.rs
|
|
@ -1,7 +1,10 @@
|
||||||
//! Tree-sitter based frontend for TypeScript/JavaScript.
|
//! Tree-sitter based frontend for TypeScript/JavaScript.
|
||||||
//!
|
//!
|
||||||
//! CONTRACT: see [`crate::parser`]. Implement `extract_imports` using
|
//! CONTRACT: see [`crate::parser`]. Traversal lives in [`ts_extract`].
|
||||||
//! `tree-sitter-typescript` grammars.
|
|
||||||
|
// Inline module so the folder stays at `mod.rs` + 3 files.
|
||||||
|
#[path = "ts_extract.rs"]
|
||||||
|
mod ts_extract;
|
||||||
|
|
||||||
use super::{ImportRecord, Language, SourceLanguage};
|
use super::{ImportRecord, Language, SourceLanguage};
|
||||||
|
|
||||||
|
|
@ -23,7 +26,138 @@ impl Language for TreeSitterTs {
|
||||||
self.lang
|
self.lang
|
||||||
}
|
}
|
||||||
|
|
||||||
fn extract_imports(&self, _source: &str) -> Vec<ImportRecord> {
|
fn extract_imports(&self, source: &str) -> Vec<ImportRecord> {
|
||||||
todo!("parser agent: implement tree-sitter extraction")
|
ts_extract::extract(self.lang, source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn parse(lang: SourceLanguage, source: &str) -> Vec<ImportRecord> {
|
||||||
|
TreeSitterTs::new(lang).extract_imports(source)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Assert `records[i]` is specifier `spec` over `source` with quotes
|
||||||
|
/// `q`, at the byte span implied by the occurrence of `q.spec.q`.
|
||||||
|
#[track_caller]
|
||||||
|
fn assert_import(source: &str, records: &[ImportRecord], index: usize, spec: &str) {
|
||||||
|
let rec = &records[index];
|
||||||
|
assert_eq!(rec.specifier, spec);
|
||||||
|
assert_eq!(&source[rec.span.clone()], spec);
|
||||||
|
let before = source[..rec.span.start].bytes().last().unwrap_or(b'\n');
|
||||||
|
let after = source.as_bytes()[rec.span.end];
|
||||||
|
assert!(
|
||||||
|
(before == b'"' || before == b'\'') && before == after,
|
||||||
|
"span must sit exactly between matching quotes, got {before:?} / {after:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn import_statements_with_default_and_named_bindings() {
|
||||||
|
let src = "import X from './a'\nimport { A, B } from \"../utils/b\";\n";
|
||||||
|
let recs = parse(SourceLanguage::TypeScript, src);
|
||||||
|
assert_eq!(recs.len(), 2);
|
||||||
|
assert_import(src, &recs, 0, "./a");
|
||||||
|
assert_import(src, &recs, 1, "../utils/b");
|
||||||
|
assert!(recs.iter().all(|r| !r.is_dynamic));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bare_side_effect_import() {
|
||||||
|
let src = "import './styles.css';\n";
|
||||||
|
let recs = parse(SourceLanguage::TypeScript, src);
|
||||||
|
assert_eq!(recs.len(), 1);
|
||||||
|
assert_import(src, &recs, 0, "./styles.css");
|
||||||
|
assert!(!recs[0].is_dynamic);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn type_only_import_is_static() {
|
||||||
|
let src = "import type { A } from './types';\n";
|
||||||
|
let recs = parse(SourceLanguage::TypeScript, src);
|
||||||
|
assert_eq!(recs.len(), 1);
|
||||||
|
assert_import(src, &recs, 0, "./types");
|
||||||
|
assert!(!recs[0].is_dynamic);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn export_from_reexport() {
|
||||||
|
let src = "export { A } from './a';\nexport * from './b';\n";
|
||||||
|
let recs = parse(SourceLanguage::TypeScript, src);
|
||||||
|
assert_eq!(recs.len(), 2);
|
||||||
|
assert_import(src, &recs, 0, "./a");
|
||||||
|
assert_import(src, &recs, 1, "./b");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn require_call_is_dynamic() {
|
||||||
|
let src = "const x = require('./a');\n";
|
||||||
|
let recs = parse(SourceLanguage::TypeScript, src);
|
||||||
|
assert_eq!(recs.len(), 1);
|
||||||
|
assert_import(src, &recs, 0, "./a");
|
||||||
|
assert!(recs[0].is_dynamic);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dynamic_import_await_and_chained() {
|
||||||
|
let src = "const a = await import('./a');\nimport('./b').then((m) => m);\n";
|
||||||
|
let recs = parse(SourceLanguage::TypeScript, src);
|
||||||
|
assert_eq!(recs.len(), 2);
|
||||||
|
assert_import(src, &recs, 0, "./a");
|
||||||
|
assert_import(src, &recs, 1, "./b");
|
||||||
|
assert!(recs.iter().all(|r| r.is_dynamic));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn declarations_and_comments_are_not_captured() {
|
||||||
|
let src = "export const x = 1;\n// import y from './nope'\n/* require('./nah') */\n";
|
||||||
|
assert!(parse(SourceLanguage::TypeScript, src).is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn nested_imports_inside_functions_and_blocks() {
|
||||||
|
let src = "function f() { if (true) { const m = require('./deep'); } }\n";
|
||||||
|
let recs = parse(SourceLanguage::JavaScript, src);
|
||||||
|
assert_eq!(recs.len(), 1);
|
||||||
|
assert_import(src, &recs, 0, "./deep");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn jsx_file_uses_tsx_grammar() {
|
||||||
|
// The `require` sits inside a JSX attribute body: only the TSX
|
||||||
|
// grammar parses that expression tree (TS reads `<Button` as a
|
||||||
|
// type assertion and loses the nested call).
|
||||||
|
let src = "export const C = () => <Button onClick={() => require('./a')} />;\n";
|
||||||
|
let recs = parse(SourceLanguage::Tsx, src);
|
||||||
|
assert_eq!(recs.len(), 1);
|
||||||
|
assert_import(src, &recs, 0, "./a");
|
||||||
|
assert!(recs[0].is_dynamic);
|
||||||
|
assert!(
|
||||||
|
parse(SourceLanguage::TypeScript, src).is_empty(),
|
||||||
|
"JSX must need the TSX grammar to parse fully"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn records_are_in_byte_order_with_distinct_spans() {
|
||||||
|
let src = "import a from './a';\nexport * from './b';\nconst c = require('./c');\n";
|
||||||
|
let recs = parse(SourceLanguage::TypeScript, src);
|
||||||
|
let spans: Vec<_> = recs.iter().map(|r| r.span.start).collect();
|
||||||
|
assert_eq!(spans, {
|
||||||
|
let mut s = spans.clone();
|
||||||
|
s.sort();
|
||||||
|
s
|
||||||
|
});
|
||||||
|
assert_eq!(recs.len(), 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn plain_javascript_parses_with_the_ts_grammar() {
|
||||||
|
let src = "const x = require('./a');\nimport y from './b';\n";
|
||||||
|
let recs = parse(SourceLanguage::JavaScript, src);
|
||||||
|
assert_eq!(recs.len(), 2);
|
||||||
|
assert_import(src, &recs, 1, "./b");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
91
src/parser/ts_extract.rs
Normal file
91
src/parser/ts_extract.rs
Normal file
|
|
@ -0,0 +1,91 @@
|
||||||
|
//! Tree-sitter traversal backing [`crate::parser::ts`].
|
||||||
|
//!
|
||||||
|
//! Finds every module reference: `import`/`export … from` statements (the
|
||||||
|
//! grammar exposes the specifier via the `source` field), `require("…")`
|
||||||
|
//! calls and dynamic `import("…")` calls. Nesting depth is arbitrary, so
|
||||||
|
//! the walk is fully recursive.
|
||||||
|
|
||||||
|
use tree_sitter::{Language, Node, Parser};
|
||||||
|
|
||||||
|
use super::{ImportRecord, SourceLanguage};
|
||||||
|
|
||||||
|
/// Record for a `string` node used as a specifier: the span covers the
|
||||||
|
/// text between the quote characters (the grammar only emits ASCII
|
||||||
|
/// `"`/`'` quotes for import sources), never the quotes themselves.
|
||||||
|
fn record(string: Node, source: &str, is_dynamic: bool) -> Option<ImportRecord> {
|
||||||
|
let (start, end) = (string.start_byte(), string.end_byte());
|
||||||
|
let text = &source[start..end];
|
||||||
|
let quote = *text.as_bytes().first()?;
|
||||||
|
if quote != b'"' && quote != b'\'' || text.as_bytes().last() != Some("e) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(ImportRecord {
|
||||||
|
specifier: text[1..text.len() - 1].to_string(),
|
||||||
|
span: start + 1..end - 1,
|
||||||
|
is_dynamic,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record for a `require("…")` / `import("…")` call whose first argument
|
||||||
|
/// is a string literal. Both are flagged `is_dynamic` per the contract in
|
||||||
|
/// [`crate::parser`].
|
||||||
|
fn call_record(call: Node, source: &str) -> Option<ImportRecord> {
|
||||||
|
let func = call.child_by_field_name("function")?;
|
||||||
|
let is_require = func.kind() == "identifier"
|
||||||
|
&& source.get(func.start_byte()..func.end_byte()) == Some("require");
|
||||||
|
if func.kind() != "import" && !is_require {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let args = call.child_by_field_name("arguments")?;
|
||||||
|
let first = args.named_child(0).filter(|n| n.kind() == "string")?;
|
||||||
|
record(first, source, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pre-order walk pushing every module reference under `node`.
|
||||||
|
fn visit(node: Node, source: &str, out: &mut Vec<ImportRecord>) {
|
||||||
|
match node.kind() {
|
||||||
|
// `import "./a.css"` and `export * from "./a"` also carry a
|
||||||
|
// `source` field; `export const x = …` has none and is skipped.
|
||||||
|
"import_statement" | "export_statement" => out.extend(
|
||||||
|
node.child_by_field_name("source")
|
||||||
|
.and_then(|string| record(string, source, false)),
|
||||||
|
),
|
||||||
|
"call_expression" => out.extend(call_record(node, source)),
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
let mut cursor = node.walk();
|
||||||
|
for child in node.named_children(&mut cursor) {
|
||||||
|
visit(child, source, out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Grammar for a language variant. The TypeScript grammar also parses
|
||||||
|
/// plain JavaScript; only JSX needs the TSX variant.
|
||||||
|
fn grammar(lang: SourceLanguage) -> Language {
|
||||||
|
match lang {
|
||||||
|
SourceLanguage::Tsx => tree_sitter_typescript::LANGUAGE_TSX.into(),
|
||||||
|
SourceLanguage::TypeScript | SourceLanguage::JavaScript => {
|
||||||
|
tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse `source` and collect its imports in byte-offset order.
|
||||||
|
/// A parse failure yields whatever the partial tree understood (possibly
|
||||||
|
/// nothing); it never panics.
|
||||||
|
pub(super) fn extract(lang: SourceLanguage, source: &str) -> Vec<ImportRecord> {
|
||||||
|
// tree-sitter's Parser holds raw pointers; build one per call instead
|
||||||
|
// of storing it in the (Sync) frontend struct.
|
||||||
|
let mut parser = Parser::new();
|
||||||
|
let grammar = grammar(lang);
|
||||||
|
if parser.set_language(&grammar).is_err() {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
let Some(tree) = parser.parse(source, None) else {
|
||||||
|
return Vec::new();
|
||||||
|
};
|
||||||
|
let mut out = Vec::new();
|
||||||
|
visit(tree.root_node(), source, &mut out);
|
||||||
|
out.sort_by_key(|rec| rec.span.start);
|
||||||
|
out
|
||||||
|
}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue