Merge pull request #6 from wldmr/synthetic-nodes

Synthetic nodes

README.md

@@ -15,7 +15,7 @@
 "Type-safe" here means that:
 
 - Instead of representing all tree-sitter nodes by [`tree_sitter::Node`](https://docs.rs/tree-sitter/latest/tree_sitter/struct.Node.html), each node type has its own data-type which wraps `tree_sitter::Node`.
-  - Nodes with supertypes are `enum`s, so their 
+  - Supertype nodes are `enum`s, so you can pattern-match their subtypes with compile-time exhaustiveness checking.
   - Each node data-type implements [`type_sitter::Node`](https://docs.rs/type-sitter-lib/latest/type_sitter_lib/trait.Node.html). You can use generics and convert to/from [`type_sitter::UntypedNode`](https://docs.rs/type-sitter-lib/latest/type_sitter_lib/struct.UntypedNode.html) to write methods that take or return arbitrary-typed nodes.
 - Instead of accessing fields by `field("field_name")`, you access by specific methods like `field_name()`.
   - These methods, and every other generated method, also return typed nodes.
@@ -26,6 +26,7 @@
 - Typed error, missing, and extra nodes.
 - From a typed node you can lookup the "extra" nodes before and after, e.g. to handle comments.
 - [`Option<NodeResult<'_>>.flatten()`](https://docs.rs/type-sitter-lib/latest/type_sitter_lib/trait.NodeResultExtraOrExt.html#tymethod.flatten).
+- Custom supertypes can be created at build time, to group nodes that are't grouped in the original grammar. You could, for instance, create create a supertype for all named nodes, all nodes that have named fields, or any other grouping that makes sense for the tool that you're building.
 
 Lastly, there's an optional feature, `yak-sitter`, which re-exports the `tree-sitter` API with a few small changes, most notably nodes being able to access their text and filepath directly. The [`yak-sitter`](./yak-sitter/README.md) library is a drop-in replacement for `tree-sitter` and can by used by itself without `type-sitter` (and `yak-sitter` is optional in `type-sitter`).
 
@@ -79,7 +80,7 @@ cargo add --build type-sitter-gen  # Notice `cargo add --build`
 Then, in `build.rs`
 
 ```rust
-use std::path::PathBuf;
+use std::path::{PathBuf, Path};
 use std::{env, fs};
 use type_sitter_gen::{generate_nodes, generate_queries, super_nodes};
 
@@ -93,11 +94,10 @@ fn main() {
     println!("cargo::rerun-if-changed=vendor/path/to/tree-sitter-foobar-lang");
 
     // To generate nodes
+    let path = Path::new("vendor/path/to/tree-sitter-foobar-lang/src/node-types.json")
     fs::write(
         out_dir.join("nodes.rs"),
-        generate_nodes(
-            "vendor/path/to/tree-sitter-foobar-lang/src/node-types.json"
-        ).unwrap().into_string()
+        generate_nodes(path).unwrap().into_string()
     ).unwrap();
 
     // To generate queries
@@ -127,6 +127,30 @@ mod queries {
 }
 ```
 
+To generate custom supertypes, modify the above to something like
+
+```rust
+    use type_sitter_gen::{NodeTypeMap, NodeName, NodeTypeKind}
+    // ...
+
+    // To generate nodes
+    let path = Path::new("vendor/path/to/tree-sitter-foobar-lang/src/node-types.json")
+    let node_type_map = NodeTypeMap::try_from(path).unwrap();
+
+    let named: Vec<NodeName> = node_type_map
+        .values()
+        .map(|node| node.name.clone())
+        .filter(|name| name.is_named);
+    node_type_map
+        .add_custom_supertype("_all_named", named)
+        .expect("this mustn't already exist");
+
+    fs::write(
+        out_dir.join("nodes.rs"),
+        generate_nodes(node_type_map).unwrap().into_string()
+    ).unwrap();
+```
+
 ### CLI tool (flexible)
 
 ```shell

type-sitter-gen/Cargo.toml

@@ -29,3 +29,8 @@ walkdir = "2.5.0"
 logos = "0.14.1"
 slice-group-by = "0.3.1"
 prettyplease = "0.2.22"
+
+[dev-dependencies]
+# to make these libraries available in doctests
+tree-sitter-rust = { path = "../vendor/tree-sitter-rust" }
+tree-sitter-json = { path = "../vendor/tree-sitter-json" }

type-sitter-gen/src/node_types/mod.rs

@@ -13,35 +13,64 @@ pub use generated_tokens::*;
 use std::fs::File;
 use std::io::BufReader;
 use std::path::Path;
-pub(crate) use types::*;
+pub use types::*;
 pub use rust_names::*;
-pub(crate) use names::*;
+pub use names::*;
 
 /// Generate source code (tokens) for typed AST node wrappers.
 ///
 /// # Parameters
-/// - `path`: Path to the `node-types.json` file of the language.
+/// - `types`: Anything that can be converted to a `NodeTypeMap`
 ///
-/// # Example
+/// # Examples
+///
+/// Using a file path:
+///
+/// ```rust
+/// use type_sitter_gen::generate_nodes;
+/// use std::path::Path;
+///
+/// fn main() {
+///     let path = Path::new("../vendor/tree-sitter-rust/src/node-types.json");
+///     let code = generate_nodes(path).unwrap().into_string();
+///     assert!(code.contains("pub struct TraitItem"));
+/// }
+/// ```
+///
+/// Using the contents of `node-types.json` directly:
 ///
 /// ```rust
 /// use type_sitter_gen::generate_nodes;
 ///
 /// fn main() {
-///     println!("{}", generate_nodes(
-///         "../vendor/tree-sitter-rust/src/node-types.json"
-///     ).unwrap().into_string());
+///     let contents: &str = tree_sitter_rust::NODE_TYPES;
+///     let code = generate_nodes(contents).unwrap().into_string();
+///     assert!(code.contains("pub struct TraitItem"));
 /// }
 /// ```
-pub fn generate_nodes(path: impl AsRef<Path>) -> Result<GeneratedNodeTokens, Error> {
-    generate_nodes_with_custom_module_paths(path, &type_sitter_raw(), &type_sitter())
+///
+/// Using a `NodeTypeMap`:
+///
+/// ```rust
+/// use type_sitter_gen::{generate_nodes, NodeTypeMap};
+///
+/// fn main() {
+///     let mut node_type_map = NodeTypeMap::try_from(tree_sitter_rust::NODE_TYPES).unwrap();
+///     // customize node_type_map
+///     let code = generate_nodes(node_type_map).unwrap().into_string();
+///     assert!(code.contains("pub struct TraitItem"));
+/// }
+/// ```
+pub fn generate_nodes<T, E>(types: T) -> Result<GeneratedNodeTokens, E>
+where T: TryInto<NodeTypeMap, Error = E> {
+    generate_nodes_with_custom_module_paths(types, &type_sitter_raw(), &type_sitter())
 }
 
 /// Generate source code (tokens) for typed AST node wrappers, and the generated code will refer to
 /// the provided modules instead of `type_sitter::raw` and `type_sitter` respectively.
 ///
 /// # Parameters
-/// - `path`: Path to the `node-types.json` file of the language.
+/// - `types`: Anything that can be converted to a `NodeTypeMap`
 /// - `tree_sitter`: Path to the crate with the tree-sitter API. In [`generate_nodes`] this is
 ///   [`type_sitter_raw`] but you can provide something else, like the re-exported [`tree_sitter`]
 ///   or [`yak_sitter`] directly.
@@ -53,17 +82,20 @@ pub fn generate_nodes(path: impl AsRef<Path>) -> Result<GeneratedNodeTokens, Err
 ///
 /// ```rust
 /// use type_sitter_gen::{generate_nodes_with_custom_module_paths, tree_sitter, type_sitter_lib};
+/// use std::path::Path;
 ///
 /// fn main() {
-///     println!("{}", generate_nodes_with_custom_module_paths(
-///         "../vendor/tree-sitter-rust/src/node-types.json",
+///     let code = generate_nodes_with_custom_module_paths(
+///         Path::new("../vendor/tree-sitter-rust/src/node-types.json"),
 ///         &tree_sitter(),
 ///         &type_sitter_lib()
-///     ).unwrap().into_string());
+///     ).unwrap().into_string();
+///     assert!(code.contains("pub struct TraitItem"));
 /// }
 /// ```
-pub fn generate_nodes_with_custom_module_paths(path: impl AsRef<Path>, tree_sitter: &syn::Path, type_sitter_lib: &syn::Path) -> Result<GeneratedNodeTokens, Error> {
-    let all_types = parse_node_type_map(path)?;
+pub fn generate_nodes_with_custom_module_paths<T, E>(all_types: T, tree_sitter: &syn::Path, type_sitter_lib: &syn::Path) -> Result<GeneratedNodeTokens, E>
+where T: TryInto<NodeTypeMap, Error = E> {
+    let all_types = all_types.try_into()?;
 
     let ctx = PrintCtx {
         all_types: &all_types,
@@ -77,10 +109,12 @@ pub fn generate_nodes_with_custom_module_paths(path: impl AsRef<Path>, tree_sitt
 }
 
 /// Parse a `node-types.json` file into a map of [SEXP name](NodeName::sexp_name) to [`NodeType`].
+#[deprecated = "use NodeTypeMap::try_from(...) instead"]
 pub(crate) fn parse_node_type_map(path: impl AsRef<Path>) -> Result<NodeTypeMap, Error> {
     let path = path.as_ref();
     let reader = BufReader::new(File::open(path)?);
     let elems = iter_json_array::<ContextFreeNodeType, _>(reader)
         .collect::<Result<Vec<_>, _>>()?;
     Ok(NodeTypeMap::new(elems))
-}
+}
+