From a7bcff1721e4053c2eb64b3601602fe02e3e81b3 Mon Sep 17 00:00:00 2001 From: Pieter Goetschalckx Date: Sun, 22 Sep 2024 20:49:33 +0200 Subject: [PATCH 1/5] Update rust binding: use tree-sitter-language --- Cargo.lock | 5 +-- Cargo.toml | 5 ++- bindings/rust/README.md | 43 +++++++++++++++---------- bindings/rust/lib.rs | 71 ++++++++++++++++++++--------------------- 4 files changed, 69 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 72fd7c5..c87f153 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.15" +version = "1.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6" +checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" dependencies = [ "shlex", ] @@ -85,4 +85,5 @@ version = "0.23.0" dependencies = [ "cc", "tree-sitter", + "tree-sitter-language", ] diff --git a/Cargo.toml b/Cargo.toml index f116b61..8600798 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,10 @@ include = [ path = "bindings/rust/lib.rs" [dependencies] -tree-sitter = ">=0.22" +tree-sitter-language = "0.1" [build-dependencies] cc = "1.0.87" + +[dev-dependencies] +tree-sitter = { version = "0.23" } diff --git a/bindings/rust/README.md b/bindings/rust/README.md index f103290..83ff7ba 100644 --- a/bindings/rust/README.md +++ b/bindings/rust/README.md @@ -1,8 +1,13 @@ # tree-sitter-ocaml -This crate provides Ocaml grammars for the [tree-sitter][] parsing library. -There are separate grammars for implementations (`.ml`), interfaces (`.mli`) -and types. +[![CI][ci]](https://github.com/tree-sitter/tree-sitter-ocaml/actions/workflows/ci.yml) +[![crates][crates]](https://crates.io/crates/tree-sitter-ocaml) + +This crate provides OCaml language support for the [tree-sitter][] parsing +library. There are separate languages for implementations (`.ml`), +interfaces (`.mli`) and types. + +## Installation To use this crate, add it to the `[dependencies]` section of your `Cargo.toml` file. (Note that you will probably also need to depend on the @@ -11,12 +16,14 @@ way.) ```toml [dependencies] -tree-sitter = "0.22" -tree-sitter-ocaml = "0.22" +tree-sitter = "0.23" +tree-sitter-ocaml = "0.23" ``` -Typically, you will use the [language][language func] function to add this -grammar to a tree-sitter [Parser][], and then use the parser to parse some code: +## Usage + +Typically, you will use [`LANGUAGE_OCAML`][LANGUAGE_OCAML] to add this language +to a tree-sitter [`Parser`][Parser], and then use the parser to parse some code: ```rust let code = r#" @@ -24,19 +31,23 @@ let code = r#" let x = 0 end "#; -let mut parser = Parser::new(); +let mut parser = tree_sitter::Parser::new(); +let language = tree_sitter_ocaml::LANGUAGE_OCAML; parser - .set_language(&tree_sitter_ocaml::language_ocaml()) - .expect("Error loading OCaml grammar"); + .set_language(&language.into()) + .expect("Error loading OCaml language"); let tree = parser.parse(code, None).unwrap(); ``` -If you have any questions, please reach out to us in the [tree-sitter -discussions] page. +Use [`LANGUAGE_OCAML_INTERFACE`][LANGUAGE_OCAML_INTERFACE] to parse interface +files (with `.mli` extension) and [`LANGUAGE_OCAML_TYPE`][LANGUAGE_OCAML_TYPE] +to parse type signatures. -[Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html -[language func]: https://docs.rs/tree-sitter-rust/*/tree_sitter_ocaml/fn.language_ocaml.html -[Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html +[ci]: https://img.shields.io/github/actions/workflow/status/tree-sitter/tree-sitter-ocaml/ci.yml?logo=github&label=CI +[crates]: https://img.shields.io/crates/v/tree-sitter-ocaml?logo=rust [tree-sitter]: https://tree-sitter.github.io/ [tree-sitter crate]: https://crates.io/crates/tree-sitter -[tree-sitter discussions]: https://github.com/tree-sitter/tree-sitter/discussions +[LANGUAGE_OCAML]: https://docs.rs/tree-sitter-ocaml/*/tree_sitter_ocaml/constant.LANGUAGE_OCAML.html +[LANGUAGE_OCAML_INTERFACE]: https://docs.rs/tree-sitter-ocaml/*/tree_sitter_ocaml/constant.LANGUAGE_OCAML_INTERFACE.html +[LANGUAGE_OCAML_TYPE]: https://docs.rs/tree-sitter-ocaml/*/tree_sitter_ocaml/constant.LANGUAGE_OCAML_TYPE.html +[Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html diff --git a/bindings/rust/lib.rs b/bindings/rust/lib.rs index 4b5cb84..92f71a1 100644 --- a/bindings/rust/lib.rs +++ b/bindings/rust/lib.rs @@ -2,9 +2,8 @@ //! library. There are separate languages for implementation, (`.ml`), //! interfaces (`.mli`) and types. //! -//! Typically, you will use the [language_ocaml][language func] function to add -//! this language to a tree-sitter [Parser][], and then use the parser to parse -//! some code: +//! Typically, you will use [`LANGUAGE_OCAML`][LANGUAGE_OCAML] to add this language +//! to a tree-sitter [`Parser`][Parser], and then use the parser to parse some code: //! //! ``` //! let code = r#" @@ -13,70 +12,70 @@ //! end //! "#; //! let mut parser = tree_sitter::Parser::new(); +//! let language = tree_sitter_ocaml::LANGUAGE_OCAML; //! parser -//! .set_language(&tree_sitter_ocaml::language_ocaml()) -//! .expect("Error loading OCaml grammar"); +//! .set_language(&language.into()) +//! .expect("Error loading OCaml language"); //! let tree = parser.parse(code, None).unwrap(); //! assert!(!tree.root_node().has_error()); //! ``` //! -//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html -//! [language func]: fn.language_ocaml.html +//! Use [`LANGUAGE_OCAML_INTERFACE`][LANGUAGE_OCAML_INTERFACE] to parse interface +//! files (with `.mli` extension) and [`LANGUAGE_OCAML_TYPE`][LANGUAGE_OCAML_TYPE] +//! to parse type signatures. +//! +//! [LANGUAGE_OCAML]: constant.LANGUAGE_OCAML.html +//! [LANGUAGE_OCAML_INTERFACE]: constant.LANGUAGE_OCAML_INTERFACE.html +//! [LANGUAGE_OCAML_TYPE]: constant.LANGUAGE_OCAML_TYPE.html //! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html //! [tree-sitter]: https://tree-sitter.github.io/ -use tree_sitter::Language; +use tree_sitter_language::LanguageFn; extern "C" { - fn tree_sitter_ocaml() -> Language; - fn tree_sitter_ocaml_interface() -> Language; - fn tree_sitter_ocaml_type() -> Language; + fn tree_sitter_ocaml() -> *const (); + fn tree_sitter_ocaml_interface() -> *const (); + fn tree_sitter_ocaml_type() -> *const (); } -/// Get the tree-sitter [Language][] for OCaml. +/// The tree-sitter [`LanguageFn`][LanguageFn] for OCaml. /// -/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html -pub fn language_ocaml() -> Language { - unsafe { tree_sitter_ocaml() } -} +/// [LanguageFn]: https://docs.rs/tree-sitter-language/*/tree_sitter_language/struct.LanguageFn.html +pub const LANGUAGE_OCAML: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_ocaml) }; -/// Get the tree-sitter [Language][] for OCaml interfaces. +/// The tree-sitter [`LanguageFn`] for OCaml interfaces. /// /// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html -pub fn language_ocaml_interface() -> Language { - unsafe { tree_sitter_ocaml_interface() } -} +pub const LANGUAGE_OCAML_INTERFACE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_ocaml_interface) }; -/// Get the tree-sitter [Language][] for OCaml types. +/// The tree-sitter [`LanguageFn`] for OCaml types. /// /// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html -pub fn language_ocaml_type() -> Language { - unsafe { tree_sitter_ocaml_type() } -} +pub const LANGUAGE_OCAML_TYPE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_ocaml_type) }; /// The content of the [`node-types.json`][] file for OCaml. /// /// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types -pub const OCAML_NODE_TYPES: &'static str = include_str!("../../grammars/ocaml/src/node-types.json"); +pub const OCAML_NODE_TYPES: &str = include_str!("../../grammars/ocaml/src/node-types.json"); /// The content of the [`node-types.json`][] file for OCaml interfaces. /// /// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types -pub const INTERFACE_NODE_TYPES: &'static str = include_str!("../../grammars/interface/src/node-types.json"); +pub const INTERFACE_NODE_TYPES: &str = include_str!("../../grammars/interface/src/node-types.json"); /// The content of the [`node-types.json`][] file for OCaml types. /// /// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types -pub const TYPE_NODE_TYPES: &'static str = include_str!("../../grammars/type/src/node-types.json"); +pub const TYPE_NODE_TYPES: &str = include_str!("../../grammars/type/src/node-types.json"); /// The syntax highlighting query for OCaml. -pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm"); +pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm"); /// The local-variable syntax highlighting query for OCaml. -pub const LOCALS_QUERY: &'static str = include_str!("../../queries/locals.scm"); +pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm"); /// The symbol tagging query for OCaml. -pub const TAGGING_QUERY: &'static str = include_str!("../../queries/tags.scm"); +pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm"); #[cfg(test)] mod tests { @@ -84,8 +83,8 @@ mod tests { fn test_ocaml() { let mut parser = tree_sitter::Parser::new(); parser - .set_language(&super::language_ocaml()) - .expect("Error loading OCaml grammar"); + .set_language(&super::LANGUAGE_OCAML.into()) + .expect("Error loading OCaml parser"); let code = r#" module M = struct @@ -102,8 +101,8 @@ mod tests { fn test_ocaml_interface() { let mut parser = tree_sitter::Parser::new(); parser - .set_language(&super::language_ocaml_interface()) - .expect("Error loading OCaml interface grammar"); + .set_language(&super::LANGUAGE_OCAML_INTERFACE.into()) + .expect("Error loading OCaml interface parser"); let code = r#" module M : sig @@ -120,8 +119,8 @@ mod tests { fn test_ocaml_type() { let mut parser = tree_sitter::Parser::new(); parser - .set_language(&super::language_ocaml_type()) - .expect("Error loading OCaml type grammar"); + .set_language(&super::LANGUAGE_OCAML_TYPE.into()) + .expect("Error loading OCaml type parser"); let code = r#"int list"#; From b0196314f352fd39860c172ca4d0a57b945fd616 Mon Sep 17 00:00:00 2001 From: Pieter Goetschalckx Date: Sun, 22 Sep 2024 20:50:10 +0200 Subject: [PATCH 2/5] Update swift binding: remove exclude --- Package.swift | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/Package.swift b/Package.swift index 45be2bb..6d8a354 100644 --- a/Package.swift +++ b/Package.swift @@ -13,31 +13,6 @@ let package = Package( .target( name: "TreeSitterOCaml", path: ".", - exclude: [ - "Cargo.toml", - "Makefile", - "binding.gyp", - "bindings/go", - "bindings/node", - "bindings/python", - "bindings/rust", - "common/common.mak", - "package.json", - "package-lock.json", - "pyproject.toml", - "setup.py", - "test", - ".editorconfig", - ".github", - ".gitignore", - ".gitattributes", - "grammars/ocaml/Makefile", - "grammars/ocaml/grammar.js", - "grammars/interface/Makefile", - "grammars/interface/grammar.js", - "grammars/type/Makefile", - "grammars/type/grammar.js", - ], sources: [ "grammars/ocaml/src/parser.c", "grammars/ocaml/src/scanner.c", From c106a0123522f4347174d8d5cc1e61497a8b2c9e Mon Sep 17 00:00:00 2001 From: Pieter Goetschalckx Date: Sun, 22 Sep 2024 21:01:08 +0200 Subject: [PATCH 3/5] Update python binding: rename language functions --- bindings/python/tree_sitter_ocaml/__init__.py | 6 +++--- bindings/python/tree_sitter_ocaml/__init__.pyi | 6 +++--- bindings/python/tree_sitter_ocaml/binding.c | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/bindings/python/tree_sitter_ocaml/__init__.py b/bindings/python/tree_sitter_ocaml/__init__.py index 5f56d7c..3dddf2a 100644 --- a/bindings/python/tree_sitter_ocaml/__init__.py +++ b/bindings/python/tree_sitter_ocaml/__init__.py @@ -23,9 +23,9 @@ def __getattr__(name): __all__ = [ - "ocaml", - "interface", - "type", + "language_ocaml", + "language_ocaml_interface", + "language_ocaml_type", "HIGHLIGHTS_QUERY", "LOCALS_QUERY", "TAGS_QUERY", diff --git a/bindings/python/tree_sitter_ocaml/__init__.pyi b/bindings/python/tree_sitter_ocaml/__init__.pyi index 5c093f5..0a4630f 100644 --- a/bindings/python/tree_sitter_ocaml/__init__.pyi +++ b/bindings/python/tree_sitter_ocaml/__init__.pyi @@ -6,6 +6,6 @@ LOCALS_QUERY: Final[str] TAGS_QUERY: Final[str] -def ocaml() -> int: ... -def interface() -> int: ... -def type() -> int: ... +def language_ocaml() -> object: ... +def language_ocaml_interface() -> object: ... +def language_ocaml_type() -> object: ... diff --git a/bindings/python/tree_sitter_ocaml/binding.c b/bindings/python/tree_sitter_ocaml/binding.c index 35d9398..af3ea88 100644 --- a/bindings/python/tree_sitter_ocaml/binding.c +++ b/bindings/python/tree_sitter_ocaml/binding.c @@ -7,23 +7,23 @@ TSLanguage *tree_sitter_ocaml_interface(void); TSLanguage *tree_sitter_ocaml_type(void); static PyObject* _binding_language_ocaml(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) { - return PyLong_FromVoidPtr(tree_sitter_ocaml()); + return PyCapsule_New(tree_sitter_ocaml(), "tree_sitter.Language", NULL); } static PyObject* _binding_language_ocaml_interface(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) { - return PyLong_FromVoidPtr(tree_sitter_ocaml_interface()); + return PyCapsule_New(tree_sitter_ocaml_interface(), "tree_sitter.Language", NULL); } static PyObject* _binding_language_ocaml_type(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) { - return PyLong_FromVoidPtr(tree_sitter_ocaml_type()); + return PyCapsule_New(tree_sitter_ocaml_type(), "tree_sitter.Language", NULL); } static PyMethodDef methods[] = { - {"ocaml", _binding_language_ocaml, METH_NOARGS, + {"language_ocaml", _binding_language_ocaml, METH_NOARGS, "Get the tree-sitter language for OCaml."}, - {"interface", _binding_language_ocaml_interface, METH_NOARGS, + {"language_ocaml_interface", _binding_language_ocaml_interface, METH_NOARGS, "Get the tree-sitter language for OCaml interfaces."}, - {"type", _binding_language_ocaml_type, METH_NOARGS, + {"language_ocaml_type", _binding_language_ocaml_type, METH_NOARGS, "Get the tree-sitter language for OCaml types."}, {NULL, NULL, 0, NULL} }; From eae9be1c406235c3e5afadf332e63547824981fa Mon Sep 17 00:00:00 2001 From: Pieter Goetschalckx Date: Sun, 22 Sep 2024 21:01:22 +0200 Subject: [PATCH 4/5] Update python binding: add README --- bindings/python/README.md | 57 +++++++++++++++++++ bindings/python/tests/test_binding.py | 6 +- bindings/python/tree_sitter_ocaml/__init__.py | 2 +- pyproject.toml | 4 +- 4 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 bindings/python/README.md diff --git a/bindings/python/README.md b/bindings/python/README.md new file mode 100644 index 0000000..38fd270 --- /dev/null +++ b/bindings/python/README.md @@ -0,0 +1,57 @@ +# tree-sitter-ocaml + +[![CI][ci]](https://github.com/tree-sitter/tree-sitter-ocaml/actions/workflows/ci.yml) +[![pypi][pypi]](https://pypi.org/project/tree-sitter-ocaml/) + +This module provides OCaml grammars for the [tree-sitter][] parsing library. +There are separate grammars for implementations (`.ml`), interfaces (`.mli`) +and types. + +## Installation + +```sh +pip install tree-sitter-ocaml +``` + +You will probably also need the [tree-sitter binding][tree-sitter binding]. + +```sh +pip install tree-sitter +``` + +## Usage + +Load the grammar as a `Language` object: + +```python +import tree_sitter_ocaml +from tree_sitter import Language, Parser + +language_ocaml = Language(tree_sitter_ocaml.language_ocaml()) +``` + +Create a `Parser` and configure it to use the language: + +```python +parser = Parser(language_ocaml) +``` + +Parse some source code: + +```python +tree = parser.parse( + b""" + module M : sig + val x : int + end + """ +) +``` + +Use `language_ocaml_interface()` to parse interface files (with `.mli` extension) +and `language_ocaml_type()` to parse type signatures. + +[ci]: https://img.shields.io/github/actions/workflow/status/tree-sitter/tree-sitter-ocaml/ci.yml?logo=github&label=CI +[pypi]: https://img.shields.io/pypi/v/tree-sitter-ocaml?logo=pypi&logoColor=white&label=PyPI +[tree-sitter]: https://tree-sitter.github.io/tree-sitter/ +[tree-sitter binding]: https://pypi.org/project/tree-sitter/ diff --git a/bindings/python/tests/test_binding.py b/bindings/python/tests/test_binding.py index 2055cf0..ad4d8b4 100644 --- a/bindings/python/tests/test_binding.py +++ b/bindings/python/tests/test_binding.py @@ -6,7 +6,7 @@ class TestLanguage(TestCase): def test_ocaml_grammar(self): - language = Language(tree_sitter_ocaml.ocaml()) + language = Language(tree_sitter_ocaml.language_ocaml()) parser = Parser(language) tree = parser.parse( b""" @@ -18,7 +18,7 @@ def test_ocaml_grammar(self): self.assertFalse(tree.root_node.has_error) def test_interface_grammar(self): - language = Language(tree_sitter_ocaml.interface()) + language = Language(tree_sitter_ocaml.language_ocaml_interface()) parser = Parser(language) tree = parser.parse( b""" @@ -30,7 +30,7 @@ def test_interface_grammar(self): self.assertFalse(tree.root_node.has_error) def test_type_grammar(self): - language = Language(tree_sitter_ocaml.type()) + language = Language(tree_sitter_ocaml.language_ocaml_type()) parser = Parser(language) tree = parser.parse(b"int list") self.assertFalse(tree.root_node.has_error) diff --git a/bindings/python/tree_sitter_ocaml/__init__.py b/bindings/python/tree_sitter_ocaml/__init__.py index 3dddf2a..8198100 100644 --- a/bindings/python/tree_sitter_ocaml/__init__.py +++ b/bindings/python/tree_sitter_ocaml/__init__.py @@ -2,7 +2,7 @@ from importlib.resources import files as _files -from ._binding import ocaml, interface, type +from ._binding import language_ocaml, language_ocaml_interface, language_ocaml_type def _get_query(name, file): diff --git a/pyproject.toml b/pyproject.toml index 67fd871..33e6ef0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,13 +16,13 @@ classifiers = [ ] requires-python = ">=3.9" license.text = "MIT" -readme = "README.md" +readme = "bindings/python/README.md" [project.urls] Homepage = "https://github.com/tree-sitter/tree-sitter-ocaml" [project.optional-dependencies] -core = ["tree-sitter~=0.22"] +core = ["tree-sitter~=0.23"] [tool.cibuildwheel] build = "cp39-*" From 6ac67312509ca32e6b04f4bf6185629d35620f59 Mon Sep 17 00:00:00 2001 From: Pieter Goetschalckx Date: Sun, 22 Sep 2024 21:02:05 +0200 Subject: [PATCH 5/5] Update README --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 92e02b6..e8d0a6d 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,29 @@ # tree-sitter-ocaml [![CI][ci]](https://github.com/tree-sitter/tree-sitter-ocaml/actions/workflows/ci.yml) -[![discord][discord]](https://discord.gg/w7nTvsVJhm) -[![matrix][matrix]](https://matrix.to/#/#tree-sitter-chat:matrix.org) -[![crates][crates]](https://crates.io/crates/tree-sitter-ocaml) [![npm][npm]](https://www.npmjs.com/package/tree-sitter-ocaml) +[![crates][crates]](https://crates.io/crates/tree-sitter-ocaml) +[![pypi][pypi]](https://pypi.org/project/tree-sitter-ocaml/) -OCaml grammar for [tree-sitter](https://github.com/tree-sitter/tree-sitter). +OCaml grammar for [tree-sitter][]. This module defines grammars for implementations (`.ml`) interfaces (`.mli`) and types. Require them as follows: ```js require('tree-sitter-ocaml').ocaml; -require('tree-sitter-ocaml').interface; -require('tree-sitter-ocaml').type; +require('tree-sitter-ocaml').ocaml_interface; +require('tree-sitter-ocaml').ocaml_type; ``` References - [OCaml language reference](https://ocaml.org/manual/language.html) - [OCaml language extensions](https://ocaml.org/manual/extn.html) +- [OCaml lexer](https://github.com/ocaml/ocaml/blob/trunk/parsing/lexer.mll) - [OCaml parser](https://github.com/ocaml/ocaml/blob/trunk/parsing/parser.mly) [ci]: https://img.shields.io/github/actions/workflow/status/tree-sitter/tree-sitter-ocaml/ci.yml?logo=github&label=CI -[discord]: https://img.shields.io/discord/1063097320771698699?logo=discord&label=discord -[matrix]: https://img.shields.io/matrix/tree-sitter-chat%3Amatrix.org?logo=matrix&label=matrix [npm]: https://img.shields.io/npm/v/tree-sitter-ocaml?logo=npm [crates]: https://img.shields.io/crates/v/tree-sitter-ocaml?logo=rust +[pypi]: https://img.shields.io/pypi/v/tree-sitter-ocaml?logo=pypi&logoColor=white&label=PyPI +[tree-sitter]: https://tree-sitter.github.io/tree-sitter/