From 3cb95be452491a72d18c98ebc619b0d2abb1b746 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bla=C5=BE=20Hrastnik?= <blaz@mxxn.io>
Date: Mon, 6 Sep 2021 13:18:16 +0900
Subject: [PATCH] Update tree-sitter to 0.20
0.20 includes querying improvements, we no longer have to convert
fragments to strings but can return an iterator of chunks instead.
---
Cargo.lock | 4 +-
helix-core/Cargo.toml | 2 +-
helix-core/src/syntax.rs | 79 +++++++++++++++++++++-------------------
helix-syntax/Cargo.toml | 2 +-
4 files changed, 45 insertions(+), 42 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 298ad15b..1b2492b8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1034,9 +1034,9 @@ dependencies = [
[[package]]
name = "tree-sitter"
-version = "0.19.5"
+version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad726ec26496bf4c083fff0f43d4eb3a2ad1bba305323af5ff91383c0b6ecac0"
+checksum = "63ec02a07a782abef91279b72fe8fd2bee4c168a22112cedec5d3b0d49b9e4f9"
dependencies = [
"cc",
"regex",
diff --git a/helix-core/Cargo.toml b/helix-core/Cargo.toml
index 8c83816c..5d2db642 100644
--- a/helix-core/Cargo.toml
+++ b/helix-core/Cargo.toml
@@ -22,7 +22,7 @@ unicode-segmentation = "1.8"
unicode-width = "0.1"
unicode-general-category = "0.4"
# slab = "0.4.2"
-tree-sitter = "0.19"
+tree-sitter = "0.20"
once_cell = "1.8"
arc-swap = "1"
regex = "1"
diff --git a/helix-core/src/syntax.rs b/helix-core/src/syntax.rs
index 64b921e6..a7a5d022 100644
--- a/helix-core/src/syntax.rs
+++ b/helix-core/src/syntax.rs
@@ -276,16 +276,6 @@ fn byte_range_to_str(range: std::ops::Range<usize>, source: RopeSlice) -> Cow<st
Cow::from(source.slice(start_char..end_char))
}
-fn node_to_bytes<'a>(node: Node, source: RopeSlice<'a>) -> Cow<'a, [u8]> {
- let start_char = source.byte_to_char(node.start_byte());
- let end_char = source.byte_to_char(node.end_byte());
- let fragment = source.slice(start_char..end_char);
- match fragment.as_str() {
- Some(fragment) => Cow::Borrowed(fragment.as_bytes()),
- None => Cow::Owned(String::from(fragment).into_bytes()),
- }
-}
-
impl Syntax {
// buffer, grammar, config, grammars, sync_timeout?
pub fn new(
@@ -380,14 +370,11 @@ impl Syntax {
// TODO: if reusing cursors this might need resetting
if let Some(range) = &range {
- cursor_ref.set_byte_range(range.start, range.end);
+ cursor_ref.set_byte_range(range.clone());
}
let captures = cursor_ref
- .captures(query_ref, tree_ref.root_node(), move |n: Node| {
- // &source[n.byte_range()]
- node_to_bytes(n, source)
- })
+ .captures(query_ref, tree_ref.root_node(), RopeProvider(source))
.peekable();
// manually craft the root layer based on the existing tree
@@ -501,10 +488,7 @@ impl LanguageLayer {
// let mut injections_by_pattern_index =
// vec![(None, Vec::new(), false); combined_injections_query.pattern_count()];
// let matches =
- // cursor.matches(combined_injections_query, tree.root_node(), |n: Node| {
- // // &source[n.byte_range()]
- // node_to_bytes(n, source)
- // });
+ // cursor.matches(combined_injections_query, tree.root_node(), RopeProvider(source));
// for mat in matches {
// let entry = &mut injections_by_pattern_index[mat.pattern_index];
// let (language_name, content_node, include_children) =
@@ -716,7 +700,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
use std::{iter, mem, ops, str, usize};
use tree_sitter::{
Language as Grammar, Node, Parser, Point, Query, QueryCaptures, QueryCursor, QueryError,
- QueryMatch, Range, Tree,
+ QueryMatch, Range, TextProvider, Tree,
};
const CANCELLATION_CHECK_INTERVAL: usize = 100;
@@ -776,7 +760,7 @@ struct LocalScope<'a> {
}
#[derive(Debug)]
-struct HighlightIter<'a, 'tree: 'a, F>
+struct HighlightIter<'a, F>
where
F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
{
@@ -784,16 +768,41 @@ where
byte_offset: usize,
injection_callback: F,
cancellation_flag: Option<&'a AtomicUsize>,
- layers: Vec<HighlightIterLayer<'a, 'tree>>,
+ layers: Vec<HighlightIterLayer<'a>>,
iter_count: usize,
next_event: Option<HighlightEvent>,
last_highlight_range: Option<(usize, usize, usize)>,
}
-struct HighlightIterLayer<'a, 'tree: 'a> {
+// Adapter to convert rope chunks to bytes
+struct ChunksBytes<'a> {
+ chunks: ropey::iter::Chunks<'a>,
+}
+impl<'a> Iterator for ChunksBytes<'a> {
+ type Item = &'a [u8];
+ fn next(&mut self) -> Option<Self::Item> {
+ self.chunks.next().map(str::as_bytes)
+ }
+}
+
+struct RopeProvider<'a>(RopeSlice<'a>);
+impl<'a> TextProvider<'a> for RopeProvider<'a> {
+ type I = ChunksBytes<'a>;
+
+ fn text(&mut self, node: Node) -> Self::I {
+ let start_char = self.0.byte_to_char(node.start_byte());
+ let end_char = self.0.byte_to_char(node.end_byte());
+ let fragment = self.0.slice(start_char..end_char);
+ ChunksBytes {
+ chunks: fragment.chunks(),
+ }
+ }
+}
+
+struct HighlightIterLayer<'a> {
_tree: Option<Tree>,
cursor: QueryCursor,
- captures: iter::Peekable<QueryCaptures<'a, 'tree, Cow<'a, [u8]>>>,
+ captures: iter::Peekable<QueryCaptures<'a, 'a, RopeProvider<'a>>>,
config: &'a HighlightConfiguration,
highlight_end_stack: Vec<usize>,
scope_stack: Vec<LocalScope<'a>>,
@@ -801,7 +810,7 @@ struct HighlightIterLayer<'a, 'tree: 'a> {
depth: usize,
}
-impl<'a, 'tree: 'a> fmt::Debug for HighlightIterLayer<'a, 'tree> {
+impl<'a> fmt::Debug for HighlightIterLayer<'a> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("HighlightIterLayer").finish()
}
@@ -972,7 +981,7 @@ impl HighlightConfiguration {
}
}
-impl<'a, 'tree: 'a> HighlightIterLayer<'a, 'tree> {
+impl<'a> HighlightIterLayer<'a> {
/// Create a new 'layer' of highlighting for this document.
///
/// In the even that the new layer contains "combined injections" (injections where multiple
@@ -1029,10 +1038,7 @@ impl<'a, 'tree: 'a> HighlightIterLayer<'a, 'tree> {
let matches = cursor.matches(
combined_injections_query,
tree.root_node(),
- |n: Node| {
- // &source[n.byte_range()]
- node_to_bytes(n, source)
- },
+ RopeProvider(source),
);
for mat in matches {
let entry = &mut injections_by_pattern_index[mat.pattern_index];
@@ -1079,10 +1085,7 @@ impl<'a, 'tree: 'a> HighlightIterLayer<'a, 'tree> {
let cursor_ref =
unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };
let captures = cursor_ref
- .captures(&config.query, tree_ref.root_node(), move |n: Node| {
- // &source[n.byte_range()]
- node_to_bytes(n, source)
- })
+ .captures(&config.query, tree_ref.root_node(), RopeProvider(source))
.peekable();
result.push(HighlightIterLayer {
@@ -1236,7 +1239,7 @@ impl<'a, 'tree: 'a> HighlightIterLayer<'a, 'tree> {
}
}
-impl<'a, 'tree: 'a, F> HighlightIter<'a, 'tree, F>
+impl<'a, F> HighlightIter<'a, F>
where
F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
{
@@ -1287,7 +1290,7 @@ where
}
}
- fn insert_layer(&mut self, mut layer: HighlightIterLayer<'a, 'tree>) {
+ fn insert_layer(&mut self, mut layer: HighlightIterLayer<'a>) {
if let Some(sort_key) = layer.sort_key() {
let mut i = 1;
while i < self.layers.len() {
@@ -1306,7 +1309,7 @@ where
}
}
-impl<'a, 'tree: 'a, F> Iterator for HighlightIter<'a, 'tree, F>
+impl<'a, F> Iterator for HighlightIter<'a, F>
where
F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
{
@@ -1570,7 +1573,7 @@ where
fn injection_for_match<'a>(
config: &HighlightConfiguration,
query: &'a Query,
- query_match: &QueryMatch<'a>,
+ query_match: &QueryMatch<'a, 'a>,
source: RopeSlice<'a>,
) -> (Option<Cow<'a, str>>, Option<Node<'a>>, bool) {
let content_capture_index = config.injection_content_capture_index;
diff --git a/helix-syntax/Cargo.toml b/helix-syntax/Cargo.toml
index 73eda472..9c2b8275 100644
--- a/helix-syntax/Cargo.toml
+++ b/helix-syntax/Cargo.toml
@@ -11,7 +11,7 @@ homepage = "https://helix-editor.com"
include = ["src/**/*", "languages/**/*", "build.rs", "!**/docs/**/*", "!**/test/**/*", "!**/examples/**/*", "!**/build/**/*"]
[dependencies]
-tree-sitter = "0.19"
+tree-sitter = "0.20"
libloading = "0.7"
anyhow = "1"