diff --git a/Cargo.lock b/Cargo.lock
index ac49da51..469a5862 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -358,11 +358,12 @@ dependencies = [
name = "helix-core"
version = "0.6.0"
dependencies = [
+ "anyhow",
"arc-swap",
"chrono",
"encoding_rs",
"etcetera",
- "helix-syntax",
+ "libloading",
"log",
"once_cell",
"quickcheck",
@@ -415,22 +416,12 @@ dependencies = [
"which",
]
-[[package]]
-name = "helix-syntax"
-version = "0.6.0"
-dependencies = [
- "anyhow",
- "cc",
- "libloading",
- "threadpool",
- "tree-sitter",
-]
-
[[package]]
name = "helix-term"
version = "0.6.0"
dependencies = [
"anyhow",
+ "cc",
"chrono",
"content_inspector",
"crossterm",
@@ -454,6 +445,7 @@ dependencies = [
"serde_json",
"signal-hook",
"signal-hook-tokio",
+ "threadpool",
"tokio",
"tokio-stream",
"toml",
diff --git a/Cargo.toml b/Cargo.toml
index 31088f98..0847e6ba 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -4,7 +4,6 @@ members = [
"helix-view",
"helix-term",
"helix-tui",
- "helix-syntax",
"helix-lsp",
"helix-dap",
"xtask",
@@ -14,10 +13,6 @@ default-members = [
"helix-term"
]
-# Build helix-syntax in release mode to make the code path faster in development.
-# [profile.dev.package."helix-syntax"]
-# opt-level = 3
-
[profile.dev]
split-debuginfo = "unpacked"
diff --git a/docs/architecture.md b/docs/architecture.md
index 40e01745..3c743eca 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -1,11 +1,10 @@
-| Crate | Description |
-| ----------- | ----------- |
-| helix-core | Core editing primitives, functional. |
-| helix-syntax | Tree-sitter grammars |
-| helix-lsp | Language server client |
-| helix-view | UI abstractions for use in backends, imperative shell. |
-| helix-term | Terminal UI |
+| Crate | Description |
+| ----------- | ----------- |
+| helix-core | Core editing primitives, functional. |
+| helix-lsp | Language server client |
+| helix-view | UI abstractions for use in backends, imperative shell. |
+| helix-term | Terminal UI |
| helix-tui | TUI primitives, forked from tui-rs, inspired by Cursive |
diff --git a/helix-core/Cargo.toml b/helix-core/Cargo.toml
index 6682c37f..5582d38b 100644
--- a/helix-core/Cargo.toml
+++ b/helix-core/Cargo.toml
@@ -13,8 +13,6 @@ include = ["src/**/*", "README.md"]
[features]
[dependencies]
-helix-syntax = { version = "0.6", path = "../helix-syntax" }
-
ropey = "1.3"
smallvec = "1.8"
smartstring = "1.0.0"
@@ -40,5 +38,8 @@ encoding_rs = "0.8"
chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] }
+libloading = "0.7"
+anyhow = "1"
+
[dev-dependencies]
quickcheck = { version = "1", default-features = false }
diff --git a/helix-core/src/syntax.rs b/helix-core/src/syntax.rs
index 53d20da3..0f7d224d 100644
--- a/helix-core/src/syntax.rs
+++ b/helix-core/src/syntax.rs
@@ -7,7 +7,9 @@ use crate::{
Rope, RopeSlice, Tendril,
};
-pub use helix_syntax::get_language;
+use anyhow::{Context, Result};
+use libloading::{Library, Symbol};
+use tree_sitter::Language;
use arc_swap::{ArcSwap, Guard};
use slotmap::{DefaultKey as LayerId, HopSlotMap};
@@ -25,6 +27,34 @@ use std::{
use once_cell::sync::{Lazy, OnceCell};
use serde::{Deserialize, Serialize};
+#[cfg(unix)]
+pub const DYLIB_EXTENSION: &str = "so";
+
+#[cfg(windows)]
+pub const DYLIB_EXTENSION: &str = "dll";
+
+fn replace_dashes_with_underscores(name: &str) -> String {
+ name.replace('-', "_")
+}
+
+pub fn get_language(runtime_path: &std::path::Path, name: &str) -> Result<Language> {
+ let name = name.to_ascii_lowercase();
+ let mut library_path = runtime_path.join("grammars").join(&name);
+ library_path.set_extension(DYLIB_EXTENSION);
+
+ let library = unsafe { Library::new(&library_path) }
+ .with_context(|| format!("Error opening dynamic library {:?}", &library_path))?;
+ let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(&name));
+ let language = unsafe {
+ let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
+ .get(language_fn_name.as_bytes())
+ .with_context(|| format!("Failed to load symbol {}", language_fn_name))?;
+ language_fn()
+ };
+ std::mem::forget(library);
+ Ok(language)
+}
+
fn deserialize_regex<'de, D>(deserializer: D) -> Result<Option<Regex>, D::Error>
where
D: serde::Deserializer<'de>,
@@ -426,7 +456,7 @@ impl LanguageConfiguration {
&injections_query,
&locals_query,
)
- .unwrap(); // TODO: avoid panic
+ .unwrap_or_else(|query_error| panic!("Could not parse queries for language {:?}. Are your grammars out of sync? Try running 'hx --fetch-grammars' and 'hx --build-grammars'. This query could not be parsed: {:?}", self.language_id, query_error));
config.configure(scopes);
Some(Arc::new(config))
@@ -2023,7 +2053,7 @@ mod test {
);
let loader = Loader::new(Configuration { language: vec![] });
- let language = get_language(&crate::RUNTIME_DIR, "Rust").unwrap();
+ let language = get_language("Rust").unwrap();
let query = Query::new(language, query_str).unwrap();
let textobject = TextObjectQuery { query };
diff --git a/helix-syntax/Cargo.toml b/helix-syntax/Cargo.toml
deleted file mode 100644
index 855839be..00000000
--- a/helix-syntax/Cargo.toml
+++ /dev/null
@@ -1,21 +0,0 @@
-[package]
-name = "helix-syntax"
-version = "0.6.0"
-authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
-edition = "2021"
-license = "MPL-2.0"
-description = "Tree-sitter grammars support"
-categories = ["editor"]
-repository = "https://github.com/helix-editor/helix"
-homepage = "https://helix-editor.com"
-include = ["src/**/*", "languages/**/*", "build.rs", "!**/docs/**/*", "!**/test/**/*", "!**/examples/**/*", "!**/build/**/*"]
-
-[dependencies]
-tree-sitter = "0.20"
-libloading = "0.7"
-anyhow = "1"
-
-[build-dependencies]
-cc = { version = "1" }
-threadpool = { version = "1.0" }
-anyhow = "1"
diff --git a/helix-syntax/README.md b/helix-syntax/README.md
deleted file mode 100644
index bba2197a..00000000
--- a/helix-syntax/README.md
+++ /dev/null
@@ -1,13 +0,0 @@
-helix-syntax
-============
-
-Syntax highlighting for helix, (shallow) submodules resides here.
-
-Differences from nvim-treesitter
---------------------------------
-
-As the syntax are commonly ported from
-<https://github.com/nvim-treesitter/nvim-treesitter>.
-
-Note that we do not support the custom `#any-of` predicate which is
-supported by neovim so one needs to change it to `#match` with regex.
diff --git a/helix-syntax/src/lib.rs b/helix-syntax/src/lib.rs
deleted file mode 100644
index b0ec48d8..00000000
--- a/helix-syntax/src/lib.rs
+++ /dev/null
@@ -1,31 +0,0 @@
-use anyhow::{Context, Result};
-use libloading::{Library, Symbol};
-use tree_sitter::Language;
-
-fn replace_dashes_with_underscores(name: &str) -> String {
- name.replace('-', "_")
-}
-#[cfg(unix)]
-const DYLIB_EXTENSION: &str = "so";
-
-#[cfg(windows)]
-const DYLIB_EXTENSION: &str = "dll";
-
-pub fn get_language(runtime_path: &std::path::Path, name: &str) -> Result<Language> {
- let name = name.to_ascii_lowercase();
- let mut library_path = runtime_path.join("grammars").join(&name);
- // TODO: duplicated under build
- library_path.set_extension(DYLIB_EXTENSION);
-
- let library = unsafe { Library::new(&library_path) }
- .with_context(|| format!("Error opening dynamic library {:?}", &library_path))?;
- let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(&name));
- let language = unsafe {
- let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
- .get(language_fn_name.as_bytes())
- .with_context(|| format!("Failed to load symbol {}", language_fn_name))?;
- language_fn()
- };
- std::mem::forget(library);
- Ok(language)
-}
diff --git a/helix-term/Cargo.toml b/helix-term/Cargo.toml
index 9f7821f6..93d50d7e 100644
--- a/helix-term/Cargo.toml
+++ b/helix-term/Cargo.toml
@@ -66,5 +66,9 @@ grep-searcher = "0.1.8"
# Remove once retain_mut lands in stable rust
retain_mut = "0.1.7"
+# compiling grammars
+cc = { version = "1" }
+threadpool = { version = "1.0" }
+
[target.'cfg(not(windows))'.dependencies] # https://github.com/vorner/signal-hook/issues/100
signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"] }
diff --git a/helix-term/build.rs b/helix-term/build.rs
index b5d62b28..7303041c 100644
--- a/helix-term/build.rs
+++ b/helix-term/build.rs
@@ -14,5 +14,10 @@ fn main() {
None => env!("CARGO_PKG_VERSION").into(),
};
+ println!(
+ "cargo:rustc-env=BUILD_TARGET={}",
+ std::env::var("TARGET").unwrap()
+ );
+
println!("cargo:rustc-env=VERSION_AND_GIT_HASH={}", version);
}
diff --git a/helix-syntax/build.rs b/helix-term/src/grammars.rs
similarity index 74%
rename from helix-syntax/build.rs
rename to helix-term/src/grammars.rs
index fa8be8b3..6a4910a3 100644
--- a/helix-syntax/build.rs
+++ b/helix-term/src/grammars.rs
@@ -6,11 +6,13 @@ use std::{
process::Command,
};
-use std::sync::mpsc::channel;
+use helix_core::syntax::DYLIB_EXTENSION;
-fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
+const BUILD_TARGET: &str = env!("BUILD_TARGET");
+
+pub fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
let mut dirs = Vec::new();
- let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("languages");
+ let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../helix-syntax/languages");
for entry in fs::read_dir(path)? {
let entry = entry?;
@@ -32,12 +34,6 @@ fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
Ok(dirs)
}
-#[cfg(unix)]
-const DYLIB_EXTENSION: &str = "so";
-
-#[cfg(windows)]
-const DYLIB_EXTENSION: &str = "dll";
-
fn build_library(src_path: &Path, language: &str) -> Result<()> {
let header_path = src_path;
// let grammar_path = src_path.join("grammar.json");
@@ -65,7 +61,12 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {
return Ok(());
}
let mut config = cc::Build::new();
- config.cpp(true).opt_level(2).cargo_metadata(false);
+ config
+ .cpp(true)
+ .opt_level(2)
+ .cargo_metadata(false)
+ .host(BUILD_TARGET)
+ .target(BUILD_TARGET);
let compiler = config.get_compiler();
let mut command = Command::new(compiler.path());
command.current_dir(src_path);
@@ -148,9 +149,10 @@ fn mtime(path: &Path) -> Result<SystemTime> {
Ok(fs::metadata(path)?.modified()?)
}
-fn build_dir(dir: &str, language: &str) {
+pub fn build_dir(dir: &str, language: &str) {
println!("Build language {}", language);
- if PathBuf::from("languages")
+ if PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+ .join("../helix-syntax/languages")
.join(dir)
.read_dir()
.unwrap()
@@ -158,49 +160,16 @@ fn build_dir(dir: &str, language: &str) {
.is_none()
{
eprintln!(
- "The directory {} is empty, you probably need to use 'git submodule update --init --recursive'?",
+ "The directory {} is empty, you probably need to use './scripts/grammars sync'?",
dir
);
std::process::exit(1);
}
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
- .join("languages")
+ .join("../helix-syntax/languages")
.join(dir)
.join("src");
build_library(&path, language).unwrap();
}
-
-fn main() {
- let ignore = vec![
- "tree-sitter-typescript".to_string(),
- "tree-sitter-ocaml".to_string(),
- ];
- let dirs = collect_tree_sitter_dirs(&ignore).unwrap();
-
- let mut n_jobs = 0;
- let pool = threadpool::Builder::new().build(); // by going through the builder, it'll use num_cpus
- let (tx, rx) = channel();
-
- for dir in dirs {
- let tx = tx.clone();
- n_jobs += 1;
-
- pool.execute(move || {
- let language = &dir.strip_prefix("tree-sitter-").unwrap();
- build_dir(&dir, language);
-
- // report progress
- tx.send(1).unwrap();
- });
- }
- pool.join();
- // drop(tx);
- assert_eq!(rx.try_iter().sum::<usize>(), n_jobs);
-
- build_dir("tree-sitter-typescript/tsx", "tsx");
- build_dir("tree-sitter-typescript/typescript", "typescript");
- build_dir("tree-sitter-ocaml/ocaml", "ocaml");
- build_dir("tree-sitter-ocaml/interface", "ocaml-interface")
-}
diff --git a/helix-term/src/lib.rs b/helix-term/src/lib.rs
index fc8e934e..22747998 100644
--- a/helix-term/src/lib.rs
+++ b/helix-term/src/lib.rs
@@ -7,6 +7,7 @@ pub mod commands;
pub mod compositor;
pub mod config;
pub mod health;
+pub mod grammars;
pub mod job;
pub mod keymap;
pub mod ui;