feat: implement reading files in different file encodings

lapce-app/Cargo.toml

@@ -54,7 +54,7 @@ unicode-width = "0.1.11"
 nucleo = "0.5.0"
 bytemuck = "1.14.3"
 config = { version = "=0.13.4", default-features = false, features = ["toml"] }
-structdesc = { git = "https://github.com/lapce/structdesc" }
+structdesc = { rev = "c6b0437cdac3d82cd531b4ecef5fb31fe5025a03", git = "https://github.com/lapce/structdesc" }
 base64 = "0.21.7"
 sha2 = "0.10.8"
 zip = { version = "0.6.6", default-features = false, features = ["deflate"] }

lapce-app/src/doc.rs

@@ -158,6 +158,7 @@ pub struct Doc {
  pub loaded: RwSignal<bool>,
  pub buffer: RwSignal<Buffer>,
  pub syntax: RwSignal<Syntax>,
+ pub encoding: RwSignal<String>,
  semantic_styles: RwSignal<Option<Spans<Style>>>,
  /// Inlay hints for the document
  pub inlay_hints: RwSignal<Option<Spans<InlayHint>>>,
@@ -211,6 +212,7 @@ impl Doc {
  buffer_id: BufferId::next(),
  buffer: cx.create_rw_signal(Buffer::new("")),
  syntax: cx.create_rw_signal(syntax),
+ encoding: cx.create_rw_signal(String::new()),
  line_styles: Rc::new(RefCell::new(HashMap::new())),
  parser: Rc::new(RefCell::new(BracketParser::new(
  String::new(),
@@ -258,6 +260,7 @@ impl Doc {
  buffer_id: BufferId::next(),
  buffer: cx.create_rw_signal(Buffer::new("")),
  syntax: cx.create_rw_signal(Syntax::plaintext()),
+ encoding: cx.create_rw_signal(String::new()),
  line_styles: Rc::new(RefCell::new(HashMap::new())),
  parser: Rc::new(RefCell::new(BracketParser::new(
  String::new(),
@@ -306,6 +309,7 @@ impl Doc {
  buffer_id: BufferId::next(),
  buffer: cx.create_rw_signal(Buffer::new("")),
  syntax: cx.create_rw_signal(syntax),
+ encoding: cx.create_rw_signal(String::new()),
  line_styles: Rc::new(RefCell::new(HashMap::new())),
  parser: Rc::new(RefCell::new(BracketParser::new(
  String::new(),

lapce-app/src/main_split.rs

@@ -602,9 +602,11 @@ impl MainSplitData {
  if let Ok(ProxyResponse::NewBufferResponse {
  content,
  read_only,
+ encoding,
  }) = result
  {
  local_doc.init_content(Rope::from(content));
+ local_doc.encoding.set(encoding);
  if read_only {
  local_doc.content.update(|content| {
  if let DocContent::File { read_only, .. } = content {

lapce-app/src/status.rs

@@ -364,7 +364,15 @@ pub fn status(
  .on_click_stop(move |_| {
  palette_clone.run(PaletteKind::Language);
  });
- (cursor_info, line_ending_info, language_info)
+ let encoding_info = status_text(config, editor, move || {
+ if let Some(editor) = editor.get() {
+ let doc = editor.doc_signal().get();
+ doc.encoding.get()
+ } else {
+ "unknown".to_string()
+ }
+ });
+ (cursor_info, line_ending_info, encoding_info, language_info)
  })
  .style(|s| {
  s.height_pct(100.0)

lapce-proxy/Cargo.toml

@@ -50,6 +50,13 @@ jsonrpc-lite = "0.6.0"
 polling = "3.5.0"
 libc = "0.2"
 
+# Encoding
+encoding_rs = { version = "0.8.34" }
+encoding_rs_io = { version = "0.1.7" }
+# charset-normalizer-rs = { version = "1.0.6" }
+# charset-normalizer-rs = { path = "../../charset-normalizer-rs/crates/charset-normalizer" }
+charset-normalizer-rs = { rev = "807868b19ef693e86fb17d0b4d93463753c3b59b", git = "https://github.com/lapce/charset-normalizer-rs" }
+
 # git
 git2 = { version = "0.18.2", features = ["vendored-openssl"] }
 

lapce-proxy/src/buffer.rs

@@ -24,24 +24,35 @@ pub struct Buffer {
  pub path: PathBuf,
  pub rev: u64,
  pub mod_time: Option<SystemTime>,
+ pub encoding: String,
 }
 
+pub const ENCODING_UTF8: &str = "UTF-8";
+
 impl Buffer {
  pub fn new(id: BufferId, path: PathBuf) -> Buffer {
- let (s, read_only) = match load_file(&path) {
- Ok(s) => (s, false),
+ let (content, encoding, read_only) = match load_file(&path) {
+ Ok((string, encoding)) => (string, encoding, false),
  Err(err) => match err.downcast_ref::<std::io::Error>() {
  Some(err) => match err.kind() {
- std::io::ErrorKind::PermissionDenied => {
- ("Permission Denied".to_string(), true)
+ std::io::ErrorKind::PermissionDenied => (
+ "Permission Denied".to_string(),
+ ENCODING_UTF8.to_owned(),
+ true,
+ ),
+ std::io::ErrorKind::NotFound => {
+ ("".to_string(), ENCODING_UTF8.to_owned(), false)
+ }
+ _ => {
+ ("Not Supported".to_string(), ENCODING_UTF8.to_owned(), true)
  }
- std::io::ErrorKind::NotFound => ("".to_string(), false),
- _ => ("Not Supported".to_string(), true),
  },
- None => ("Not Supported".to_string(), true),
+ None => {
+ ("Not Supported".to_string(), ENCODING_UTF8.to_owned(), true)
+ }
  },
  };
- let rope = Rope::from(s);
+ let rope = Rope::from(content);
  let rev = u64::from(!rope.is_empty());
  let language_id = language_id_from_path(&path).unwrap_or("");
  let mod_time = get_mod_time(&path);
@@ -53,6 +64,7 @@ impl Buffer {
  language_id,
  rev,
  mod_time,
+ encoding,
  }
  }
 
@@ -181,22 +193,50 @@ impl Buffer {
  }
 }
 
-pub fn load_file(path: &Path) -> Result<String> {
+pub fn load_file(path: &Path) -> Result<(String, String)> {
  read_path_to_string(path)
 }
 
-pub fn read_path_to_string<P: AsRef<Path>>(path: P) -> Result<String> {
+pub fn read_path_to_string<P: AsRef<Path>>(path: P) -> Result<(String, String)> {
  let path = path.as_ref();
+ let mut encoding = ENCODING_UTF8.to_string();
 
  let mut file = File::open(path)?;
  // Read the file in as bytes
  let mut buffer = Vec::new();
  file.read_to_end(&mut buffer)?;
 
+ let mut contents = String::new();
  // Parse the file contents as utf8
- let contents = String::from_utf8(buffer)?;
+ match String::from_utf8(buffer.clone()) {
+ Ok(s) => {
+ contents = s;
+ }
+ Err(e) => {
+ tracing::error!("Failed to decode from `utf-8`: {e}");
+
+ use charset_normalizer_rs as chardet;
+ use encoding_rs::Encoding;
+ use encoding_rs_io::DecodeReaderBytesBuilder;
+
+ let mut enc_reader = DecodeReaderBytesBuilder::new();
+
+ let chardet = chardet::from_bytes(&buffer, None);
+
+ if let Some(charset) = chardet.get_best() {
+ let enc = charset.encoding();
+ tracing::info!("Detected encoding: {enc}");
+ enc_reader.encoding(Encoding::for_label(enc.as_bytes()));
+ encoding = enc.to_owned();
+ };
+
+ let mut decoder = enc_reader.build(buffer.as_slice());
+
+ decoder.read_to_string(&mut contents)?;
+ }
+ };
 
- Ok(contents.to_string())
+ Ok((contents.to_owned(), encoding))
 }
 
 pub fn language_id_from_path(path: &Path) -> Option<&'static str> {

lapce-proxy/src/dispatch.rs

@@ -116,7 +116,7 @@ impl ProxyHandler for Dispatcher {
  if get_mod_time(&buffer.path) == buffer.mod_time {
  return;
  }
- if let Ok(content) = load_file(&buffer.path) {
+ if let Ok((content, _)) = load_file(&buffer.path) {
  self.core_rpc.open_file_changed(path, content);
  }
  }
@@ -341,6 +341,7 @@ impl ProxyHandler for Dispatcher {
  NewBuffer { buffer_id, path } => {
  let buffer = Buffer::new(buffer_id, path.clone());
  let content = buffer.rope.to_string();
+ let encoding = buffer.encoding.clone();
  let read_only = buffer.read_only;
  self.catalog_rpc.did_open_document(
  &path,
@@ -352,7 +353,11 @@ impl ProxyHandler for Dispatcher {
  self.buffers.insert(path, buffer);
  self.respond_rpc(
  id,
- Ok(ProxyResponse::NewBufferResponse { content, read_only }),
+ Ok(ProxyResponse::NewBufferResponse {
+ content,
+ read_only,
+ encoding,
+ }),
  );
  }
  BufferHead { path } => {

lapce-rpc/src/proxy.rs

@@ -333,6 +333,7 @@ pub enum ProxyResponse {
  NewBufferResponse {
  content: String,
  read_only: bool,
+ encoding: String,
  },
  BufferHeadResponse {
  version: String,