perf: reduce memory allocations by using Lua 5.5 external strings (#3634)

CHANGELOG.md

@@ -25,6 +25,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/):
 
 - Archive extraction fails for target paths with non-ASCII characters on Windows ([#3607])
 
+### Improved
+
+- Reduce memory allocations by using Lua 5.5 external strings ([#3634])
+
 ## [v26.1.22]
 
 ### Added
@@ -1629,3 +1633,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/):
 [#3608]: https://github.com/sxyazi/yazi/pull/3608
 [#3617]: https://github.com/sxyazi/yazi/pull/3617
 [#3633]: https://github.com/sxyazi/yazi/pull/3633
+[#3634]: https://github.com/sxyazi/yazi/pull/3634

yazi-binding/src/error.rs

@@ -74,7 +74,7 @@ impl UserData for Error {
 	fn add_methods<M: UserDataMethods<Self>>(methods: &mut M) {
 		methods.add_meta_method(MetaMethod::ToString, |lua, me, ()| {
 			Ok(match me {
-				Self::Io(_) | Self::Fs(_) | Self::Serde(_) => lua.create_string(me.to_string()),
+				Self::Io(_) | Self::Fs(_) | Self::Serde(_) => lua.create_external_string(me.to_string()),
 				Self::Custom(s) => lua.create_string(&**s),
 			})
 		});
@@ -82,11 +82,11 @@ impl UserData for Error {
 			match (lhs, rhs) {
 				(Value::String(l), Value::UserData(r)) => {
 					let r = r.borrow::<Self>()?;
-					lua.create_string([&l.as_bytes(), r.to_string().as_bytes()].concat())
+					lua.create_external_string([&l.as_bytes(), r.to_string().as_bytes()].concat())
 				}
 				(Value::UserData(l), Value::String(r)) => {
 					let l = l.borrow::<Self>()?;
-					lua.create_string([l.to_string().as_bytes(), &r.as_bytes()].concat())
+					lua.create_external_string([l.to_string().as_bytes(), &r.as_bytes()].concat())
 				}
 				_ => Err("only string can be concatenated with Error".into_lua_err()),
 			}

yazi-binding/src/macros.rs

@@ -50,6 +50,28 @@ macro_rules! cached_field {
 	};
 }
 
+#[macro_export]
+macro_rules! cached_field_mut {
+	($fields:ident, $key:ident, $value:expr) => {
+		$fields.add_field_function_get(stringify!($key), |lua, ud| {
+			use mlua::{Error::UserDataDestructed, IntoLua, Lua, Result, Value, Value::UserData};
+			ud.borrow_mut_scoped::<Self, Result<Value>>(|me| match paste::paste! { &me.[<v_ $key>] } {
+				Some(Ok(v)) if !v.is_userdata() => Ok(v.clone()),
+				Some(Ok(v @ UserData(ud))) if !matches!(ud.borrow::<()>(), Err(UserDataDestructed)) => {
+					Ok(v.clone())
+				}
+				Some(Err(e)) => Err(e.clone()),
+				_ => {
+					let v =
+						($value as fn(&Lua, &mut Self) -> Result<_>)(lua, me).and_then(|v| v.into_lua(lua));
+					paste::paste! { me.[<v_ $key>] = Some(v.clone()) };
+					v
+				}
+			})?
+		});
+	};
+}
+
 #[macro_export]
 macro_rules! impl_area_method {
 	($methods:ident) => {

yazi-binding/src/path.rs

@@ -132,7 +132,7 @@ impl UserData for Path {
 		methods.add_method("strip_prefix", |_, me, base: Value| me.strip_prefix(base));
 
 		methods.add_meta_method(MetaMethod::Concat, |lua, lhs, rhs: mlua::String| {
-			lua.create_string([lhs.encoded_bytes(), &rhs.as_bytes()].concat())
+			lua.create_external_string([lhs.encoded_bytes(), &rhs.as_bytes()].concat())
 		});
 		methods.add_meta_method(MetaMethod::Eq, |_, me, other: PathRef| Ok(me.inner == other.inner));
 		methods

yazi-binding/src/url.rs

@@ -213,7 +213,7 @@ impl UserData for Url {
 			lua.create_string(me.to_strand().encoded_bytes())
 		});
 		methods.add_meta_method(MetaMethod::Concat, |lua, lhs, rhs: mlua::String| {
-			lua.create_string([lhs.to_strand().encoded_bytes(), &rhs.as_bytes()].concat())
+			lua.create_external_string([lhs.to_strand().encoded_bytes(), &rhs.as_bytes()].concat())
 		});
 	}
 }

yazi-dds/src/sendable.rs

@@ -70,6 +70,7 @@ impl Sendable {
 
 	pub fn data_to_value(lua: &Lua, data: Data) -> mlua::Result<Value> {
 		Ok(match data {
+			Data::String(Cow::Owned(s)) => Value::String(lua.create_external_string(s)?),
 			Data::List(l) => {
 				let mut vec = Vec::with_capacity(l.len());
 				for v in l.into_iter() {
@@ -87,6 +88,7 @@ impl Sendable {
 			}
 			Data::Url(u) => yazi_binding::Url::new(u).into_lua(lua)?,
 			Data::Path(u) => yazi_binding::Path::new(u).into_lua(lua)?,
+			Data::Bytes(b) => Value::String(lua.create_external_string(b)?),
 			Data::Any(b) => {
 				let mut b = b.into_any();
 				macro_rules! try_cast {
@@ -104,7 +106,7 @@ impl Sendable {
 				try_cast!(|v: yazi_binding::ChordCow| v.into_lua(lua));
 				Err("unsupported DataAny included".into_lua_err())?
 			}
-			data => Self::data_to_value_ref(lua, &data)?,
+			_ => Self::data_to_value_ref(lua, &data)?,
 		})
 	}
 
@@ -245,8 +247,10 @@ impl Sendable {
 
 	fn key_to_value(lua: &Lua, key: DataKey) -> mlua::Result<Value> {
 		match key {
+			DataKey::String(Cow::Owned(s)) => lua.create_external_string(s).map(Value::String),
 			DataKey::Url(u) => yazi_binding::Url::new(u).into_lua(lua),
 			DataKey::Path(u) => yazi_binding::Path::new(u).into_lua(lua),
+			DataKey::Bytes(b) => lua.create_external_string(b).map(Value::String),
 			_ => Self::key_to_value_ref(lua, &key),
 		}
 	}

yazi-parser/src/app/quit.rs

@@ -5,8 +5,8 @@ use yazi_shared::{event::CmdCow, strand::StrandBuf};
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct QuitOpt {
 	pub code:        i32,
-	#[serde(skip)] // FIXME
-	pub selected: Option<StrandBuf>,
+	#[serde(skip)]
+	pub selected:    Option<StrandBuf>,
 	pub no_cwd_file: bool,
 }
 
@@ -20,7 +20,7 @@ impl TryFrom<CmdCow> for QuitOpt {
 
 		Ok(Self {
 			code:        c.get("code").unwrap_or_default(),
-			selected:    None, // FIXME
+			selected:    None,
 			no_cwd_file: c.bool("no-cwd-file"),
 		})
 	}

yazi-plugin/src/elements/elements.rs

@@ -66,7 +66,7 @@ pub(super) fn printable(lua: &Lua) -> mlua::Result<Value> {
 	let f = lua.create_function(|lua, s: mlua::String| {
 		Ok(match replace_to_printable(&s.as_bytes(), false, 1, true) {
 			Cow::Borrowed(_) => s,
-			Cow::Owned(new) => lua.create_string(&new)?,
+			Cow::Owned(new) => lua.create_external_string(new)?,
 		})
 	})?;
 
@@ -162,7 +162,7 @@ pub(super) fn truncate(lua: &Lua) -> mlua::Result<Value> {
 				"…".bytes().chain(lossy[idx + len..].bytes()).collect()
 			}
 		};
-		lua.create_string(result)
+		lua.create_external_string(result)
 	})?;
 
 	f.into_lua(lua)

yazi-plugin/src/process/child.rs

@@ -118,7 +118,7 @@ impl UserData for Child {
 		});
 		methods.add_async_method_mut("read_line", |lua, mut me, ()| async move {
 			match me.read_line().await {
-				(Some(b), event) => (lua.create_string(b)?, event).into_lua_multi(&lua),
+				(Some(b), event) => (lua.create_external_string(b)?, event).into_lua_multi(&lua),
 				(None, event) => (Value::Nil, event).into_lua_multi(&lua),
 			}
 		});
@@ -129,7 +129,7 @@ impl UserData for Child {
 				return (Value::Nil, 3u8).into_lua_multi(&lua);
 			};
 			match result {
-				(Some(b), event) => (lua.create_string(b)?, event).into_lua_multi(&lua),
+				(Some(b), event) => (lua.create_external_string(b)?, event).into_lua_multi(&lua),
 				(None, event) => (Value::Nil, event).into_lua_multi(&lua),
 			}
 		});

yazi-plugin/src/process/output.rs

@@ -1,5 +1,7 @@
+use std::mem;
+
 use mlua::{UserData, Value};
-use yazi_binding::cached_field;
+use yazi_binding::{cached_field, cached_field_mut};
 
 use super::Status;
 
@@ -7,8 +9,8 @@ pub struct Output {
 	inner: std::process::Output,
 
 	v_status: Option<Value>,
-	v_stdout: Option<Value>,
-	v_stderr: Option<Value>,
+	v_stdout: Option<mlua::Result<Value>>,
+	v_stderr: Option<mlua::Result<Value>>,
 }
 
 impl Output {
@@ -20,7 +22,11 @@ impl Output {
 impl UserData for Output {
 	fn add_fields<F: mlua::UserDataFields<Self>>(fields: &mut F) {
 		cached_field!(fields, status, |_, me| Ok(Status::new(me.inner.status)));
-		cached_field!(fields, stdout, |lua, me| lua.create_string(&me.inner.stdout));
-		cached_field!(fields, stderr, |lua, me| lua.create_string(&me.inner.stderr));
+		cached_field_mut!(fields, stdout, |lua, me| {
+			lua.create_external_string(mem::take(&mut me.inner.stdout))
+		});
+		cached_field_mut!(fields, stderr, |lua, me| {
+			lua.create_external_string(mem::take(&mut me.inner.stderr))
+		});
 	}
 }

yazi-plugin/src/utils/text.rs

@@ -19,7 +19,7 @@ impl Utils {
 				Some(false) => yazi_shared::shell::windows::escape_os_bytes(&b),
 				None => yazi_shared::shell::escape_os_bytes(&b),
 			};
-			lua.create_string(&*s)
+			lua.create_external_string(s)
 		})
 	}
 
@@ -29,7 +29,7 @@ impl Utils {
 				CLIPBOARD.set(text).await;
 				Ok(None)
 			} else {
-				Some(lua.create_string(CLIPBOARD.get().await)).transpose()
+				Some(lua.create_external_string(CLIPBOARD.get().await)).transpose()
 			}
 		})
 	}