Initialize Luau inliner flags before VM creation and compilation

Author: khvzak

src/chunk.rs

@@ -396,6 +396,9 @@ impl Compiler {
     ///
     /// Returns [`Error::SyntaxError`] if the source code is invalid.
     pub fn compile(&self, source: impl AsRef<[u8]>) -> Result<Vec<u8>> {
+        #[cfg(feature = "luau-jit")]
+        crate::luau::init_jit_flags();
+
         use std::cell::RefCell;
         use std::ffi::CStr;
         use std::os::raw::{c_char, c_int};

src/luau/mod.rs

@@ -19,6 +19,19 @@ use crate::types::MaybeSend;
 pub use heap_dump::HeapDump;
 pub use require::{FsRequirer, NavigateError, Require};
 
+#[cfg(feature = "luau-jit")]
+pub(crate) fn init_jit_flags() {
+    static INIT: std::sync::Once = std::sync::Once::new();
+    INIT.call_once(|| {
+        // These process-global flags must be set before any VM or compiler uses them.
+        let _ = Lua::set_fflag("LuauCallFeedback", true);
+        let _ = Lua::set_fflag("LuauEmitCallFeedback", true);
+        let _ = Lua::set_fflag("LuauCIProto", true);
+        let _ = Lua::set_fflag("LuauPromoteProto", true);
+        let _ = Lua::set_fflag("LuauVirtualBcBuilder", true);
+    });
+}
+
 // Since Luau has some missing standard functions, we re-implement them here
 
 impl Lua {

src/state.rs

@@ -288,7 +288,7 @@ impl JitOptions {
 
     /// Toggles the runtime bytecode inliner.
     ///
-    /// Disabled by default. Changing this option does not affect already loaded functions.
+    /// Disabled by default. Enable before compiling and executing code.
     #[must_use]
     pub const fn inliner(mut self, enabled: bool) -> Self {
         self.inliner = enabled;
@@ -1325,8 +1325,6 @@ impl Lua {
         unsafe {
             let state = lua.main_state();
             if options.inliner {
-                let _ = Self::set_fflag("LuauCallFeedback", true);
-                let _ = Self::set_fflag("LuauEmitCallFeedback", true);
                 ffi::luau_enable_jit_inliner(state);
             } else {
                 ffi::luau_disable_jit_inliner(state);
@@ -1336,11 +1334,16 @@ impl Lua {
 
     /// Sets Luau feature flag (global setting).
     ///
+    /// Flags must be configured before creating any Lua VMs or compiling any Luau code.
+    /// This function must not run concurrently with any other Luau operation, including itself.
+    /// Changing flags later can cause data races or invalidate compiled bytecode.
+    ///
     /// See https://github.com/luau-lang/luau/blob/master/CONTRIBUTING.md#feature-flags for details.
     #[cfg(feature = "luau")]
     #[doc(hidden)]
     #[allow(clippy::result_unit_err)]
     pub fn set_fflag(name: &str, enabled: bool) -> StdResult<(), ()> {
+        // TODO: Make this function unsafe in the next breaking release.
         if let Ok(name) = std::ffi::CString::new(name)
             && unsafe { ffi::luau_setfflag(name.as_ptr(), enabled as c_int) != 0 }
         {

src/state/raw.rs

@@ -130,6 +130,9 @@ impl RawLua {
     }
 
     pub(super) unsafe fn new(libs: StdLib, options: &LuaOptions) -> XRc<ReentrantMutex<Self>> {
+        #[cfg(feature = "luau-jit")]
+        crate::luau::init_jit_flags();
+
         let mem_state: *mut MemoryState = Box::into_raw(Box::default());
         #[cfg(feature = "lua55")]
         let mut state = {

tests/luau.rs

@@ -390,19 +390,31 @@ fn test_fflags() {
 #[cfg(feature = "luau-jit")]
 #[test]
 fn test_jit_inliner() -> Result<()> {
+    let compiler = Compiler::new();
+    let bytecode = compiler.compile("return 42")?;
+
     let lua = Lua::new();
+    lua.enable_jit(false);
+    lua.sandbox(true)?;
     lua.set_jit_options(mlua::state::JitOptions::new().inliner(true));
 
+    // Enabling the inliner must not change global compiler flags.
+    assert_eq!(compiler.compile("return 42")?, bytecode);
+
     // An inlinable helper called in a hot loop.
     let sum = lua
         .load(
             r#"
             local function add(a, b)
-                return a + b
+                return a + b + 0.5
+            end
+            local function caller(a, b)
+                local sum = add(a, b)
+                return sum - 0.5
             end
             local sum = 0
             for i = 1, 1000 do
-                sum = add(sum, i)
+                sum = caller(sum, i)
             end
             return sum
         "#,
@@ -508,11 +520,23 @@ fn test_heap_dump() -> Result<()> {
     Ok(())
 }
 
+#[cfg(not(target_arch = "wasm32"))]
 #[test]
 fn test_integer64_type() -> Result<()> {
-    let lua = Lua::new();
+    // Isolate process-global flag changes from other tests' VMs and compilers.
+    if std::env::var_os("MLUA_TEST_INTEGER64_TYPE").is_none() {
+        assert!(
+            std::process::Command::new(std::env::current_exe()?)
+                .args(["--exact", "test_integer64_type"])
+                .env("MLUA_TEST_INTEGER64_TYPE", "1")
+                .status()?
+                .success()
+        );
+        return Ok(());
+    }
 
     _ = Lua::set_fflag("LuauIntegerType2", true);
+    let lua = Lua::new();
 
     let integer_lib = lua.globals().get::<Table>("integer")?;
     let n = integer_lib.call_function::<i64>("create", 42)?;