fix(lua): don't strip debuginfo in precompile module #39191

Problem:
debug.getinfo on bytecode module/func don't give you detail source info.

Solution:
- Use `loadstring`+`string.dump` to replace LUAC_PRG(`luac`/`luajit -b`)
- `string.dump(…,false)` to generate non-strip version bytecode
- `loadstring(…,fname)` to specify the full source name

BEFORE:

    $ nvim --clean +'=debug.getinfo(vim.fn.maparg("]<Space>", "n", 0, 1).callback, "Sl")' --headless +q
    {
      currentline = -1,
      lastlinedefined = 456,
      linedefined = 452,
      short_src = "?",
      source = "=?",
      what = "Lua"
    }

AFTER:

    $ nvim --clean +'=debug.getinfo(vim.fn.maparg("]<Space>", "n", 0, 1).callback, "Sl")' --headless +q
    {
      currentline = -1,
      lastlinedefined = 456,
      linedefined = 452,
      short_src = "/home/xx/b/neovim/runtime/lua/vim/_core/defaults.lua",
      source = "@/home/xx/b/neovim/runtime/lua/vim/_core/defaults.lua",
      what = "Lua"
    }

CMakeLists.txt

@@ -208,30 +208,6 @@ mark_as_advanced(LUA_GEN_PRG)
 message(STATUS "Using Lua interpreter for code generation: ${LUA_GEN_PRG}")
 
 option(COMPILE_LUA "Pre-compile Lua sources into bytecode (for sources that are included in the binary)" ON)
-if(COMPILE_LUA AND NOT WIN32)
-  if(PREFER_LUA)
-    foreach(CURRENT_LUAC_PRG luac5.1 luac)
-      find_program(_CHECK_LUAC_PRG ${CURRENT_LUAC_PRG})
-      if(_CHECK_LUAC_PRG)
-        # Escape "%" for string.format() (in gen_char_blob.lua). #39271
-        string(REPLACE "%" "%%" _LUAC_PRG_ESCAPED "${_CHECK_LUAC_PRG}")
-        set(LUAC_PRG "${_LUAC_PRG_ESCAPED} -s -o - %q" CACHE STRING "Format for compiling to Lua bytecode")
-        break()
-      endif()
-    endforeach()
-  elseif(LUA_PRG MATCHES "luajit")
-    check_lua_module(${LUA_PRG} "jit.bcsave" LUAJIT_HAS_JIT_BCSAVE)
-    if(LUAJIT_HAS_JIT_BCSAVE)
-      # Escape "%" for string.format() (in gen_char_blob.lua). #39271
-      string(REPLACE "%" "%%" _LUA_PRG_ESCAPED "${LUA_PRG}")
-      set(LUAC_PRG "${_LUA_PRG_ESCAPED} -b -s %q -" CACHE STRING "Format for compiling to Lua bytecode")
-    endif()
-  endif()
-endif()
-mark_as_advanced(LUAC_PRG)
-if(LUAC_PRG)
-  message(STATUS "Using Lua compiler: ${LUAC_PRG}")
-endif()
 
 # Lint
 option(CI_LINT "Abort if lint programs not found" OFF)

src/gen/gen_char_blob.lua

@@ -12,6 +12,12 @@ end
 --   -c   compile Lua bytecode
 local options = {}
 
+local ignorelist = {
+  ['vim._core.editor'] = true,
+  ['vim._core.options'] = true,
+  ['vim.keymap'] = true,
+}
+
 while true do
   local opt = string.match(arg[1], '^-(%w)')
   if not opt then
@@ -31,7 +37,6 @@ target:write('#include <stdint.h>\n\n')
 
 local index_items = {}
 
-local warn_on_missing_compiler = true
 local modnames = {}
 for argi = 2, #arg, 2 do
   local source_file = arg[argi]
@@ -44,22 +49,14 @@ for argi = 2, #arg, 2 do
   local varname = string.gsub(modname, '%.', '_dot_') .. '_module'
   target:write(('static const uint8_t %s[] = {\n'):format(varname))
 
-  local output
+  local source = io.open(source_file, 'r')
+    or error(string.format("source_file %q doesn't exist", source_file))
+  local output = source:read('*a')
+  source:close()
   if options.c then
-    local luac = os.getenv('LUAC_PRG')
-    if luac and luac ~= '' then
-      output = io.popen(luac:format(source_file), 'r'):read('*a')
-    elseif warn_on_missing_compiler then
-      print('LUAC_PRG is missing, embedding raw source')
-      warn_on_missing_compiler = false
-    end
-  end
-
-  if not output then
-    local source = io.open(source_file, 'r')
-      or error(string.format("source_file %q doesn't exist", source_file))
-    output = source:read('*a')
-    source:close()
+    local prefix = ignorelist[modname] and '' or '@'
+    local relpath = modname:gsub('%.', '/')
+    output = string.dump(assert((loadstring or load)(output, prefix .. relpath)), false)
   end
 
   local num_bytes = 0

src/gen/gen_steps.zig

@@ -65,8 +65,7 @@ pub fn nvim_gen_sources(
     {
         const gen_step = b.addRunArtifact(nlua0);
         gen_step.addFileArg(b.path("src/gen/gen_char_blob.lua"));
-        // TODO(bfredl): LUAC_PRG is missing. tricky with cross-compiling..
-        // gen_step.addArg("-c");
+        gen_step.addArg("-c");
         _ = gen_header(b, gen_step, "lua/vim_module.generated.h", gen_headers);
         // NB: vim._init_packages and vim.inspect must be be first and second ones
         // respectively, otherwise --luamod-dev won't work properly.

src/nvim/CMakeLists.txt

@@ -362,6 +362,11 @@ foreach(core_source ${LUA_CORE_MODULE_SOURCES})
   list(APPEND LUA_CORE_COMMAND_ARGS ${core_source} "vim._core.${core_filename}")
 endforeach()
 
+set(LUA_BLOB_COMPILE_FLAG)
+if(COMPILE_LUA AND NOT WIN32)
+  set(LUA_BLOB_COMPILE_FLAG -c)
+endif()
+
 file(GLOB API_HEADERS CONFIGURE_DEPENDS api/*.h)
 list(REMOVE_ITEM API_HEADERS ${CMAKE_CURRENT_LIST_DIR}/api/ui_events.in.h)
 file(GLOB MSGPACK_RPC_HEADERS CONFIGURE_DEPENDS msgpack_rpc/*.h)
@@ -636,9 +641,7 @@ add_custom_command(
 
 add_custom_command(
   OUTPUT ${VIM_MODULE_FILE}
-  COMMAND ${CMAKE_COMMAND} -E env
-      "LUAC_PRG=${LUAC_PRG}"
-      ${LUA_PRG} ${CHAR_BLOB_GENERATOR} -c ${VIM_MODULE_FILE}
+  COMMAND ${LUA_PRG} ${CHAR_BLOB_GENERATOR} ${LUA_BLOB_COMPILE_FLAG} ${VIM_MODULE_FILE}
       # NB: vim._init_packages and vim.inspect must be be first and second ones
       # respectively, otherwise --luamod-dev won't work properly.
       ${LUA_INIT_PACKAGES_MODULE_SOURCE} "vim._init_packages"

src/nvim/lua/executor.c

@@ -769,13 +769,7 @@ static int nlua_module_preloader(lua_State *lstate)
 {
   size_t i = (size_t)lua_tointeger(lstate, lua_upvalueindex(1));
   ModuleDef def = builtin_modules[i];
-  char name[256];
-  name[0] = '@';
-  size_t off = xstrlcpy(name + 1, def.name, (sizeof name) - 2);
-  strchrsub(name + 1, '.', '/');
-  xstrlcpy(name + 1 + off, ".lua", (sizeof name) - 2 - off);
-
-  if (luaL_loadbuffer(lstate, (const char *)def.data, def.size - 1, name)) {
+  if (luaL_loadbuffer(lstate, (const char *)def.data, def.size - 1, NULL)) {
     return lua_error(lstate);
   }
 
@@ -2372,15 +2366,6 @@ void nlua_set_sctx(sctx_T *current)
   lua_State *const lstate = active_lstate;
   lua_Debug *info = (lua_Debug *)xmalloc(sizeof(lua_Debug));
 
-  // Files where internal wrappers are defined so we can ignore them
-  // like vim.o/opt etc are defined in _options.lua
-  char *ignorelist[] = {
-    "vim/_core/editor.lua",
-    "vim/_core/options.lua",
-    "vim/keymap.lua",
-  };
-  int ignorelist_size = sizeof(ignorelist) / sizeof(ignorelist[0]);
-
   for (int level = 1; true; level++) {
     if (lua_getstack(lstate, level, info) != 1) {
       goto cleanup;
@@ -2388,19 +2373,7 @@ void nlua_set_sctx(sctx_T *current)
     if (lua_getinfo(lstate, "nSl", info) == 0) {
       goto cleanup;
     }
-
-    bool is_ignored = false;
     if (info->what[0] == 'C' || info->source[0] != '@') {
-      is_ignored = true;
-    } else {
-      for (int i = 0; i < ignorelist_size; i++) {
-        if (strncmp(ignorelist[i], info->source + 1, strlen(ignorelist[i])) == 0) {
-          is_ignored = true;
-          break;
-        }
-      }
-    }
-    if (is_ignored) {
       continue;
     }
     break;