From 55f9c2136e52d8719495b6021ce7e8d64c5141fe Mon Sep 17 00:00:00 2001 From: Lewis Russell Date: Wed, 25 Mar 2026 13:33:17 +0000 Subject: [PATCH] test: replace busted with local harness Replace the busted-based Lua test runner with a repo-local harness. The new harness runs spec files directly under `nvim -ll`, ships its own reporter and lightweight `luassert` shim, and keeps the helper/preload flow used by the functional and unit test suites. Keep the file boundary model shallow and busted-like by restoring `_G`, `package.loaded`, `package.preload`, `arg`, and the process environment between files, without carrying extra reset APIs or custom assertion machinery. Update the build and test entrypoints to use the new runner, add black-box coverage for the harness itself, and drop the bundled busted/luacheck dependency path. AI-assisted: Codex --- .github/workflows/test.yml | 4 - .luarc.json | 3 - BUILD.md | 2 +- CMakeLists.txt | 26 +- build.zig | 5 +- build.zig.zon | 4 - cmake.deps/deps.txt | 3 - cmake/Deps.cmake | 1 - cmake/RunTests.cmake | 16 +- contrib/local.mk.example | 1 - runtime/doc/dev_test.txt | 38 +- runtime/doc/dev_tools.txt | 4 +- runtime/lua/vim/_core/shared.lua | 27 + test/.emmyrc.json | 25 + test/.luarc.json | 7 +- test/CMakeLists.txt | 9 +- test/_meta.lua | 31 +- test/assert.lua | 102 ++ test/benchmark/preload.lua | 6 +- test/busted/outputHandlers/TAP.lua | 16 - test/busted/outputHandlers/nvim.lua | 359 ---- test/format_string.lua | 4 +- test/functional/autocmd/filetype_spec.lua | 2 +- test/functional/harness/assert_spec.lua | 28 + test/functional/harness/harness_spec.lua | 1382 +++++++++++++++ test/functional/lua/buffer_updates_spec.lua | 3 +- test/functional/lua/net_spec.lua | 2 +- test/functional/lua/vim_spec.lua | 26 +- test/functional/preload.lua | 6 +- test/functional/terminal/parser_spec.lua | 4 +- test/functional/testnvim.lua | 5 +- test/functional/ui/screen.lua | 7 +- test/harness.lua | 1694 +++++++++++++++++++ test/lua_runner.lua | 91 - test/reporter.lua | 461 +++++ test/run_tests.zig | 18 +- test/runner.lua | 35 + test/testutil.lua | 51 +- test/unit/os/fs_spec.lua | 4 +- test/unit/path_spec.lua | 8 - test/unit/preload.lua | 6 +- test/unit/tempfile_spec.lua | 2 +- test/unit/testtest_spec.lua | 3 +- test/unit/testutil.lua | 20 +- test/unit/vterm_spec.lua | 7 + 45 files changed, 3899 insertions(+), 659 deletions(-) create mode 100644 test/.emmyrc.json create mode 100644 test/assert.lua delete mode 100644 test/busted/outputHandlers/TAP.lua delete mode 100644 test/busted/outputHandlers/nvim.lua create mode 100644 test/functional/harness/assert_spec.lua create mode 100644 test/functional/harness/harness_spec.lua create mode 100644 test/harness.lua delete mode 100644 test/lua_runner.lua create mode 100644 test/reporter.lua create mode 100644 test/runner.lua diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9402bfa252..45e44416fe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,10 +74,6 @@ jobs: name: luals run: cmake --build build --target luals - - if: success() || failure() && steps.abort_job.outputs.status == 'success' - name: luacheck - run: cmake --build build --target lintlua-luacheck - - if: success() || failure() && steps.abort_job.outputs.status == 'success' name: lintsh run: cmake --build build --target lintsh diff --git a/.luarc.json b/.luarc.json index c897b8d9a2..87eaa43928 100644 --- a/.luarc.json +++ b/.luarc.json @@ -4,9 +4,6 @@ "version": "LuaJIT" }, "workspace": { - "library": [ - "${3rd}/busted/library" - ], "ignoreDir": [ ".deps", "build", diff --git a/BUILD.md b/BUILD.md index 6a9d58e7e2..1e10863bc1 100644 --- a/BUILD.md +++ b/BUILD.md @@ -569,7 +569,7 @@ Note that the C++ compiler is explicitly set so that it can be found when the de See "Available System Integrations" in `zig build -h` to see available system integrations. Enabling an integration, e.g. `zig build -fsys=utf8proc` will use the system's installation of utf8proc. -`zig build --system deps_dir` will enable all integrations and turn off dependency fetching. This requires you to pre-download the dependencies which don't have a system integration into `deps_dir` (at the time of writing these are ziglua, [`lua_dev_deps`](https://github.com/neovim/deps/blob/master/opt/lua-dev-deps.tar.gz), and the built-in tree-sitter parsers). You have to create subdirectories whose names are the respective package's hash under `deps_dir` and unpack the dependencies inside that directory - ziglua should go under `deps_dir/zlua-0.1.0-hGRpC1dCBQDf-IqqUifYvyr8B9-4FlYXqY8cl7HIetrC` and so on. Hashes should be taken from `build.zig.zon`. +`zig build --system deps_dir` will enable all integrations and turn off dependency fetching. This requires you to pre-download the dependencies which don't have a system integration into `deps_dir` (at the time of writing these are ziglua and the built-in tree-sitter parsers). You have to create subdirectories whose names are the respective package's hash under `deps_dir` and unpack the dependencies inside that directory - ziglua should go under `deps_dir/zlua-0.1.0-hGRpC1dCBQDf-IqqUifYvyr8B9-4FlYXqY8cl7HIetrC` and so on. Hashes should be taken from `build.zig.zon`. See the `prepare` function of [this `PKGBUILD`](https://git.sr.ht/~chinmay/nvim_build/tree/26364a4cf9b4819f52a3e785fa5a43285fb9cea2/item/PKGBUILD#L90) for an example. diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a4026a83c..84db914661 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -243,15 +243,6 @@ mark_as_advanced(TS_QUERY_LS_PRG) set(STYLUA_DIRS runtime scripts src test contrib) -add_glob_target( - TARGET lintlua-luacheck - COMMAND $ - FLAGS -ll ${PROJECT_SOURCE_DIR}/test/lua_runner.lua ${CMAKE_BINARY_DIR}/usr/share/lua/5.1 luacheck -q - GLOB_DIRS runtime scripts src test - GLOB_PAT *.lua - TOUCH_STRATEGY PER_DIR) -add_dependencies(lintlua-luacheck lua_dev_deps) - add_glob_target( TARGET lintlua-stylua COMMAND ${STYLUA_PRG} @@ -271,7 +262,7 @@ add_custom_target(lintlua-stylua2 add_dependencies(lintlua-stylua lintlua-stylua2) add_custom_target(lintlua) -add_dependencies(lintlua lintlua-luacheck lintlua-stylua) +add_dependencies(lintlua lintlua-stylua) add_glob_target( TARGET lintsh @@ -355,21 +346,6 @@ ExternalProject_Add(uncrustify EXCLUDE_FROM_ALL TRUE ${EXTERNALPROJECT_OPTIONS}) -option(USE_BUNDLED_BUSTED "Use bundled busted" ON) -if(USE_BUNDLED_BUSTED) - get_externalproject_options(lua_dev_deps ${DEPS_IGNORE_SHA}) - ExternalProject_Add(lua_dev_deps - DOWNLOAD_DIR ${DEPS_DOWNLOAD_DIR}/lua_dev_deps - SOURCE_DIR ${DEPS_SHARE_DIR} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - EXCLUDE_FROM_ALL TRUE - ${EXTERNALPROJECT_OPTIONS}) -else() - add_custom_target(lua_dev_deps) -endif() - if (CMAKE_SYSTEM_PROCESSOR MATCHES "arm|aarch") set(LUALS_ARCH arm64) else() diff --git a/build.zig b/build.zig index 6192f6c998..8743b66e56 100644 --- a/build.zig +++ b/build.zig @@ -545,8 +545,6 @@ pub fn build(b: *std.Build) !void { const gen_runtime = try runtime.nvim_gen_runtime(b, nlua0, funcs_data); - const lua_dev_deps = b.dependency("lua_dev_deps", .{}); - const test_deps = b.step("test_deps", "test prerequisites"); test_deps.dependOn(&nvim_exe_install.step); // running tests doesn't require copying the static runtime, only the generated stuff @@ -673,7 +671,6 @@ pub fn build(b: *std.Build) !void { b, nvim_exe, test_deps, - lua_dev_deps.path("."), test_config_step.getDirectory(), unit_headers, ); @@ -804,7 +801,7 @@ pub fn test_config(b: *std.Build) ![]u8 { \\M.test_source_path = "{[src_path]s}" \\M.test_lua_prg = "" \\M.test_luajit_prg = "" - \\ -- include path passed on the cmdline, see test/lua_runner.lua + \\ -- include path passed on the cmdline, see test/runner.lua \\M.include_paths = _G.c_include_path or {{}} \\ \\return M diff --git a/build.zig.zon b/build.zig.zon index 1b913016fc..3e88ac7d59 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -41,10 +41,6 @@ .hash = "libiconv-1.18.0-p9sJwWnqAACzVYeWgXB5r5lOQ74XwTPlptixV0JPRO28", .lazy = true, }, - .lua_dev_deps = .{ - .url = "https://github.com/neovim/deps/raw/06ef2b58b0876f8de1a3f5a710473dcd7afff251/opt/lua-dev-deps.tar.gz", - .hash = "N-V-__8AAGevEQCHAkCozca5AIdN9DFc3Luf3g3r2AcbyOrm", - }, .treesitter_c = .{ .url = "git+https://github.com/tree-sitter/tree-sitter-c?ref=v0.24.1#7fa1be1b694b6e763686793d97da01f36a0e5c12", .hash = "N-V-__8AANxPSABzw3WBTSH_YkwaGAfrK6PBqAMqQedkDDim", diff --git a/cmake.deps/deps.txt b/cmake.deps/deps.txt index 4609af08f3..d607fdc5e8 100644 --- a/cmake.deps/deps.txt +++ b/cmake.deps/deps.txt @@ -51,6 +51,3 @@ WASMTIME_SHA256 c4a3c596a07c02ba6adce503154a2095fd98037a1e50d56add9773f0269ec9b7 UNCRUSTIFY_URL https://github.com/uncrustify/uncrustify/archive/uncrustify-0.82.0.tar.gz UNCRUSTIFY_SHA256 e05f8d5ee36aaa1acfa032fe97546b7be46b1f4620e7c38037f8a42e25fe676f -# This is where we get busted, luassert, ... -LUA_DEV_DEPS_URL https://github.com/neovim/deps/raw/06ef2b58b0876f8de1a3f5a710473dcd7afff251/opt/lua-dev-deps.tar.gz -LUA_DEV_DEPS_SHA256 49f8399e453103064a23c65534f266f3067cda716b6502f016bfafeed5799354 diff --git a/cmake/Deps.cmake b/cmake/Deps.cmake index f8b7dfd01f..38ce9c9cc9 100644 --- a/cmake/Deps.cmake +++ b/cmake/Deps.cmake @@ -1,7 +1,6 @@ set(DEPS_INSTALL_DIR "${CMAKE_BINARY_DIR}/usr") set(DEPS_BIN_DIR "${DEPS_INSTALL_DIR}/bin") set(DEPS_LIB_DIR "${DEPS_INSTALL_DIR}/lib") -set(DEPS_SHARE_DIR "${DEPS_INSTALL_DIR}/share/lua/5.1") set(DEPS_BUILD_DIR "${CMAKE_BINARY_DIR}/build") set(DEPS_DOWNLOAD_DIR "${DEPS_BUILD_DIR}/downloads") diff --git a/cmake/RunTests.cmake b/cmake/RunTests.cmake index 62cff6eebd..c5c15bb08c 100644 --- a/cmake/RunTests.cmake +++ b/cmake/RunTests.cmake @@ -60,18 +60,18 @@ if(IS_ABSOLUTE ${TEST_PATH}) file(RELATIVE_PATH TEST_PATH "${ROOT_DIR}" "${TEST_PATH}") endif() -separate_arguments(BUSTED_ARGS NATIVE_COMMAND $ENV{BUSTED_ARGS}) +separate_arguments(TEST_ARGS NATIVE_COMMAND $ENV{TEST_ARGS}) if(DEFINED ENV{TEST_TAG} AND NOT "$ENV{TEST_TAG}" STREQUAL "") - list(APPEND BUSTED_ARGS --tags $ENV{TEST_TAG}) + list(APPEND TEST_ARGS --tags $ENV{TEST_TAG}) endif() if(DEFINED ENV{TEST_FILTER} AND NOT "$ENV{TEST_FILTER}" STREQUAL "") - list(APPEND BUSTED_ARGS --filter $ENV{TEST_FILTER}) + list(APPEND TEST_ARGS --filter $ENV{TEST_FILTER}) endif() if(DEFINED ENV{TEST_FILTER_OUT} AND NOT "$ENV{TEST_FILTER_OUT}" STREQUAL "") - list(APPEND BUSTED_ARGS --filter-out $ENV{TEST_FILTER_OUT}) + list(APPEND TEST_ARGS --filter-out $ENV{TEST_FILTER_OUT}) endif() # TMPDIR: for testutil.tmpname() and Nvim tempname(). @@ -95,14 +95,14 @@ endif() execute_process( # Note: because of "-ll" (low-level interpreter mode), some modules like # _core/editor.lua are not loaded. - COMMAND ${NVIM_PRG} -ll ${ROOT_DIR}/test/lua_runner.lua ${DEPS_INSTALL_DIR}/share/lua/5.1/ busted -v -o test.busted.outputHandlers.nvim - -Xoutput "{\"test_path\": \"${TEST_PATH}\", \"summary_file\": \"${TEST_SUMMARY_FILE}\"}" - --lazy --helper=${TEST_DIR}/${TEST_TYPE}/preload.lua + COMMAND ${NVIM_PRG} -ll ${ROOT_DIR}/test/runner.lua -v + --summary-file=${TEST_SUMMARY_FILE} + --helper=${TEST_DIR}/${TEST_TYPE}/preload.lua --lpath=${BUILD_DIR}/?.lua --lpath=${ROOT_DIR}/src/?.lua --lpath=${ROOT_DIR}/runtime/lua/?.lua --lpath=?.lua - ${BUSTED_ARGS} + ${TEST_ARGS} ${TEST_PATH} TIMEOUT $ENV{TEST_TIMEOUT} WORKING_DIRECTORY ${TEST_XDG_PREFIX} diff --git a/contrib/local.mk.example b/contrib/local.mk.example index 223602fddc..faade3935c 100644 --- a/contrib/local.mk.example +++ b/contrib/local.mk.example @@ -41,7 +41,6 @@ # Uncomment these entries to instead use system-wide installations of # them. # -# DEPS_CMAKE_FLAGS += -DUSE_BUNDLED_BUSTED=OFF # DEPS_CMAKE_FLAGS += -DUSE_BUNDLED_GETTEXT=OFF # DEPS_CMAKE_FLAGS += -DUSE_BUNDLED_LIBICONV=OFF # DEPS_CMAKE_FLAGS += -DUSE_BUNDLED_LIBUV=OFF diff --git a/runtime/doc/dev_test.txt b/runtime/doc/dev_test.txt index 9dffba743c..2b16a9f65f 100644 --- a/runtime/doc/dev_test.txt +++ b/runtime/doc/dev_test.txt @@ -37,12 +37,25 @@ You can learn [Lua concepts 15 minutes](https://learnxinyminutes.com/docs/lua/), see also |lua-guide|. Use any existing test as a template to start writing new tests, or see |dev-quickstart|. -Tests are run by the `/cmake/RunTests.cmake` script using `busted` (a Lua test-runner). +Tests are run by the `/cmake/RunTests.cmake` script using the repo-local Lua +test harness in `/test/harness.lua`. For some failures, `./build/nvim.log` (or `$NVIM_LOG_FILE`) may provide insight. Depending on the presence of binaries (e.g., `xclip`) some tests will be skipped. +Harness isolation *dev-test-harness-isolation* + +The Lua harness runs all selected spec files in one low-level `nvim -ll` +process, then restores a baseline before each file. For each suite iteration, +the file boundary baseline is captured after `--helper` is loaded, so +helper-provided modules and defaults persist across files. +Helper files are preload-only: they may require modules, set defaults, and +register suite-end cleanup, but they do not define tests or hooks. + +The harness restores `_G`, `package.loaded`, `package.preload`, `arg`, and the +process environment with the same shallow file boundary model that busted uses. + ============================================================================== Test Layout @@ -56,7 +69,8 @@ Test Layout - `/test/includes` : include-files for use by luajit `ffi.cdef` C definitions parser: normally used to make macros not accessible via this mechanism accessible the other way. -- `/test/*/preload.lua` : modules preloaded by busted `--helper` option +- `/test/*/preload.lua` : modules and defaults preloaded by the harness + `--helper` option - `/test/**/testutil.lua` : common utility functions in the context of the test runner - `/test/**/testnvim.lua` : common utility functions in the context of the @@ -145,9 +159,7 @@ need to install and configure: 2. [local-lua-debugger-vscode](https://github.com/mfussenegger/nvim-dap/wiki/Debug-Adapter-installation#local-lua-debugger-vscode) 3. [nlua](https://github.com/mfussenegger/nlua) 4. [one-small-step-for-vimkind](https://github.com/jbyuki/one-small-step-for-vimkind) (called `osv`) -5. A `nbusted` command in `$PATH`. This command can be a copy of `busted` with - `exec '/usr/bin/lua5.1'"` replaced with `"exec '/usr/bin/nlua'"` (or the - path to your `nlua`) +5. An `nlua` command in `$PATH`. The setup roughly looks like this: > @@ -163,7 +175,7 @@ The setup roughly looks like this: > │ │ ▼ │ ┌─────────┐ │ - │ nbusted │ │ + │ nlua │ │ └─────────┘ │ │ │ ▼ │ @@ -206,11 +218,11 @@ With these installed you can use a configuration like this: > request = "launch", cwd = "${workspaceFolder}", program = { - command = "nbusted", + command = "nlua", }, args = { - "--ignore-lua", - "--lazy", + "-ll", + "test/runner.lua", "--helper=test/functional/preload.lua", "--lpath=build/?.lua", "--lpath=?.lua", @@ -274,7 +286,7 @@ Limitations: if os.getenv("LOCAL_LUA_DEBUGGER_VSCODE") == "1" then require("lldebugger").start() end -< This is a [local-lua-debugger limitation](https://github.com/tomblind/local-lua-debugger-vscode?tab=readme-ov-file#busted) +< This is a local-lua-debugger limitation. - You cannot step into code of files which get baked into the nvim binary (such as `_core/*.lua` and `inspect.lua`). @@ -314,10 +326,10 @@ or > cmake -E env "TEST_FILE=test/functional/foo.lua" cmake --build build --target functionaltest To _repeat_ a test: > - BUSTED_ARGS="--repeat=100 --no-keep-going" TEST_FILE=test/functional/foo_spec.lua make functionaltest + TEST_ARGS="--repeat=100 --no-keep-going" TEST_FILE=test/functional/foo_spec.lua make functionaltest or > - cmake -E env "TEST_FILE=test/functional/foo_spec.lua" cmake -E env BUSTED_ARGS="--repeat=100 --no-keep-going" cmake --build build --target functionaltest + cmake -E env "TEST_FILE=test/functional/foo_spec.lua" cmake -E env TEST_ARGS="--repeat=100 --no-keep-going" cmake --build build --target functionaltest FILTER BY TAG @@ -430,7 +442,7 @@ Test behaviour is affected by environment variables. Currently supported treated as Integer; when defined, treated as String; when defined, treated as Number; !must be defined to function properly): -- `BUSTED_ARGS` (F) (U): arguments forwarded to `busted`. +- `TEST_ARGS` (F) (U): arguments forwarded to the local test harness. - `CC` (U) (S): specifies which C compiler to use to preprocess files. Currently only compilers with gcc-compatible arguments are supported. diff --git a/runtime/doc/dev_tools.txt b/runtime/doc/dev_tools.txt index 5234ce0fc5..94cb67b1fa 100644 --- a/runtime/doc/dev_tools.txt +++ b/runtime/doc/dev_tools.txt @@ -388,7 +388,7 @@ Gdb *dev-tools-gdb* USING GDB TO STEP THROUGH FUNCTIONAL TESTS -Use `TEST_TAG` to run tests matching busted tags (of the form `#foo` e.g. +Use `TEST_TAG` to run tests matching `#foo` tags (e.g. `it("test #foo ...", ...)`): >bash GDB=1 TEST_TAG=foo make functionaltest @@ -403,7 +403,7 @@ Then, in another terminal: USING LLDB TO STEP THROUGH UNIT TESTS > - lldb .deps/usr/bin/luajit -- .deps/usr/bin/busted --lpath="./build/?.lua" test/unit/ + lldb build/bin/nvim -- -ll test/runner.lua --lpath=./build/?.lua test/unit/ < USING GDB diff --git a/runtime/lua/vim/_core/shared.lua b/runtime/lua/vim/_core/shared.lua index f390e3c1e9..5237063a8d 100644 --- a/runtime/lua/vim/_core/shared.lua +++ b/runtime/lua/vim/_core/shared.lua @@ -64,6 +64,31 @@ function vim.deepcopy(orig, noref) return deepcopy(orig, not noref and {} or nil) end +--- Returns a shallow copy of `orig`. +--- +--- Non-table values are returned as-is. Table keys and values are copied by +--- reference, and the original metatable is preserved. Use |vim.deepcopy()| +--- for a recursive copy. +--- +--- @nodoc +--- @generic T +--- @param orig T +--- @return T +function vim._copy(orig) + if type(orig) ~= 'table' then + return orig + end + + --- @cast orig table + + local copy = {} --- @type table + for k, v in pairs(orig) do + copy[k] = v + end + + return setmetatable(copy, getmetatable(orig)) +end + --- @class vim.gsplit.Opts --- @inlinedoc --- @@ -678,6 +703,8 @@ local function deep_equal(left, right, seen) return false end + ---@cast left table + ---@cast right table seen = seen or {} local seen_left = seen[left] if seen_left and seen_left[right] ~= nil then diff --git a/test/.emmyrc.json b/test/.emmyrc.json new file mode 100644 index 0000000000..34d1e0b860 --- /dev/null +++ b/test/.emmyrc.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://raw.githubusercontent.com/EmmyLuaLs/emmylua-analyzer-rust/refs/heads/main/crates/emmylua_code_analysis/resources/schema.json", + "format": { + "externalTool": { + "program": "stylua", + "args": [ + "-", + "--stdin-filepath", + "${file}" + ] + } + }, + "diagnostics": { + "disable": [ + "unnecessary-if" + ] + }, + "codeAction": { + "insertSpace": true + }, + "strict": { + "typeCall": true, + "arrayIndex": true + } +} diff --git a/test/.luarc.json b/test/.luarc.json index f63e15bb41..343ed38a53 100644 --- a/test/.luarc.json +++ b/test/.luarc.json @@ -6,12 +6,7 @@ "workspace": { "library": [ "../runtime/lua", - "../src", - "../build/usr/share/lua/5.1", - "../build", - "${3rd}/busted/library", - "${3rd}/luassert/library", - "${3rd}/luv/library" + "../src" ], "checkThirdParty": "Disable" }, diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1efc87cb4d..cb367b9330 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -7,7 +7,6 @@ set(TEST_OPTIONS -D BUILD_DIR=${CMAKE_BINARY_DIR} -D CIRRUS_CI=$ENV{CIRRUS_CI} -D CI_BUILD=${CI_BUILD} - -D DEPS_INSTALL_DIR=${DEPS_INSTALL_DIR} -D NVIM_PRG=$ -D TEST_DIR=${TEST_DIR} -D ROOT_DIR=${PROJECT_SOURCE_DIR}) @@ -20,7 +19,7 @@ if(LUA_HAS_FFI) ${TEST_OPTIONS} -P ${PROJECT_SOURCE_DIR}/cmake/RunTests.cmake USES_TERMINAL) - add_dependencies(unittest lua_dev_deps nvim) + add_dependencies(unittest nvim) else() message(WARNING "disabling unit tests: no Luajit FFI in ${LUA_PRG}") endif() @@ -36,7 +35,7 @@ add_custom_target(benchmark -P ${PROJECT_SOURCE_DIR}/cmake/RunTests.cmake DEPENDS tty-test USES_TERMINAL) -add_dependencies(benchmark lua_dev_deps nvim) +add_dependencies(benchmark nvim) add_custom_target(functionaltest COMMAND ${CMAKE_COMMAND} @@ -45,7 +44,7 @@ add_custom_target(functionaltest -P ${PROJECT_SOURCE_DIR}/cmake/RunTests.cmake DEPENDS printenv-test printargs-test shell-test pwsh-test streams-test tty-test USES_TERMINAL) -add_dependencies(functionaltest lua_dev_deps nvim) +add_dependencies(functionaltest nvim) # Create multiple targets for groups of functional tests to enable parallel testing. set(group_targets "") @@ -66,7 +65,7 @@ foreach(test_file ${test_files}) ${TEST_OPTIONS} -P ${PROJECT_SOURCE_DIR}/cmake/RunTests.cmake DEPENDS printenv-test printargs-test shell-test pwsh-test streams-test tty-test) - add_dependencies(${group_target} lua_dev_deps nvim) + add_dependencies(${group_target} nvim) list(APPEND group_targets ${group_target}) list(APPEND summary_files ${summary_file}) endif() diff --git a/test/_meta.lua b/test/_meta.lua index b736bcc6d9..480aa40d26 100644 --- a/test/_meta.lua +++ b/test/_meta.lua @@ -1,10 +1,25 @@ --- @meta -do -- Mark block as optional - ---Mark a test as placeholder. - --- - ---This will not fail or pass, it will simply be marked as "pending". - ---@param name string - ---@param block? fun() - function pending(name, block) end -end +--- @param name string +--- @param fn? fun() +function it(name, fn) end + +--- @param name string +--- @param fn fun() +function describe(name, fn) end + +--- @param name? string +--- @param block? fun()|string +function pending(name, block) end + +--- @param fn fun() +function setup(fn) end + +--- @param fn fun() +function before_each(fn) end + +--- @param fn fun() +function after_each(fn) end + +--- @param fn fun() +function teardown(fn) end diff --git a/test/assert.lua b/test/assert.lua new file mode 100644 index 0000000000..47eb2e4c85 --- /dev/null +++ b/test/assert.lua @@ -0,0 +1,102 @@ +--- @class test.assert +local M = {} + +local FORMAT_DEPTH = 100 + +--- @param value any +--- @return string +local function format_value(value) + if type(value) == 'string' then + return string.format('%q', value) + end + + local ok, inspected = pcall(vim.inspect, value, { depth = FORMAT_DEPTH }) + if ok then + return inspected + end + + return tostring(value) +end + +--- @param condition boolean +--- @param value any +--- @param context any +--- @param fallback string +local function assert_value(condition, value, context, fallback) + if not condition then + local message = context ~= nil and tostring(context) or fallback + error(message, 0) + end + return value +end + +--- @param expected any +--- @param actual any +--- @param comparator string +--- @return string +local function comparison_message(expected, actual, comparator) + return ('Expected values to be %s.\nExpected:\n%s\nActual:\n%s'):format( + comparator, + format_value(expected), + format_value(actual) + ) +end + +--- @param expected any +--- @param actual any +--- @param context? any +--- @return any +function M.eq(expected, actual, context) + return assert_value( + vim.deep_equal(expected, actual), + actual, + context, + comparison_message(expected, actual, 'equal') + ) +end + +--- @param expected any +--- @param actual any +--- @param context? any +--- @return any +function M.neq(expected, actual, context) + return assert_value( + not vim.deep_equal(expected, actual), + actual, + context, + ('Expected values to differ.\nValue:\n%s'):format(format_value(actual)) + ) +end + +--- @param value any +--- @param context? any +--- @return any +function M.is_true(value, context) + return M.eq(true, value, context) +end + +--- @param value any +--- @param context? any +--- @return any +function M.is_false(value, context) + return M.eq(false, value, context) +end + +-- TODO(lewis6991): remove these aliases +M.True = M.is_true +M.False = M.is_false +M.equals = M.eq +M.Equal = M.eq + +return setmetatable(M, { + --- @param condition any + --- @param message? string + --- @param level? integer + __call = function(_, condition, message, level, ...) + if condition then + return condition, message, level, ... + end + + error(message or 'assertion failed!', (type(level) == 'number' and level or 1) + 1) + end, +}) diff --git a/test/benchmark/preload.lua b/test/benchmark/preload.lua index 8d0b65e2c8..8f03718745 100644 --- a/test/benchmark/preload.lua +++ b/test/benchmark/preload.lua @@ -1,4 +1,4 @@ --- Modules loaded here will not be cleared and reloaded by Busted. --- Busted started doing this to help provide more isolation. See issue #62 --- for more information about this. +-- Modules loaded here will not be cleared and reloaded by the local harness. +-- Keeping these preloaded preserves cross-file setup while still resetting +-- non-helper modules between files. require('test.functional.testnvim')() diff --git a/test/busted/outputHandlers/TAP.lua b/test/busted/outputHandlers/TAP.lua deleted file mode 100644 index 2439547050..0000000000 --- a/test/busted/outputHandlers/TAP.lua +++ /dev/null @@ -1,16 +0,0 @@ --- Extends the upstream TAP handler, to display the log with suiteEnd. - -local t_global = require('test.testutil') - -return function(options) - local busted = require 'busted' - local handler = require 'busted.outputHandlers.TAP'(options) - - local suiteEnd = function() - io.write(t_global.read_nvim_log(nil, true)) - return nil, true - end - busted.subscribe({ 'suite', 'end' }, suiteEnd) - - return handler -end diff --git a/test/busted/outputHandlers/nvim.lua b/test/busted/outputHandlers/nvim.lua deleted file mode 100644 index 532449965c..0000000000 --- a/test/busted/outputHandlers/nvim.lua +++ /dev/null @@ -1,359 +0,0 @@ -local pretty = require 'pl.pretty' -local t_global = require('test.testutil') - -local colors = setmetatable({}, { - __index = function() - return function(s) - return s == nil and '' or tostring(s) - end - end, -}) - -local enable_colors = true -if os.getenv 'TEST_COLORS' then - local test_colors = os.getenv('TEST_COLORS'):lower() - local disable_colors = test_colors == 'false' - or test_colors == '0' - or test_colors == 'no' - or test_colors == 'off' - enable_colors = not disable_colors -end -if enable_colors then - colors = require 'term.colors' -end - -return function(options) - local busted = require 'busted' - local handler = require 'busted.outputHandlers.base'() - local args = options.arguments - args = vim.json.decode(#args > 0 and table.concat(args, ',') or '{}') - - local c = { - succ = function(s) - return colors.bright(colors.green(s)) - end, - skip = function(s) - return colors.bright(colors.yellow(s)) - end, - fail = function(s) - return colors.bright(colors.magenta(s)) - end, - errr = function(s) - return colors.bright(colors.red(s)) - end, - test = tostring, - file = colors.cyan, - time = colors.dim, - note = colors.yellow, - sect = function(s) - return colors.green(colors.dim(s)) - end, - nmbr = colors.bright, - } - - local repeatSuiteString = '\nRepeating all tests (run %d of %d) . . .\n\n' - local randomizeString = c.note('Note: Randomizing test order with a seed of %d.\n') - local globalSetup = c.sect('--------') .. ' Global test environment setup.\n' - local fileStartString = c.sect('--------') .. ' Running tests from ' .. c.file('%s') .. '\n' - local runString = c.sect('RUN ') .. ' ' .. c.test('%s') .. ': ' - local successString = c.succ('OK') .. '\n' - local skippedString = c.skip('SKIP') .. '\n' - local failureString = c.fail('FAIL') .. '\n' - local errorString = c.errr('ERR') .. '\n' - local fileEndString = c.sect('--------') - .. ' ' - .. c.nmbr('%d') - .. ' %s from ' - .. c.file('%s') - .. ' ' - .. c.time('(%.2f ms total)') - .. '\n\n' - local globalTeardown = c.sect('--------') .. ' Global test environment teardown.\n' - local suiteEndString = c.sect('========') - .. ' ' - .. c.nmbr('%d') - .. ' %s from ' - .. c.nmbr('%d') - .. ' test %s ran. ' - .. c.time('(%.2f ms total)') - .. '\n' - local successStatus = c.succ('PASSED ') .. ' ' .. c.nmbr('%d') .. ' %s.\n' - local timeString = c.time('%.2f ms') - - local summaryStrings = { - skipped = { - header = c.skip('SKIPPED ') .. ' ' .. c.nmbr('%d') .. ' %s, listed below:\n', - test = c.skip('SKIPPED ') .. ' %s\n', - footer = ' ' .. c.nmbr('%d') .. ' SKIPPED %s\n', - }, - - failure = { - header = c.fail('FAILED ') .. ' ' .. c.nmbr('%d') .. ' %s, listed below:\n', - test = c.fail('FAILED ') .. ' %s\n', - footer = ' ' .. c.nmbr('%d') .. ' FAILED %s\n', - }, - - error = { - header = c.errr('ERROR ') .. ' ' .. c.nmbr('%d') .. ' %s, listed below:\n', - test = c.errr('ERROR ') .. ' %s\n', - footer = ' ' .. c.nmbr('%d') .. ' %s\n', - }, - } - - local fileCount = 0 - local fileTestCount = 0 - local testCount = 0 - local successCount = 0 - local skippedCount = 0 - local failureCount = 0 - local errorCount = 0 - - local naCheck = function(pending) - if vim.list_contains(vim.split(pending.name, '[ :]'), 'N/A') then - return true - end - if type(pending.message) ~= 'string' then - return false - end - if vim.list_contains(vim.split(pending.message, '[ :]'), 'N/A') then - return true - end - return false - end - - local pendingDescription = function(pending) - local string = '' - - if type(pending.message) == 'string' then - string = string .. pending.message .. '\n' - elseif pending.message ~= nil then - string = string .. pretty.write(pending.message) .. '\n' - end - - return string - end - - local failureDescription = function(failure) - local string = failure.randomseed and ('Random seed: ' .. failure.randomseed .. '\n') or '' - if type(failure.message) == 'string' then - string = string .. failure.message - elseif failure.message == nil then - string = string .. 'Nil error' - else - string = string .. pretty.write(failure.message) - end - - string = string .. '\n' - - if options.verbose and failure.trace and failure.trace.traceback then - string = string .. failure.trace.traceback .. '\n' - end - - return string - end - - local getFileLine = function(element) - local fileline = '' - if element.trace or element.trace.short_src then - local fname = vim.fs.normalize(element.trace.short_src) - fileline = colors.cyan(fname) .. ' @ ' .. colors.cyan(element.trace.currentline) .. ': ' - end - return fileline - end - - local getTestList = function(status, count, list, getDescription) - local string = '' - local header = summaryStrings[status].header - if count > 0 and header then - local tests = (count == 1 and 'test' or 'tests') - local errors = (count == 1 and 'error' or 'errors') - string = header:format(count, status == 'error' and errors or tests) - - local testString = summaryStrings[status].test - if testString then - local naCount = 0 - for _, t in ipairs(list) do - if status == 'skipped' and naCheck(t) then - naCount = naCount + 1 - else - local fullname = getFileLine(t.element) .. colors.bright(t.name) - string = string .. testString:format(fullname) - string = string .. getDescription(t) - end - end - if naCount > 0 then - string = string - .. colors.bright( - ('%d N/A %s not shown\n'):format(naCount, naCount == 1 and 'test' or 'tests') - ) - end - end - end - return string - end - - local getSummary = function(status, count) - local string = '' - local footer = summaryStrings[status].footer - if count > 0 and footer then - local tests = (count == 1 and 'TEST' or 'TESTS') - local errors = (count == 1 and 'ERROR' or 'ERRORS') - string = footer:format(count, status == 'error' and errors or tests) - end - return string - end - - local getSummaryString = function() - local tests = (successCount == 1 and 'test' or 'tests') - local string = successStatus:format(successCount, tests) - - string = string .. getTestList('skipped', skippedCount, handler.pendings, pendingDescription) - string = string .. getTestList('failure', failureCount, handler.failures, failureDescription) - string = string .. getTestList('error', errorCount, handler.errors, failureDescription) - - string = string .. ((skippedCount + failureCount + errorCount) > 0 and '\n' or '') - string = string .. getSummary('skipped', skippedCount) - string = string .. getSummary('failure', failureCount) - string = string .. getSummary('error', errorCount) - - return string - end - - handler.suiteReset = function() - fileCount = 0 - fileTestCount = 0 - testCount = 0 - successCount = 0 - skippedCount = 0 - failureCount = 0 - errorCount = 0 - - return nil, true - end - - handler.suiteStart = function(_suite, count, total, randomseed) - if total > 1 then - io.write(repeatSuiteString:format(count, total)) - end - if randomseed then - io.write(randomizeString:format(randomseed)) - end - io.write(globalSetup) - io.flush() - - return nil, true - end - - local function getElapsedTime(tbl) - if tbl.duration then - return tbl.duration * 1000 - else - return tonumber('nan') - end - end - - handler.suiteEnd = function(suite, _count, _total) - local elapsedTime_ms = getElapsedTime(suite) - local tests = (testCount == 1 and 'test' or 'tests') - local files = (fileCount == 1 and 'file' or 'files') - if type(args.test_path) == 'string' then - files = files .. ' of ' .. args.test_path - end - local sf = type(args.summary_file) == 'string' - and args.summary_file ~= '-' - and io.open(args.summary_file, 'w') - or io.stdout - io.write(globalTeardown) - io.flush() - sf:write('\n') - sf:write(suiteEndString:format(testCount, tests, fileCount, files, elapsedTime_ms)) - sf:write(getSummaryString()) - if failureCount > 0 or errorCount > 0 then - sf:write(t_global.read_nvim_log(nil, true)) - end - sf:flush() - if sf ~= io.stdout then - sf:close() - end - - return nil, true - end - - handler.fileStart = function(file) - fileTestCount = 0 - io.write(fileStartString:format(vim.fs.normalize(file.name))) - io.flush() - return nil, true - end - - handler.fileEnd = function(file) - local elapsedTime_ms = getElapsedTime(file) - local tests = (fileTestCount == 1 and 'test' or 'tests') - fileCount = fileCount + 1 - io.write( - fileEndString:format(fileTestCount, tests, vim.fs.normalize(file.name), elapsedTime_ms) - ) - io.flush() - return nil, true - end - - handler.testStart = function(element, _parent) - local testid = _G._nvim_test_id or '' - local desc = ('%s %s'):format(testid, handler.getFullName(element)) - io.write(runString:format(desc)) - io.flush() - - return nil, true - end - - local function write_status(element, string) - io.write(timeString:format(getElapsedTime(element)) .. ' ' .. string) - io.flush() - end - - handler.testEnd = function(element, _parent, status, _debug) - local string - - fileTestCount = fileTestCount + 1 - testCount = testCount + 1 - if status == 'success' then - successCount = successCount + 1 - string = successString - elseif status == 'pending' then - skippedCount = skippedCount + 1 - string = skippedString - elseif status == 'failure' then - failureCount = failureCount + 1 - string = failureString .. failureDescription(handler.failures[#handler.failures]) - elseif status == 'error' then - errorCount = errorCount + 1 - string = errorString .. failureDescription(handler.errors[#handler.errors]) - else - string = 'unexpected test status! (' .. status .. ')' - end - write_status(element, string) - - return nil, true - end - - handler.error = function(element, _parent, _message, _debug) - if element.descriptor ~= 'it' then - write_status(element, failureDescription(handler.errors[#handler.errors])) - errorCount = errorCount + 1 - end - - return nil, true - end - - busted.subscribe({ 'suite', 'reset' }, handler.suiteReset) - busted.subscribe({ 'suite', 'start' }, handler.suiteStart) - busted.subscribe({ 'suite', 'end' }, handler.suiteEnd) - busted.subscribe({ 'file', 'start' }, handler.fileStart) - busted.subscribe({ 'file', 'end' }, handler.fileEnd) - busted.subscribe({ 'test', 'start' }, handler.testStart, { predicate = handler.cancelOnPending }) - busted.subscribe({ 'test', 'end' }, handler.testEnd, { predicate = handler.cancelOnPending }) - busted.subscribe({ 'failure' }, handler.error) - busted.subscribe({ 'error' }, handler.error) - - return handler -end diff --git a/test/format_string.lua b/test/format_string.lua index 777fb652e8..0e4ed9ab97 100644 --- a/test/format_string.lua +++ b/test/format_string.lua @@ -1,5 +1,3 @@ -local luaassert = require('luassert') - local M = {} local SUBTBL = { @@ -124,7 +122,7 @@ function M.format_luav(v, indent, opts) else print(type(v)) -- Not implemented yet - luaassert(false) + assert(false) end return ret end diff --git a/test/functional/autocmd/filetype_spec.lua b/test/functional/autocmd/filetype_spec.lua index 91843c7910..da2d2050fb 100644 --- a/test/functional/autocmd/filetype_spec.lua +++ b/test/functional/autocmd/filetype_spec.lua @@ -12,6 +12,6 @@ describe('autocmd FileType', function() command('let g:foo = 0') command('autocmd FileType help let g:foo = g:foo + 1') command('help help') - assert.same(1, eval('g:foo')) + assert.eq(1, eval('g:foo')) end) end) diff --git a/test/functional/harness/assert_spec.lua b/test/functional/harness/assert_spec.lua new file mode 100644 index 0000000000..df15a8a05d --- /dev/null +++ b/test/functional/harness/assert_spec.lua @@ -0,0 +1,28 @@ +local assert = require('test.assert') + +describe('test.assert', function() + it('ignores aliasing differences', function() + local shared = {} + + assert.eq({ 1, shared, 1, shared }, { 1, {}, 1, {} }) + assert.eq({ 1, {}, 1, {} }, { 1, shared, 1, shared }) + end) + + it('handles cyclic tables', function() + local expected = {} + local actual = {} + + expected[1] = expected + actual[1] = actual + + assert.eq(expected, actual) + end) + + it('still rejects different structures', function() + local expected = {} + + expected[1] = expected + + assert.neq(expected, { {} }) + end) +end) diff --git a/test/functional/harness/harness_spec.lua b/test/functional/harness/harness_spec.lua new file mode 100644 index 0000000000..57e5943258 --- /dev/null +++ b/test/functional/harness/harness_spec.lua @@ -0,0 +1,1382 @@ +-- Black-box tests for the local Lua harness itself. These spawn a separate +-- low-level Nvim process instead of driving an embedded instance via testnvim. +local t = require('test.testutil') +local uv = vim.uv + +local eq = t.eq + +local root = t.paths.test_source_path +local build_dir = t.paths.test_build_dir +local nvim_prog = build_dir .. '/bin/nvim' +local runner = root .. '/test/runner.lua' + +---@param path string +local function mkdir(path) + assert(t.mkdir(path), ('failed to create directory: %s'):format(path)) +end + +---@param suite_files table +---@return string +local function write_suite(suite_files) + local dir = t.tmpname(false) + mkdir(dir) + + for name, contents in pairs(suite_files) do + t.write_file(dir .. '/' .. name, contents) + end + + return dir +end + +---@param overrides table +---@return string[] +local function make_env(overrides) + local env = uv.os_environ() + for k, v in pairs(overrides) do + env[k] = v + end + + local items = {} + for k, v in pairs(env) do + if v ~= nil then + items[#items + 1] = k .. '=' .. v + end + end + + return items +end + +---@param suite_dir string +---@param extra_args? string[] +---@return integer, string +local function run_harness(suite_dir, extra_args) + local env_root = t.tmpname(false) + mkdir(env_root) + mkdir(env_root .. '/config') + mkdir(env_root .. '/share') + mkdir(env_root .. '/state') + mkdir(env_root .. '/tmp') + + local stdout = assert(uv.new_pipe(false)) + local stderr = assert(uv.new_pipe(false)) + local out = {} + local err = {} + local exit_code --- @type integer? + + local args = { + '-ll', + runner, + '-v', + '--lpath=' .. build_dir .. '/?.lua', + '--lpath=' .. root .. '/src/?.lua', + '--lpath=' .. root .. '/runtime/lua/?.lua', + '--lpath=' .. suite_dir .. '/?.lua', + '--lpath=?.lua', + } + for _, arg in ipairs(extra_args or {}) do + args[#args + 1] = arg + end + args[#args + 1] = suite_dir + + local handle = assert(uv.spawn(nvim_prog, { + args = args, + env = make_env({ + NVIM_TEST = '1', + TEST_COLORS = '0', + VIMRUNTIME = root .. '/runtime', + XDG_CONFIG_HOME = env_root .. '/config', + XDG_DATA_HOME = env_root .. '/share', + XDG_STATE_HOME = env_root .. '/state', + NVIM_RPLUGIN_MANIFEST = env_root .. '/rplugin_manifest', + NVIM_LOG_FILE = env_root .. '/nvim.log', + TMPDIR = env_root .. '/tmp', + SYSTEM_NAME = os.getenv('SYSTEM_NAME') or uv.os_uname().sysname, + SHELL = os.getenv('SHELL') or 'sh', + }), + stdio = { nil, stdout, stderr }, + hide = true, + }, function(code, _signal) + exit_code = code + end)) + + local function read_all(pipe, chunks) + pipe:read_start(function(read_err, chunk) + assert(not read_err, read_err) + if chunk == nil then + pipe:read_stop() + pipe:close() + else + chunks[#chunks + 1] = chunk + end + end) + end + + read_all(stdout, out) + read_all(stderr, err) + + while exit_code == nil or not stdout:is_closing() or not stderr:is_closing() do + uv.run('once') + end + + handle:close() + while not handle:is_closing() do + uv.run('once') + end + + return exit_code, table.concat(out) .. table.concat(err) +end + +---@param suite_dir string +---@param extra_args? string[] +local function assert_harness_passes(suite_dir, extra_args) + local code, output = run_harness(suite_dir, extra_args) + eq(0, code) + eq(true, output:find('PASSED', 1, true) ~= nil) +end + +describe('test harness', function() + it('restores package.preload between files', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + package.preload.leak_mod = function() + return 'leak' + end + + describe('one', function() + it('defines a preload entry', function() + assert.Equal('leak', require('leak_mod')) + end) + end) + ]], + ['two_spec.lua'] = [[ + describe('two', function() + it('does not see the preload entry from another file', function() + local ok = pcall(require, 'leak_mod') + assert.False(ok) + end) + end) + ]], + }) + + assert_harness_passes(suite_dir) + end) + + it('restores package.loaded between files', function() + local suite_dir = write_suite({ + ['loaded_mod.lua'] = [[ + return { source = 'file' } + ]], + ['one_spec.lua'] = [[ + describe('one', function() + it('mutates package.loaded', function() + assert.Equal('file', require('loaded_mod').source) + package.loaded.loaded_mod = 'leak' + assert.Equal('leak', require('loaded_mod')) + end) + end) + ]], + ['two_spec.lua'] = [[ + describe('two', function() + it('does not see package.loaded leaks from another file', function() + local mod = require('loaded_mod') + assert.Equal('file', mod.source) + end) + end) + ]], + }) + + assert_harness_passes(suite_dir) + end) + + it('restores environment variables between files', function() + local env_name = '__NVIM_HARNESS_TEST_ENV__' + local suite_dir = write_suite({ + ['one_spec.lua'] = string.format( + [[ + describe('one', function() + it('mutates the environment', function() + vim.uv.os_setenv(%q, 'leak') + assert.Equal('leak', vim.uv.os_getenv(%q)) + end) + end) + ]], + env_name, + env_name + ), + ['two_spec.lua'] = string.format( + [[ + describe('two', function() + it('does not see environment leaks from another file', function() + assert.Equal(nil, vim.uv.os_getenv(%q)) + end) + end) + ]], + env_name + ), + }) + + assert_harness_passes(suite_dir) + end) + + it('restores globals between files', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + it('mutates a global', function() + _G.__harness_leak = 'leak' + assert.Equal('leak', _G.__harness_leak) + end) + end) + ]], + ['two_spec.lua'] = [[ + describe('two', function() + it('does not see global leaks from another file', function() + assert.Equal(nil, _G.__harness_leak) + end) + end) + ]], + }) + + assert_harness_passes(suite_dir) + end) + + it('restores arg between files', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + it('mutates arg', function() + _G.arg.__leak = 'leak' + assert.Equal('leak', _G.arg.__leak) + end) + end) + ]], + ['two_spec.lua'] = [[ + describe('two', function() + it('does not see arg leaks from another file', function() + assert.Equal(nil, _G.arg.__leak) + end) + end) + ]], + }) + + assert_harness_passes(suite_dir) + end) + + it('restores helper-provided globals to their baseline between files', function() + local suite_dir = write_suite({ + ['helper.lua'] = [[ + _G.helper_value = 'baseline' + ]], + ['one_spec.lua'] = [[ + describe('one', function() + it('mutates a helper-provided global', function() + assert.Equal('baseline', _G.helper_value) + _G.helper_value = 'leak' + assert.Equal('leak', _G.helper_value) + end) + end) + ]], + ['two_spec.lua'] = [[ + describe('two', function() + it('restores the helper baseline between files', function() + assert.Equal('baseline', _G.helper_value) + end) + end) + ]], + }) + + assert_harness_passes(suite_dir, { + '--helper=' .. suite_dir .. '/helper.lua', + }) + end) + + it('restores helper-provided globals shallowly between files', function() + local suite_dir = write_suite({ + ['helper.lua'] = [[ + _G.helper_value = { nested = 'baseline' } + ]], + ['one_spec.lua'] = [[ + describe('one', function() + it('mutates nested helper state in place', function() + assert.Equal('baseline', _G.helper_value.nested) + _G.helper_value.nested = 'leak' + assert.Equal('leak', _G.helper_value.nested) + end) + end) + ]], + ['two_spec.lua'] = [[ + describe('two', function() + it('keeps the same nested helper table', function() + assert.Equal('leak', _G.helper_value.nested) + end) + end) + ]], + }) + + assert_harness_passes(suite_dir, { + '--helper=' .. suite_dir .. '/helper.lua', + }) + end) + + it('keeps helper-loaded modules across files', function() + local suite_dir = write_suite({ + ['helper_mod.lua'] = [[ + _G.helper_module_loads = (_G.helper_module_loads or 0) + 1 + return { loads = _G.helper_module_loads } + ]], + ['helper.lua'] = [[ + require('helper_mod') + ]], + ['one_spec.lua'] = [[ + describe('one', function() + it('uses the helper-loaded module', function() + assert.Equal(1, require('helper_mod').loads) + end) + end) + ]], + ['two_spec.lua'] = [[ + describe('two', function() + it('does not reload the helper module', function() + assert.Equal(1, require('helper_mod').loads) + end) + end) + ]], + }) + + assert_harness_passes(suite_dir, { + '--helper=' .. suite_dir .. '/helper.lua', + }) + end) + + it('rejects helpers that register hooks', function() + local suite_dir = write_suite({ + ['helper.lua'] = [[ + before_each(function() end) + ]], + ['one_spec.lua'] = [[ + describe('real', function() + it('passes', function() end) + end) + ]], + }) + + local code, output = run_harness(suite_dir, { + '--helper=' .. suite_dir .. '/helper.lua', + }) + + eq(1, code) + eq(true, not not output:find("attempt to call global 'before_each'", 1, true), output) + end) + + it('rejects helpers that define suites or tests', function() + local suite_dir = write_suite({ + ['helper.lua'] = [[ + describe('helper suite', function() + it('should fail', function() end) + end) + ]], + ['one_spec.lua'] = [[ + describe('real', function() + it('passes', function() end) + end) + ]], + }) + + local code, output = run_harness(suite_dir, { + '--helper=' .. suite_dir .. '/helper.lua', + }) + + eq(1, code) + eq(true, not not output:find("attempt to call global 'describe'", 1, true), output) + end) + + it('deduplicates suite-end callbacks registered from the same module', function() + local marker = t.tmpname(false) + local suite_dir = write_suite({ + ['cbmod.lua'] = string.format( + [[ + local harness = require('test.harness') + + harness.on_suite_end(function() + local file = assert(io.open(%q, 'ab')) + file:write('hit\n') + file:close() + end) + + return true + ]], + marker + ), + ['one_spec.lua'] = [[ + require('cbmod') + + describe('one', function() + it('works', function() + assert.True(true) + end) + end) + ]], + ['two_spec.lua'] = [[ + require('cbmod') + + describe('two', function() + it('works', function() + assert.True(true) + end) + end) + ]], + }) + + assert_harness_passes(suite_dir) + eq('hit\n', t.read_file(marker)) + end) + + it( + 'keeps distinct suite-end callbacks from long paths with the same truncated short_src tail', + function() + local marker = t.tmpname(false) + local suite_dir = t.tmpname(false) + local left = suite_dir + .. '/' + .. string.rep('L', 100) + .. '/common/common/common/common/common/same_suffix' + local right = suite_dir + .. '/' + .. string.rep('R', 100) + .. '/common/common/common/common/common/same_suffix' + + mkdir(suite_dir) + for _, path in ipairs({ left, right }) do + local parents = {} + local dir = path + while dir ~= suite_dir and not uv.fs_stat(dir) do + table.insert(parents, 1, dir) + dir = vim.fs.dirname(dir) + end + for _, parent in ipairs(parents) do + if not uv.fs_stat(parent) then + mkdir(parent) + end + end + end + + t.write_file( + left .. '/same_spec.lua', + string.format( + [[ + local harness = require('test.harness') + + harness.on_suite_end(function() + local file = assert(io.open(%q, 'ab')) + file:write('left\n') + file:close() + end) + + describe('left', function() + it('works', function() end) + end) + ]], + marker + ) + ) + t.write_file( + right .. '/same_spec.lua', + string.format( + [[ + local harness = require('test.harness') + + harness.on_suite_end(function() + local file = assert(io.open(%q, 'ab')) + file:write('right\n') + file:close() + end) + + describe('right', function() + it('works', function() end) + end) + ]], + marker + ) + ) + + assert_harness_passes(suite_dir) + eq('left\nright\n', t.read_file(marker)) + end + ) + + it('reports suite-end callback failures without crashing the runner', function() + local suite_dir = write_suite({ + ['cbmod.lua'] = [[ + local harness = require('test.harness') + + harness.on_suite_end(function() + error('boom from suite_end') + end) + + return true + ]], + ['one_spec.lua'] = [[ + require('cbmod') + + describe('one', function() + it('works', function() + assert.True(true) + end) + end) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(true, not not output:find('[suite_end 1]', 1, true)) + eq(true, not not output:find('suite_end', 1, true)) + eq(true, not not output:find('boom from suite_end', 1, true)) + eq(true, not not output:find('ERROR ', 1, true)) + eq(false, not not output:find('FAILED ', 1, true)) + eq(true, not not output:find('1 test from 1 test file ran.', 1, true)) + eq(true, not not output:find('Global test environment teardown.', 1, true)) + end) + + it('reports wrapped suite-end callback failures at the callback definition line', function() + local suite_dir = write_suite({ + ['wrapper.lua'] = [[ + local M = {} + + function M.fail() + error('boom from wrapper') + end + + return M + ]], + ['cbmod.lua'] = [[ + local harness = require('test.harness') + local wrapper = require('wrapper') + + harness.on_suite_end(function() + wrapper.fail() + end) + + return true + ]], + ['one_spec.lua'] = [[ + require('cbmod') + + describe('one', function() + it('works', function() + assert.True(true) + end) + end) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(true, not not output:find('cbmod.lua @ 4: [suite_end 1]', 1, true), output) + eq(false, not not output:find('wrapper.lua @ 4:', 1, true), output) + end) + + it('reloads helper suite-end callbacks between repeats', function() + local marker = t.tmpname(false) + local suite_dir = write_suite({ + ['helper.lua'] = string.format( + [[ + local harness = require('test.harness') + local hits = 0 + + harness.on_suite_end(function() + hits = hits + 1 + local file = assert(io.open(%q, 'ab')) + file:write(hits .. '\n') + file:close() + end) + ]], + marker + ), + ['one_spec.lua'] = [[ + describe('one', function() + it('works', function() + assert.True(true) + end) + end) + ]], + }) + + assert_harness_passes(suite_dir, { + '--helper=' .. suite_dir .. '/helper.lua', + '--repeat=2', + }) + eq('1\n1\n', t.read_file(marker)) + end) + + it('restores process state before each repeat', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + local harness = require('test.harness') + + harness.on_suite_end(function() + _G.__repeat_leak = 'leak' + end) + + describe('one', function() + it('starts clean', function() + assert.Equal(nil, _G.__repeat_leak) + end) + end) + ]], + }) + + assert_harness_passes(suite_dir, { + '--repeat=2', + }) + end) + + it('filters tests by tags across files', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + it('fast #fast', function() end) + end) + ]], + ['two_spec.lua'] = [[ + describe('two', function() + it('slow #slow', function() end) + end) + ]], + }) + + local code, output = run_harness(suite_dir, { + '--tags=fast', + }) + + eq(0, code) + eq(true, not not output:find('1 test from 1 test file ran.', 1, true), output) + eq(true, not not output:find('one fast #fast', 1, true), output) + eq(false, not not output:find('two slow #slow', 1, true), output) + end) + + it('filters tests by suite tags across files', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one #fast', function() + it('works', function() end) + end) + ]], + ['two_spec.lua'] = [[ + describe('two #slow', function() + it('works', function() end) + end) + ]], + }) + + local code, output = run_harness(suite_dir, { + '--tags=fast', + }) + + eq(0, code) + eq(true, not not output:find('1 test from 1 test file ran.', 1, true), output) + eq(true, not not output:find('one #fast works', 1, true), output) + eq(false, not not output:find('two #slow works', 1, true), output) + end) + + it('filters tests by name across files', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + it('chosen', function() end) + end) + ]], + ['two_spec.lua'] = [[ + describe('two', function() + it('skipped', function() end) + end) + ]], + }) + + local code, output = run_harness(suite_dir, { + '--filter=chosen', + }) + + eq(0, code) + eq(true, not not output:find('1 test from 1 test file ran.', 1, true), output) + eq(true, not not output:find('one chosen', 1, true), output) + eq(false, not not output:find('two skipped', 1, true), output) + end) + + it('filters tests by suite name across files', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('chosen suite', function() + it('works', function() end) + end) + ]], + ['two_spec.lua'] = [[ + describe('skipped suite', function() + it('works', function() end) + end) + ]], + }) + + local code, output = run_harness(suite_dir, { + '--filter', + 'chosen suite', + }) + + eq(0, code) + eq(true, not not output:find('1 test from 1 test file ran.', 1, true), output) + eq(true, not not output:find('chosen suite works', 1, true), output) + eq(false, not not output:find('skipped suite works', 1, true), output) + end) + + it('does not keep suite-end callbacks from filtered-out files', function() + local marker = t.tmpname(false) + local suite_dir = write_suite({ + ['one_spec.lua'] = string.format( + [[ + local harness = require('test.harness') + + harness.on_suite_end(function() + local file = assert(io.open(%q, 'ab')) + file:write('filtered\n') + file:close() + end) + + describe('one', function() + it('skipped', function() + assert.True(true) + end) + end) + ]], + marker + ), + ['two_spec.lua'] = [[ + describe('two', function() + it('chosen', function() + assert.True(true) + end) + end) + ]], + }) + + local code, output = run_harness(suite_dir, { + '--filter=chosen', + }) + + eq(0, code) + eq(nil, uv.fs_stat(marker)) + eq(true, not not output:find('1 test from 1 test file ran.', 1, true), output) + end) + + it('filters tests out by name across files', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + it('chosen', function() end) + end) + ]], + ['two_spec.lua'] = [[ + describe('two', function() + it('skipped', function() end) + end) + ]], + }) + + local code, output = run_harness(suite_dir, { + '--filter-out=skipped', + }) + + eq(0, code) + eq(true, not not output:find('1 test from 1 test file ran.', 1, true), output) + eq(true, not not output:find('one chosen', 1, true), output) + eq(false, not not output:find('two skipped', 1, true), output) + end) + + it('filters tests out by suite name across files', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('chosen suite', function() + it('works', function() end) + end) + ]], + ['two_spec.lua'] = [[ + describe('skipped suite', function() + it('works', function() end) + end) + ]], + }) + + local code, output = run_harness(suite_dir, { + '--filter-out', + 'skipped suite', + }) + + eq(0, code) + eq(true, not not output:find('1 test from 1 test file ran.', 1, true), output) + eq(true, not not output:find('chosen suite works', 1, true), output) + eq(false, not not output:find('skipped suite works', 1, true), output) + end) + + it('reports when filters exclude all tests', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + it('skipped', function() end) + end) + ]], + }) + + local code, output = run_harness(suite_dir, { + '--filter-out=.', + }) + + eq(1, code) + eq(true, not not output:find('No tests matched the current selection.', 1, true), output) + eq(false, not not output:find('Running tests from', 1, true), output) + end) + + it('reports malformed filter patterns clearly before selection runs', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + it('works', function() end) + end) + ]], + }) + + for _, case in ipairs({ + { '--filter=[', 'invalid value for --filter: malformed pattern' }, + { '--filter-out=[', 'invalid value for --filter-out: malformed pattern' }, + }) do + local code, output = run_harness(suite_dir, { case[1] }) + + eq(1, code) + eq(true, not not output:find(case[2], 1, true), output) + eq(false, not not output:find('test_selected', 1, true), output) + eq(false, not not output:find('Running tests from', 1, true), output) + eq(false, not not output:find('test harness failed with exit code', 1, true), output) + end + end) + + it('skips unreadable test directories and keeps running readable files', function() + if t.is_os('win') then + pending('N/A: permission denied directory scan depends on POSIX chmod') + end + + local suite_dir = t.tmpname(false) + mkdir(suite_dir) + t.write_file( + suite_dir .. '/one_spec.lua', + [[ + describe('one', function() + it('works', function() end) + end) + ]] + ) + local blocked = suite_dir .. '/blocked' + + mkdir(blocked) + t.write_file( + blocked .. '/two_spec.lua', + [[ + describe('two', function() + it('is hidden behind permissions', function() end) + end) + ]] + ) + finally(function() + assert(uv.fs_chmod(blocked, 448)) + end) + assert(uv.fs_chmod(blocked, 0)) + + local code, output = run_harness(suite_dir) + + eq(0, code) + eq(true, not not output:find('Running tests from', 1, true), output) + eq(true, not not output:find('one works', 1, true), output) + eq(false, not not output:find('two is hidden behind permissions', 1, true), output) + eq(true, not not output:find('1 test from 1 test file ran.', 1, true), output) + end) + + it('reports missing test paths clearly before loading suites', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + it('works', function() end) + end) + ]], + }) + local missing = suite_dir .. '/missing' + + local code, output = run_harness(suite_dir, { missing }) + + eq(1, code) + eq(true, not not output:find('test path not found: ' .. missing, 1, true), output) + eq(false, not not output:find('collect_test_files', 1, true), output) + eq(false, not not output:find('Running tests from', 1, true), output) + eq(false, not not output:find('test harness failed with exit code', 1, true), output) + end) + + it('reports missing helpers clearly before running suites', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + it('works', function() end) + end) + ]], + }) + local missing = suite_dir .. '/missing.lua' + + local code, output = run_harness(suite_dir, { + '--helper=' .. missing, + }) + + eq(1, code) + eq(true, not not output:find('cannot open ' .. missing, 1, true), output) + eq(false, not not output:find('load_helper', 1, true), output) + eq(false, not not output:find('Running tests from', 1, true), output) + eq(false, not not output:find('test harness failed with exit code', 1, true), output) + end) + + it('reports empty value options before running suites', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + it('works', function() end) + end) + ]], + }) + + for _, case in ipairs({ + { '--helper=', 'missing value for --helper' }, + { '--summary-file=', 'missing value for --summary-file' }, + }) do + local code, output = run_harness(suite_dir, { + case[1], + }) + + eq(1, code) + eq(true, not not output:find(case[2], 1, true), output) + eq(false, not not output:find('open_summary_file', 1, true), output) + eq(false, not not output:find('Running tests from', 1, true), output) + eq(false, not not output:find('test harness failed with exit code', 1, true), output) + end + end) + + it('treats the next argv item as the value even when it starts with dashes', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + it('works', function() end) + end) + ]], + }) + + local code, output = run_harness(suite_dir, { '--filter', '--verbose' }) + + eq(1, code) + eq(true, not not output:find('No tests matched the current selection.', 1, true), output) + eq(false, not not output:find('missing value for --filter', 1, true), output) + + code, output = run_harness(suite_dir, { '--repeat', '-1' }) + + eq(1, code) + eq(true, not not output:find('invalid value for --repeat: -1', 1, true), output) + eq(false, not not output:find('missing value for --repeat', 1, true), output) + end) + + it('reports test-body errors as failures', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + it('fails', function() + error('boom from test') + end) + end) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(true, not not output:find('boom from test', 1, true)) + eq(true, not not output:find('FAILED ', 1, true)) + eq(false, not not output:find('ERROR ', 1, true)) + end) + + it('reports test-body failures at the failing line', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + it('fails', function() + error('boom from test') + end) + end) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(true, not not output:find('one_spec.lua @ 3: one fails', 1, true), output) + end) + + it('ignores fake trace lines embedded in multiline error messages', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + it('fake trace', function() + error('boom\n' .. debug.getinfo(1, 'S').source:sub(2) .. ':999: fake trace', 0) + end) + end) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(true, not not output:find('one_spec.lua @ 3: one fake trace', 1, true), output) + eq(false, not not output:find('one_spec.lua @ 999: one fake trace', 1, true), output) + end) + + it('reports wrapped assertion failures at the test definition line', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + it('equal fail', function() + assert.Equal(1, 2) + end) + end) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(true, not not output:find('one_spec.lua @ 2: one equal fail', 1, true), output) + eq(false, not not output:find(root .. '/test/assert.lua @', 1, true), output) + end) + + it('reports local wrapper failures at the test definition line', function() + local suite_dir = write_suite({ + ['wrapper.lua'] = [[ + local M = {} + + function M.fail() + error('boom from wrapper') + end + + return M + ]], + ['one_spec.lua'] = [[ + local wrapper = require('wrapper') + + describe('one', function() + it('wrapper fail', function() + wrapper.fail() + end) + end) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(true, not not output:find('one_spec.lua @ 4: one wrapper fail', 1, true), output) + eq(false, not not output:find('wrapper.lua @ 4:', 1, true), output) + end) + + it('reports failing finally cleanup at the cleanup line', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + it('finfail', function() + finally(function() + error('boom fin') + end) + pending('later') + end) + end) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(true, not not output:find('one_spec.lua @ 4: one finfail', 1, true), output) + end) + + it('rejects finally in setup hooks', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + setup(function() + finally(function() end) + end) + + it('works', function() end) + end) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(true, not not output:find('one_spec.lua @ 2: one [setup]', 1, true), output) + eq( + true, + not not output:find('finally() must be called while a test body is running', 1, true), + output + ) + end) + + it('rejects finally in teardown hooks', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + teardown(function() + finally(function() end) + end) + + it('works', function() end) + end) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(true, not not output:find('one_spec.lua @ 2: one [teardown]', 1, true), output) + eq( + true, + not not output:find('finally() must be called while a test body is running', 1, true), + output + ) + end) + + it('reports failing after_each cleanup at the cleanup line', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('one', function() + after_each(function() + error('boom after') + end) + + it('afterfail', function() + pending('later') + end) + end) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(true, not not output:find('one_spec.lua @ 3: one afterfail', 1, true), output) + end) + + it('reports cross-file after_each wrapper failures at the hook definition line', function() + local suite_dir = write_suite({ + ['wrapper.lua'] = [[ + local M = {} + + function M.fail() + error('boom from wrapper') + end + + return M + ]], + ['one_spec.lua'] = [[ + local wrapper = require('wrapper') + + describe('one', function() + after_each(function() + wrapper.fail() + end) + + it('afterwrap', function() + assert.True(true) + end) + end) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(true, not not output:find('one_spec.lua @ 4: one afterwrap', 1, true), output) + eq(false, not not output:find('one_spec.lua @ 8: one afterwrap', 1, true), output) + eq(false, not not output:find('wrapper.lua @ 4:', 1, true), output) + end) + + it( + 'reports pending cross-file after_each wrapper failures at the hook definition line', + function() + local suite_dir = write_suite({ + ['wrapper.lua'] = [[ + local M = {} + + function M.fail() + error('boom from wrapper') + end + + return M + ]], + ['one_spec.lua'] = [[ + local wrapper = require('wrapper') + + describe('one', function() + after_each(function() + wrapper.fail() + end) + + it('afterwrap pending', function() + pending('later') + end) + end) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(true, not not output:find('one_spec.lua @ 4: one afterwrap pending', 1, true), output) + eq(false, not not output:find('one_spec.lua @ 8: one afterwrap pending', 1, true), output) + eq(false, not not output:find('wrapper.lua @ 4:', 1, true), output) + end + ) + + it('reports cross-file finally wrapper failures at the cleanup definition line', function() + local suite_dir = write_suite({ + ['wrapper.lua'] = [[ + local M = {} + + function M.fail() + error('boom from wrapper') + end + + return M + ]], + ['one_spec.lua'] = [[ + local wrapper = require('wrapper') + + describe('one', function() + it('finwrap', function() + finally(function() + wrapper.fail() + end) + end) + end) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(true, not not output:find('one_spec.lua @ 5: one finwrap', 1, true), output) + eq(false, not not output:find('one_spec.lua @ 4: one finwrap', 1, true), output) + eq(false, not not output:find('wrapper.lua @ 4:', 1, true), output) + end) + + it('does not count synthetic hook failures as executed tests', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('trace', function() + setup(function() + error('boom setup') + end) + + it('passes', function() end) + end) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(true, not not output:find('0 tests from 1 test file ran.', 1, true)) + eq(true, not not output:find('1 ERROR', 1, true)) + end) + + it('reports setup failures at the hook definition site', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + describe('trace', function() + setup(function() + error('boom setup') + end) + + it('passes', function() end) + end) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(true, not not output:find('one_spec.lua @ 2: trace [setup]', 1, true), output) + end) + + it('does not keep suite-end callbacks from load-error files', function() + local marker = t.tmpname(false) + local suite_dir = write_suite({ + ['one_spec.lua'] = string.format( + [[ + local harness = require('test.harness') + + harness.on_suite_end(function() + local file = assert(io.open(%q, 'ab')) + file:write('load\n') + file:close() + end) + + error('boom during load') + ]], + marker + ), + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(nil, uv.fs_stat(marker)) + eq(true, not not output:find('boom during load', 1, true), output) + end) + + it('reports load failures at the failing line', function() + local suite_dir = write_suite({ + ['one_spec.lua'] = [[ + local ok = true + + local broken = ) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(true, output:match('one_spec%.lua @ 3: .-one_spec%.lua %[load%]') ~= nil, output) + end) + + it('reports required-module syntax errors at the real module line', function() + local suite_dir = write_suite({ + ['broken.lua'] = [[ + local ok = true + + local broken = ) + ]], + ['one_spec.lua'] = [[ + require('broken') + + describe('one', function() + it('works', function() end) + end) + ]], + }) + + local code, output = run_harness(suite_dir) + + eq(1, code) + eq(true, output:match('broken%.lua @ 3: .-one_spec%.lua %[load%]') ~= nil, output) + end) +end) diff --git a/test/functional/lua/buffer_updates_spec.lua b/test/functional/lua/buffer_updates_spec.lua index e1a46589f2..bf2b8e2626 100644 --- a/test/functional/lua/buffer_updates_spec.lua +++ b/test/functional/lua/buffer_updates_spec.lua @@ -8,7 +8,6 @@ local api = n.api local fn = n.fn local clear = n.clear local eq = t.eq -local fail = t.fail local exec_lua = n.exec_lua local feed = n.feed local expect_events = t.expect_events @@ -507,7 +506,7 @@ describe('lua: nvim_buf_attach on_bytes', function() for _, event in ipairs(events) do for _, elem in ipairs(event) do if type(elem) == 'number' and elem < 0 then - fail(string.format('Received event has negative values')) + error('Received event has negative values') end end diff --git a/test/functional/lua/net_spec.lua b/test/functional/lua/net_spec.lua index d8fe345a80..3c5547f7da 100644 --- a/test/functional/lua/net_spec.lua +++ b/test/functional/lua/net_spec.lua @@ -199,7 +199,7 @@ describe('vim.net.request', function() return result ]]) - assert.is_table(headers) + t.eq('table', type(headers), 'Expected headers to be a table') -- httpbingo.org/request returns each header as a list in the returned value t.eq(headers.Authorization[1], 'Bearer test-token', 'Expected Authorization header') t.eq(headers['X-Custom-Header'][1], 'custom-value', 'Expected X-Custom-Header') diff --git a/test/functional/lua/vim_spec.lua b/test/functional/lua/vim_spec.lua index e9a1779367..2fde79a806 100644 --- a/test/functional/lua/vim_spec.lua +++ b/test/functional/lua/vim_spec.lua @@ -834,6 +834,26 @@ describe('lua stdlib', function() ) end) + it('vim._copy', function() + ok(exec_lua([[ + local inner = { x = 1 } + local mt = { tag = true } + local a = setmetatable({ inner = inner }, mt) + local b = vim._copy(a) + + local c = vim.empty_dict() + c.inner = inner + local d = vim._copy(c) + + return b ~= a + and b.inner == inner + and getmetatable(b) == mt + and d ~= c + and d.inner == inner + and not vim.islist(d) + ]])) + end) + it('vim.pesc', function() eq('foo%-bar', exec_lua([[return vim.pesc('foo-bar')]])) eq('foo%%%-bar', exec_lua([[return vim.pesc(vim.pesc('foo-bar'))]])) @@ -1249,7 +1269,11 @@ describe('lua stdlib', function() eq(true, exec_lua [[ return vim.deep_equal({a={b=1}}, {a={b=1}}) ]]) eq(true, exec_lua [[ return vim.deep_equal({a={b={nil}}}, {a={b={}}}) ]]) eq(true, exec_lua [[ return vim.deep_equal({a=1, [5]=5}, {nil,nil,nil,nil,5,a=1}) ]]) - eq(true, exec_lua [[ local shared = {}; return vim.deep_equal({ 1, shared, 1, shared }, { 1, {}, 1, {} }) ]]) + eq( + true, + exec_lua [[ local shared = {}; return vim.deep_equal({ 1, shared, 1, shared }, { 1, {}, 1, {} }) ]] + ) + -- cyclic table eq(true, exec_lua [[ local a,b={},{}; a[1]=a; b[1]=b; return vim.deep_equal(a, b) ]]) eq(false, exec_lua [[ return vim.deep_equal(1, {nil,nil,nil,nil,5,a=1}) ]]) eq(false, exec_lua [[ return vim.deep_equal(1, 3) ]]) diff --git a/test/functional/preload.lua b/test/functional/preload.lua index 4aef9eb70b..bcaf5e5a6d 100644 --- a/test/functional/preload.lua +++ b/test/functional/preload.lua @@ -1,6 +1,6 @@ --- Modules loaded here will NOT be cleared and reloaded by Busted. --- Busted started doing this to help provide more isolation. See issue #62 --- for more information about this. +-- Modules loaded here will not be cleared and reloaded by the local harness. +-- Keeping these preloaded preserves cross-file setup while still resetting +-- non-helper modules between files. local t = require('test.testutil') require('test.functional.testnvim')() require('test.functional.ui.screen') diff --git a/test/functional/terminal/parser_spec.lua b/test/functional/terminal/parser_spec.lua index 62292eb31c..64aa3ea134 100644 --- a/test/functional/terminal/parser_spec.lua +++ b/test/functional/terminal/parser_spec.lua @@ -80,14 +80,14 @@ describe(':terminal', function() send_osc_with_terminator(BEL) --- @type string - assert.same( + assert.eq( { sequence = OSC_PREFIX .. '10;?', terminator = BEL }, exec_lua([[return _G.osc10_response]]) ) send_osc_with_terminator(ST) --- @type string - assert.same( + assert.eq( { sequence = OSC_PREFIX .. '10;?', terminator = ST }, exec_lua([[return _G.osc10_response]]) ) diff --git a/test/functional/testnvim.lua b/test/functional/testnvim.lua index 5743331032..a1515b458b 100644 --- a/test/functional/testnvim.lua +++ b/test/functional/testnvim.lua @@ -1,6 +1,7 @@ local uv = vim.uv local t = require('test.testutil') -local busted = require('busted') +---@type test.harness +local harness = require('test.harness') local Session = require('test.client.session') local uv_stream = require('test.client.uv_stream') @@ -528,7 +529,7 @@ function M.new_session(keep, ...) return new_session end -busted.subscribe({ 'suite', 'end' }, function() +harness.on_suite_end(function() M.check_close(true) local timed_out = false local timer = assert(vim.uv.new_timer()) diff --git a/test/functional/ui/screen.lua b/test/functional/ui/screen.lua index 25351499cc..64701d2175 100644 --- a/test/functional/ui/screen.lua +++ b/test/functional/ui/screen.lua @@ -47,7 +47,6 @@ local t = require('test.testutil') local n = require('test.functional.testnvim')() -local busted = require('busted') local uv = vim.uv local deepcopy = vim.deepcopy @@ -911,7 +910,7 @@ between asynchronous (feed(), nvim_input()) and synchronous API calls. if eof then err = err .. '\n\n' .. eof[2] end - busted.fail(err .. '\n\nSnapshot:\n' .. self:_print_snapshot(), 3) + error(err .. '\n\nSnapshot:\n' .. self:_print_snapshot(), 3) elseif did_warn then if eof then print(eof[2]) @@ -922,9 +921,9 @@ between asynchronous (feed(), nvim_input()) and synchronous API calls. end if flags.intermediate and not intermediate_seen then - busted.fail('Expected intermediate screen state before final screen state', 3) + error('Expected intermediate screen state before final screen state', 3) elseif flags.unchanged and intermediate_seen then - busted.fail( + error( 'Expected screen state to be unchanged.\nIntermediate screen state:\n' .. intermediate_state_snapshot, 3 diff --git a/test/harness.lua b/test/harness.lua new file mode 100644 index 0000000000..77d6194a66 --- /dev/null +++ b/test/harness.lua @@ -0,0 +1,1694 @@ +--- Result statuses emitted for test, hook, and synthetic records. +--- @alias test.harness.ResultStatus +---| 'success' +---| 'pending' +---| 'failure' +---| 'error' + +--- Hook phases supported by the chunk environment. +--- @alias test.harness.HookKind +---| 'setup' +---| 'teardown' +---| 'before_each' +---| 'after_each' + +--- Execution scopes that can run under harness error handling. +--- @alias test.harness.ExecutionScope +---| 'test' +---| 'setup' +---| 'teardown' +---| 'before_each' +---| 'after_each' +---| 'suite_end' + +--- Source location recorded for defined suites, tests, and hooks. +--- @class test.harness.Trace +--- @field short_src string +--- @field currentline integer + +--- Structured error payload used internally by the harness. +--- @class test.harness.ErrorPayload +--- @field __harness_pending? boolean +--- @field message string +--- @field trace? test.harness.Trace +--- @field traceback? string + +--- @alias test.harness.Element +--- | test.harness.Suite +--- | test.harness.Test + +--- Base node shared by suite and test definitions. +--- @class test.harness.ElementBase +--- @field name string +--- @field parent? test.harness.Suite +--- @field trace? test.harness.Trace +--- @field duration? number +--- @field full_name? string + +--- Registered callback together with the location where it was registered. +--- @class test.harness.RegisteredCallback +--- @field fn fun() +--- @field trace? test.harness.Trace + +--- Hook callbacks grouped by phase. +--- @class test.harness.HookSet +--- @field setup test.harness.RegisteredCallback[] +--- @field teardown test.harness.RegisteredCallback[] +--- @field before_each test.harness.RegisteredCallback[] +--- @field after_each test.harness.RegisteredCallback[] + +--- Suite node containing hooks and nested children. +--- @class test.harness.Suite : test.harness.ElementBase +--- @field kind 'suite' +--- @field is_file boolean +--- @field children test.harness.Element[] +--- @field selected_count integer +--- @field hooks test.harness.HookSet + +--- Test node containing an optional runnable body. +--- @class test.harness.Test : test.harness.ElementBase +--- @field kind 'test' +--- @field fn? fun() +--- @field parent test.harness.Suite +--- @field pending_message? string +--- @field selected? boolean + +--- Normalized result returned from running a test or hook. +--- @class test.harness.Result +--- @field status test.harness.ResultStatus +--- @field message? string +--- @field traceback? string +--- @field trace? test.harness.Trace + +--- Collected test file path plus its display label. +--- @class test.harness.FileEntry +--- @field path string +--- @field display_name string + +--- Shallow process baseline restored between test files. +--- Mutable tables intentionally preserve identity here; deeper isolation +--- requires separate processes rather than table cloning inside one Lua state. +--- @class test.harness.RuntimeBaseline +--- @field cwd string +--- @field package_path string +--- @field package_cpath string +--- @field package_preload table +--- @field globals table +--- @field loaded table +--- @field env table +--- @field arg table + +--- Parsed CLI options controlling one harness run. +--- @class test.harness.Options +--- @field keep_going boolean +--- @field verbose boolean +--- @field repeat_count integer +--- @field summary_file string +--- @field helper? string +--- @field tags string[] +--- @field filter? string +--- @field filter_out? string +--- @field lpaths string[] +--- @field cpaths string[] +--- @field paths string[] + +--- Stored suite-end callback together with its registration site. +--- @class test.harness.SuiteEndRegistration : test.harness.RegisteredCallback +--- @field key string + +--- Active execution context for one running hook or test. +--- @class test.harness.Execution +--- @field scope test.harness.ExecutionScope +--- @field finalizers test.harness.RegisteredCallback[] + +--- Mutable harness state shared across definition and execution. +--- @class test.harness.State +--- @field suite_end_callbacks test.harness.SuiteEndRegistration[] +--- @field current_define_suite? test.harness.Suite +--- @field current_execution? test.harness.Execution + +local uv = vim.uv + +--- Public test harness module surface. +--- @class test.harness +--- @field is_ci fun(name?: 'cirrus'|'github'): boolean +--- @field on_suite_end fun(callback: fun()): fun() +--- @field read_nvim_log fun(logfile?: string, ci_rename?: boolean): string? +local M = {} + +--- @type test.harness.State +local state = { + suite_end_callbacks = {}, + current_define_suite = nil, + current_execution = nil, +} + +--- Return the current wall-clock time in seconds. +--- @return number +local function now_seconds() + local sec, usec = assert(uv.gettimeofday()) + return sec + usec * 1e-6 +end + +--- Check whether the harness is running in CI, optionally for one provider. +--- @param name? 'cirrus'|'github' +--- @return boolean +function M.is_ci(name) + local any_provider = (name == nil) + assert(any_provider or name == 'github' or name == 'cirrus') + local github_actions = ((any_provider or name == 'github') and nil ~= os.getenv('GITHUB_ACTIONS')) + local cirrus_ci = ((any_provider or name == 'cirrus') and nil ~= os.getenv('CIRRUS_CI')) + return github_actions or cirrus_ci +end + +--- Read the last `keep` lines from a file. +--- @param path string +--- @param keep integer +--- @return string[]? +local function read_tail_lines(path, keep) + local file = io.open(path, 'r') + if not file then + return nil + end + + local lines = {} + for line in file:lines() do + lines[#lines + 1] = line + if #lines > keep then + table.remove(lines, 1) + end + end + file:close() + return lines +end + +-- TODO(lewis6991): move out of harness +--- Read and optionally rename the current Nvim log for failure output. +--- @param logfile? string +--- @param ci_rename? boolean +--- @return string? +function M.read_nvim_log(logfile, ci_rename) + logfile = logfile or os.getenv('NVIM_LOG_FILE') or 'nvim.log' + if not uv.fs_stat(logfile) then + return + end + + local ci = M.is_ci() + local keep = ci and 100 or 10 + local lines = read_tail_lines(logfile, keep) or {} + local separator = ('-'):rep(78) + local parts = { + separator, + '\n', + string.format('$NVIM_LOG_FILE: %s\n', logfile), + #lines > 0 and string.format('(last %d lines)\n', keep) or '(empty)\n', + } + for _, line in ipairs(lines) do + parts[#parts + 1] = line + parts[#parts + 1] = '\n' + end + if ci and ci_rename then + os.rename(logfile, logfile .. '.displayed') + end + parts[#parts + 1] = separator + parts[#parts + 1] = '\n' + return table.concat(parts) +end + +--- Normalize a path relative to the current working directory. +--- @param path string +--- @return string +local function normalize_path(path) + return vim.fs.normalize(vim.fs.abspath(path)) +end + +--- Render a path relative to the current working directory when possible. +--- @param path string +--- @return string +local function display_path(path) + path = normalize_path(path) + local relative = vim.fs.relpath('.', path) + if relative then + return relative + end + return path +end + +--- Restore a table to a previously captured shallow snapshot. +--- @generic K, V +--- @param current table +--- @param snapshot table +--- @param unset? fun(key: K) +--- @param set? fun(key: K, value: V) +local function restore_snapshot(current, snapshot, unset, set) + unset = unset or function(k) + rawset(current, k, nil) + end + set = set or function(k, v) + rawset(current, k, v) + end + for k in pairs(current) do + if rawget(snapshot, k) == nil then + unset(k) + end + end + for k, v in pairs(snapshot) do + set(k, v) + end +end + +--- Restore the process state to a captured baseline. +--- @param baseline test.harness.RuntimeBaseline +local function restore_runtime_baseline(baseline) + if uv.cwd() ~= baseline.cwd then + uv.chdir(baseline.cwd) + end + + package.path = baseline.package_path + package.cpath = baseline.package_cpath + restore_snapshot(package.preload, baseline.package_preload) + restore_snapshot(package.loaded, baseline.loaded) + restore_snapshot(_G, baseline.globals) + restore_snapshot(uv.os_environ(), baseline.env, uv.os_unsetenv, uv.os_setenv) + _G.arg = vim._copy(baseline.arg) + state.current_define_suite = nil + state.current_execution = nil +end + +--- Restore the baseline and run GC cleanup. +--- @param baseline test.harness.RuntimeBaseline +local function cleanup_runtime_baseline(baseline) + restore_runtime_baseline(baseline) + -- One full cycle may only run __gc/finalizers for dead userdata/cdata. + -- Those finalizers can release the last references to more uv/mpack objects, + -- which do not become collectible until the next cycle. Collect twice before + -- switching files or ending the harness so leak checks see only live state. + collectgarbage('collect') + collectgarbage('collect') +end + +local harness_source = debug.getinfo(1, 'S').source +local test_assert = require('test.assert') +local assert_source = debug.getinfo(test_assert.eq, 'S').source + +--- @param info? debug.Info +--- @return test.harness.Trace +local function trace_from_info(info) + return { + short_src = info and vim.fs.normalize(info.short_src or '') or '', + currentline = info and info.currentline or 0, + } +end + +--- Capture the source location of a caller frame. +--- Walk upward until we find a user Lua frame so Lua 5.1 tail-call elision +--- does not collapse `it()`/hook registrations into `(tail call) @ -1`. +--- @param level? integer +--- @return test.harness.Trace +local function caller_trace(level) + local frame = level or 3 + local fallback + + while true do + local info = debug.getinfo(frame, 'Sln') + if not info then + return trace_from_info(fallback) + end + + fallback = fallback or info + if + info.what ~= 'C' + and info.source ~= harness_source + and info.short_src ~= '(tail call)' + and info.currentline > 0 + then + return trace_from_info(info) + end + + frame = frame + 1 + end +end + +--- Register a suite-end callback, deduplicated by callsite. +--- @param callback fun() +--- @return fun() +function M.on_suite_end(callback) + assert(type(callback) == 'function', 'on_suite_end() expects a function') + local trace = caller_trace(3) + local caller_info = debug.getinfo(2, 'S') + local key_source = caller_info and caller_info.source + if type(key_source) == 'string' and vim.startswith(key_source, '@') then + key_source = vim.fs.normalize(key_source:sub(2)) + else + key_source = trace.short_src + end + local key = ('%s:%d'):format(key_source, trace.currentline) + for _, registration in ipairs(state.suite_end_callbacks) do + if registration.key == key then + return registration.fn + end + end + table.insert(state.suite_end_callbacks, { + fn = callback, + trace = trace, + key = key, + }) + return callback +end + +--- Create a suite node for the definition tree. +--- @param name string? +--- @param parent? test.harness.Suite +--- @param trace? test.harness.Trace +--- @param is_file? boolean +--- @return test.harness.Suite +local function create_suite(name, parent, trace, is_file) + return { + kind = 'suite', + name = name or '', + parent = parent, + trace = trace, + is_file = is_file or false, + hooks = { + setup = {}, + teardown = {}, + before_each = {}, + after_each = {}, + }, + children = {}, + selected_count = 0, + } +end + +--- Return the suite currently receiving test definitions. +--- @return test.harness.Suite +local function current_suite() + assert(state.current_define_suite, 'test definition is not active') + return state.current_define_suite +end + +--- Add a test node to the current suite. +--- @param name string +--- @param fn? fun() +--- @param pending_message? string +--- @return test.harness.Test +local function register_test(name, fn, pending_message) + assert(type(name) == 'string' and name ~= '', 'test name must be a non-empty string') + if fn ~= nil then + assert(type(fn) == 'function', 'test body must be a function') + end + + local suite = current_suite() + local test = { + kind = 'test', + name = name, + fn = fn, + parent = suite, + trace = caller_trace(3), + pending_message = pending_message, + } + table.insert(suite.children, test) + return test +end + +--- Build a hook registrar exposed in the test chunk environment. +--- @param kind test.harness.HookKind +--- @return fun(fn: fun()) +local function chunk_hook(kind) + return function(fn) + assert(type(fn) == 'function', ('%s expects a function'):format(kind)) + table.insert(current_suite().hooks[kind], { + fn = fn, + trace = caller_trace(3), + }) + end +end + +-- Chunk environment +local chunk_env = { + _G = _G, + assert = test_assert, + setup = chunk_hook('setup'), + teardown = chunk_hook('teardown'), + before_each = chunk_hook('before_each'), + after_each = chunk_hook('after_each'), +} + +--- Define a nested suite in the chunk environment. +--- @param name string +--- @param fn fun() +--- @return test.harness.Suite +function chunk_env.describe(name, fn) + assert(type(name) == 'string', 'describe() expects a string') + assert(type(fn) == 'function', 'describe() expects a function body') + + local parent = current_suite() + local suite = create_suite(name, parent, caller_trace(3), false) + table.insert(parent.children, suite) + + local previous_define_suite = state.current_define_suite + state.current_define_suite = suite + local ok, err = xpcall(fn, debug.traceback) + state.current_define_suite = previous_define_suite + + if not ok then + error(err, 0) + end + + return suite +end + +--- Define a test in the chunk environment. +--- @param name string +--- @param fn? fun() +--- @return test.harness.Test +function chunk_env.it(name, fn) + return register_test(name, fn, nil) +end + +--- Mark the current test as pending or define a pending test. +--- When called while a test or hook is running, this aborts the current +--- execution and reports the current test as pending with `name` as the +--- pending message. +--- When called during file definition, this registers a new pending test in +--- the current suite. In that form, `block` may be a string used as the +--- pending message. +--- @param name? string +--- @param block? fun()|string +--- @return boolean +function chunk_env.pending(name, block) + if state.current_execution then + error({ + __harness_pending = true, + message = name or 'pending', + }, 0) + end + local pending_message = type(block) == 'string' and block or nil + register_test(name or 'pending', nil, pending_message) + return false +end + +--- Register a finalizer to run after the current test body. +--- @param fn fun() +function chunk_env.finally(fn) + assert(type(fn) == 'function', 'finally() expects a function') + assert( + state.current_execution and state.current_execution.scope == 'test', + 'finally() must be called while a test body is running' + ) + table.insert(state.current_execution.finalizers, { + fn = fn, + trace = caller_trace(3), + }) +end + +--- Convert an arbitrary error value into printable text. +--- @param err any +--- @return string +local function format_error_value(err) + if type(err) == 'string' then + return err + end + + local ok, inspected = pcall(vim.inspect, err) + if ok then + return inspected + end + + return tostring(err) +end + +--- Parse one traceback line into a source location. +--- @param text? string +--- @return test.harness.Trace? +local function parse_trace_line(text) + if type(text) ~= 'string' then + return + end + + local src, line = text:match('^(.+):(%d+):') + if not src or not line then + return + end + + return { + short_src = vim.fs.normalize(src), + currentline = tonumber(line) or 0, + } +end + +--- @param source string +--- @param candidate string +--- @return boolean +local function source_matches(source, candidate) + if source == candidate then + return true + end + + local tail = source:match('^%.%.%.(.+)$') + return tail ~= nil and vim.endswith(candidate, tail) +end + +--- @param left? test.harness.Trace +--- @param right? test.harness.Trace +--- @return boolean +local function same_trace_source(left, right) + if left == nil or right == nil then + return false + end + return source_matches(left.short_src, right.short_src) + or source_matches(right.short_src, left.short_src) +end + +--- Extract a source location from an error message or traceback. +--- @param message? string +--- @param traceback? string +--- @return test.harness.Trace? +local function parse_error_trace(message, traceback) + if type(message) == 'string' then + local first_line = message:match('^[^\n]+') + local trace = parse_trace_line(first_line) + if trace then + return trace + end + + local load_path = first_line + and first_line:match("^error loading module .+ from file '([^']+)':$") + if load_path == nil and first_line then + load_path = first_line:match('^error loading module .+ from file "([^"]+)":$') + end + if load_path then + local detail_line = message:match('\n([^\n]+)') + trace = parse_trace_line(detail_line and detail_line:gsub('^%s+', '')) + local load_trace = { + short_src = vim.fs.normalize(load_path), + currentline = 0, + } + if trace and same_trace_source(trace, load_trace) then + return trace + end + end + end + + if type(traceback) == 'string' then + for line in traceback:gmatch('[^\n]+') do + local trace = parse_trace_line(line:gsub('^%s+', '')) + if trace then + return trace + end + end + end +end + +--- Capture only user-visible Lua frames for test and hook failures. +--- Once execution returns to the harness, the rest of the stack is framework +--- noise and should not be printed in verbose failure output. +--- @return string? +local function build_error_traceback() + local lines = {} + + for level = 3, math.huge do + local info = debug.getinfo(level, 'Sln') + if not info then + break + end + + if info.source == harness_source then + break + end + + if (info.what == 'Lua' or info.what == 'main') and info.source ~= assert_source then + local trace = trace_from_info(info) + if trace.currentline > 0 and trace.short_src ~= '' then + local location = ('%s:%d'):format(trace.short_src, trace.currentline) + if info.what == 'main' then + lines[#lines + 1] = ('\t%s: in main chunk'):format(location) + elseif info.name and info.name ~= '' then + lines[#lines + 1] = ("\t%s: in function '%s'"):format(location, info.name) + elseif (info.linedefined or 0) > 0 then + lines[#lines + 1] = ('\t%s: in function <%s:%d>'):format( + location, + trace.short_src, + info.linedefined + ) + else + lines[#lines + 1] = ('\t%s: in function ?'):format(location) + end + end + end + end + + if #lines == 0 then + return + end + + return 'stack traceback:\n' .. table.concat(lines, '\n') +end + +--- Normalize thrown values into harness error payloads. +--- @param err any +--- @return test.harness.ErrorPayload +local function exception_handler(err) + if type(err) == 'table' and err.__harness_pending then + --- @cast err test.harness.ErrorPayload + return err + end + + local message = format_error_value(err) + local raw_traceback = debug.traceback('', 2) + return { + message = message, + trace = parse_error_trace(message, raw_traceback), + traceback = build_error_traceback() or raw_traceback, + } +end + +--- Convert a handled error payload into a test result. +--- @param err test.harness.ErrorPayload +--- @param fallback_status? test.harness.ResultStatus +--- @return test.harness.Result +local function decode_error(err, fallback_status) + return { + status = err.__harness_pending and 'pending' or fallback_status or 'failure', + message = err.message, + traceback = err.traceback, + trace = err.trace or parse_error_trace(err.message, err.traceback), + } +end + +--- Run a callable under harness error handling and finalizer cleanup. +--- @param scope test.harness.ExecutionScope +--- @param callable test.harness.RegisteredCallback +--- @param fallback_status? test.harness.ResultStatus +--- @return test.harness.Result, test.harness.Trace? +local function run_callable(scope, callable, fallback_status) + local previous_execution = state.current_execution + --- @type test.harness.Execution + local execution = { + scope = scope, + finalizers = {}, + } + state.current_execution = execution + + local ok, err = xpcall(callable.fn, exception_handler) + local finalizer_err + local finalizer_trace + for i = #execution.finalizers, 1, -1 do + local finalizer = execution.finalizers[i] + local finalizer_ok, ferr = xpcall(finalizer.fn, exception_handler) + if not finalizer_ok and not finalizer_err then + finalizer_err = ferr + finalizer_trace = finalizer.trace + end + end + + state.current_execution = previous_execution + + local result = ok and { status = 'success' } or decode_error(err, fallback_status) + local report_trace = not ok and callable.trace or nil + if not finalizer_err then + return result, report_trace + end + + local finalizer_result = decode_error(finalizer_err, 'error') + if result.status == 'success' then + finalizer_result.status = 'error' + return finalizer_result, finalizer_trace + end + + if result.status == 'pending' then + return { + status = 'error', + message = ('finally: %s'):format(finalizer_result.message), + traceback = finalizer_result.traceback, + trace = finalizer_result.trace, + }, + finalizer_trace + end + + result.message = result.message .. '\n\nfinally: ' .. finalizer_result.message + if not result.traceback then + result.traceback = finalizer_result.traceback + end + if not result.trace then + result.trace = finalizer_result.trace + end + return result, report_trace +end + +--- Prefer the parsed trace only when it points at the same source file as the +--- owning test or callback. Otherwise, fall back to the definition site. +--- @param trace? test.harness.Trace +--- @param fallback? test.harness.Trace +--- @return test.harness.Trace? +local function summary_trace(trace, fallback) + if same_trace_source(trace, fallback) then + return trace + end + return fallback or trace +end + +--- Build the suite and test name parts used for reporting. +--- @param element test.harness.Element? +--- @return string[] +local function full_name_parts(element) + local parts = {} + local node = element + while node do + if node.kind == 'test' and node.name ~= '' then + table.insert(parts, 1, node.name) + elseif node.kind == 'suite' and not node.is_file and node.name ~= '' then + table.insert(parts, 1, node.name) + end + node = node.parent + end + return parts +end + +--- Return the full hierarchical name for a suite or test. +--- @param element test.harness.Element +--- @return string +function M.get_full_name(element) + return table.concat(full_name_parts(element), ' ') +end + +--- Check whether a test matches the current selection options. +--- @param test test.harness.Test +--- @param opts test.harness.Options +--- @return boolean +local function test_selected(test, opts) + test.full_name = M.get_full_name(test) + + if #opts.tags > 0 then + local tagged = false + for tag in test.full_name:gmatch('#([%w_%-]+)') do + if vim.list_contains(opts.tags, tag) then + tagged = true + break + end + end + + if not tagged then + return false + end + end + + if opts.filter and not test.full_name:match(opts.filter) then + return false + end + + if opts.filter_out and test.full_name:match(opts.filter_out) then + return false + end + + return true +end + +--- Mark selected tests in a suite subtree and count them. +--- @param node test.harness.Suite +--- @param opts test.harness.Options +--- @return integer +local function mark_selected(node, opts) + local selected_count = 0 + for _, child in ipairs(node.children) do + if child.kind == 'suite' then + selected_count = selected_count + mark_selected(child, opts) + else + --- @cast child test.harness.Test + child.selected = test_selected(child, opts) + if child.selected then + selected_count = selected_count + 1 + end + end + end + + node.selected_count = selected_count + return selected_count +end + +--- Collect inherited `before_each` hooks from outermost to innermost. +--- @param suite test.harness.Suite +--- @return test.harness.RegisteredCallback[] +local function gather_before_each(suite) + local hooks = {} + if suite.parent then + vim.list_extend(hooks, gather_before_each(suite.parent)) + end + for _, hook in ipairs(suite.hooks.before_each) do + hooks[#hooks + 1] = hook + end + return hooks +end + +--- Collect inherited `after_each` hooks from innermost to outermost. +--- @param suite test.harness.Suite +--- @return test.harness.RegisteredCallback[] +local function gather_after_each(suite) + local hooks = {} + for _, hook in ipairs(suite.hooks.after_each) do + hooks[#hooks + 1] = hook + end + if suite.parent then + vim.list_extend(hooks, gather_after_each(suite.parent)) + end + return hooks +end + +--- Finalized result record passed to the reporter and stored in summaries. +--- @class test.harness.Record +--- @field name string +--- @field status test.harness.ResultStatus +--- @field trace? test.harness.Trace +--- @field duration number +--- @field message? string +--- @field traceback? string + +--- Execution summary accumulated by the harness for one suite iteration. +--- @class test.harness.RunSummary +--- @field file_count integer +--- @field result_count integer +--- @field test_count integer +--- @field success_count integer +--- @field skipped_count integer +--- @field failure_count integer +--- @field error_count integer +--- @field pendings test.harness.Record[] +--- @field failures test.harness.Record[] +--- @field errors test.harness.Record[] + +--- Per-file summary accumulated by the harness while one file runs. +--- @class test.harness.FileRunSummary +--- @field test_count integer + +--- Build the reporter record for a completed result. +--- @param name string +--- @param result test.harness.Result +--- @param trace? test.harness.Trace +--- @param duration number +--- @return test.harness.Record +local function build_record(name, result, trace, duration) + return { + name = name, + status = result.status, + trace = trace, + duration = duration, + message = result.message, + traceback = result.traceback, + } +end + +--- Record a completed test or synthetic result into the harness summary. +--- @param summary test.harness.RunSummary +--- @param file_summary? test.harness.FileRunSummary +--- @param record test.harness.Record +--- @param count_as_test? boolean +local function record_result(summary, file_summary, record, count_as_test) + summary.result_count = summary.result_count + 1 + if count_as_test ~= false then + summary.test_count = summary.test_count + 1 + if file_summary then + file_summary.test_count = file_summary.test_count + 1 + end + end + + if record.status == 'success' then + summary.success_count = summary.success_count + 1 + elseif record.status == 'pending' then + summary.skipped_count = summary.skipped_count + 1 + table.insert(summary.pendings, record) + elseif record.status == 'failure' then + summary.failure_count = summary.failure_count + 1 + table.insert(summary.failures, record) + else -- error + summary.error_count = summary.error_count + 1 + table.insert(summary.errors, record) + end +end + +--- Report a synthetic result as a test-shaped record. +--- @param reporter test.base_reporter +--- @param summary test.harness.RunSummary +--- @param file_summary? test.harness.FileRunSummary +--- @param parent test.harness.Suite +--- @param phase string +--- @param result test.harness.Result +--- @param trace? test.harness.Trace +--- @return test.harness.ResultStatus +local function run_synthetic_result(reporter, summary, file_summary, parent, phase, result, trace) + local name = M.get_full_name(parent) + if name == '' then + name = parent.name ~= '' and parent.name or 'suite' + end + name = ('%s [%s]'):format(name, phase) + local record_trace = trace or result.trace or parent.trace + local record = build_record(name, result, record_trace, 0) + reporter:test_start(record.name) + record_result(summary, file_summary, record, false) + reporter:test_end(record) + return record.status +end + +--- Run registered suite-end callbacks and report failures. +--- @param reporter test.base_reporter +--- @param summary test.harness.RunSummary +--- @return boolean +local function run_suite_end_callbacks(reporter, summary) + local suite_end_callbacks = vim._copy(state.suite_end_callbacks) + local callback_failed = false + + for index, callback in ipairs(suite_end_callbacks) do + local result, report_trace = run_callable('suite_end', callback, 'error') + if result.status ~= 'success' then + callback_failed = true + result.status = 'error' + local name = ('[suite_end %d]'):format(index) + local record_trace = summary_trace(result.trace, report_trace or callback.trace) + local record = build_record(name, result, record_trace, 0) + reporter:test_start(record.name) + record_result(summary, nil, record, false) + reporter:test_end(record) + end + end + + return callback_failed +end + +--- Run a single test with its surrounding before and after hooks. +--- @param test test.harness.Test +--- @param reporter test.base_reporter +--- @param summary test.harness.RunSummary +--- @param file_summary test.harness.FileRunSummary +--- @return test.harness.ResultStatus +local function run_test(test, reporter, summary, file_summary) + local name = test.full_name or M.get_full_name(test) + + local start_time = now_seconds() + --- @type test.harness.Result + local result + local report_trace = test.trace + if test.fn == nil then + reporter:test_start(name) + result = { + status = 'pending', + message = test.pending_message, + } + else + result = { status = 'success' } + + for _, hook in ipairs(gather_before_each(test.parent)) do + result, report_trace = run_callable('before_each', hook, 'failure') + if result.status ~= 'success' then + break + end + end + + reporter:test_start(name) + + if result.status == 'success' then + result, report_trace = run_callable('test', { fn = test.fn, trace = test.trace }, 'failure') + end + + for _, hook in ipairs(gather_after_each(test.parent)) do + local hook_result, hook_trace = run_callable('after_each', hook, 'failure') + if result.status == 'success' then + result = hook_result + report_trace = hook_trace + elseif hook_result.status ~= 'success' then + local hook_report_trace = hook_trace or hook.trace + result.message = (result.message or '') + .. (result.message and result.message ~= '' and '\n\n' or '') + .. 'after_each: ' + .. hook_result.message + if not result.traceback then + result.traceback = hook_result.traceback + end + if not result.trace then + result.trace = hook_result.trace + end + if result.status == 'pending' then + result.status = 'error' + report_trace = hook_report_trace + elseif not report_trace then + report_trace = hook_report_trace + end + end + end + end + + test.duration = now_seconds() - start_time + local record = build_record( + name, + result, + summary_trace(result.trace, report_trace or test.trace), + test.duration + ) + record_result(summary, file_summary, record) + reporter:test_end(record) + return record.status +end + +--- Run a suite subtree until completion or a stop condition. +--- @param suite test.harness.Suite +--- @param reporter test.base_reporter +--- @param summary test.harness.RunSummary +--- @param file_summary test.harness.FileRunSummary +--- @param opts test.harness.Options +--- @return boolean +local function run_suite(suite, reporter, summary, file_summary, opts) + if suite.selected_count == 0 then + return false + end + + local stop_requested = false + local run_children = true + + -- Run setup() hooks + for _, hook in ipairs(suite.hooks.setup) do + local result = run_callable('setup', hook, 'error') + if result.status ~= 'success' then + run_synthetic_result(reporter, summary, file_summary, suite, 'setup', result, hook.trace) + run_children = false + if result.status ~= 'pending' and not opts.keep_going then + stop_requested = true + end + break + end + end + + if run_children and not stop_requested then + for _, child in ipairs(suite.children) do + if child.kind == 'suite' then + stop_requested = run_suite(child, reporter, summary, file_summary, opts) or stop_requested + elseif child.selected then + local status = run_test(child, reporter, summary, file_summary) + if status ~= 'success' and status ~= 'pending' and not opts.keep_going then + stop_requested = true + end + end + + if stop_requested then + break + end + end + end + + -- Run teardown() hooks + for _, hook in ipairs(suite.hooks.teardown) do + local result = run_callable('teardown', hook, 'error') + if result.status ~= 'success' then + run_synthetic_result(reporter, summary, file_summary, suite, 'teardown', result, hook.trace) + if result.status ~= 'pending' and not opts.keep_going then + stop_requested = true + end + end + end + + return stop_requested +end + +--- Collect test files from a file or directory path. +--- @param path string +--- @param files test.harness.FileEntry[] +--- @param seen_files table +--- @return boolean?, string? +local function collect_test_files(path, files, seen_files) + --- @param file string + local function add_test_file(file) + local abs_file = normalize_path(file) + if seen_files[abs_file] then + return + end + + seen_files[abs_file] = true + files[#files + 1] = { + path = abs_file, + display_name = display_path(abs_file), + } + end + + local abs = normalize_path(path) + local stat = uv.fs_stat(abs) + if not stat then + return nil, ('test path not found: %s'):format(path) + end + + if stat.type == 'file' then + add_test_file(abs) + return true + end + + if stat.type ~= 'directory' then + return nil, ('unsupported test path: %s'):format(path) + end + + for _, file in + ipairs(vim.fs.find(function(name) + return name:match('_spec%.lua$') ~= nil + end, { + path = abs, + type = 'file', + limit = math.huge, + })) + do + add_test_file(file) + end + + return true +end + +--- Parse harness CLI arguments into execution options. +--- @param argv string[] +--- @return test.harness.Options?, string? +local function parse_args(argv) + --- @type test.harness.Options + local opts = { + keep_going = true, + verbose = false, + repeat_count = 1, + summary_file = '-', + tags = {}, + lpaths = {}, + cpaths = {}, + paths = {}, + } + + --- @type table + local seen_tags = {} + + --- @param flag string + --- @return nil, string + local function missing_value(flag) + return nil, 'missing value for ' .. flag + end + + --- Parse and validate the `--repeat` argument. + --- @param value? string + --- @return integer?, string? + local function parse_repeat_count(value) + if type(value) ~= 'string' or value == '' then + return missing_value('--repeat') + end + + local count = tonumber(value) + if count == nil or count < 1 or count ~= math.floor(count) then + return nil, ('invalid value for --repeat: %s'):format(value) + end + --- @cast count integer + + return count + end + + --- @type table + local switch_options = { + ['-v'] = function() + opts.verbose = true + end, + ['--verbose'] = function() + opts.verbose = true + end, + ['--no-keep-going'] = function() + opts.keep_going = false + end, + } + + --- @param flag string + --- @param value string + --- @return string?, string? + local function require_nonempty(flag, value) + if value == '' then + return missing_value(flag) + end + return value + end + + --- @param setter fun(value: any) + --- @return fun(value: any): boolean?, string? + local function set_value(setter) + return function(value) + setter(value) + return true + end + end + + --- @param parse fun(value: string): any?, string? + --- @param setter fun(value: any) + --- @return fun(value: string): boolean?, string? + local function set_parsed_value(parse, setter) + return function(value) + local parsed, err = parse(value) + if parsed == nil then + return nil, err + end + setter(parsed) + return true + end + end + + --- @param flag string + --- @param setter fun(value: string) + --- @return fun(value: string): boolean?, string? + local function set_nonempty_value(flag, setter) + return set_parsed_value(function(value) + return require_nonempty(flag, value) + end, setter) + end + + --- Validate that a filter option contains a valid Lua pattern. + --- @param flag string + --- @param value string + --- @return string?, string? + local function validate_pattern(flag, value) + local ok, err = pcall(string.match, '', value) + if not ok then + local message = tostring(err) + local detail = message:match('malformed pattern.*') or message + return nil, ('invalid value for %s: %s'):format(flag, detail) + end + return value + end + + --- @param flag string + --- @param setter fun(value: string) + --- @return fun(value: string): boolean?, string? + local function set_pattern_value(flag, setter) + return set_parsed_value(function(value) + return validate_pattern(flag, value) + end, setter) + end + + --- @param values string[] + --- @return fun(value: string): boolean?, string? + local function append_value(values) + return set_value(function(value) + table.insert(values, value) + end) + end + + --- @type table + local value_options = { + ['--repeat'] = set_parsed_value(parse_repeat_count, function(count) + opts.repeat_count = count + end), + ['--helper'] = set_nonempty_value('--helper', function(value) + opts.helper = value + end), + ['--summary-file'] = set_nonempty_value('--summary-file', function(value) + opts.summary_file = value + end), + ['--tags'] = function(value) + for token in value:gmatch('[^,%s]+') do + local tag = token:gsub('^#', '') + if tag ~= '' and not seen_tags[tag] then + seen_tags[tag] = true + table.insert(opts.tags, tag) + end + end + return true + end, + ['--filter'] = set_pattern_value('--filter', function(pattern) + opts.filter = pattern + end), + ['--filter-out'] = set_pattern_value('--filter-out', function(pattern) + opts.filter_out = pattern + end), + ['--lpath'] = append_value(opts.lpaths), + ['--cpath'] = append_value(opts.cpaths), + } + + local i = 1 + + --- @param arg string + --- @return string, string? + local function split_option(arg) + local eq = arg:find('=', 1, true) + if not eq then + return arg, nil + end + return arg:sub(1, eq - 1), arg:sub(eq + 1) + end + + --- Consume the next argv item as the value for `flag`. + --- @param flag string + --- @return string?, string? + local function take_value(flag) + i = i + 1 + local value = argv[i] + if type(value) ~= 'string' then + return missing_value(flag) + end + return value + end + + --- Parse one named option and apply it to `opts`. + --- @param arg string + --- @return boolean, string? + local function apply_named_option(arg) + local switch_handler = switch_options[arg] + if switch_handler then + switch_handler() + return true + end + + local flag, value = split_option(arg) + local handler = value_options[flag] + if handler then + if value == nil then + local err + value, err = take_value(flag) + if not value then + return false, err + end + end + local ok, handler_err = handler(value) + if not ok then + return false, handler_err + end + return true + end + + return false + end + + while i <= #argv do + local arg = assert(argv[i]) + local handled, err = apply_named_option(arg) + if handled then + elseif err then + return nil, err + elseif vim.startswith(arg, '-') then + return nil, 'unknown test harness option: ' .. arg + else + opts.paths[#opts.paths + 1] = arg + end + i = i + 1 + end + + if #opts.paths == 0 then + return nil, 'no test paths provided' + end + + return opts +end + +--- Load a Lua chunk and bind it to a shallow copy of the given environment. +--- @param path string +--- @param env table +--- @return function?, string? +local function load_chunk(path, env) + local chunk, err = loadfile(path) + if not chunk then + return nil, err + end + + return setfenv(chunk, setmetatable(vim._copy(env), { __index = _G })) +end + +--- Load a helper file before the test baseline is captured. +--- Helper files are preload-only: they may require modules, set defaults, +--- and register suite-end callbacks, but they do not define tests or hooks. +--- @param path string +--- @return boolean?, string? +local function load_helper(path) + local helper_path = normalize_path(path) + local chunk, err = load_chunk(helper_path, { + _G = _G, + assert = test_assert, + }) + if not chunk then + return nil, err + end + local ok, load_err = xpcall(chunk, debug.traceback) + if not ok then + return nil, load_err + end + + return true +end + +--- Evaluate a test file into a per-file root suite. +--- @param file test.harness.FileEntry +--- @param root_suite test.harness.Suite +--- @return test.harness.Suite, test.harness.Result? +local function evaluate_test_file(file, root_suite) + local file_suite = create_suite(file.display_name, root_suite, { + short_src = file.display_name, + currentline = 1, + }, true) + table.insert(root_suite.children, file_suite) + + state.current_define_suite = file_suite + local chunk, load_err = load_chunk(file.path, chunk_env) + if not chunk then + state.current_define_suite = nil + return file_suite, + { + status = 'error', + message = load_err, + trace = parse_error_trace(load_err, nil), + } + end + + local ok, runtime_err = xpcall(chunk, exception_handler) + state.current_define_suite = nil + if not ok then + local load_error = decode_error(runtime_err, 'error') + load_error.status = 'error' + return file_suite, load_error + end + + return file_suite +end + +--- Run a single file in the current prepared runtime state. +--- @param file test.harness.FileEntry +--- @param reporter test.base_reporter +--- @param summary test.harness.RunSummary +--- @param opts test.harness.Options +--- @return boolean, boolean +local function run_test_file(file, reporter, summary, opts) + local root_suite = create_suite('') + local saved_suite_end_callbacks = vim._copy(state.suite_end_callbacks) + + local file_suite, load_error = evaluate_test_file(file, root_suite) + + local selected_count = mark_selected(root_suite, opts) + + if load_error then + state.suite_end_callbacks = saved_suite_end_callbacks + elseif selected_count == 0 then + state.suite_end_callbacks = saved_suite_end_callbacks + return false, false + end + + --- @type test.reporter.FileElement + local file_element = { name = file.display_name, duration = 0 } + + --- @type test.harness.FileRunSummary + local file_summary = { test_count = 0 } + + reporter:file_start(file_element) + + local start = now_seconds() + local stop_requested = false + if load_error then + local status = run_synthetic_result( + reporter, + summary, + file_summary, + file_suite, + 'load', + load_error, + load_error.trace + ) + if status ~= 'success' and status ~= 'pending' and not opts.keep_going then + stop_requested = true + end + else + stop_requested = run_suite(root_suite, reporter, summary, file_summary, opts) + end + + file_element.duration = now_seconds() - start + summary.file_count = summary.file_count + 1 + + reporter:file_end(file_element, file_summary.test_count) + + return true, stop_requested +end + +--- Aggregate outcome from running one suite iteration. +--- @class test.harness.IterationResult +--- @field ran_any boolean +--- @field stop_requested boolean +--- @field summary test.harness.RunSummary + +--- Run one full suite iteration across the selected files. +--- @param Reporter test.base_reporter +--- @param opts test.harness.Options +--- @param files test.harness.FileEntry[] +--- @param pre_helper_baseline test.harness.RuntimeBaseline +--- @param repeat_index integer +--- @return test.harness.IterationResult?, string? +local function run_iteration(Reporter, opts, files, pre_helper_baseline, repeat_index) + local reporter = Reporter.new({ + verbose = opts.verbose, + summary_file = opts.summary_file, + }) + --- @type test.harness.RunSummary + local summary = { + file_count = 0, + result_count = 0, + test_count = 0, + success_count = 0, + skipped_count = 0, + failure_count = 0, + error_count = 0, + pendings = {}, + failures = {}, + errors = {}, + } + restore_runtime_baseline(pre_helper_baseline) + state.suite_end_callbacks = {} + if opts.helper then + local ok, err = load_helper(opts.helper) + if not ok then + return nil, err + end + end + --- @type test.harness.RuntimeBaseline + local file_baseline = { + cwd = assert(uv.cwd()), + package_path = package.path, + package_cpath = package.cpath, + package_preload = vim._copy(package.preload), + globals = vim._copy(_G), + loaded = vim._copy(package.loaded), + env = vim._copy(uv.os_environ()), + arg = vim._copy(_G.arg or {}), + } + reporter:suite_start(repeat_index, opts.repeat_count) + + local start_time = now_seconds() + local ran_any = false + local stop_requested = false + + for _, file in ipairs(files) do + restore_runtime_baseline(file_baseline) + local ran_file, stop = run_test_file(file, reporter, summary, opts) + cleanup_runtime_baseline(file_baseline) + ran_any = ran_any or ran_file + if stop then + stop_requested = true + break + end + end + + local duration = now_seconds() - start_time + + run_suite_end_callbacks(reporter, summary) + cleanup_runtime_baseline(file_baseline) + + state.suite_end_callbacks = {} + + local failure_output + if summary.failure_count > 0 or summary.error_count > 0 then + failure_output = M.read_nvim_log(nil, true) + end + + reporter:suite_end(duration, summary, failure_output) + + return { + ran_any = ran_any, + stop_requested = stop_requested, + summary = summary, + } +end + +--- Run the test harness CLI entrypoint. +--- @param argv string[] +--- @return integer +function M.main(argv) + if os.getenv('BUSTED_ARGS') ~= nil then + io.stderr:write('$BUSTED_ARGS is no longer supported; use $TEST_ARGS instead.\n') + return 1 + end + + local opts, err = parse_args(argv) + if not opts then + io.stderr:write(err .. '\n') + return 1 + end + + if #opts.lpaths > 0 then + package.path = table.concat(opts.lpaths, ';') .. ';' .. package.path + end + + if #opts.cpaths > 0 then + package.cpath = table.concat(opts.cpaths, ';') .. ';' .. package.cpath + end + + --- @type test.harness.RuntimeBaseline + local pre_helper_baseline = { + cwd = assert(uv.cwd()), + package_path = package.path, + package_cpath = package.cpath, + package_preload = vim._copy(package.preload), + globals = vim._copy(_G), + loaded = vim._copy(package.loaded), + env = vim._copy(uv.os_environ()), + arg = vim._copy(_G.arg or {}), + } + local files = {} --- @type test.harness.FileEntry[] + local seen_files = {} --- @type table + for _, path in ipairs(opts.paths) do + local ok, collect_err = collect_test_files(path, files, seen_files) + if not ok then + io.stderr:write(collect_err .. '\n') + return 1 + end + end + + table.sort(files, function(a, b) + return a.display_name < b.display_name + end) + + if #files == 0 then + io.stderr:write('No test files found.\n') + return 1 + end + + local ReporterModule = require('reporter') + local exit_code = 0 + local ran_any = false + for repeat_index = 1, opts.repeat_count do + local result, run_err = + run_iteration(ReporterModule, opts, files, pre_helper_baseline, repeat_index) + if not result then + io.stderr:write(run_err .. '\n') + return 1 + end + + ran_any = ran_any or result.ran_any + + if not result.ran_any and result.summary.result_count == 0 then + io.stderr:write('No tests matched the current selection.\n') + exit_code = 1 + break + end + + if result.summary.failure_count > 0 or result.summary.error_count > 0 then + exit_code = 1 + end + + if result.stop_requested then + break + end + end + + if not ran_any then + exit_code = 1 + end + + return exit_code +end + +return M diff --git a/test/lua_runner.lua b/test/lua_runner.lua deleted file mode 100644 index d57bcb9319..0000000000 --- a/test/lua_runner.lua +++ /dev/null @@ -1,91 +0,0 @@ -local platform = vim.uv.os_uname() -local deps_install_dir = table.remove(_G.arg, 1) -_G.c_include_path = {} -while vim.startswith(_G.arg[1], '-I') do - table.insert(_G.c_include_path, string.sub(table.remove(_G.arg, 1), 3)) -end -local subcommand = table.remove(_G.arg, 1) -local suffix = (platform and platform.sysname:lower():find 'windows') and '.dll' or '.so' -package.path = (deps_install_dir .. '/?.lua;') - .. (deps_install_dir .. '/?/init.lua;') - .. package.path -package.cpath = deps_install_dir .. '/?' .. suffix .. ';' .. package.cpath - -local uv = vim.uv - --- we use busted and luacheck and their lua dependencies --- But installing their binary dependencies with luarocks is very --- slow, replace them with vim.uv wrappers - -local system = {} -package.loaded['system.core'] = system -function system.monotime() - uv.update_time() - return uv.now() * 1e-3 -end -function system.gettime() - local sec, usec = uv.gettimeofday() - return sec + usec * 1e-6 -end -function system.sleep(sec) - uv.sleep(sec * 1e3) -end - -local term = {} -package.loaded['term.core'] = term -function term.isatty(_) - return uv.guess_handle(1) == 'tty' -end - -local lfs = { _VERSION = 'fake' } -package.loaded['lfs'] = lfs - -function lfs.attributes(path, attr) - local stat = uv.fs_stat(path) - if attr == 'mode' then - return stat and stat.type or '' - elseif attr == 'modification' then - if not stat then - return nil - end - local mtime = stat.mtime - return mtime.sec + mtime.nsec * 1e-9 - else - error('not implemented') - end -end - -function lfs.currentdir() - return uv.cwd() -end - -function lfs.chdir(dir) - local status, err = pcall(uv.chdir, dir) - if status then - return true - else - return nil, err - end -end - -function lfs.dir(path) - local fs = uv.fs_scandir(path) - return function() - if not fs then - return - end - return uv.fs_scandir_next(fs) - end -end - -function lfs.mkdir(dir) - return uv.fs_mkdir(dir, 493) -- octal 755 -end - -if subcommand == 'busted' then - require 'busted.runner'({ standalone = false }) -elseif subcommand == 'luacheck' then - require 'luacheck.main' -else - error 'unknown subcommand' -end diff --git a/test/reporter.lua b/test/reporter.lua new file mode 100644 index 0000000000..3b42992ef4 --- /dev/null +++ b/test/reporter.lua @@ -0,0 +1,461 @@ +--- @alias test.reporter.SummaryStatus 'skipped' | 'failure' | 'error' + +--- @class test.reporter.ColorMap +--- @field bright fun(s: any): string +--- @field green fun(s: any): string +--- @field yellow fun(s: any): string +--- @field magenta fun(s: any): string +--- @field red fun(s: any): string +--- @field cyan fun(s: any): string +--- @field dim fun(s: any): string + +--- @class test.reporter.FileElement +--- @field name string +--- @field duration? number + +--- @class test.reporter.Options +--- @field verbose boolean +--- @field summary_file string + +--- @class test.base_reporter +--- @field new fun(opts: test.reporter.Options): test.base_reporter +--- @field suite_start fun(self: test.base_reporter, repeat_index?: integer, repeat_count?: integer) +--- @field file_start fun(self: test.base_reporter, file: test.reporter.FileElement) +--- @field test_start fun(self: test.base_reporter, name: string) +--- @field test_end fun(self: test.base_reporter, record: test.harness.Record) +--- @field file_end fun(self: test.base_reporter, file: test.reporter.FileElement, test_count: integer) +--- @field suite_end fun(self: test.base_reporter, duration: number, run_summary: test.harness.RunSummary, failure_output?: string) + +--- @class test.reporter : test.base_reporter +--- @field opts test.reporter.Options +--- @field colors test.reporter.ColorMap +local M = {} +M.__index = M + +--- @return test.reporter.ColorMap +local function identity_colors() + return setmetatable({}, { + __index = function() + return function(s) + return s == nil and '' or tostring(s) + end + end, + }) +end + +--- @param code string +--- @return fun(s: any): string +local function ansi_color(code) + return function(s) + return ('\27[%sm%s\27[0m'):format(code, s == nil and '' or tostring(s)) + end +end + +local ansi_colors = { + bright = ansi_color('1'), + green = ansi_color('32'), + yellow = ansi_color('33'), + magenta = ansi_color('35'), + red = ansi_color('31'), + cyan = ansi_color('36'), + dim = ansi_color('2'), +} + +--- @return boolean +local function use_colors() + local test_colors = os.getenv('TEST_COLORS') + if not test_colors then + return true + end + + local value = test_colors:lower() + return not (value == 'false' or value == '0' or value == 'no' or value == 'off') +end + +--- @param path? string +--- @return file +local function open_summary_file(path) + if type(path) ~= 'string' or path == '-' then + return io.stdout + end + + return (assert(io.open(path, 'w'))) +end + +--- @param opts test.reporter.Options +--- @return test.reporter +function M.new(opts) + local colors = identity_colors() + if use_colors() then + colors = ansi_colors + end + + --- @type test.reporter + local self = setmetatable({ + opts = opts, + colors = colors, + }, M) + + return self +end + +--- @param element? { duration?: number } +--- @return number +local function get_elapsed_time_ms(element) + if element and element.duration then + return element.duration * 1000 + end + + return tonumber('nan') +end + +--- @private +--- @param s any +--- @return string +function M:succ(s) + return self.colors.bright(self.colors.green(s)) +end + +--- @private +--- @param s any +--- @return string +function M:skip(s) + return self.colors.bright(self.colors.yellow(s)) +end + +--- @private +--- @param s any +--- @return string +function M:fail(s) + return self.colors.bright(self.colors.magenta(s)) +end + +--- @private +--- @param s any +--- @return string +function M:errr(s) + return self.colors.bright(self.colors.red(s)) +end + +--- @private +--- @param s any +--- @return string +function M:fpath(s) + return self.colors.cyan(s) +end + +--- @private +--- @param s any +--- @return string +function M:time(s) + return self.colors.dim(s) +end + +--- @private +--- @param s any +--- @return string +function M:sect(s) + return self.colors.green(self.colors.dim(s)) +end + +--- @private +--- @param s any +--- @return string +function M:nmbr(s) + return self.colors.bright(s) +end + +--- @private +--- @param status test.harness.ResultStatus +--- @return string +function M:result_text(status) + if status == 'success' then + return ('%s\n'):format(self:succ('OK')) + elseif status == 'pending' then + return ('%s\n'):format(self:skip('SKIP')) + elseif status == 'failure' then + return ('%s\n'):format(self:fail('FAIL')) + end + + return ('%s\n'):format(self:errr('ERR')) +end + +--- @private +--- @param status test.reporter.SummaryStatus +--- @return string +function M:summary_label(status) + if status == 'skipped' then + return self:skip('SKIPPED ') + elseif status == 'failure' then + return self:fail('FAILED ') + end + + return self:errr('ERROR ') +end + +--- @private +--- @param status test.reporter.SummaryStatus +--- @param count integer +--- @return string +function M:summary_header_noun(status, count) + if status == 'error' then + return count == 1 and 'error' or 'errors' + end + + return count == 1 and 'test' or 'tests' +end + +--- @private +--- @param status test.reporter.SummaryStatus +--- @param count integer +--- @return string +function M:summary_footer_noun(status, count) + if status == 'skipped' then + return count == 1 and 'SKIPPED TEST' or 'SKIPPED TESTS' + elseif status == 'failure' then + return count == 1 and 'FAILED TEST' or 'FAILED TESTS' + elseif status == 'error' then + return count == 1 and 'ERROR' or 'ERRORS' + end + + return count == 1 and 'TEST' or 'TESTS' +end + +--- @param message any +--- @return string +local function stringify_message(message) + if type(message) == 'string' then + return message + elseif message == nil then + return '' + end + + return vim.inspect(message) +end + +--- @private +--- @param pending test.harness.Record +--- @return boolean +function M:na_check(pending) + if pending.name and vim.list_contains(vim.split(pending.name, '[ :]'), 'N/A') then + return true + end + + if type(pending.message) == 'string' then + return vim.list_contains(vim.split(pending.message, '[ :]'), 'N/A') + end + + return false +end + +--- @private +--- @param pending test.harness.Record +--- @return string +function M:pending_description(pending) + local message = stringify_message(pending.message) + if message == '' then + return '' + end + + return table.concat({ message, '\n' }) +end + +--- @private +--- @param failure test.harness.Record +--- @return string +function M:failure_description(failure) + local message = stringify_message(failure.message) + if message == '' then + message = 'Nil error' + end + + local parts = { message, '\n' } + if self.opts.verbose and failure.traceback then + parts[#parts + 1] = failure.traceback + parts[#parts + 1] = '\n' + end + + return table.concat(parts) +end + +--- @private +--- @param trace? test.harness.Trace +--- @return string +function M:get_file_line(trace) + if not trace or not trace.short_src then + return '' + end + + local source = vim.fs.normalize(trace.short_src) + local line = trace.currentline or 0 + return self:fpath(source) .. ' @ ' .. self:fpath(line) .. ': ' +end + +--- @private +--- @param status test.reporter.SummaryStatus +--- @param count integer +--- @param list test.harness.Record[] +--- @param describe fun(self: test.reporter, item: test.harness.Record): string +--- @return string +function M:get_test_list(status, count, list, describe) + if count == 0 then + return '' + end + + local label = self:summary_label(status) + local parts = { + ('%s %s %s, listed below:\n'):format( + label, + self:nmbr(count), + self:summary_header_noun(status, count) + ), + } + local na_count = 0 + + for _, item in ipairs(list) do + if status == 'skipped' and self:na_check(item) then + na_count = na_count + 1 + else + local fullname = self:get_file_line(item.trace) .. self:nmbr(item.name) + parts[#parts + 1] = ('%s %s\n'):format(label, fullname) + parts[#parts + 1] = describe(self, item) + end + end + + if na_count > 0 then + parts[#parts + 1] = + self:nmbr(('%d N/A %s not shown\n'):format(na_count, na_count == 1 and 'test' or 'tests')) + end + + return table.concat(parts) +end + +--- @private +--- @param status test.reporter.SummaryStatus +--- @param count integer +--- @return string +function M:get_summary(status, count) + if count == 0 then + return '' + end + + return (' %s %s\n'):format(self:nmbr(count), self:summary_footer_noun(status, count)) +end + +--- @private +--- @return string +--- @param summary test.harness.RunSummary +function M:get_summary_string(summary) + local tests = summary.success_count == 1 and 'test' or 'tests' + local parts = { + ('%s %s %s.\n'):format(self:succ('PASSED '), self:nmbr(summary.success_count), tests), + self:get_test_list('skipped', summary.skipped_count, summary.pendings, M.pending_description), + self:get_test_list('failure', summary.failure_count, summary.failures, M.failure_description), + self:get_test_list('error', summary.error_count, summary.errors, M.failure_description), + } + + if (summary.skipped_count + summary.failure_count + summary.error_count) > 0 then + parts[#parts + 1] = '\n' + end + + parts[#parts + 1] = self:get_summary('skipped', summary.skipped_count) + parts[#parts + 1] = self:get_summary('failure', summary.failure_count) + parts[#parts + 1] = self:get_summary('error', summary.error_count) + return table.concat(parts) +end + +--- @return nil +--- @param repeat_index? integer +--- @param repeat_count? integer +function M:suite_start(repeat_index, repeat_count) + if repeat_count and repeat_count > 1 and repeat_index then + io.write(('\nRepeating all tests (run %d of %d) . . .\n\n'):format(repeat_index, repeat_count)) + end + io.write(('%s Global test environment setup.\n'):format(self:sect('--------'))) + io.flush() +end + +--- @param file test.reporter.FileElement +function M:file_start(file) + io.write( + ('%s Running tests from %s\n'):format( + self:sect('--------'), + self:fpath(vim.fs.normalize(file.name)) + ) + ) + io.flush() +end + +--- @param name string +function M:test_start(name) + local desc = ('%s %s'):format(_G._nvim_test_id or '', name) + io.write(('%s %s: '):format(self:sect('RUN '), desc)) + io.flush() +end + +--- @private +--- @param record { duration?: number } +--- @param text string +function M:write_status(record, text) + io.write(('%s %s'):format(self:time(('%.2f ms'):format(get_elapsed_time_ms(record))), text)) + io.flush() +end + +--- @param record test.harness.Record +function M:test_end(record) + local text = self:result_text(record.status) + if record.status == 'failure' or record.status == 'error' then + text = text .. self:failure_description(record) + end + + self:write_status(record, text) +end + +--- @param file test.reporter.FileElement +--- @param test_count integer +function M:file_end(file, test_count) + io.write( + ('%s %s %s from %s %s\n\n'):format( + self:sect('--------'), + self:nmbr(test_count), + test_count == 1 and 'test' or 'tests', + self:fpath(vim.fs.normalize(file.name)), + self:time(('(%.2f ms total)'):format(get_elapsed_time_ms(file))) + ) + ) + io.flush() +end + +--- @param duration number +--- @param run_summary test.harness.RunSummary +--- @param failure_output? string +function M:suite_end(duration, run_summary, failure_output) + local tests = run_summary.test_count == 1 and 'test' or 'tests' + local files = run_summary.file_count == 1 and 'file' or 'files' + + io.write(('%s Global test environment teardown.\n'):format(self:sect('--------'))) + io.flush() + + local summary_file = open_summary_file(self.opts.summary_file) + summary_file:write('\n') + summary_file:write( + ('%s %s %s from %s test %s ran. %s\n'):format( + self:sect('========'), + self:nmbr(run_summary.test_count), + tests, + self:nmbr(run_summary.file_count), + files, + self:time(('(%.2f ms total)'):format(duration * 1000)) + ) + ) + summary_file:write(self:get_summary_string(run_summary)) + if failure_output then + summary_file:write(failure_output) + end + summary_file:flush() + if summary_file ~= io.stdout then + summary_file:close() + end +end + +return M diff --git a/test/run_tests.zig b/test/run_tests.zig index 80a7494140..70d6688176 100644 --- a/test/run_tests.zig +++ b/test/run_tests.zig @@ -1,24 +1,22 @@ const std = @import("std"); const LazyPath = std.Build.LazyPath; -pub fn testStep(b: *std.Build, kind: []const u8, nvim_bin: *std.Build.Step.Compile, lua_deps: LazyPath, config_dir: LazyPath, include_path: ?[]const LazyPath) !*std.Build.Step.Run { +pub fn testStep(b: *std.Build, kind: []const u8, nvim_bin: *std.Build.Step.Compile, config_dir: LazyPath, include_path: ?[]const LazyPath) !*std.Build.Step.Run { const test_step = b.addRunArtifact(nvim_bin); test_step.addArg("-ll"); - test_step.addFileArg(b.path("./test/lua_runner.lua")); - test_step.addDirectoryArg(lua_deps); + test_step.addFileArg(b.path("./test/runner.lua")); if (include_path) |paths| { for (paths) |path| { test_step.addPrefixedDirectoryArg("-I", path); } } - test_step.addArgs(&.{ "busted", "-v", "-o", "test.busted.outputHandlers.nvim", "--lazy" }); - // TODO(bfredl): a bit funky with paths, should work even if we run "zig build" in a nested dir - test_step.addArg(b.fmt("./test/{s}/preload.lua", .{kind})); + test_step.addArg("-v"); + test_step.addArg(b.fmt("--helper=./test/{s}/preload.lua", .{kind})); test_step.addArg("--lpath=./src/?.lua"); test_step.addArg("--lpath=./runtime/lua/?.lua"); test_step.addArg("--lpath=./?.lua"); test_step.addPrefixedFileArg("--lpath=", config_dir.path(b, "?.lua")); // FULING: not a real file but works anyway? - // TODO(bfredl): look into $BUSTED_ARGS user hook, TEST_TAG, TEST_FILTER + // TODO(bfredl): look into a TEST_ARGS user hook, TEST_TAG, TEST_FILTER. if (b.args) |args| { test_step.addArgs(args); // accept TEST_FILE as a positional argument } else { @@ -40,12 +38,12 @@ pub fn testStep(b: *std.Build, kind: []const u8, nvim_bin: *std.Build.Step.Compi return test_step; } -pub fn test_steps(b: *std.Build, nvim_bin: *std.Build.Step.Compile, depend_on: *std.Build.Step, lua_deps: LazyPath, config_dir: LazyPath, unit_paths: ?[]const LazyPath) !void { +pub fn test_steps(b: *std.Build, nvim_bin: *std.Build.Step.Compile, depend_on: *std.Build.Step, config_dir: LazyPath, unit_paths: ?[]const LazyPath) !void { const empty_dir = b.addWriteFiles(); _ = empty_dir.add(".touch", ""); const tmpdir_create = b.addInstallDirectory(.{ .source_dir = empty_dir.getDirectory(), .install_dir = .prefix, .install_subdir = "Xtest_tmpdir/" }); - const functional_tests = try testStep(b, "functional", nvim_bin, lua_deps, config_dir, null); + const functional_tests = try testStep(b, "functional", nvim_bin, config_dir, null); functional_tests.step.dependOn(depend_on); functional_tests.step.dependOn(&tmpdir_create.step); @@ -53,7 +51,7 @@ pub fn test_steps(b: *std.Build, nvim_bin: *std.Build.Step.Compile, depend_on: * functionaltest_step.dependOn(&functional_tests.step); if (unit_paths) |paths| { - const unit_tests = try testStep(b, "unit", nvim_bin, lua_deps, config_dir, paths); + const unit_tests = try testStep(b, "unit", nvim_bin, config_dir, paths); unit_tests.step.dependOn(depend_on); unit_tests.step.dependOn(&tmpdir_create.step); diff --git a/test/runner.lua b/test/runner.lua new file mode 100644 index 0000000000..a73719893d --- /dev/null +++ b/test/runner.lua @@ -0,0 +1,35 @@ +local uv = vim.uv + +---@return string +local function repo_root() + local source = debug.getinfo(1, 'S').source + assert(type(source) == 'string' and vim.startswith(source, '@'), 'failed to resolve runner path') + local script_path = assert(uv.fs_realpath(source:sub(2)), 'failed to resolve runner path') + return vim.fs.dirname(vim.fs.dirname(script_path)) +end + +---@param roots string[] +local function prepend_package_roots(roots) + local entries = {} + for _, root in ipairs(roots) do + entries[#entries + 1] = root .. '/?.lua' + entries[#entries + 1] = root .. '/?/init.lua' + end + + package.path = table.concat(entries, ';') .. ';' .. package.path +end + +_G.c_include_path = {} +while _G.arg[1] and vim.startswith(_G.arg[1], '-I') do + table.insert(_G.c_include_path, string.sub(table.remove(_G.arg, 1), 3)) +end + +local root = repo_root() +prepend_package_roots({ root, root .. '/test', '.', './test' }) + +local exit_code = require('test.harness').main(_G.arg) +io.stdout:flush() +io.stderr:flush() + +-- Close the standalone Lua state before exit so sanitizers see Lua-owned cleanup. +os.exit(exit_code, true) diff --git a/test/testutil.lua b/test/testutil.lua index b605318074..ac072e6bb8 100644 --- a/test/testutil.lua +++ b/test/testutil.lua @@ -1,10 +1,9 @@ -local luaassert = require('luassert') -local busted = require('busted') +local test_assert = require('test.assert') +---@type test.harness +local harness = require('test.harness') local uv = vim.uv local Paths = require('test.cmakeconfig.paths') -luaassert:set_parameter('TableFormatLevel', 100) - --- Functions executing in the context of the test runner (not the current nvim test session). --- @class test.testutil local M = { @@ -86,7 +85,7 @@ function M.retry(max, max_ms, fn) end uv.update_time() -- Update cached value of luv.now() (libuv: uv_now()). if (max and tries >= max) or (uv.now() - start_time > timeout) then - busted.fail(string.format('retry() attempts: %d\n%s', tries, tostring(result)), 2) + error(string.format('retry() attempts: %d\n%s', tries, tostring(result)), 2) end tries = tries + 1 uv.sleep(20) -- Avoid hot loop... @@ -100,10 +99,10 @@ local check_logs_useless_lines = { } function M.eq(expected, actual, context) - return luaassert.are.same(expected, actual, context) + return test_assert.eq(expected, actual, context) end function M.neq(expected, actual, context) - return luaassert.are_not.same(expected, actual, context) + return test_assert.neq(expected, actual, context) end --- Compare paths after resolving symlinks with realpath. @@ -125,15 +124,6 @@ function M.ok(cond, expected, actual) return assert(cond, msg) end -local function epicfail(state, arguments, _) - state.failure_message = arguments[1] - return false -end -luaassert:register('assertion', 'epicfail', epicfail) -function M.fail(msg) - return luaassert.epicfail(msg) -end - --- @param pat string --- @param actual string --- @return boolean @@ -365,7 +355,7 @@ function M.check_logs() end end end - luaassert( + test_assert( 0 == #runtime_errors, string.format('Found runtime errors in logfile(s): %s', table.concat(runtime_errors, ', ')) ) @@ -789,34 +779,15 @@ end --- @param name? 'cirrus'|'github' --- @return boolean function M.is_ci(name) - local any = (name == nil) - assert(any or name == 'github' or name == 'cirrus') - local gh = ((any or name == 'github') and nil ~= os.getenv('GITHUB_ACTIONS')) - local cirrus = ((any or name == 'cirrus') and nil ~= os.getenv('CIRRUS_CI')) - return gh or cirrus + return harness.is_ci(name) end -- Gets the (tail) contents of `logfile`. -- Also moves the file to "${NVIM_LOG_FILE}.displayed" on CI environments. function M.read_nvim_log(logfile, ci_rename) logfile = logfile or os.getenv('NVIM_LOG_FILE') or 'nvim.log' - assert(uv.fs_stat(logfile), ('logfile not found: %q'):format(logfile)) - local is_ci = M.is_ci() - local keep = is_ci and 100 or 10 - local lines = M.read_file_list(logfile, -keep) or {} - local log = ( - ('-'):rep(78) - .. '\n' - .. string.format('$NVIM_LOG_FILE: %s\n', logfile) - .. (#lines > 0 and '(last ' .. tostring(keep) .. ' lines)\n' or '(empty)\n') - ) - for _, line in ipairs(lines) do - log = log .. line .. '\n' - end - log = log .. ('-'):rep(78) .. '\n' - if is_ci and ci_rename then - os.rename(logfile, logfile .. '.displayed') - end + local log = harness.read_nvim_log(logfile, ci_rename) + assert(log, ('logfile not found: %q'):format(logfile)) return log end @@ -843,7 +814,7 @@ function M.expect_events(expected, received, kind) for _, e in ipairs(expected) do msg = msg .. ' ' .. vim.inspect(e) .. ';\n' end - M.fail(msg) + error(msg, 2) end return received end diff --git a/test/unit/os/fs_spec.lua b/test/unit/os/fs_spec.lua index 5b6d7301a4..399b1f89ac 100644 --- a/test/unit/os/fs_spec.lua +++ b/test/unit/os/fs_spec.lua @@ -76,8 +76,8 @@ describe('fs.c', function() uv.fs_symlink('test.file', 'unit-test-directory/test_link.file') uv.fs_symlink('non_existing_file.file', 'unit-test-directory/test_broken_link.file') - -- The tests are invoked with an absolute path to `busted` executable. - absolute_executable = arg[0] + -- The tests are invoked with an absolute executable path in arg[0]. + absolute_executable = vim.fs.normalize(assert(uv.exepath())) -- Split the absolute_executable path into a directory and filename. directory, executable_name = string.match(absolute_executable, '^(.*)/(.*)$') end) diff --git a/test/unit/path_spec.lua b/test/unit/path_spec.lua index 21f0556aed..676364bd4e 100644 --- a/test/unit/path_spec.lua +++ b/test/unit/path_spec.lua @@ -397,14 +397,6 @@ describe('path.c', function() setup(function() mkdir('unit-test-directory') io.open('unit-test-directory/test.file', 'w'):close() - - -- Since the tests are executed, they are called by an executable. We use - -- that executable for several asserts. - local absolute_executable = arg[0] - - -- Split absolute_executable into a directory and the actual file name for - -- later usage. - local directory, executable_name = string.match(absolute_executable, '^(.*)/(.*)$') -- luacheck: ignore end) teardown(function() diff --git a/test/unit/preload.lua b/test/unit/preload.lua index 7c88007e14..920f949edf 100644 --- a/test/unit/preload.lua +++ b/test/unit/preload.lua @@ -1,6 +1,6 @@ --- Modules loaded here will not be cleared and reloaded by Busted. --- Busted started doing this to help provide more isolation. See issue #62 --- for more information about this. +-- Modules loaded here will not be cleared and reloaded by the local harness. +-- Keeping these preloaded preserves cross-file setup while still resetting +-- non-helper modules between files. require('ffi') require('test.unit.testutil') require('test.unit.preprocess') diff --git a/test/unit/tempfile_spec.lua b/test/unit/tempfile_spec.lua index 1a42eed17e..74073b4854 100644 --- a/test/unit/tempfile_spec.lua +++ b/test/unit/tempfile_spec.lua @@ -46,7 +46,7 @@ describe('tempfile related functions', function() itp('generate name of non-existing file', function() local file = vim_tempname() - assert.truthy(file) + assert(file) assert.False(lib.os_path_exists(file)) end) diff --git a/test/unit/testtest_spec.lua b/test/unit/testtest_spec.lua index 0750cac5cc..dc63af0546 100644 --- a/test/unit/testtest_spec.lua +++ b/test/unit/testtest_spec.lua @@ -1,5 +1,4 @@ local t = require('test.unit.testutil') -local assert = require('luassert') local itp = t.gen_itp(it) @@ -11,7 +10,7 @@ if os.getenv('NVIM_TEST_RUN_TESTTEST') ~= '1' then end describe('test code', function() itp('does not hang when working with lengthy errors', function() - assert.just_fail(('x'):rep(65536)) + error(('x'):rep(65536), 0) end) itp('shows trace after exiting abnormally', function() sc.exit(0) diff --git a/test/unit/testutil.lua b/test/unit/testutil.lua index b23c85d3b0..8604cb27bb 100644 --- a/test/unit/testutil.lua +++ b/test/unit/testutil.lua @@ -4,8 +4,7 @@ local Set = require('test.unit.set') local Preprocess = require('test.unit.preprocess') local t_global = require('test.testutil') local paths = t_global.paths -local assert = require('luassert') -local say = require('say') +local assert = require('test.assert') local check_cores = t_global.check_cores local dedent = t_global.dedent @@ -538,19 +537,6 @@ if os.getenv('NVIM_TEST_PRINT_SYSCALLS') == '1' then end end -local function just_fail(_) - return false -end -say:set('assertion.just_fail.positive', '%s') -say:set('assertion.just_fail.negative', '%s') -assert:register( - 'assertion', - 'just_fail', - just_fail, - 'assertion.just_fail.positive', - 'assertion.just_fail.negative' -) - local hook_fnamelen = 30 local hook_sfnamelen = 30 local hook_numlen = 5 @@ -769,7 +755,7 @@ local function check_child_err(rd) end end if err ~= '' then - assert.just_fail(err) + error(err, 0) end end end @@ -810,7 +796,7 @@ local function gen_itp(it) end -- Pre-emptively calculating error location, wasteful, ugh! - -- But the way this code messes around with busted implies the real location is strictly + -- But the way this code wraps the local harness means the real location is strictly -- not available in the parent when an actual error occurs. so we have to do this here. local location = debug.traceback() it(name, function() diff --git a/test/unit/vterm_spec.lua b/test/unit/vterm_spec.lua index 41c05edd58..3492487eff 100644 --- a/test/unit/vterm_spec.lua +++ b/test/unit/vterm_spec.lua @@ -275,6 +275,13 @@ local function push(input, vt) vterm.vterm_input_write(vt, input, string.len(input)) end +-- vterm_input_write() can synchronously invoke the Lua callbacks installed +-- above. LuaJIT must not JIT-compile an FFI call that re-enters Lua, or the +-- test can panic with "bad callback". +if jit then + jit.off(push, true) +end + local function expect(expected) local actual = read_rm() t.eq(expected .. '\n', actual)