PublicArchive/valkey
2
0
Fork 0
mirror of https://github.com/valkey-io/valkey.git synced 2026-05-06 05:26:42 -04:00
Code Issues Packages Projects Releases Wiki Activity

Add Static Module Support (#3392)

Browse Source 
Add a build option to compile the Lua scripting engine as a static
module and wire the server to load it directly at startup when enabled.
The module load path now resolves on-load and on-unload entry points
from the main binary, and the module lifecycle keeps those callbacks so
unload works without a shared library handle.

The Lua module build was updated to support both static and shared
variants, with the static path exporting visible wrapper symbols and
linking the server with the module archive. While touching the Lua code,
a few internal symbols were renamed for consistency and the monotonic
time helper was clarified.

Note that this PR addresses the LUA module, but it can be applied to
other "core" modules (like: Bloom, Json, Search and others). With this
change, it will be easier to ship Valkey bundle with modules.

Areas touched:

* CMake
* Makefile
* Lua scripting module
* Core module loading

**Generated by CodeLite**

---------

Signed-off-by: Eran Ifrah <eifrah@amazon.com>
This commit is contained in:
eifrah-aws
2026-04-20 14:45:57 +03:00
committed by GitHub
parent 269b1c5eda
commit 0327c27131
13 changed files with 311 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+1 -1
View File
@@ -13,7 +13,7 @@ if (APPLE)
endif ()
# Options
option(BUILD_LUA "Build Valkey Lua scripting engine" ON)
set(BUILD_LUA "static" CACHE STRING "Build Valkey Lua scripting engine: static (default), module, no")
option(BUILD_UNIT_GTESTS "Build valkey-unit-gtests" OFF)
option(BUILD_TEST_MODULES "Build all test modules" OFF)
option(BUILD_EXAMPLE_MODULES "Build example modules" OFF)
src/CMakeLists.txt
+24 -15
View File
@@ -17,24 +17,34 @@ add_dependencies(valkey-server generate_commands_def)
add_dependencies(valkey-server generate_fmtargs_h)
add_dependencies(valkey-server release_header)
if (BUILD_LUA)
if (NOT BUILD_LUA STREQUAL "no")
message(STATUS "Build Lua scripting engine module")
if (BUILD_LUA STREQUAL "static")
add_compile_definitions(STATIC_LUA=1)
message(STATUS "Building LUA as a STATIC module")
else ()
add_compile_definitions(STATIC_LUA=0)
message(STATUS "Building LUA as a DYNAMIC module")
endif ()
add_subdirectory(modules/lua)
add_dependencies(valkey-server valkeylua)
if (BUILD_LUA STREQUAL "static")
target_link_libraries(valkey-server $<LINK_LIBRARY:WHOLE_ARCHIVE,valkeylua>)
else ()
add_dependencies(valkey-server valkeylua)
endif ()
target_compile_definitions(valkey-server PRIVATE LUA_ENABLED)
if (UNIX AND NOT APPLE)
target_compile_definitions(valkey-server PRIVATE LUA_LIB=libvalkeylua.so)
target_link_options(valkey-server PRIVATE -Wl,--disable-new-dtags)
else ()
target_compile_definitions(valkey-server PRIVATE LUA_LIB=libvalkeylua.dylib)
endif ()
set(VALKEY_INSTALL_RPATH "")
set_target_properties(valkey-server PROPERTIES
INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR};${CMAKE_LIBRARY_OUTPUT_DIRECTORY}"
INSTALL_RPATH_USE_LINK_PATH TRUE
BUILD_WITH_INSTALL_RPATH TRUE
)
endif()
set_target_properties(
valkey-server
PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR};${CMAKE_LIBRARY_OUTPUT_DIRECTORY}"
INSTALL_RPATH_USE_LINK_PATH TRUE
BUILD_WITH_INSTALL_RPATH TRUE)
endif ()
unset(BUILD_LUA CACHE)
if (VALKEY_RELEASE_BUILD)
@@ -49,9 +59,8 @@ if (DEBUG_FORCE_DEFRAG)
endif ()
if (BUILD_SANITIZER)
# 'BUILD_SANITIZER' is defined in ValkeySetup module (based on user input)
# If defined, the variables 'VALKEY_SANITAIZER_CFLAGS' and 'VALKEY_SANITAIZER_LDFLAGS'
# are set with the link & compile flags required
# 'BUILD_SANITIZER' is defined in ValkeySetup module (based on user input) If defined, the variables
# 'VALKEY_SANITAIZER_CFLAGS' and 'VALKEY_SANITAIZER_LDFLAGS' are set with the link & compile flags required
message(STATUS "Adding sanitizer flags for target valkey-server")
target_compile_options(valkey-server PRIVATE ${VALKEY_SANITAIZER_CFLAGS})
target_link_options(valkey-server PRIVATE ${VALKEY_SANITAIZER_LDFLAGS})
@@ -118,10 +127,10 @@ endif ()
# Friendly hint like the Makefile one
file(RELATIVE_PATH _CMAKE_DIR_RELATIVE_PATH "${CMAKE_SOURCE_DIR}" "${CMAKE_BINARY_DIR}")
add_custom_target(hint ALL
add_custom_target(
hint ALL
DEPENDS valkey-server valkey-cli valkey-benchmark
COMMAND ${CMAKE_COMMAND} -E echo ""
COMMAND ${CMAKE_COMMAND} -E echo "Hint: It is a good idea to run tests with your CMake-built binaries \\;\\)"
COMMAND ${CMAKE_COMMAND} -E echo " ./${_CMAKE_DIR_RELATIVE_PATH}/runtest"
COMMAND ${CMAKE_COMMAND} -E echo ""
)
COMMAND ${CMAKE_COMMAND} -E echo "")
src/Makefile
+25 -10
View File
@@ -259,22 +259,37 @@ endif
FINAL_CFLAGS+= -I../deps/libvalkey/include -I../deps/linenoise -I../deps/hdr_histogram -I../deps/fpconv -I../deps/fast_float
# Lua scripting engine module
LUA_MODULE_NAME:=modules/lua/libvalkeylua.so
ifeq ($(BUILD_LUA),no)
LUA_MODULE_NAME=
LUA_MODULE=
LUA_MODULE_INSTALL=
else
FINAL_CFLAGS+=-DSTATIC_LUA=0
else ifeq ($(BUILD_LUA),module)
LUA_MODULE_NAME=modules/lua/libvalkeylua.so
LUA_MODULE=$(LUA_MODULE_NAME)
LUA_MODULE_INSTALL=install-lua-module
current_dir = $(shell pwd)
FINAL_CFLAGS+=-DLUA_ENABLED -DLUA_LIB=libvalkeylua.so
ifeq ($(uname_S),Darwin)
FINAL_LDFLAGS+= -Wl,-rpath,$(PREFIX)/lib
FINAL_LDFLAGS+= -Wl,-rpath,$(current_dir)/modules/lua
FINAL_CFLAGS+=-DLUA_ENABLED -DLUA_LIB=libvalkeylua.so -DSTATIC_LUA=0
ifeq ($(uname_S),Darwin)
FINAL_LDFLAGS+= -Wl,-rpath,$(PREFIX)/lib
FINAL_LDFLAGS+= -Wl,-rpath,$(current_dir)/modules/lua
else
FINAL_LDFLAGS+= -Wl,-rpath,$(PREFIX)/lib:$(current_dir)/modules/lua -Wl,--disable-new-dtags
endif
else
FINAL_LDFLAGS+= -Wl,-rpath,$(PREFIX)/lib:$(current_dir)/modules/lua -Wl,--disable-new-dtags
endif
# The default: building Lua as a static module.
LUA_MODULE_NAME=modules/lua/libvalkeylua.a
LUA_MODULE=$(LUA_MODULE_NAME)
current_dir = $(shell pwd)
FINAL_CFLAGS+=-DLUA_ENABLED -DSTATIC_LUA=1
ifeq ($(uname_S),Darwin)
LUA_LDFLAGS=-Wl,-export_dynamic -Wl,-force_load,$(current_dir)/modules/lua/libvalkeylua.a ../deps/lua/src/liblua.a
else ifeq ($(uname_S),FreeBSD)
LUA_LDFLAGS=-Wl,--export-dynamic -Wl,--whole-archive $(current_dir)/modules/lua/libvalkeylua.a -Wl,--no-whole-archive ../deps/lua/src/liblua.a
else
LUA_LDFLAGS=-Wl,--whole-archive $(current_dir)/modules/lua/libvalkeylua.a -Wl,--no-whole-archive ../deps/lua/src/liblua.a
endif
endif
# Determine systemd support and/or build preference (defaulting to auto-detection)
@@ -693,7 +708,7 @@ endif
# valkey-server
$(SERVER_NAME): $(ENGINE_SERVER_OBJ) $(LUA_MODULE)
$(SERVER_LD) -o $@ $(ENGINE_SERVER_OBJ) ../deps/libvalkey/lib/libvalkey.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a $(FINAL_LIBS)
$(SERVER_LD) -o $@ $(ENGINE_SERVER_OBJ) ../deps/libvalkey/lib/libvalkey.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a $(FINAL_LIBS) $(LUA_LDFLAGS)
# Valkey static library, used to compile against for unit testing
$(ENGINE_LIB_NAME): $(ENGINE_SERVER_OBJ)
@@ -721,7 +736,7 @@ $(RDMA_MODULE_NAME): $(SERVER_NAME)
# engine_lua.so
$(LUA_MODULE_NAME): .make-prerequisites
cd modules/lua && $(MAKE) OPTIMIZATION="$(OPTIMIZATION)"
$(MAKE) -C modules/lua OPTIMIZATION="$(OPTIMIZATION)" BUILD_LUA="$(BUILD_LUA)"
# valkey-cli
$(ENGINE_CLI_NAME): $(ENGINE_CLI_OBJ)
src/config.c
+1
View File
@@ -1619,6 +1619,7 @@ void rewriteConfigLoadmoduleOption(struct rewriteConfigState *state) {
dictEntry *de;
while ((de = dictNext(di)) != NULL) {
struct ValkeyModule *module = dictGetVal(de);
if (module->is_static_module) continue;
line = moduleLoadQueueEntryToLoadmoduleOptionStr(module, "loadmodule");
rewriteConfigRewriteLine(state, "loadmodule", line, 1);
}
src/eval.c
-1
View File
@@ -181,7 +181,6 @@ void evalRelease(int async) {
}
}
void evalReset(int async) {
evalRelease(async);
evalInit();
src/module.c
+168 -67
View File
@@ -55,18 +55,17 @@
* replacements are done, such as the replacement of RM with ValkeyModule in
* function names. For details, see the script src/modules/gendoc.rb.
* -------------------------------------------------------------------------- */
#include "server.h"
#include "cluster.h"
#include "commandlog.h"
#include "rdb.h"
#include "monotonic.h"
#include "script.h"
#include "call_reply.h"
#include "hdr_histogram.h"
#include "crc16_slottable.h"
#include "valkeymodule.h"
#include "module.h"
#include "call_reply.h"
#include "io_threads.h"
#include "scripting_engine.h"
#include "cluster_migrateslots.h"
@@ -13075,10 +13074,100 @@ void moduleUnregisterCleanup(ValkeyModule *module) {
moduleUnregisterAuthCBs(module);
}
/* Common helper for moduleLoad and moduleLoadStatic.
* Invokes the onload callback, registers the module, and performs post-load
* validation. 'display_name' is used in log messages, 'handle' is the
* dlopen handle (NULL for static modules), and 'is_static' controls the
* is_static_module flag and handle ownership semantics. */
static int moduleInitPostOnLoadResolved(ModuleLoadFunc onload,
void *handle,
const char *display_name,
void **module_argv,
int module_argc,
int is_loadex,
int is_static) {
ValkeyModuleCtx ctx;
moduleCreateContext(&ctx, NULL, VALKEYMODULE_CTX_TEMP_CLIENT); /* We pass NULL since we don't have a module yet. */
if (onload((void *)&ctx, module_argv, module_argc) == VALKEYMODULE_ERR) {
if (ctx.module) {
serverLog(LL_WARNING, "%sModule %s initialization failed. Module not loaded.",
is_static ? "Static " : "", display_name);
moduleUnregisterCleanup(ctx.module);
moduleRemoveCateogires(ctx.module);
moduleFreeModuleStructure(ctx.module);
} else {
/* If there is no ctx.module, this means that our ValkeyModule_Init call failed,
* and currently init will only fail on busy name. */
serverLog(LL_WARNING, "%sModule %s initialization failed. Module name is busy.",
is_static ? "Static " : "", display_name);
}
moduleFreeContext(&ctx);
if (handle) {
dlclose(handle);
}
return C_ERR;
}
if (is_static && handle) {
dlclose(handle);
handle = NULL;
}
/* Module loaded! Register it. */
dictAdd(modules, ctx.module->name, ctx.module);
ctx.module->blocked_clients = 0;
ctx.module->handle = handle;
ctx.module->is_static_module = is_static;
ctx.module->loadmod = zmalloc(sizeof(struct moduleLoadQueueEntry));
ctx.module->loadmod->path = sdsnew(display_name);
ctx.module->loadmod->argv = module_argc ? zmalloc(sizeof(robj *) * module_argc) : NULL;
ctx.module->loadmod->argc = module_argc;
for (int i = 0; i < module_argc; i++) {
ctx.module->loadmod->argv[i] = module_argv[i];
incrRefCount(ctx.module->loadmod->argv[i]);
}
/* If module commands have ACL categories, recompute command bits
* for all existing users once the modules has been registered. */
if (ctx.module->num_commands_with_acl_categories) {
ACLRecomputeCommandBitsFromCommandRulesAllUsers();
}
if (is_static) {
serverLog(LL_NOTICE, "Static Module '%s' successfully loaded", ctx.module->name);
} else {
serverLog(LL_NOTICE, "Module '%s' loaded from %s", ctx.module->name, display_name);
}
ctx.module->onload = 0;
int post_load_err = 0;
if (listLength(ctx.module->module_configs) && !ctx.module->configs_initialized) {
serverLogRaw(LL_WARNING,
"Module Configurations were not set, likely a missing LoadConfigs call. Unloading the module.");
post_load_err = 1;
}
if (is_loadex && dictSize(server.module_configs_queue)) {
serverLogRaw(LL_WARNING,
"Loadex configurations were not applied, likely due to invalid arguments. Unloading the module.");
post_load_err = 1;
}
if (post_load_err) {
moduleUnload(ctx.module->name, NULL);
moduleFreeContext(&ctx);
return C_ERR;
}
/* Fire the loaded modules event. */
moduleFireServerEvent(VALKEYMODULE_EVENT_MODULE_CHANGE, VALKEYMODULE_SUBEVENT_MODULE_LOADED, ctx.module);
moduleFreeContext(&ctx);
return C_OK;
}
/* Load a module and initialize it. On success C_OK is returned, otherwise
* C_ERR is returned. */
int moduleLoad(const char *path, void **module_argv, int module_argc, int is_loadex) {
int (*onload)(void *, void **, int);
ModuleLoadFunc onload;
void *handle;
struct stat st;
@@ -13109,7 +13198,7 @@ int moduleLoad(const char *path, void **module_argv, int module_argc, int is_loa
const char *onLoadNames[] = {"ValkeyModule_OnLoad", "RedisModule_OnLoad"};
for (size_t i = 0; i < sizeof(onLoadNames) / sizeof(onLoadNames[0]); i++) {
onload = (int (*)(void *, void **, int))(unsigned long)dlsym(handle, onLoadNames[i]);
onload = (ModuleLoadFunc)(unsigned long)dlsym(handle, onLoadNames[i]);
if (onload != NULL) {
if (i != 0) {
serverLog(LL_NOTICE, "Legacy Redis Module %s found", path);
@@ -13126,71 +13215,75 @@ int moduleLoad(const char *path, void **module_argv, int module_argc, int is_loa
path);
return C_ERR;
}
ValkeyModuleCtx ctx;
moduleCreateContext(&ctx, NULL, VALKEYMODULE_CTX_TEMP_CLIENT); /* We pass NULL since we don't have a module yet. */
if (onload((void *)&ctx, module_argv, module_argc) == VALKEYMODULE_ERR) {
if (ctx.module) {
serverLog(LL_WARNING, "Module %s initialization failed. Module not loaded.", path);
moduleUnregisterCleanup(ctx.module);
moduleRemoveCateogires(ctx.module);
moduleFreeModuleStructure(ctx.module);
} else {
/* If there is no ctx.module, this means that our ValkeyModule_Init call failed,
* and currently init will only fail on busy name. */
serverLog(LL_WARNING, "Module %s initialization failed. Module name is busy.", path);
}
moduleFreeContext(&ctx);
dlclose(handle);
return moduleInitPostOnLoadResolved(onload, handle, path, module_argv, module_argc, is_loadex, 0);
}
/* Resolve a symbol from a statically linked module. The symbol is looked up
* by constructing the name "<symbol_name>_<module_name>" and searching for it
* in the current process via dlopen(NULL)/dlsym(). On success, '*out' is set
* to the symbol address, '*handle' is set to the dlopen handle, and C_OK is
* returned. On failure C_ERR is returned and an appropriate warning is logged. */
static int moduleLoadStaticSymbol(void **out, void **handle, const char *symbol_name, const char *module_name) {
char symbol_full_name[128];
int n = snprintf(symbol_full_name, sizeof(symbol_full_name), "%s_%s", symbol_name, module_name);
if (n >= (int)sizeof(symbol_full_name)) {
serverLog(LL_WARNING, "Module name is too long");
return C_ERR;
}
/* Module loaded! Register it. */
dictAdd(modules, ctx.module->name, ctx.module);
ctx.module->blocked_clients = 0;
ctx.module->handle = handle;
ctx.module->loadmod = zmalloc(sizeof(struct moduleLoadQueueEntry));
ctx.module->loadmod->path = sdsnew(path);
ctx.module->loadmod->argv = module_argc ? zmalloc(sizeof(robj *) * module_argc) : NULL;
ctx.module->loadmod->argc = module_argc;
for (int i = 0; i < module_argc; i++) {
ctx.module->loadmod->argv[i] = module_argv[i];
incrRefCount(ctx.module->loadmod->argv[i]);
}
/* Open a handle to self */
*handle = dlopen(NULL, RTLD_NOW);
if (*handle == NULL) {
char *error = dlerror();
if (error == NULL) error = "Unknown error";
/* If module commands have ACL categories, recompute command bits
* for all existing users once the modules has been registered. */
if (ctx.module->num_commands_with_acl_categories) {
ACLRecomputeCommandBitsFromCommandRulesAllUsers();
}
serverLog(LL_NOTICE, "Module '%s' loaded from %s", ctx.module->name, path);
ctx.module->onload = 0;
int post_load_err = 0;
if (listLength(ctx.module->module_configs) && !ctx.module->configs_initialized) {
serverLogRaw(LL_WARNING,
"Module Configurations were not set, likely a missing LoadConfigs call. Unloading the module.");
post_load_err = 1;
}
if (is_loadex && dictSize(server.module_configs_queue)) {
serverLogRaw(LL_WARNING,
"Loadex configurations were not applied, likely due to invalid arguments. Unloading the module.");
post_load_err = 1;
}
if (post_load_err) {
moduleUnload(ctx.module->name, NULL);
moduleFreeContext(&ctx);
serverLog(LL_WARNING, "Failed to load static module: %s. %s", module_name, error);
return C_ERR;
}
/* Fire the loaded modules event. */
moduleFireServerEvent(VALKEYMODULE_EVENT_MODULE_CHANGE, VALKEYMODULE_SUBEVENT_MODULE_LOADED, ctx.module);
*out = dlsym(*handle, symbol_full_name);
if (*out == NULL) {
char *error = dlerror();
if (error == NULL) error = "Unknown error";
moduleFreeContext(&ctx);
serverLog(LL_WARNING,
"Failed to load static module: %s. Could not load method: %s. %s", module_name,
symbol_full_name, error);
dlclose(*handle);
return C_ERR;
}
return C_OK;
}
/* Load a statically linked module and initialize it. This is the static
* counterpart of moduleLoad(): instead of dlopen()ing a shared object from
* a file path, it resolves the module's entry point from the running
* executable itself.
*
* The entry point is located by constructing the symbol name
* "ValkeyModule_OnLoad_<module_name>" and resolving it with
* moduleLoadStaticSymbol(). For example, a module named "mymodule" must
* provide a function called ValkeyModule_OnLoad_mymodule.
*
* Once the entry point is found, the function creates a temporary module
* context, invokes the OnLoad callback, and on success registers the module
* in the global modules dictionary with is_static_module set to 1 and handle
* set to NULL (since there is no shared-object handle to keep open).
*
* If 'is_loadex' is true, the function also validates that all queued module
* configurations were consumed; otherwise the module is unloaded.
*
* On success C_OK is returned, otherwise C_ERR is returned. */
int moduleLoadStatic(const char *module_name, void **module_argv, int module_argc, int is_loadex) {
ModuleLoadFunc onload;
void *handle = NULL;
if (moduleLoadStaticSymbol((void **)&onload, &handle, "ValkeyModule_OnLoad", module_name) != C_OK) {
return C_ERR;
}
return moduleInitPostOnLoadResolved(onload, handle, module_name, module_argv, module_argc,
is_loadex, 1);
}
static int moduleUnloadInternal(struct ValkeyModule *module, const char **errmsg) {
if (listLength(module->types)) {
*errmsg = "the module exports one or more module-side data "
@@ -13223,15 +13316,23 @@ static int moduleUnloadInternal(struct ValkeyModule *module, const char **errmsg
}
/* Give module a chance to clean up. */
const char *onUnloadNames[] = {"ValkeyModule_OnUnload", "RedisModule_OnUnload"};
int (*onunload)(void *) = NULL;
for (size_t i = 0; i < sizeof(onUnloadNames) / sizeof(onUnloadNames[0]); i++) {
onunload = (int (*)(void *))(unsigned long)dlsym(module->handle, onUnloadNames[i]);
if (onunload) {
if (i != 0) {
serverLog(LL_NOTICE, "Legacy Redis Module %s found", module->name);
ModuleUnLoadFunc onunload = NULL;
if (module->is_static_module == 1) {
if (moduleLoadStaticSymbol((void **)&onunload, &module->handle, "ValkeyModule_OnUnload", module->name) != C_OK) {
serverLog(LL_WARNING, "Module %s OnUnload failed. Unload canceled.", module->name);
errno = ECANCELED;
return C_ERR;
}
} else {
const char *onUnloadNames[] = {"ValkeyModule_OnUnload", "RedisModule_OnUnload"};
for (size_t i = 0; i < sizeof(onUnloadNames) / sizeof(onUnloadNames[0]); i++) {
onunload = (int (*)(void *))(unsigned long)dlsym(module->handle, onUnloadNames[i]);
if (onunload) {
if (i != 0) {
serverLog(LL_NOTICE, "Legacy Redis Module %s found", module->name);
}
break;
}
break;
}
}
@@ -13251,7 +13352,7 @@ static int moduleUnloadInternal(struct ValkeyModule *module, const char **errmsg
moduleUnregisterCleanup(module);
/* Unload the dynamic library. */
if (dlclose(module->handle) == -1) {
if (module->handle != NULL && dlclose(module->handle) == -1) {
char *error = dlerror();
if (error == NULL) error = "Unknown error";
serverLog(LL_WARNING, "Error when trying to close the %s module: %s", module->name, error);
src/module.h
+5
View File
@@ -95,6 +95,9 @@ typedef struct moduleValue {
void *value;
} moduleValue;
typedef int (*ModuleLoadFunc)(void *, void **, int);
typedef int (*ModuleUnLoadFunc)(void *);
/* This structure represents a module inside the system. */
typedef struct ValkeyModule {
void *handle; /* Module dlopen() handle. */
@@ -117,6 +120,7 @@ typedef struct ValkeyModule {
int num_commands_with_acl_categories; /* Number of commands in this module included in acl categories */
int onload; /* Flag to identify if the call is being made from Onload (0 or 1) */
size_t num_acl_categories_added; /* Number of acl categories added by this module. */
int is_static_module; /* 1 if this is a static module, 0 otherwise */
} ValkeyModule;
/* This is a wrapper for the 'rio' streams used inside rdb.c in the server, so that
@@ -182,6 +186,7 @@ void moduleInitModulesSystem(void);
void moduleInitModulesSystemLast(void);
void modulesCron(void);
int moduleLoad(const char *path, void **argv, int argc, int is_loadex);
int moduleLoadStatic(const char *path, void **argv, int argc, int is_loadex);
int moduleUnload(sds name, const char **errmsg);
void moduleUnloadAllModules(void);
void moduleLoadFromQueue(void);
src/modules/lua/CMakeLists.txt
+18 -10
View File
@@ -4,22 +4,30 @@ if (VALKEY_DEBUG_BUILD)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -W -Wall -fno-common -g -ggdb -std=c99 -O2 -D_GNU_SOURCE")
else ()
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -W -Wall -fno-common -O3 -std=c99 -D_GNU_SOURCE")
endif()
endif ()
set(LUA_ENGINE_SRCS
engine_lua.c
script_lua.c
function_lua.c
debug_lua.c
list.c
../../sha1.c
../../rand.c)
list.c)
add_library(valkeylua SHARED "${LUA_ENGINE_SRCS}")
if (BUILD_LUA STREQUAL "module")
list(APPEND LUA_ENGINE_SRCS ../../sha1.c)
list(APPEND LUA_ENGINE_SRCS ../../rand.c)
endif ()
target_link_libraries(valkeylua PRIVATE lualib fpconv)
target_include_directories(valkeylua PRIVATE ../../../deps/lua/src)
if (BUILD_LUA STREQUAL "static")
message(STATUS "Building STATIC LUA module")
add_library(valkeylua STATIC "${LUA_ENGINE_SRCS}")
target_link_libraries(valkeylua PUBLIC lualib fpconv)
target_include_directories(valkeylua PUBLIC ../../../deps/lua/src)
else ()
message(STATUS "Building DYNAMIC LUA module")
add_library(valkeylua SHARED "${LUA_ENGINE_SRCS}")
target_link_libraries(valkeylua PRIVATE lualib fpconv)
target_include_directories(valkeylua PRIVATE ../../../deps/lua/src)
endif ()
install(TARGETS valkeylua
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
)
install(TARGETS valkeylua LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
src/modules/lua/Makefile
+26 -9
View File
@@ -3,11 +3,19 @@ CLANG := $(findstring clang,$(shell sh -c '$(CC) --version | head -1'))
DEPS_DIR=../../../deps
ifeq ($(BUILD_LUA),module)
STATIC_LUA_FLAG=-DSTATIC_LUA=0
LUA_TARGET=libvalkeylua.so
else
STATIC_LUA_FLAG=-DSTATIC_LUA=1
LUA_TARGET=libvalkeylua.a
endif
ifeq ($(uname_S),Darwin)
SHOBJ_CFLAGS= -I. -I$(DEPS_DIR)/lua/src -I$(DEPS_DIR)/fpconv -fPIC -W -Wall -dynamic -fno-common $(OPTIMIZATION) -std=gnu11 -D_GNU_SOURCE $(CFLAGS)
SHOBJ_CFLAGS= -I. -I$(DEPS_DIR)/lua/src -I$(DEPS_DIR)/fpconv -fPIC -W -Wall -dynamic -fno-common $(OPTIMIZATION) -std=gnu11 -D_GNU_SOURCE $(STATIC_LUA_FLAG) $(CFLAGS)
SHOBJ_LDFLAGS= -bundle -undefined dynamic_lookup $(LDFLAGS)
else
SHOBJ_CFLAGS= -I. -I$(DEPS_DIR)/lua/src -I$(DEPS_DIR)/fpconv -fPIC -W -Wall -fno-common $(OPTIMIZATION) -std=gnu11 -D_GNU_SOURCE $(CFLAGS)
SHOBJ_CFLAGS= -I. -I$(DEPS_DIR)/lua/src -I$(DEPS_DIR)/fpconv -fPIC -W -Wall -fno-common $(OPTIMIZATION) -std=gnu11 -D_GNU_SOURCE $(STATIC_LUA_FLAG) $(CFLAGS)
SHOBJ_LDFLAGS= -shared $(LDFLAGS)
# Pretty-printing setup for module build
@@ -43,10 +51,15 @@ LIBS = \
$(DEPS_DIR)/lua/src/liblua.a \
$(DEPS_DIR)/fpconv/libfpconv.a
SRCS= $(wildcard *.c)
ifeq ($(BUILD_LUA),module)
OBJS = \
$(SRCS:.c=.o) \
sha1.o \
rand.o
$(SRCS:.c=.o) \
sha1.o \
rand.o
else
OBJS = $(SRCS:.c=.o)
endif
# OS X 11.x doesn't have /usr/lib/libSystem.dylib and needs an explicit setting.
ifeq ($(uname_S),Darwin)
@@ -55,10 +68,14 @@ ifeq ("$(wildcard /usr/lib/libSystem.dylib)","")
endif
endif
all: libvalkeylua.so
libvalkeylua.so: $(OBJS) $(LIBS)
all: $(LUA_TARGET)
ifeq ($(BUILD_LUA),module)
$(LUA_TARGET): $(OBJS) $(LIBS)
$(QUIET_LINK_MOD)$(CC) -o $@ $(SHOBJ_LDFLAGS) $^
else
$(LUA_TARGET): $(OBJS) $(LIBS)
$(QUIET_LINK_MOD)$(AR) rcs $@ $(OBJS)
endif
sha1.o: ../../sha1.c
$(QUIET_CC_MOD)$(CC) $(SHOBJ_CFLAGS) -c $< -o $@
@@ -76,4 +93,4 @@ $(DEPS_DIR)/fpconv/libfpconv.a:
cd $(DEPS_DIR) && $(MAKE) fpconv
clean:
rm -f *.so $(OBJS)
rm -f *.so *.a $(OBJS)
src/modules/lua/engine_lua.c
+29 -4
View File
@@ -480,9 +480,20 @@ static void luaEngineDebuggerEnd(ValkeyModuleCtx *module_ctx,
static struct luaEngineCtx *engine_ctx = NULL;
int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx,
ValkeyModuleString **argv,
int argc) {
#if STATIC_LUA
/*
* When building Lua as a static library, hide the generic module entry
* points, ValkeyModule_OnLoad and ValkeyModule_OnLoad, to avoid multiple
* symbol definitions. This is done by declaring these functions as static.
*/
#define LUA_MODULE_VISIBILITY static
#else
#define LUA_MODULE_VISIBILITY
#endif
LUA_MODULE_VISIBILITY int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx,
ValkeyModuleString **argv,
int argc) {
VALKEYMODULE_NOT_USED(argv);
VALKEYMODULE_NOT_USED(argc);
@@ -533,7 +544,8 @@ int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx,
return VALKEYMODULE_OK;
}
int ValkeyModule_OnUnload(ValkeyModuleCtx *ctx) {
LUA_MODULE_VISIBILITY int ValkeyModule_OnUnload(ValkeyModuleCtx *ctx) {
if (ValkeyModule_UnregisterScriptingEngine(ctx, LUA_ENGINE_NAME) != VALKEYMODULE_OK) {
ValkeyModule_Log(ctx, "error", "Failed to unregister engine");
return VALKEYMODULE_ERR;
@@ -544,3 +556,16 @@ int ValkeyModule_OnUnload(ValkeyModuleCtx *ctx) {
return VALKEYMODULE_OK;
}
#if STATIC_LUA
/* Unique entry points (Load and Unload) used by the Lua module when linked statically */
int ValkeyModule_OnLoad_lua(ValkeyModuleCtx *ctx,
ValkeyModuleString **argv,
int argc) {
return ValkeyModule_OnLoad(ctx, argv, argc);
}
int ValkeyModule_OnUnload_lua(ValkeyModuleCtx *ctx) {
return ValkeyModule_OnUnload(ctx);
}
#endif
src/modules/lua/function_lua.c
+10 -6
View File
@@ -55,7 +55,11 @@
typedef uint64_t monotime;
static monotime getMonotonicUs(void) {
#if STATIC_LUA
/* Use the engine's version */
#include "../../monotonic.h"
#else
monotime getMonotonicUs(void) {
/* clock_gettime() is specified in POSIX.1b (1993). Even so, some systems
* did not support this until much later. CLOCK_MONOTONIC is technically
* optional and may not be supported - but it appears to be universal.
@@ -64,14 +68,14 @@ static monotime getMonotonicUs(void) {
clock_gettime(CLOCK_MONOTONIC, &ts);
return ((uint64_t)ts.tv_sec) * 1000000 + ts.tv_nsec / 1000;
}
static inline uint64_t elapsedUs(monotime start_time) {
inline uint64_t elapsedUs(monotime start_time) {
return getMonotonicUs() - start_time;
}
static inline uint64_t elapsedMs(monotime start_time) {
inline uint64_t elapsedMs(monotime start_time) {
return elapsedUs(start_time) / 1000;
}
#endif
typedef struct loadCtx {
List *functions;
@@ -213,7 +217,7 @@ typedef struct flagStr {
const char *str;
} flagStr;
flagStr scripts_flags_def[] = {
static flagStr lua_scripts_flags[] = {
{.flag = VMSE_SCRIPT_FLAG_NO_WRITES, .str = "no-writes"},
{.flag = VMSE_SCRIPT_FLAG_ALLOW_OOM, .str = "allow-oom"},
{.flag = VMSE_SCRIPT_FLAG_ALLOW_STALE, .str = "allow-stale"},
@@ -244,7 +248,7 @@ static int luaRegisterFunctionReadFlags(lua_State *lua, uint64_t *flags) {
const char *flag_str = lua_tostring(lua, -1);
int found = 0;
for (flagStr *flag = scripts_flags_def; flag->str; ++flag) {
for (flagStr *flag = lua_scripts_flags; flag->str; ++flag) {
if (!strcasecmp(flag->str, flag_str)) {
f_flags |= flag->flag;
found = 1;
src/modules/lua/script_lua.c
+1 -21
View File
@@ -176,26 +176,6 @@ static void _serverPanic(const char *file, int line, const char *msg, ...) {
#define serverPanic(...) _serverPanic(__FILE__, __LINE__, __VA_ARGS__)
typedef uint64_t monotime;
monotime getMonotonicUs(void) {
/* clock_gettime() is specified in POSIX.1b (1993). Even so, some systems
* did not support this until much later. CLOCK_MONOTONIC is technically
* optional and may not be supported - but it appears to be universal.
* If this is not supported, provide a system-specific alternate version. */
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return ((uint64_t)ts.tv_sec) * 1000000 + ts.tv_nsec / 1000;
}
inline uint64_t elapsedUs(monotime start_time) {
return getMonotonicUs() - start_time;
}
inline uint64_t elapsedMs(monotime start_time) {
return elapsedUs(start_time) / 1000;
}
static int server_math_random(lua_State *L);
static int server_math_randomseed(lua_State *L);
@@ -1280,7 +1260,7 @@ static int luaRedisPCallCommand(lua_State *lua) {
*
* 'digest' should point to a 41 bytes buffer: 40 for SHA1 converted into an
* hexadecimal number, plus 1 byte for null term. */
void sha1hex(char *digest, char *script, size_t len) {
__attribute__((weak)) void sha1hex(char *digest, char *script, size_t len) {
SHA1_CTX ctx;
unsigned char hash[20];
char *cset = "0123456789abcdef";
src/server.c
+3 -5
View File
@@ -7697,17 +7697,15 @@ __attribute__((weak)) int main(int argc, char **argv) {
clusterInitLast();
}
/* Initialize the LUA scripting engine. */
#ifdef LUA_ENABLED
#define LUA_LIB_STR STRINGIFY(LUA_LIB)
#if defined(LUA_ENABLED) && STATIC_LUA
/* Initialize the LUA scripting engine on-startup only when LUA is built statically */
if (scriptingEngineManagerFind("lua") == NULL) {
if (moduleLoad(LUA_LIB_STR, NULL, 0, 0) != C_OK) {
if (moduleLoadStatic("lua", NULL, 0, 0) != C_OK) {
serverPanic("Lua engine initialization failed, check the server logs.");
}
}
#endif
InitServerLast();
if (!server.sentinel_mode) {