From 222e486a4b6a53c70d164ee6ec2f6c25e701b189 Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Wed, 11 Mar 2026 05:10:32 -0700 Subject: [PATCH 01/18] Major lljson rework Of note, there's now a notion of replacer and reviver callbacks as in JS' `JSON` APIs. An example of their use is in the tests folder as `lljson_typedjson.lua`. We went with this form since it allows constructing a different representation of the object before serializing without requiring you to construct an entire, serializable copy before calling `lljson.encode()`. That allows you to save memory, since the serializable version of each object only need to be alive as long as we're still traversing the object. Additionally, an empty table is now encoded as `[]` by default. This is probably the most common meaning for an empty table, but you can also apply `object_mt` as a metatable or add `__jsontype="object"` to your own metatable to force serialization as an object. Similarly, `array_mt` or `__jsontype="array"` will act as a hint to treat your object as an array. `__len` should no longer be used as a hint that the object should be treated as an array, that's what `__jsontype` is for. Also added a new options table format to `lljson.encode()` and friends. The table now allows you to specify that `__tojson` hooks should be skipped, so you can manually invoke them at your leisure in your replacer hooks. --- VM/include/lljson.h | 3 +- VM/src/cjson/lua_cjson.cpp | 1277 ++++++++--------- tests/SLConformance.test.cpp | 98 +- tests/conformance/lljson.lua | 431 ++++-- tests/conformance/lljson_replacer.lua | 559 ++++++++ tests/conformance/lljson_typedjson.lua | 395 +++++ .../metamethod_and_callback_interrupts.lua | 2 +- tests/conformance/sl_ares.lua | 2 - 8 files changed, 1948 insertions(+), 819 deletions(-) create mode 100644 tests/conformance/lljson_replacer.lua create mode 100644 tests/conformance/lljson_typedjson.lua diff --git a/VM/include/lljson.h b/VM/include/lljson.h index 5d932f70..60fcf2c7 100644 --- a/VM/include/lljson.h +++ b/VM/include/lljson.h @@ -3,7 +3,8 @@ #define UTAG_JSON_CONFIG 27 #define JSON_NULL ((void *)3) -#define JSON_EMPTY_ARRAY ((void *)4) #define JSON_ARRAY ((void *)5) +#define JSON_REMOVE ((void *)6) +#define JSON_OBJECT ((void *)7) constexpr int LU_TAG_JSON_INTERNAL = 60; diff --git a/VM/src/cjson/lua_cjson.cpp b/VM/src/cjson/lua_cjson.cpp index 6bce34f9..c05e192c 100644 --- a/VM/src/cjson/lua_cjson.cpp +++ b/VM/src/cjson/lua_cjson.cpp @@ -61,10 +61,11 @@ typedef lua_YieldSafeStrBuf strbuf_t; #include "../lapi.h" #include "../lstate.h" #include "../ltable.h" +#include "../lstring.h" // ServerLua: yieldable infrastructure for encode/decode #include "lyieldablemacros.h" -// ServerLua: Shims restoring original cjson strbuf API — captures `l` from enclosing scope +// ServerLua: Shims restoring original cjson strbuf API - captures `l` from enclosing scope #define strbuf_init(s, len) luaYB_init(l, (s), (len)) #define strbuf_free(s) luaYB_free(l, (s)) #define strbuf_resize(s, len) luaYB_resize(l, (s), (len)) @@ -89,17 +90,7 @@ typedef lua_YieldSafeStrBuf strbuf_t; // ServerLua: internal configuration #define CJSON_MODNAME "lljson" -// Ehhh, close enough. Luau is sort of a mutant Lua 5.1 with extras. -#define LUA_VERSION_NUM 501 - - -#ifndef CJSON_MODNAME -#define CJSON_MODNAME "cjson" -#endif - -#ifndef CJSON_VERSION #define CJSON_VERSION "2.1.0.11" -#endif #ifdef _MSC_VER #define snprintf sprintf_s @@ -109,19 +100,8 @@ typedef lua_YieldSafeStrBuf strbuf_t; #define isnan(x) _isnan(x) #endif -#endif - -#ifdef _MSC_VER -#define CJSON_EXPORT __declspec(dllexport) #define strncasecmp(x,y,z) _strnicmp(x,y,z) #define strcasecmp _stricmp -#else -#define CJSON_EXPORT extern -#endif - -/* Workaround for Solaris platforms missing isinf() */ -#if !defined(isinf) && (defined(USE_INTERNAL_ISINF) || defined(MISSING_ISINF)) -#define isinf(x) (!isnan(x) && isnan((x) - (x))) #endif #ifdef __clang__ @@ -130,37 +110,30 @@ typedef lua_YieldSafeStrBuf strbuf_t; #endif #endif -#define DEFAULT_SPARSE_CONVERT 0 -#define DEFAULT_SPARSE_RATIO 2 -#define DEFAULT_SPARSE_SAFE 10 -#define DEFAULT_ENCODE_MAX_DEPTH 100 -#define DEFAULT_DECODE_MAX_DEPTH 100 -#define DEFAULT_ENCODE_INVALID_NUMBERS 1 -#define DEFAULT_DECODE_INVALID_NUMBERS 1 -#define DEFAULT_ENCODE_NUMBER_PRECISION 14 -#define DEFAULT_ENCODE_EMPTY_TABLE_AS_OBJECT 1 -#define DEFAULT_DECODE_ARRAY_WITH_ARRAY_MT 0 -#define DEFAULT_ENCODE_ESCAPE_FORWARD_SLASH 0 -#define DEFAULT_ENCODE_SKIP_UNSUPPORTED_VALUE_TYPES 0 +#define JSON_SPARSE_RATIO 2 +#define JSON_SPARSE_SAFE 10 +#define JSON_MAX_DEPTH 100 +#define JSON_NUMBER_PRECISION 14 #define DEFAULT_MAX_SIZE 60000 -#ifdef DISABLE_INVALID_NUMBERS -#undef DEFAULT_DECODE_INVALID_NUMBERS -#define DEFAULT_DECODE_INVALID_NUMBERS 0 -#endif - -#if LONG_MAX > ((1UL << 31) - 1) && FALSE -// This is unnecessary because we correctly tag lightuserdata under Luau -#define json_lightudata_mask(ludata) \ - ((void *) ((uintptr_t) (ludata) & ((1UL << 47) - 1))) - -#else -#define json_lightudata_mask(ludata) (ludata) -#endif +// ServerLua: Fixed Lua stack positions for encode/decode. +// SlotManager's opaque state always occupies position 1. +enum class EncodeStack +{ + OPAQUE = 1, + STRBUF = 2, + CTX = 3, + REPLACER = 4, + VALUE = 5, +}; -#if LUA_VERSION_NUM >= 502 -#define lua_objlen(L,i) luaL_len(L, (i)) -#endif +enum class DecodeStack +{ + OPAQUE = 1, + STRBUF = 2, + INPUT = 3, + REVIVER = 4, +}; typedef enum { T_OBJ_BEGIN, @@ -256,20 +229,11 @@ static void json_init_lookup_tables() } struct json_config_t { - int encode_sparse_convert = DEFAULT_SPARSE_CONVERT; - int encode_sparse_ratio = DEFAULT_SPARSE_RATIO; - int encode_sparse_safe = DEFAULT_SPARSE_SAFE; - int encode_max_depth = DEFAULT_ENCODE_MAX_DEPTH; - int encode_invalid_numbers = DEFAULT_ENCODE_INVALID_NUMBERS; - int encode_number_precision = DEFAULT_ENCODE_NUMBER_PRECISION; - int encode_empty_table_as_object = DEFAULT_ENCODE_EMPTY_TABLE_AS_OBJECT; - int encode_escape_forward_slash = DEFAULT_ENCODE_ESCAPE_FORWARD_SLASH; - int decode_invalid_numbers = DEFAULT_DECODE_INVALID_NUMBERS; - int decode_max_depth = DEFAULT_DECODE_MAX_DEPTH; - int decode_array_with_array_mt = DEFAULT_DECODE_ARRAY_WITH_ARRAY_MT; - int encode_skip_unsupported_value_types = DEFAULT_ENCODE_SKIP_UNSUPPORTED_VALUE_TYPES; - bool sl_tagged_types = false; // When true, use !v, !q, !u, !i, !f tagging - bool sl_tight_encoding = false; // When true, use compact format (no brackets, base64 UUIDs) + int encode_sparse_convert = 0; + bool sl_tagged_types = false; + bool sl_tight_encoding = false; + bool has_replacer = false; + bool skip_tojson = false; }; typedef struct { @@ -279,6 +243,7 @@ typedef struct { lua_YieldSafeStrBuf *tmp; json_config_t *cfg; int current_depth; + bool has_reviver; // When true, a reviver function is on the decode stack } json_parse_t; typedef struct { @@ -303,7 +268,7 @@ static const char *char2escape[256] = { "\\u0018", "\\u0019", "\\u001a", "\\u001b", "\\u001c", "\\u001d", "\\u001e", "\\u001f", NULL, NULL, "\\\"", NULL, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, NULL, NULL, NULL, "\\/", + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, @@ -332,271 +297,6 @@ static const char *char2escape[256] = { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, }; -#if 0 // ServerLua: config functions unused, config is now stack-local RAII - -/* ===== CONFIGURATION ===== */ - -static json_config_t *json_fetch_config(lua_State *l) -{ - json_config_t *cfg; - - cfg = (json_config_t *)lua_touserdata(l, lua_upvalueindex(1)); - if (!cfg) - luaL_error(l, "BUG: Unable to fetch CJSON configuration"); - - return cfg; -} - -/* Ensure the correct number of arguments have been provided. - * Pad with nil to allow other functions to simply check arg[i] - * to find whether an argument was provided */ -static json_config_t *json_arg_init(lua_State *l, int args) -{ - luaL_argcheck(l, lua_gettop(l) <= args, args + 1, - "found too many arguments"); - - while (lua_gettop(l) < args) - lua_pushnil(l); - - return json_fetch_config(l); -} - -/* Process integer options for configuration functions */ -static int json_integer_option(lua_State *l, int optindex, int *setting, - int min, int max) -{ - char errmsg[64]; - int value; - - if (!lua_isnil(l, optindex)) { - value = luaL_checkinteger(l, optindex); - snprintf(errmsg, sizeof(errmsg), "expected integer between %d and %d", min, max); - luaL_argcheck(l, min <= value && value <= max, 1, errmsg); - *setting = value; - } - - lua_pushinteger(l, *setting); - - return 1; -} - -/* Process enumerated arguments for a configuration function */ -static int json_enum_option(lua_State *l, int optindex, int *setting, - const char **options, int bool_true) -{ - static const char *bool_options[] = { "off", "on", NULL }; - - if (!options) { - options = bool_options; - bool_true = 1; - } - - if (!lua_isnil(l, optindex)) { - if (bool_true && lua_isboolean(l, optindex)) - *setting = lua_toboolean(l, optindex) * bool_true; - else - *setting = luaL_checkoption(l, optindex, NULL, options); - } - - if (bool_true && (*setting == 0 || *setting == bool_true)) - lua_pushboolean(l, *setting); - else - lua_pushstring(l, options[*setting]); - - return 1; -} - -/* Configures handling of extremely sparse arrays: - * convert: Convert extremely sparse arrays into objects? Otherwise error. - * ratio: 0: always allow sparse; 1: never allow sparse; >1: use ratio - * safe: Always use an array when the max index <= safe */ -static int json_cfg_encode_sparse_array(lua_State *l) -{ - json_config_t *cfg = json_arg_init(l, 3); - - json_enum_option(l, 1, &cfg->encode_sparse_convert, NULL, 1); - json_integer_option(l, 2, &cfg->encode_sparse_ratio, 0, INT_MAX); - json_integer_option(l, 3, &cfg->encode_sparse_safe, 0, INT_MAX); - - return 3; -} - -/* Configures the maximum number of nested arrays/objects allowed when - * encoding */ -static int json_cfg_encode_max_depth(lua_State *l) -{ - json_config_t *cfg = json_arg_init(l, 1); - - return json_integer_option(l, 1, &cfg->encode_max_depth, 1, INT_MAX); -} - -/* Configures the maximum number of nested arrays/objects allowed when - * encoding */ -static int json_cfg_decode_max_depth(lua_State *l) -{ - json_config_t *cfg = json_arg_init(l, 1); - - return json_integer_option(l, 1, &cfg->decode_max_depth, 1, INT_MAX); -} - -/* Configures number precision when converting doubles to text */ -static int json_cfg_encode_number_precision(lua_State *l) -{ - json_config_t *cfg = json_arg_init(l, 1); - - return json_integer_option(l, 1, &cfg->encode_number_precision, 1, 16); -} - -/* Configures how to treat empty table when encode lua table */ -static int json_cfg_encode_empty_table_as_object(lua_State *l) -{ - json_config_t *cfg = json_arg_init(l, 1); - - return json_enum_option(l, 1, &cfg->encode_empty_table_as_object, NULL, 1); -} - -/* Configures how to decode arrays */ -static int json_cfg_decode_array_with_array_mt(lua_State *l) -{ - json_config_t *cfg = json_arg_init(l, 1); - - json_enum_option(l, 1, &cfg->decode_array_with_array_mt, NULL, 1); - - return 1; -} - -/* Configure how to treat invalid types */ -static int json_cfg_encode_skip_unsupported_value_types(lua_State *l) -{ - json_config_t *cfg = json_arg_init(l, 1); - - json_enum_option(l, 1, &cfg->encode_skip_unsupported_value_types, NULL, 1); - - return 1; -} - - -#if defined(DISABLE_INVALID_NUMBERS) && !defined(USE_INTERNAL_FPCONV) -void json_verify_invalid_number_setting(lua_State *l, int *setting) -{ - if (*setting == 1) { - *setting = 0; - luaL_error(l, "Infinity, NaN, and/or hexadecimal numbers are not supported."); - } -} -#else -#define json_verify_invalid_number_setting(l, s) do { } while(0) -#endif - -static int json_cfg_encode_invalid_numbers(lua_State *l) -{ - static const char *options[] = { "off", "on", "null", NULL }; - json_config_t *cfg = json_arg_init(l, 1); - - json_enum_option(l, 1, &cfg->encode_invalid_numbers, options, 1); - - json_verify_invalid_number_setting(l, &cfg->encode_invalid_numbers); - - return 1; -} - -static int json_cfg_decode_invalid_numbers(lua_State *l) -{ - json_config_t *cfg = json_arg_init(l, 1); - - json_enum_option(l, 1, &cfg->decode_invalid_numbers, NULL, 1); - - json_verify_invalid_number_setting(l, &cfg->encode_invalid_numbers); - - return 1; -} - -static int json_cfg_encode_escape_forward_slash(lua_State *l) -{ - int ret; - json_config_t *cfg = json_arg_init(l, 1); - - ret = json_enum_option(l, 1, &cfg->encode_escape_forward_slash, NULL, 1); - if (cfg->encode_escape_forward_slash) { - char2escape['/'] = "\\/"; - } else { - char2escape['/'] = NULL; - } - return ret; -} - -static void json_create_config(lua_State *l) -{ - json_config_t *cfg; - int i; - - cfg = (json_config_t *)lua_newuserdatatagged(l, sizeof(*cfg), UTAG_JSON_CONFIG); - if (!cfg) - abort(); - - memset(cfg, 0, sizeof(*cfg)); - - cfg->encode_sparse_convert = DEFAULT_SPARSE_CONVERT; - cfg->encode_sparse_ratio = DEFAULT_SPARSE_RATIO; - cfg->encode_sparse_safe = DEFAULT_SPARSE_SAFE; - cfg->encode_max_depth = DEFAULT_ENCODE_MAX_DEPTH; - cfg->decode_max_depth = DEFAULT_DECODE_MAX_DEPTH; - cfg->encode_invalid_numbers = DEFAULT_ENCODE_INVALID_NUMBERS; - cfg->decode_invalid_numbers = DEFAULT_DECODE_INVALID_NUMBERS; - cfg->encode_number_precision = DEFAULT_ENCODE_NUMBER_PRECISION; - cfg->encode_empty_table_as_object = DEFAULT_ENCODE_EMPTY_TABLE_AS_OBJECT; - cfg->decode_array_with_array_mt = DEFAULT_DECODE_ARRAY_WITH_ARRAY_MT; - cfg->encode_escape_forward_slash = DEFAULT_ENCODE_ESCAPE_FORWARD_SLASH; - cfg->encode_skip_unsupported_value_types = DEFAULT_ENCODE_SKIP_UNSUPPORTED_VALUE_TYPES; - - /* Decoding init */ - - /* Tag all characters as an error */ - for (i = 0; i < 256; i++) - cfg->ch2token[i] = T_ERROR; - - /* Set tokens that require no further processing */ - cfg->ch2token['{'] = T_OBJ_BEGIN; - cfg->ch2token['}'] = T_OBJ_END; - cfg->ch2token['['] = T_ARR_BEGIN; - cfg->ch2token[']'] = T_ARR_END; - cfg->ch2token[','] = T_COMMA; - cfg->ch2token[':'] = T_COLON; - cfg->ch2token['\0'] = T_END; - cfg->ch2token[' '] = T_WHITESPACE; - cfg->ch2token['\t'] = T_WHITESPACE; - cfg->ch2token['\n'] = T_WHITESPACE; - cfg->ch2token['\r'] = T_WHITESPACE; - - /* Update characters that require further processing */ - cfg->ch2token['f'] = T_UNKNOWN; /* false? */ - cfg->ch2token['i'] = T_UNKNOWN; /* inf, ininity? */ - cfg->ch2token['I'] = T_UNKNOWN; - cfg->ch2token['n'] = T_UNKNOWN; /* null, nan? */ - cfg->ch2token['N'] = T_UNKNOWN; - cfg->ch2token['t'] = T_UNKNOWN; /* true? */ - cfg->ch2token['"'] = T_UNKNOWN; /* string? */ - cfg->ch2token['+'] = T_UNKNOWN; /* number? */ - cfg->ch2token['-'] = T_UNKNOWN; - for (i = 0; i < 10; i++) - cfg->ch2token['0' + i] = T_UNKNOWN; - - /* Lookup table for parsing escape characters */ - for (i = 0; i < 256; i++) - cfg->escape2char[i] = 0; /* String error */ - cfg->escape2char['"'] = '"'; - cfg->escape2char['\\'] = '\\'; - cfg->escape2char['/'] = '/'; - cfg->escape2char['b'] = '\b'; - cfg->escape2char['t'] = '\t'; - cfg->escape2char['n'] = '\n'; - cfg->escape2char['f'] = '\f'; - cfg->escape2char['r'] = '\r'; - cfg->escape2char['u'] = 'u'; /* Unicode parsing required */ -} - -#endif // ServerLua: config functions unused - /* ===== ENCODING ===== */ static void json_encode_exception(lua_State *l, json_config_t *cfg, strbuf_t *json, int lindex, @@ -682,11 +382,11 @@ static int lua_array_length(lua_State *l, json_config_t *cfg, strbuf_t *json) while (lua_next(l, -2) != 0) { /* table, key, value */ if (lua_type(l, -2) == LUA_TNUMBER && - (k = lua_tonumber(l, -2))) { + (k = lua_tonumber(l, -2)) != 0.0) { /* Integer >= 1 and in int range? (floor(inf)==inf, so check upper bound) */ if (floor(k) == k && k >= 1 && k <= INT_MAX) { if (k > max) - max = k; + max = (int32_t)k; items++; lua_pop(l, 1); continue; @@ -699,9 +399,8 @@ static int lua_array_length(lua_State *l, json_config_t *cfg, strbuf_t *json) } /* Encode excessively sparse arrays as objects (if enabled) */ - if (cfg->encode_sparse_ratio > 0 && - max > items * cfg->encode_sparse_ratio && - max > cfg->encode_sparse_safe) { + if (max > items * JSON_SPARSE_RATIO && + max > JSON_SPARSE_SAFE) { if (!cfg->encode_sparse_convert) json_encode_exception(l, cfg, json, -1, "excessively sparse array"); @@ -724,7 +423,7 @@ static void json_check_encode_depth(lua_State *l, json_config_t *cfg, * * While this won't cause a crash due to the EXTRA_STACK reserve * slots, it would still be an improper use of the API. */ - if (current_depth <= cfg->encode_max_depth && lua_checkstack(l, 3)) + if (current_depth <= JSON_MAX_DEPTH && lua_checkstack(l, 7)) return; luaL_error(l, "Cannot serialise, excessive nesting (%d)", @@ -757,6 +456,10 @@ static void json_append_array(lua_State* l, SlotManager& parent_slots, DEFAULT = 0, ELEMENT = 1, NEXT_ELEMENT = 2, + REPLACER_CHECK = 3, + REPLACER_CALL = 4, + TOJSON_CHECK = 5, + TOJSON_CALL = 6, }; SlotManager slots(parent_slots); @@ -764,6 +467,7 @@ static void json_append_array(lua_State* l, SlotManager& parent_slots, DEFINE_SLOT(int32_t, i, 1); DEFINE_SLOT(int32_t, comma, 0); DEFINE_SLOT(int32_t, json_pos, 0); + DEFINE_SLOT(bool, replacer_removed, false); slots.finalize(); json = json_get_strbuf(l); @@ -774,12 +478,14 @@ static void json_append_array(lua_State* l, SlotManager& parent_slots, YIELD_DISPATCH_BEGIN(phase, slots); YIELD_DISPATCH(ELEMENT); YIELD_DISPATCH(NEXT_ELEMENT); + YIELD_DISPATCH(REPLACER_CHECK); + YIELD_DISPATCH(REPLACER_CALL); + YIELD_DISPATCH(TOJSON_CHECK); + YIELD_DISPATCH(TOJSON_CALL); YIELD_DISPATCH_END(); for (; i <= array_length; ++i) { - json_pos = strbuf_length(json); - if (comma++ > 0) - strbuf_append_char(json, ','); + replacer_removed = false; if (raw) { lua_rawgeti(l, -1, i); @@ -787,19 +493,54 @@ static void json_append_array(lua_State* l, SlotManager& parent_slots, lua_pushinteger(l, i); lua_gettable(l, -2); } + /* table, value */ + + if (cfg->has_replacer) { + // Resolve __tojson before replacer (JS compat: toJSON -> replacer) + if (!cfg->skip_tojson && lua_istable(l, -1) && luaL_getmetafield(l, -1, "__tojson")) { + // Stack: table, value, __tojson_fn + lua_pushvalue(l, -2); // self + lua_pushvalue(l, (int)EncodeStack::CTX); // ctx + YIELD_CHECK(l, TOJSON_CHECK, LUA_INTERRUPT_LLLIB); + YIELD_CALL(l, 2, 1, TOJSON_CALL); + // Stack: table, value, resolved + lua_remove(l, -2); // remove original value + // Stack: table, resolved + } - // Not a slot: assigned by YIELD_HELPER before read, no goto crosses this decl. - bool skip; - YIELD_HELPER(l, ELEMENT, - skip = (bool)json_append_data(l, slots, cfg, current_depth, json)); - if (skip) { - strbuf_set_length(json, json_pos); - if (comma == 1) { - comma = 0; + lua_pushvalue(l, (int)EncodeStack::REPLACER); + lua_pushinteger(l, i); // key (1-based index) + lua_pushvalue(l, -3); // value (possibly __tojson-resolved) + lua_pushvalue(l, -5); // parent (the array table) + YIELD_CHECK(l, REPLACER_CHECK, LUA_INTERRUPT_LLLIB); + YIELD_CALL(l, 3, 1, REPLACER_CALL); + // Stack: table, value, result + if (lua_tolightuserdatatagged(l, -1, LU_TAG_JSON_INTERNAL) == JSON_REMOVE) { + lua_pop(l, 2); // pop result + value -> table + replacer_removed = true; + } else { + lua_remove(l, -2); // remove original value -> table, result } } - lua_pop(l, 1); + if (!replacer_removed) { + json_pos = strbuf_length(json); + if (comma++ > 0) + strbuf_append_char(json, ','); + + // Not a slot: assigned by YIELD_HELPER before read, no goto crosses this decl. + bool skip; + YIELD_HELPER(l, ELEMENT, + skip = (bool)json_append_data(l, slots, cfg, current_depth, json)); + if (skip) { + strbuf_set_length(json, json_pos); + if (comma == 1) { + comma = 0; + } + } + + lua_pop(l, 1); + } YIELD_CHECK(l, NEXT_ELEMENT, LUA_INTERRUPT_LLLIB); } @@ -807,39 +548,23 @@ static void json_append_array(lua_State* l, SlotManager& parent_slots, strbuf_append_char(json, ']'); } +// ServerLua: forward declaration for NaN handling in json_append_number +static void json_append_tagged_float(lua_State *l, strbuf_t *json, double num, int precision); + static void json_append_number(lua_State *l, json_config_t *cfg, strbuf_t *json, int lindex) { int len; -#if LUA_VERSION_NUM >= 503 - if (lua_isinteger(l, lindex)) { - lua_Integer num = lua_tointeger(l, lindex); - strbuf_ensure_empty_length(json, FPCONV_G_FMT_BUFSIZE); /* max length of int64 is 19 */ - len = sprintf(strbuf_empty_ptr(json), LUA_INTEGER_FMT, num); - strbuf_extend_length(json, len); - return; - } -#endif double num = lua_tonumber(l, lindex); - if (cfg->encode_invalid_numbers == 0) { - /* Prevent encoding invalid numbers */ - if (isnan(num)) - json_encode_exception(l, cfg, json, lindex, - "must not be NaN"); - } else if (cfg->encode_invalid_numbers == 1) { - /* Encode NaN/Infinity separately to ensure Javascript compatible - * values are used. */ - if (isnan(num)) { - strbuf_append_mem(json, "NaN", 3); - return; - } - } else { - /* Encode invalid numbers as "null" */ - if ( isnan(num)) { + // NaN has no JSON representation. + // slencode: tagged float for round-trip. encode: null (matches JSON.stringify). + if (isnan(num)) { + if (cfg->sl_tagged_types) + json_append_tagged_float(l, json, num, JSON_NUMBER_PRECISION); + else strbuf_append_mem(json, "null", 4); - return; - } + return; } if (isinf(num)) { @@ -856,13 +581,16 @@ static void json_append_number(lua_State *l, json_config_t *cfg, } strbuf_ensure_empty_length(json, FPCONV_G_FMT_BUFSIZE); - len = fpconv_g_fmt(strbuf_empty_ptr(json), num, cfg->encode_number_precision); + len = fpconv_g_fmt(strbuf_empty_ptr(json), num, JSON_NUMBER_PRECISION); strbuf_extend_length(json, len); } +static const float DEFAULT_VECTOR[3] = {0.0f, 0.0f, 0.0f}; +static const float DEFAULT_QUATERNION[4] = {0.0f, 0.0f, 0.0f, 1.0f}; + static void json_append_coordinate_component(lua_State *l, strbuf_t *json, float val, bool tight = false) { - if (tight && val == 0.0f) - return; // Omit zeros in tight mode + if (tight && val == 0.0f && !signbit(val)) + return; // Omit positive zeros in tight mode char format_buf[256] = {}; // Use shared helper to ensure consistent normalization of non-finite values size_t str_len = luai_formatfloat(format_buf, sizeof(format_buf), "%.6f", val); @@ -870,9 +598,14 @@ static void json_append_coordinate_component(lua_State *l, strbuf_t *json, float strbuf_append_mem(json, format_buf, str_len); } -// Helper to append a tagged vector value: !v or tight: !v1,2,3 +// Helper to append a tagged vector value: !v or tight: !v1,,3 +// ZERO_VECTOR in tight mode -> "!v" static void json_append_tagged_vector(lua_State *l, strbuf_t *json, const float *a, bool tight = false) { strbuf_append_string(json, tight ? "\"!v" : "\"!v<"); + if (tight && memcmp(a, DEFAULT_VECTOR, sizeof(DEFAULT_VECTOR)) == 0) { + strbuf_append_char(json, '"'); + return; + } json_append_coordinate_component(l, json, a[0], tight); strbuf_append_char(json, ','); json_append_coordinate_component(l, json, a[1], tight); @@ -882,8 +615,13 @@ static void json_append_tagged_vector(lua_State *l, strbuf_t *json, const float } // Helper to append a tagged quaternion value: !q or tight: !q,,,1 +// ZERO_ROTATION (0,0,0,1) in tight mode -> "!q" static void json_append_tagged_quaternion(lua_State *l, strbuf_t *json, const float *a, bool tight = false) { strbuf_append_string(json, tight ? "\"!q" : "\"!q<"); + if (tight && memcmp(a, DEFAULT_QUATERNION, sizeof(DEFAULT_QUATERNION)) == 0) { + strbuf_append_char(json, '"'); + return; + } json_append_coordinate_component(l, json, a[0], tight); strbuf_append_char(json, ','); json_append_coordinate_component(l, json, a[1], tight); @@ -1042,12 +780,17 @@ static void json_append_object(lua_State* l, SlotManager& parent_slots, DEFAULT = 0, VALUE = 1, NEXT_PAIR = 2, + REPLACER_CHECK = 3, + REPLACER_CALL = 4, + TOJSON_CHECK = 5, + TOJSON_CALL = 6, }; SlotManager slots(parent_slots); DEFINE_SLOT(Phase, phase, Phase::DEFAULT); DEFINE_SLOT(int32_t, comma, 0); DEFINE_SLOT(int32_t, json_pos, 0); + DEFINE_SLOT(bool, replacer_removed, false); slots.finalize(); json = json_get_strbuf(l); @@ -1061,89 +804,125 @@ static void json_append_object(lua_State* l, SlotManager& parent_slots, YIELD_DISPATCH_BEGIN(phase, slots); YIELD_DISPATCH(VALUE); YIELD_DISPATCH(NEXT_PAIR); + YIELD_DISPATCH(REPLACER_CHECK); + YIELD_DISPATCH(REPLACER_CALL); + YIELD_DISPATCH(TOJSON_CHECK); + YIELD_DISPATCH(TOJSON_CALL); YIELD_DISPATCH_END(); /* table, startkey */ while (lua_next(l, -2) != 0) { - json_pos = strbuf_length(json); - if (comma++ > 0) - strbuf_append_char(json, ','); - /* table, key, value */ - int keytype; - keytype = lua_type(l, -2); - - if (cfg->sl_tagged_types) { - // SL tagged mode: accept any key type and tag appropriately - switch (keytype) { - case LUA_TSTRING: - json_append_string_sl(l, json, -2); - break; - case LUA_TNUMBER: - json_append_tagged_float(l, json, lua_tonumber(l, -2), cfg->encode_number_precision); - break; - case LUA_TVECTOR: { - const float* a = lua_tovector(l, -2); - json_append_tagged_vector(l, json, a, cfg->sl_tight_encoding); - break; + replacer_removed = false; + + if (cfg->has_replacer) { + // Resolve __tojson before replacer (JS compat: toJSON -> replacer) + if (!cfg->skip_tojson && lua_istable(l, -1) && luaL_getmetafield(l, -1, "__tojson")) { + // Stack: table, key, value, __tojson_fn + lua_pushvalue(l, -2); // self + lua_pushvalue(l, (int)EncodeStack::CTX); // ctx + YIELD_CHECK(l, TOJSON_CHECK, LUA_INTERRUPT_LLLIB); + YIELD_CALL(l, 2, 1, TOJSON_CALL); + // Stack: table, key, value, resolved + lua_remove(l, -2); // remove original value + // Stack: table, key, resolved } - case LUA_TBUFFER: { - strbuf_append_string(json, "\"!d"); - json_append_buffer(l, json, -2); - strbuf_append_char(json, '"'); - break; + + lua_pushvalue(l, (int)EncodeStack::REPLACER); + lua_pushvalue(l, -3); // key + lua_pushvalue(l, -3); // value (possibly __tojson-resolved) + lua_pushvalue(l, -6); // parent (the object table) + YIELD_CHECK(l, REPLACER_CHECK, LUA_INTERRUPT_LLLIB); + YIELD_CALL(l, 3, 1, REPLACER_CALL); + // Stack: table, key, value, result + if (lua_tolightuserdatatagged(l, -1, LU_TAG_JSON_INTERNAL) == JSON_REMOVE) { + lua_pop(l, 2); // pop result + value -> table, key + replacer_removed = true; + } else { + lua_remove(l, -2); // remove original value -> table, key, result } - case LUA_TUSERDATA: { - int tag = lua_userdatatag(l, -2); - if (tag == UTAG_UUID) { - json_append_tagged_uuid(l, json, -2, cfg->sl_tight_encoding); - } else if (tag == UTAG_QUATERNION) { - const float *a = luaSL_checkquaternion(l, -2); - json_append_tagged_quaternion(l, json, a, cfg->sl_tight_encoding); - } else { + } + + if (!replacer_removed) { + json_pos = strbuf_length(json); + if (comma++ > 0) + strbuf_append_char(json, ','); + + int keytype; + keytype = lua_type(l, -2); + + if (cfg->sl_tagged_types) { + // SL tagged mode: accept any key type and tag appropriately + switch (keytype) { + case LUA_TSTRING: + json_append_string_sl(l, json, -2); + break; + case LUA_TNUMBER: + json_append_tagged_float(l, json, lua_tonumber(l, -2), JSON_NUMBER_PRECISION); + break; + case LUA_TVECTOR: { + const float* a = lua_tovector(l, -2); + json_append_tagged_vector(l, json, a, cfg->sl_tight_encoding); + break; + } + case LUA_TBUFFER: { + strbuf_append_string(json, "\"!d"); + json_append_buffer(l, json, -2); + strbuf_append_char(json, '"'); + break; + } + case LUA_TUSERDATA: { + int tag = lua_userdatatag(l, -2); + if (tag == UTAG_UUID) { + json_append_tagged_uuid(l, json, -2, cfg->sl_tight_encoding); + } else if (tag == UTAG_QUATERNION) { + const float *a = luaSL_checkquaternion(l, -2); + json_append_tagged_quaternion(l, json, a, cfg->sl_tight_encoding); + } else { + json_encode_exception(l, cfg, json, -2, + "unsupported userdata type as table key"); + } + break; + } + case LUA_TBOOLEAN: + strbuf_append_string(json, lua_toboolean(l, -2) ? "\"!b1\"" : "\"!b0\""); + break; + default: json_encode_exception(l, cfg, json, -2, - "unsupported userdata type as table key"); + "unsupported table key type"); + /* never returns */ } - break; - } - case LUA_TBOOLEAN: - strbuf_append_string(json, lua_toboolean(l, -2) ? "\"!b1\"" : "\"!b0\""); - break; - default: - json_encode_exception(l, cfg, json, -2, - "unsupported table key type"); - /* never returns */ - } - strbuf_append_char(json, ':'); - } else { - // Standard JSON mode: only string and number keys - if (keytype == LUA_TNUMBER) { - strbuf_append_char(json, '"'); - json_append_number(l, cfg, json, -2); - strbuf_append_mem(json, "\":", 2); - } else if (keytype == LUA_TSTRING) { - json_append_string(l, json, -2); strbuf_append_char(json, ':'); } else { - json_encode_exception(l, cfg, json, -2, - "table key must be a number or string"); - /* never returns */ + // Standard JSON mode: only string and number keys + if (keytype == LUA_TNUMBER) { + strbuf_append_char(json, '"'); + json_append_number(l, cfg, json, -2); + strbuf_append_mem(json, "\":", 2); + } else if (keytype == LUA_TSTRING) { + json_append_string(l, json, -2); + strbuf_append_char(json, ':'); + } else { + json_encode_exception(l, cfg, json, -2, + "table key must be a number or string"); + /* never returns */ + } } - } - /* table, key, value */ - // Not a slot: assigned by YIELD_HELPER before read, no goto crosses this decl. - bool skip; - YIELD_HELPER(l, VALUE, - skip = (bool)json_append_data(l, slots, cfg, current_depth, json)); - if (skip) { - strbuf_set_length(json, json_pos); - if (comma == 1) { - comma = 0; + /* table, key, value */ + // Not a slot: assigned by YIELD_HELPER before read, no goto crosses this decl. + bool skip; + YIELD_HELPER(l, VALUE, + skip = (bool)json_append_data(l, slots, cfg, current_depth, json)); + if (skip) { + strbuf_set_length(json, json_pos); + if (comma == 1) { + comma = 0; + } } - } - lua_pop(l, 1); + lua_pop(l, 1); + } /* table, key */ YIELD_CHECK(l, NEXT_PAIR, LUA_INTERRUPT_LLLIB); @@ -1166,11 +945,11 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, TOJSON_CALL = 3, TOJSON_RECURSE = 4, APPEND_ARRAY_AUTO = 5, - APPEND_ARRAY_EMPTY = 6, - APPEND_ARRAY_LUD = 7, - TOJSON_CHECK = 8, - LEN_CHECK = 9, - LEN_CALL = 10, + APPEND_ARRAY_LUD = 6, + TOJSON_CHECK = 7, + LEN_CHECK = 8, + LEN_CALL = 9, + APPEND_OBJECT_MT = 10, }; SlotManager slots(parent_slots); @@ -1180,9 +959,9 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, DEFINE_SLOT(bool, raw, true); DEFINE_SLOT(uint8_t, type, LUA_TNIL); DEFINE_SLOT(bool, as_array, false); + DEFINE_SLOT(bool, force_object, false); DEFINE_SLOT(bool, has_metatable, false); DEFINE_SLOT(int32_t, len, 0); - DEFINE_SLOT(bool, is_empty_array, false); slots.finalize(); json = json_get_strbuf(l); @@ -1194,10 +973,10 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, YIELD_DISPATCH(TOJSON_CALL); YIELD_DISPATCH(TOJSON_RECURSE); YIELD_DISPATCH(APPEND_ARRAY_AUTO); - YIELD_DISPATCH(APPEND_ARRAY_EMPTY); YIELD_DISPATCH(APPEND_ARRAY_LUD); YIELD_DISPATCH(LEN_CHECK); YIELD_DISPATCH(LEN_CALL); + YIELD_DISPATCH(APPEND_OBJECT_MT); YIELD_DISPATCH_END(); type = lua_type(l, -1); @@ -1227,32 +1006,68 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, has_metatable = lua_getmetatable(l, -1); if (has_metatable) { - lua_pushlightuserdatatagged(l, json_lightudata_mask(JSON_ARRAY), LU_TAG_JSON_INTERNAL); - lua_rawget(l, LUA_REGISTRYINDEX); - as_array = lua_rawequal(l, -1, -2); - if (as_array) { - raw = true; - lua_pop(l, 2); - array_length = lua_objlen(l, -1); - } else { - raw = false; - lua_pop(l, 2); - if (luaL_getmetafield(l, -1, "__tojson")) { - lua_pushvalue(l, -2); - YIELD_CHECK(l, TOJSON_CHECK, LUA_INTERRUPT_LLLIB); - YIELD_CALL(l, 1, 1, TOJSON_CALL); + if (!cfg->sl_tagged_types) { + // ServerLua: Check __jsontype metamethod for shape control + lua_rawgetfield(l, -1, "__jsontype"); + if (!lua_isnil(l, -1)) { + if (!lua_isstring(l, -1)) + luaL_error(l, "invalid __jsontype value (expected string)"); + const char* jsontype; + jsontype = lua_tostring(l, -1); + if (strcmp(jsontype, "object") == 0) { + force_object = true; + } else if (strcmp(jsontype, "array") == 0) { + as_array = true; + raw = false; + } else { + luaL_error(l, "invalid __jsontype value: '%s' (expected \"array\" or \"object\")", jsontype); + } + } + lua_pop(l, 1); // pop __jsontype (or nil) + } + + lua_pop(l, 1); // pop metatable + + // __tojson provides content, __jsontype provides shape + if (!cfg->skip_tojson && luaL_getmetafield(l, -1, "__tojson")) { + lua_pushvalue(l, -2); // self + lua_pushvalue(l, (int)EncodeStack::CTX); // ctx table + YIELD_CHECK(l, TOJSON_CHECK, LUA_INTERRUPT_LLLIB); + YIELD_CALL(l, 2, 1, TOJSON_CALL); + // Stack: ..., original_table, tojson_result + if (lua_istable(l, -1)) { + // Table result: replace original, fall through to shape handling + lua_remove(l, -2); + } else { + // Non-table result: encode directly, shape hints don't apply YIELD_HELPER(l, TOJSON_RECURSE, json_append_data(l, slots, cfg, depth, json)); lua_pop(l, 1); return 0; } + } + + if (force_object) { + YIELD_HELPER(l, APPEND_OBJECT_MT, + json_append_object(l, slots, cfg, depth, json)); + break; + } + + if (as_array) { + // Validate: __jsontype="array" requires all keys to be positive integers + len = lua_array_length(l, cfg, json); + if (len < 0) + luaL_error(l, "cannot encode as array: table has non-integer keys"); + + // __len overrides the detected length (validation already passed) if (luaL_getmetafield(l, -1, "__len")) { lua_pushvalue(l, -2); YIELD_CHECK(l, LEN_CHECK, LUA_INTERRUPT_LLLIB); YIELD_CALL(l, 1, 1, LEN_CALL); array_length = lua_tonumber(l, -1); lua_pop(l, 1); - as_array = true; + } else { + array_length = len; } } } @@ -1263,25 +1078,11 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, } else { len = lua_array_length(l, cfg, json); - if (len > 0 || (len == 0 && !cfg->encode_empty_table_as_object)) { + if (len >= 0) { array_length = len; YIELD_HELPER(l, APPEND_ARRAY_AUTO, json_append_array(l, slots, cfg, depth, json, array_length, raw)); } else { - if (has_metatable) { - lua_getmetatable(l, -1); - lua_pushlightuserdatatagged(l, json_lightudata_mask(JSON_EMPTY_ARRAY), LU_TAG_JSON_INTERNAL); - lua_rawget(l, LUA_REGISTRYINDEX); - is_empty_array = lua_rawequal(l, -1, -2); - lua_pop(l, 2); /* pop pointer + metatable */ - if (is_empty_array) { - array_length = lua_objlen(l, -1); - raw = true; - YIELD_HELPER(l, APPEND_ARRAY_EMPTY, - json_append_array(l, slots, cfg, depth, json, array_length, raw)); - return 0; - } - } YIELD_HELPER(l, APPEND_OBJECT, json_append_object(l, slots, cfg, depth, json)); } @@ -1289,16 +1090,19 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, break; } case LUA_TNIL: - strbuf_append_mem(json, "null", 4); + if (cfg->sl_tagged_types) + strbuf_append_mem(json, "\"!n\"", 4); + else + strbuf_append_mem(json, "null", 4); break; case LUA_TLIGHTUSERDATA: { void* json_internal_val; json_internal_val = lua_tolightuserdatatagged(l, -1, LU_TAG_JSON_INTERNAL); if (json_internal_val) { - if (json_internal_val == json_lightudata_mask(JSON_NULL)) { + if (json_internal_val == JSON_NULL) { strbuf_append_mem(json, "null", 4); break; - } else if (json_internal_val == json_lightudata_mask(JSON_ARRAY)) { + } else if (json_internal_val == JSON_ARRAY) { YIELD_HELPER(l, APPEND_ARRAY_LUD, json_append_array(l, slots, cfg, depth, json, 0, 1)); break; @@ -1309,11 +1113,7 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, break; } - if (cfg->encode_skip_unsupported_value_types) { - return 1; - } else { - json_encode_exception(l, cfg, json, -1, "type not supported"); - } + json_encode_exception(l, cfg, json, -1, "type not supported"); // Should never reach here. break; } @@ -1364,8 +1164,6 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, json_append_coordinate_component(l, json, a[3]); strbuf_append_string(json, ">\""); } - } else if (cfg->encode_skip_unsupported_value_types) { - return 1; } else { json_encode_exception(l, cfg, json, -1, "type not supported"); } @@ -1373,12 +1171,7 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, } default: /* Remaining types (LUA_TFUNCTION, LUA_TTHREAD) cannot be serialised */ - if (cfg->encode_skip_unsupported_value_types) { - return 1; - } else { - json_encode_exception(l, cfg, json, -1, "type not supported"); - } - + json_encode_exception(l, cfg, json, -1, "type not supported"); /* never returns */ } return 0; @@ -1393,14 +1186,20 @@ static int json_encode_common(lua_State* l, bool is_init, bool sl_tagged) { DEFAULT = 0, APPEND_DATA = 1, + ROOT_TOJSON_CHECK = 2, + ROOT_TOJSON_CALL = 3, + ROOT_REPLACER_CHECK = 4, + ROOT_REPLACER_CALL = 5, }; SlotManager slots(l, is_init); DEFINE_SLOT(Phase, phase, Phase::DEFAULT); DEFINE_SLOT(bool, tight_encoding, false); + DEFINE_SLOT(bool, skip_tojson, false); slots.finalize(); json_config_t cfg; + cfg.skip_tojson = skip_tojson; if (sl_tagged) { cfg.sl_tagged_types = true; cfg.encode_sparse_convert = 1; @@ -1410,30 +1209,101 @@ static int json_encode_common(lua_State* l, bool is_init, bool sl_tagged) if (is_init) { // Args already validated by init wrapper. // SlotManager inserted nil at pos 1, original args shifted to pos 2+. - if (sl_tagged) { - tight_encoding = luaL_optboolean(l, 3, false); - cfg.sl_tight_encoding = tight_encoding; - lua_settop(l, 2); + // Stack: [opaque(1), value(2), opts_table?(3)] + + // Extract all options from the table while it's at position 3. + // The table is never read again after init. + if (lua_istable(l, 3)) { + if (sl_tagged) { + lua_rawgetfield(l, 3, "tight"); + tight_encoding = lua_toboolean(l, -1); + cfg.sl_tight_encoding = tight_encoding; + lua_pop(l, 1); + } + lua_rawgetfield(l, 3, "skip_tojson"); + skip_tojson = lua_toboolean(l, -1); + cfg.skip_tojson = skip_tojson; + lua_pop(l, 1); + lua_rawgetfield(l, 3, "replacer"); + if (!lua_isfunction(l, -1)) { + lua_pop(l, 1); + lua_pushnil(l); + } + // Stack: [opaque(1), value(2), opts(3), replacer_or_nil(4)] + lua_remove(l, 3); + } else { + lua_pushnil(l); } + // Stack: [opaque(1), value(2), replacer_or_nil(3)] luaYB_push(l); - lua_insert(l, 2); - /* Stack: [opaque(1), strbuf(2), value(3)] */ + lua_insert(l, (int)EncodeStack::STRBUF); + // Stack: [opaque(1), strbuf(2), value(3), replacer_or_nil(4)] + + // Create frozen context table for __tojson(self, ctx) + lua_newtable(l); + lua_pushstring(l, sl_tagged ? "sljson" : "json"); + lua_rawsetfield(l, -2, "mode"); + lua_pushboolean(l, tight_encoding); + lua_rawsetfield(l, -2, "tight"); + lua_setreadonly(l, -1, true); + lua_insert(l, (int)EncodeStack::CTX); + // Stack: [opaque(1), strbuf(2), ctx(3), value(4), replacer_or_nil(5)] + + // Insert replacer at its slot, pushing value to the top + lua_insert(l, (int)EncodeStack::REPLACER); + /* Stack: [opaque(1), strbuf(2), ctx(3), replacer_or_nil(4), value(5)] */ lua_hardenstack(l, 1); } + cfg.has_replacer = !lua_isnil(l, (int)EncodeStack::REPLACER); strbuf_t* buf = json_get_strbuf(l); YIELD_DISPATCH_BEGIN(phase, slots); YIELD_DISPATCH(APPEND_DATA); + YIELD_DISPATCH(ROOT_TOJSON_CHECK); + YIELD_DISPATCH(ROOT_TOJSON_CALL); + YIELD_DISPATCH(ROOT_REPLACER_CHECK); + YIELD_DISPATCH(ROOT_REPLACER_CALL); YIELD_DISPATCH_END(); + // Resolve __tojson on root value before replacer (JS compat: toJSON -> replacer) + lua_checkstack(l, 4); + if (cfg.has_replacer && !cfg.skip_tojson && lua_istable(l, (int)EncodeStack::VALUE) + && luaL_getmetafield(l, (int)EncodeStack::VALUE, "__tojson")) { + lua_pushvalue(l, (int)EncodeStack::VALUE); // self + lua_pushvalue(l, (int)EncodeStack::CTX); // ctx + YIELD_CHECK(l, ROOT_TOJSON_CHECK, LUA_INTERRUPT_LLLIB); + YIELD_CALL(l, 2, 1, ROOT_TOJSON_CALL); + // Stack: [..., resolved] + lua_replace(l, (int)EncodeStack::VALUE); + } + + // Call replacer on root value: replacer(nil, value, nil) + if (cfg.has_replacer) { + lua_pushvalue(l, (int)EncodeStack::REPLACER); + lua_pushnil(l); // key = nil (root) + lua_pushvalue(l, (int)EncodeStack::VALUE); // value (possibly __tojson-resolved) + lua_pushnil(l); // parent = nil (root) + YIELD_CHECK(l, ROOT_REPLACER_CHECK, LUA_INTERRUPT_LLLIB); + YIELD_CALL(l, 3, 1, ROOT_REPLACER_CALL); + // Stack: [..., result] + if (lua_tolightuserdatatagged(l, -1, LU_TAG_JSON_INTERNAL) == JSON_REMOVE) { + lua_pop(l, 1); + // Replace value with JSON null + lua_pushlightuserdatatagged(l, JSON_NULL, LU_TAG_JSON_INTERNAL); + } + lua_replace(l, (int)EncodeStack::VALUE); + } + + lua_pushvalue(l, (int)EncodeStack::VALUE); + lua_hardenstack(l, 1); YIELD_HELPER(l, APPEND_DATA, json_append_data(l, slots, &cfg, 0, buf)); - strbuf_tostring_inplace(2, true); - lua_settop(l, 2); + lua_settop(l, (int)EncodeStack::STRBUF); + strbuf_tostring_inplace((int)EncodeStack::STRBUF, true); luau_interruptoncalltail(l); return 1; } @@ -1441,7 +1311,10 @@ static int json_encode_common(lua_State* l, bool is_init, bool sl_tagged) // ServerLua: init / continuation wrappers for json_encode static int json_encode_v0(lua_State* l) { - luaL_argcheck(l, lua_gettop(l) == 1, 1, "expected 1 argument"); + int nargs = lua_gettop(l); + luaL_argcheck(l, nargs >= 1 && nargs <= 2, 1, "expected 1-2 arguments"); + if (nargs >= 2) + luaL_checktype(l, 2, LUA_TTABLE); return json_encode_common(l, true, false); } static int json_encode_v0_k(lua_State* l, int) @@ -1451,7 +1324,10 @@ static int json_encode_v0_k(lua_State* l, int) } static int json_encode_sl_v0(lua_State* l) { + int nargs = lua_gettop(l); luaL_checkany(l, 1); + if (nargs >= 2) + luaL_checktype(l, 2, LUA_TTABLE); return json_encode_common(l, true, true); } static int json_encode_sl_v0_k(lua_State* l, int) @@ -1522,16 +1398,29 @@ static bool json_parse_tagged_string(lua_State *l, const char *str, size_t len) size_t payload_len = len - 2; switch (tag) { + case 'n': + // Nil: !n + if (payload_len != 0) + luaL_error(l, "malformed tagged nil: %s", str); + lua_pushnil(l); + return true; + case '!': // Escaped '!' - push string with leading '!' stripped lua_pushlstring(l, str + 1, len - 1); return true; case 'v': { - // Vector: !v (normal) or !v1,2,3 (tight) + // Vector: !v (normal) or !v1,2,3 (tight) or !v (ZERO_VECTOR) float x, y, z; - if (payload_len > 0 && payload[0] == '<') { + if (payload_len == 0) { + // ZERO_VECTOR shorthand + lua_pushvector(l, 0.0f, 0.0f, 0.0f); + return true; + } + + if (payload[0] == '<') { // Normal format with brackets if (payload_len < 5 || payload[payload_len - 1] != '>') luaL_error(l, "malformed tagged vector: %s", str); @@ -1565,10 +1454,16 @@ static bool json_parse_tagged_string(lua_State *l, const char *str, size_t len) } case 'q': { - // Quaternion: !q (normal) or !q,,,1 (tight) + // Quaternion: !q (normal) or !q,,,1 (tight) or !q (ZERO_ROTATION) float x, y, z, w; - if (payload_len > 0 && payload[0] == '<') { + if (payload_len == 0) { + // ZERO_ROTATION shorthand (0,0,0,1) + luaSL_pushquaternion(l, 0.0f, 0.0f, 0.0f, 1.0f); + return true; + } + + if (payload[0] == '<') { // Normal format with brackets if (payload_len < 7 || payload[payload_len - 1] != '>') luaL_error(l, "malformed tagged quaternion: %s", str); @@ -1995,10 +1890,6 @@ static void json_next_token(json_parse_t *json, json_token_t *token) json_next_string_token(json, token); return; } else if (ch == '-' || ('0' <= ch && ch <= '9')) { - if (!json->cfg->decode_invalid_numbers && json_is_invalid_number(json)) { - json_set_token_error(token, json, "invalid number"); - return; - } json_next_number_token(json, token); return; } else if (!strncmp(json->ptr, "true", 4)) { @@ -2015,13 +1906,10 @@ static void json_next_token(json_parse_t *json, json_token_t *token) token->type = T_NULL; json->ptr += 4; return; - } else if (json->cfg->decode_invalid_numbers && - json_is_invalid_number(json)) { - /* When decode_invalid_numbers is enabled, only attempt to process - * numbers we know are invalid JSON (Inf, NaN, hex) + } else if (json_is_invalid_number(json)) { + /* Only attempt to process numbers we know are invalid JSON (Inf, NaN, hex). * This is required to generate an appropriate token error, - * otherwise all bad tokens will register as "invalid number" - */ + * otherwise all bad tokens will register as "invalid number" */ json_next_number_token(json, token); return; } @@ -2060,7 +1948,7 @@ static void json_decode_descend(lua_State *l, json_parse_t *json, int slots) { json->current_depth++; - if (json->current_depth <= json->cfg->decode_max_depth && + if (json->current_depth <= JSON_MAX_DEPTH && lua_checkstack(l, slots)) { return; } @@ -2112,6 +2000,8 @@ static void json_parse_object_context(lua_State* l, SlotManager& parent_slots, j DEFAULT = 0, VALUE = 1, NEXT_PAIR = 2, + REVIVER_CHECK = 3, + REVIVER_CALL = 4, }; SlotManager slots(parent_slots); @@ -2120,7 +2010,8 @@ static void json_parse_object_context(lua_State* l, SlotManager& parent_slots, j slots.finalize(); json_restore_offset(json, ptr_offset); - json_decode_descend(l, json, 3); + // ServerLua: 7 slots for table, key, value + reviver call args + json_decode_descend(l, json, 7); if (slots.isInit()) { lua_newtable(l); @@ -2134,29 +2025,56 @@ static void json_parse_object_context(lua_State* l, SlotManager& parent_slots, j return; } - /* Rewind — let json_parse_object_key re-parse from the key token */ + /* Rewind - let json_parse_object_key re-parse from the key token */ json->ptr = json->data + ptr_offset; json_parse_object_key(l, json); - /* Save offset after colon — process_value will parse the value token */ + /* Save offset after colon - process_value will parse the value token */ ptr_offset = json_get_offset(json); } YIELD_DISPATCH_BEGIN(phase, slots); YIELD_DISPATCH(VALUE); YIELD_DISPATCH(NEXT_PAIR); + YIELD_DISPATCH(REVIVER_CHECK); + YIELD_DISPATCH(REVIVER_CALL); YIELD_DISPATCH_END(); while (1) { /* Fetch value */ + /* Stack before: [..., table, key] */ { json_token_t token; YIELD_HELPER(l, VALUE, json_process_value(l, slots, json, &token)); } + /* Stack after: [..., table, key, value] */ - /* Set key = value */ - lua_rawset(l, -3); + // Save offset past the parsed value so reviver yields resume correctly + ptr_offset = json_get_offset(json); + + if (json->has_reviver) { + // Call reviver(key, value, parent) + lua_pushvalue(l, (int)DecodeStack::REVIVER); + lua_pushvalue(l, -3); // key + lua_pushvalue(l, -3); // value + lua_pushvalue(l, -6); // parent (the object table) + YIELD_CHECK(l, REVIVER_CHECK, LUA_INTERRUPT_LLLIB); + YIELD_CALL(l, 3, 1, REVIVER_CALL); + // Stack: [..., table, key, value, result] + if (lua_tolightuserdatatagged(l, -1, LU_TAG_JSON_INTERNAL) == JSON_REMOVE) { + // Omit this key/value pair entirely + lua_pop(l, 3); // pop result, value, key + } else { + // Replace value with result, then rawset + lua_remove(l, -2); // remove original value + // Stack: [..., table, key, result] + lua_rawset(l, -3); + } + } else { + /* Set key = value */ + lua_rawset(l, -3); + } json_token_t token; json_next_token(json, &token); @@ -2171,7 +2089,7 @@ static void json_parse_object_context(lua_State* l, SlotManager& parent_slots, j json_parse_object_key(l, json); - /* Save offset after colon — process_value will parse the value */ + /* Save offset after colon - process_value will parse the value */ ptr_offset = json_get_offset(json); YIELD_CHECK(l, NEXT_PAIR, LUA_INTERRUPT_LLLIB); @@ -2186,28 +2104,26 @@ static void json_parse_array_context(lua_State* l, SlotManager& parent_slots, js DEFAULT = 0, ELEMENT = 1, NEXT_ELEMENT = 2, + REVIVER_CHECK = 3, + REVIVER_CALL = 4, }; SlotManager slots(parent_slots); DEFINE_SLOT(Phase, phase, Phase::DEFAULT); DEFINE_SLOT(int32_t, ptr_offset, json_get_offset(json)); DEFINE_SLOT(int32_t, i, 1); + // Track the next index for insertion (may differ from i when reviver removes elements) + DEFINE_SLOT(int32_t, insert_idx, 1); slots.finalize(); json_restore_offset(json, ptr_offset); - json_decode_descend(l, json, 2); + // ServerLua: 6 slots for table, value + reviver call args + json_decode_descend(l, json, 6); if (slots.isInit()) { lua_newtable(l); - /* set array_mt on the table at the top of the stack */ - if (json->cfg->decode_array_with_array_mt) { - lua_pushlightuserdatatagged(l, json_lightudata_mask(JSON_ARRAY), LU_TAG_JSON_INTERNAL); - lua_rawget(l, LUA_REGISTRYINDEX); - lua_setmetatable(l, -2); - } - - /* Peek at first token — check for empty array */ + /* Peek at first token - check for empty array */ const char* before = json->ptr; json_token_t token; json_next_token(json, &token); @@ -2218,7 +2134,7 @@ static void json_parse_array_context(lua_State* l, SlotManager& parent_slots, js return; } - /* Restore ptr — process_value will parse the first element token */ + /* Restore ptr - process_value will parse the first element token */ json->ptr = before; ptr_offset = json_get_offset(json); } @@ -2226,6 +2142,8 @@ static void json_parse_array_context(lua_State* l, SlotManager& parent_slots, js YIELD_DISPATCH_BEGIN(phase, slots); YIELD_DISPATCH(ELEMENT); YIELD_DISPATCH(NEXT_ELEMENT); + YIELD_DISPATCH(REVIVER_CHECK); + YIELD_DISPATCH(REVIVER_CALL); YIELD_DISPATCH_END(); for (; ; i++) { @@ -2234,18 +2152,43 @@ static void json_parse_array_context(lua_State* l, SlotManager& parent_slots, js YIELD_HELPER(l, ELEMENT, json_process_value(l, slots, json, &token)); } + /* Stack: [..., table, value] */ + + // Save offset past the parsed value so reviver yields resume correctly + ptr_offset = json_get_offset(json); - lua_rawseti(l, -2, i); /* arr[i] = value */ + if (json->has_reviver) { + lua_pushvalue(l, (int)DecodeStack::REVIVER); + lua_pushinteger(l, i); // key (1-based source index) + lua_pushvalue(l, -3); // value + lua_pushvalue(l, -5); // parent (the array table) + YIELD_CHECK(l, REVIVER_CHECK, LUA_INTERRUPT_LLLIB); + YIELD_CALL(l, 3, 1, REVIVER_CALL); + // Stack: [..., table, value, result] + if (lua_tolightuserdatatagged(l, -1, LU_TAG_JSON_INTERNAL) == JSON_REMOVE) { + // Omit this element - don't insert, don't increment insert_idx + lua_pop(l, 2); // pop result and value + } else { + // Replace value with result + lua_remove(l, -2); // remove original value + // Stack: [..., table, result] + lua_rawseti(l, -2, insert_idx); + insert_idx++; + } + } else { + lua_rawseti(l, -2, i); /* arr[i] = value */ + } json_token_t token; json_next_token(json, &token); if (token.type == T_ARR_END) { // ServerLua: shrink the array to fit the contents, if necessary + int final_len = json->has_reviver ? (insert_idx - 1) : i; LuaTable *t = hvalue(luaA_toobject(l, -1)); - if (t->sizearray != i) + if (t->sizearray != final_len && !t->readonly) { - luaH_resizearray(l, t, i); + luaH_resizearray(l, t, final_len); } json_decode_ascend(json); @@ -2255,7 +2198,7 @@ static void json_parse_array_context(lua_State* l, SlotManager& parent_slots, js if (token.type != T_COMMA) json_throw_parse_error(l, json, "comma or array end", &token); - /* Save offset after comma — process_value will parse the next element */ + /* Save offset after comma - process_value will parse the next element */ ptr_offset = json_get_offset(json); YIELD_CHECK(l, NEXT_ELEMENT, LUA_INTERRUPT_LLLIB); @@ -2337,6 +2280,8 @@ static int json_decode_common(lua_State* l, bool is_init, bool sl_tagged) { DEFAULT = 0, PROCESS_VALUE = 1, + ROOT_REVIVER_CHECK = 2, + ROOT_REVIVER_CALL = 3, }; SlotManager slots(l, is_init); @@ -2345,27 +2290,20 @@ static int json_decode_common(lua_State* l, bool is_init, bool sl_tagged) slots.finalize(); json_config_t cfg; - if (sl_tagged) + if (sl_tagged) { cfg.sl_tagged_types = true; + } if (is_init) { /* Args already validated by init wrapper. - * SlotManager inserted nil at pos 1, input string shifted to pos 2. */ - size_t json_len; - const char* json_data = lua_tolstring(l, 2, &json_len); + * SlotManager inserted nil at pos 1, original args shifted to pos 2+. + * Stack: [opaque(1), input_string(2), reviver?(3)] */ - /* Detect Unicode other than UTF-8 (see RFC 4627, Sec 3) - * - * CJSON can support any simple data type, hence only the first - * character is guaranteed to be ASCII (at worst: '"'). This is - * still enough to detect whether the wrong encoding is in use. */ - if (json_len >= 2 && (!json_data[0] || !json_data[1])) - luaL_error(l, "JSON parser does not support UTF-16 or UTF-32"); + // Ensure reviver or nil occupies a stack slot (will become pos 4 after strbuf insert) + if (lua_gettop(l) < 3) + lua_pushnil(l); - if (json_len > DEFAULT_MAX_SIZE) - luaL_errorL(l, "JSON too large to decode"); - - // ServerLua: Create decode scratch buffer in memcat 1 — it's an + // ServerLua: Create decode scratch buffer in memcat 1 - it's an // internal intermediary, not user-visible output. Pre-size to // json_len so _unsafe appends can't overflow (decoded <= input). // This is only safe because JSON escapes can never produce output @@ -2377,16 +2315,31 @@ static int json_decode_common(lua_State* l, bool is_init, bool sl_tagged) [[maybe_unused]] MemcatGuard mcg(l, 1); luaYB_push(l); } - lua_insert(l, 2); + lua_insert(l, (int)DecodeStack::STRBUF); + /* Stack: [opaque(1), strbuf(2), input_string(3), reviver_or_nil(4)] */ + + size_t json_len; + const char* json_data = lua_tolstring(l, (int)DecodeStack::INPUT, &json_len); + + /* Detect Unicode other than UTF-8 (see RFC 4627, Sec 3) + * + * CJSON can support any simple data type, hence only the first + * character is guaranteed to be ASCII (at worst: '"'). This is + * still enough to detect whether the wrong encoding is in use. */ + if (json_len >= 2 && (!json_data[0] || !json_data[1])) + luaL_error(l, "JSON parser does not support UTF-16 or UTF-32"); + + if (json_len > DEFAULT_MAX_SIZE) + luaL_errorL(l, "JSON too large to decode"); + strbuf_ensure_empty_length(json_get_strbuf(l), json_len); - /* Stack: [opaque(1), strbuf(2), input_string(3)] */ lua_hardenstack(l, 1); } /* Reconstruct json_parse_t from stack and slots */ size_t json_len; - const char* json_data = lua_tolstring(l, 3, &json_len); + const char* json_data = lua_tolstring(l, (int)DecodeStack::INPUT, &json_len); strbuf_t* tmp = json_get_strbuf(l); json_parse_t json; @@ -2395,9 +2348,12 @@ static int json_decode_common(lua_State* l, bool is_init, bool sl_tagged) json.ptr = json_data + ptr_offset; json.current_depth = 0; json.tmp = tmp; + json.has_reviver = !lua_isnil(l, (int)DecodeStack::REVIVER); YIELD_DISPATCH_BEGIN(phase, slots); YIELD_DISPATCH(PROCESS_VALUE); + YIELD_DISPATCH(ROOT_REVIVER_CHECK); + YIELD_DISPATCH(ROOT_REVIVER_CALL); YIELD_DISPATCH_END(); { @@ -2409,10 +2365,33 @@ static int json_decode_common(lua_State* l, bool is_init, bool sl_tagged) ptr_offset = (int32_t)(json.ptr - json.data); /* Ensure there is no more input left */ - json_token_t token; - json_next_token(&json, &token); - if (token.type != T_END) - json_throw_parse_error(l, &json, "the end", &token); + { + json_token_t token; + json_next_token(&json, &token); + if (token.type != T_END) + json_throw_parse_error(l, &json, "the end", &token); + } + + // Call reviver on root value if present + if (json.has_reviver) { + // Stack: [..., decoded_value] + lua_checkstack(l, 4); + lua_pushvalue(l, (int)DecodeStack::REVIVER); // reviver + lua_pushnil(l); // key = nil (root) + lua_pushvalue(l, -3); // decoded value + lua_pushnil(l); // parent = nil (root) + YIELD_CHECK(l, ROOT_REVIVER_CHECK, LUA_INTERRUPT_LLLIB); + YIELD_CALL(l, 3, 1, ROOT_REVIVER_CALL); + // Stack: [..., decoded_value, result] + if (lua_tolightuserdatatagged(l, -1, LU_TAG_JSON_INTERNAL) == JSON_REMOVE) { + lua_pop(l, 2); + // Return lljson.null - decode must return a value + lua_pushlightuserdatatagged(l, JSON_NULL, LU_TAG_JSON_INTERNAL); + } else { + // Replace decoded value with result + lua_remove(l, -2); + } + } luau_interruptoncalltail(l); return 1; @@ -2421,8 +2400,11 @@ static int json_decode_common(lua_State* l, bool is_init, bool sl_tagged) // ServerLua: init / continuation wrappers for json_decode static int json_decode_v0(lua_State* l) { - luaL_argcheck(l, lua_gettop(l) == 1, 1, "expected 1 argument"); + int nargs = lua_gettop(l); + luaL_argcheck(l, nargs >= 1 && nargs <= 2, 1, "expected 1-2 arguments"); luaL_checkstring(l, 1); + if (nargs >= 2) + luaL_checktype(l, 2, LUA_TFUNCTION); return json_decode_common(l, true, false); } static int json_decode_v0_k(lua_State* l, int) @@ -2432,8 +2414,11 @@ static int json_decode_v0_k(lua_State* l, int) } static int json_decode_sl_v0(lua_State* l) { - luaL_argcheck(l, lua_gettop(l) == 1, 1, "expected 1 argument"); + int nargs = lua_gettop(l); + luaL_argcheck(l, nargs >= 1 && nargs <= 2, 1, "expected 1-2 arguments"); luaL_checkstring(l, 1); + if (nargs >= 2) + luaL_checktype(l, 2, LUA_TFUNCTION); return json_decode_common(l, true, true); } static int json_decode_sl_v0_k(lua_State* l, int) @@ -2444,60 +2429,6 @@ static int json_decode_sl_v0_k(lua_State* l, int) /* ===== INITIALISATION ===== */ -#if 0 // ServerLua: unused -#if !defined(LUA_VERSION_NUM) || LUA_VERSION_NUM < 502 -/* Compatibility for Lua 5.1 and older LuaJIT. - * - * compat_luaL_setfuncs() is used to create a module table where the functions - * have json_config_t as their first upvalue. Code borrowed from Lua 5.2 - * source's luaL_setfuncs(). - */ -static void compat_luaL_setfuncs(lua_State *l, const luaL_Reg *reg, int nup) -{ - int i; - - luaL_checkstack(l, nup, "too many upvalues"); - for (; reg->name != NULL; reg++) { /* fill the table with given functions */ - for (i = 0; i < nup; i++) /* copy upvalues to the top */ - lua_pushvalue(l, -nup); - lua_pushcclosure(l, reg->func, reg->name, nup); /* closure with those upvalues */ - lua_setfield(l, -(nup + 2), reg->name); - } - lua_pop(l, nup); /* remove upvalues */ -} -#else -#define compat_luaL_setfuncs(L, reg, nup) luaL_setfuncs(L, reg, nup) -#endif - -/* Call target function in protected mode with all supplied args. - * Assumes target function only returns a single non-nil value. - * Convert and return thrown errors as: nil, "error message" */ -static int json_protect_conversion(lua_State *l) -{ - int err; - - /* Deliberately throw an error for invalid arguments */ - luaL_argcheck(l, lua_gettop(l) == 1, 1, "expected 1 argument"); - - /* pcall() the function stored as upvalue(1) */ - lua_pushvalue(l, lua_upvalueindex(1)); - lua_insert(l, 1); - err = lua_pcall(l, 1, 1, 0); - if (!err) - return 1; - - if (err == LUA_ERRRUN) { - lua_pushnil(l); - lua_insert(l, -2); - return 2; - } - - /* Since we are not using a custom error handler, the only remaining - * errors are memory related */ - luaL_error(l, "Memory allocation error in CJSON protected call"); -} -#endif - /* Return cjson module table */ static int lua_cjson_new(lua_State *l) { @@ -2507,11 +2438,22 @@ static int lua_cjson_new(lua_State *l) lua_setlightuserdataname(l, LU_TAG_JSON_INTERNAL, "lljson_constant"); - /* Test if array metatables are in registry */ - lua_pushlightuserdatatagged(l, json_lightudata_mask(JSON_EMPTY_ARRAY), LU_TAG_JSON_INTERNAL); + // ServerLua: intern strings used per-call during encode/decode init + luaS_fix(luaS_newliteral(l, "mode")); + luaS_fix(luaS_newliteral(l, "json")); + luaS_fix(luaS_newliteral(l, "sljson")); + luaS_fix(luaS_newliteral(l, "tight")); + luaS_fix(luaS_newliteral(l, "replacer")); + luaS_fix(luaS_newliteral(l, "__tojson")); + luaS_fix(luaS_newliteral(l, "__jsontype")); + luaS_fix(luaS_newliteral(l, "array")); + luaS_fix(luaS_newliteral(l, "object")); + + /* Test if array/object metatables are in registry */ + lua_pushlightuserdatatagged(l, JSON_ARRAY, LU_TAG_JSON_INTERNAL); lua_rawget(l, LUA_REGISTRYINDEX); if (lua_isnil(l, -1)) { - /* Create array metatables. + /* Create shape metatables. * * If multiple calls to lua_cjson_new() are made, * this prevents overriding the tables at the given @@ -2519,16 +2461,20 @@ static int lua_cjson_new(lua_State *l) */ lua_pop(l, 1); - /* empty_array_mt */ - lua_pushlightuserdatatagged(l, json_lightudata_mask(JSON_EMPTY_ARRAY), LU_TAG_JSON_INTERNAL); + /* array_mt */ + lua_pushlightuserdatatagged(l, JSON_ARRAY, LU_TAG_JSON_INTERNAL); lua_newtable(l); + lua_pushliteral(l, "array"); + lua_setfield(l, -2, "__jsontype"); lua_setreadonly(l, -1, true); lua_fixvalue(l, -1); lua_rawset(l, LUA_REGISTRYINDEX); - /* array_mt */ - lua_pushlightuserdatatagged(l, json_lightudata_mask(JSON_ARRAY), LU_TAG_JSON_INTERNAL); + /* object_mt */ + lua_pushlightuserdatatagged(l, JSON_OBJECT, LU_TAG_JSON_INTERNAL); lua_newtable(l); + lua_pushliteral(l, "object"); + lua_setfield(l, -2, "__jsontype"); lua_setreadonly(l, -1, true); lua_fixvalue(l, -1); lua_rawset(l, LUA_REGISTRYINDEX); @@ -2553,51 +2499,46 @@ static int lua_cjson_new(lua_State *l) lua_pushlightuserdatatagged(l, JSON_NULL, LU_TAG_JSON_INTERNAL); lua_setfield(l, -2, "null"); - /* Set cjson.empty_array_mt */ - lua_pushlightuserdatatagged(l, json_lightudata_mask(JSON_EMPTY_ARRAY), LU_TAG_JSON_INTERNAL); - lua_rawget(l, LUA_REGISTRYINDEX); - lua_setfield(l, -2, "empty_array_mt"); + /* Set cjson.remove - sentinel to omit values in reviver/replacer */ + lua_pushlightuserdatatagged(l, JSON_REMOVE, LU_TAG_JSON_INTERNAL); + lua_setfield(l, -2, "remove"); /* Set cjson.array_mt */ - lua_pushlightuserdatatagged(l, json_lightudata_mask(JSON_ARRAY), LU_TAG_JSON_INTERNAL); + lua_pushlightuserdatatagged(l, JSON_ARRAY, LU_TAG_JSON_INTERNAL); lua_rawget(l, LUA_REGISTRYINDEX); lua_setfield(l, -2, "array_mt"); - /* Set cjson.empty_array, this is just the lightuserdata, not a table! */ - lua_pushlightuserdatatagged(l, json_lightudata_mask(JSON_ARRAY), LU_TAG_JSON_INTERNAL); - lua_setfield(l, -2, "empty_array"); - - /* Set module name / version fields */ - lua_pushliteral(l, CJSON_MODNAME); - lua_setfield(l, -2, "_NAME"); - lua_pushliteral(l, CJSON_VERSION); - lua_setfield(l, -2, "_VERSION"); - - return 1; -} - -#if 0 // ServerLua: unused -/* Return cjson.safe module table */ -static int lua_cjson_safe_new(lua_State *l) -{ - const char *func[] = { "decode", "encode", NULL }; - int i; - - lua_cjson_new(l); + /* Set cjson.object_mt */ + lua_pushlightuserdatatagged(l, JSON_OBJECT, LU_TAG_JSON_INTERNAL); + lua_rawget(l, LUA_REGISTRYINDEX); + lua_setfield(l, -2, "object_mt"); - /* Fix new() method */ - lua_pushcfunction(l, lua_cjson_safe_new, "new"); - lua_setfield(l, -2, "new"); + /* Set cjson.empty_array / empty_object - frozen tables with cloned shape metatables. + * Cloned (not shared with array_mt/object_mt) to avoid Ares duplicate-permanent-object errors. */ + lua_newtable(l); + lua_newtable(l); + lua_pushliteral(l, "array"); + lua_setfield(l, -2, "__jsontype"); + lua_setreadonly(l, -1, true); + lua_fixvalue(l, -1); + lua_setmetatable(l, -2); + lua_setreadonly(l, -1, true); + lua_fixvalue(l, -1); + lua_setfield(l, -2, "empty_array"); - for (i = 0; func[i]; i++) { - lua_getfield(l, -1, func[i]); - lua_pushcclosure(l, json_protect_conversion, func[i], 1); - lua_setfield(l, -2, func[i]); - } + lua_newtable(l); + lua_newtable(l); + lua_pushliteral(l, "object"); + lua_setfield(l, -2, "__jsontype"); + lua_setreadonly(l, -1, true); + lua_fixvalue(l, -1); + lua_setmetatable(l, -2); + lua_setreadonly(l, -1, true); + lua_fixvalue(l, -1); + lua_setfield(l, -2, "empty_object"); return 1; } -#endif int luaopen_cjson(lua_State *l) { @@ -2611,15 +2552,5 @@ int luaopen_cjson(lua_State *l) return 1; } -#if 0 // ServerLua: unused -int luaopen_cjson_safe(lua_State *l) -{ - lua_cjson_safe_new(l); - - /* Return cjson.safe table */ - return 1; -} -#endif - /* vi:ai et sw=4 ts=4: */ diff --git a/tests/SLConformance.test.cpp b/tests/SLConformance.test.cpp index e98659e6..aaf5baa3 100644 --- a/tests/SLConformance.test.cpp +++ b/tests/SLConformance.test.cpp @@ -735,60 +735,68 @@ TEST_CASE("bit32.s32") runConformance("bit32_s32.lua"); } -// ServerLua: interrupt state for lljson yield tests +// ServerLua: shared interrupt infrastructure for lljson yield tests static bool jsonInterruptEnabled = false; static int jsonYieldCount = 0; -TEST_CASE("lljson") +static void setupJsonInterruptInfra(lua_State* L) { jsonInterruptEnabled = false; jsonYieldCount = 0; - runConformance("lljson.lua", nullptr, [](lua_State* L) { - lua_pushcfunction( - L, - [](lua_State* L) -> int - { - jsonYieldCount = 0; - return 0; - }, - "clear_check_count" - ); - lua_setglobal(L, "clear_check_count"); - - lua_pushcfunction( - L, - [](lua_State* L) -> int - { - lua_pushinteger(L, jsonYieldCount); - return 1; - }, - "get_check_count" - ); - lua_setglobal(L, "get_check_count"); - - lua_pushcfunction( - L, - [](lua_State* L) -> int - { - jsonInterruptEnabled = true; - return 0; - }, - "enable_check_interrupt" - ); - lua_setglobal(L, "enable_check_interrupt"); + lua_pushcfunction( + L, + [](lua_State* L) -> int + { + jsonYieldCount = 0; + return 0; + }, + "clear_check_count" + ); + lua_setglobal(L, "clear_check_count"); - // ServerLua: Interrupt handler that yields on every YIELD_CHECK hit - lua_callbacks(L)->interrupt = [](lua_State* L, int gc) + lua_pushcfunction( + L, + [](lua_State* L) -> int { - if (gc != LUA_INTERRUPT_LLLIB) - return; - if (!jsonInterruptEnabled) - return; - jsonYieldCount++; - lua_yield(L, 0); - }; - }); + lua_pushinteger(L, jsonYieldCount); + return 1; + }, + "get_check_count" + ); + lua_setglobal(L, "get_check_count"); + + lua_pushcfunction( + L, + [](lua_State* L) -> int + { + jsonInterruptEnabled = true; + return 0; + }, + "enable_check_interrupt" + ); + lua_setglobal(L, "enable_check_interrupt"); + + lua_callbacks(L)->interrupt = [](lua_State* L, int gc) + { + if (gc != LUA_INTERRUPT_LLLIB) + return; + if (!jsonInterruptEnabled) + return; + jsonYieldCount++; + lua_yield(L, 0); + }; +} + +TEST_CASE("lljson") +{ + runConformance("lljson.lua", nullptr, setupJsonInterruptInfra); +} + +TEST_CASE("lljson_replacer") +{ + runConformance("lljson_replacer.lua", nullptr, setupJsonInterruptInfra); + runConformance("lljson_typedjson.lua", nullptr, setupJsonInterruptInfra); } TEST_CASE("llbase64") diff --git a/tests/conformance/lljson.lua b/tests/conformance/lljson.lua index 6e1497a7..1023629e 100644 --- a/tests/conformance/lljson.lua +++ b/tests/conformance/lljson.lua @@ -1,14 +1,14 @@ assert(lljson.encode(lljson.null) == "null") -- nil is unambiguous at the top level, so we can treat it as `null` assert(lljson.encode(nil) == "null") --- cjson encodes empty tables as objects by default --- TODO: Is that what we want? You have to pick one or the other. -assert(lljson.encode({}) == "{}") +-- Empty tables default to arrays +assert(lljson.encode({}) == "[]") -- But you can specify you _really_ want a table to be treated as an array -- by setting the array_mt metatable assert(lljson.encode(setmetatable({}, lljson.array_mt)) == "[]") --- `lljson.empty_array` is the same. +-- Sentinel frozen tables for empty array/object assert(lljson.encode(lljson.empty_array) == "[]") +assert(lljson.encode(lljson.empty_object) == "{}") assert(lljson.encode({1}) == "[1]") assert(lljson.encode({integer(1)}) == "[1]") assert(lljson.encode(true) == "true") @@ -19,13 +19,13 @@ assert(lljson.encode({foo="bar"}) == '{"foo":"bar"}') -- key -> nil is the same as deleting a key in Lua, we have no -- way to distinguish between a key that has a nil value and -- a non-existent key. -assert(lljson.encode({foo=nil}) == '{}') +assert(lljson.encode({foo=nil}) == '[]') -- But we can represent it explicitly with `lljson.null` assert(lljson.encode({foo=lljson.null}) == '{"foo":null}') assert(lljson.encode(vector(1, 2.5, 22.0 / 7.0)) == '"<1,2.5,3.142857>"') assert(lljson.encode(quaternion(1, 2.5, 22.0 / 7.0, 4)) == '"<1,2.5,3.142857,4>"') --- metatables are totally ignored +-- metatables without __jsontype/__tojson are ignored local SomeMT = {} function SomeMT.whatever(...) error("Placeholder function called") @@ -81,8 +81,16 @@ assert(not pcall(function() lljson.encode(self_ref) end)) local long_str = '"' .. string.rep("a", 100000) .. '"' assert(not pcall(function() lljson.decode(long_str) end)) --- Can encode NaNs (non-standard, NaN literal has no JSON representation) -assert(lljson.encode(0/0) == "NaN") +-- NaN encodes as null in standard JSON (matches JSON.stringify) +assert(lljson.encode(0/0) == "null") +-- slencode uses tagged float for NaN round-trip +assert(lljson.slencode(0/0) == '"!fNaN"') +do + local nan_rt = lljson.sldecode(lljson.slencode(0/0)) + assert(nan_rt ~= nan_rt, "NaN should round-trip through slencode/sldecode") +end +-- NaN in table values: encodes as null +assert(lljson.encode({0/0}) == '[null]') -- We can also decode them local nan_decoded = lljson.decode("nan") assert(nan_decoded ~= nan_decoded) @@ -170,18 +178,31 @@ assert(lljson.sldecode(lljson.slencode(test_quat)) == test_quat) local test_uuid = uuid("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee") assert(lljson.sldecode(lljson.slencode(test_uuid)) == test_uuid) --- Tagged keys: numeric keys become !f -local float_key_table = {[3.14] = "pi"} -local float_key_json = lljson.slencode(float_key_table) -assert(float_key_json == '{"!f3.14":"pi"}') -local float_key_decoded = lljson.sldecode(float_key_json) -assert(float_key_decoded[3.14] == "pi") +-- Tagged key round-trip: encode key, check JSON, decode and verify lookup +local function check_key_roundtrip(key, expected_json) + local json = lljson.slencode({[key] = "value"}) + assert(json == expected_json, `expected {expected_json}, got {json}`) + local decoded = lljson.sldecode(json) + -- Try direct lookup first, fall back to iteration (for quaternion reference identity) + if decoded[key] == "value" then return end + for k, v in decoded do + if k == key and v == "value" then return end + end + error(`key round-trip failed for {expected_json}`) +end + +check_key_roundtrip(3.14, '{"!f3.14":"value"}') +check_key_roundtrip(vector(1, 2, 3), '{"!v<1,2,3>":"value"}') +check_key_roundtrip(quaternion(0, 0, 0, 1), '{"!q<0,0,0,1>":"value"}') +check_key_roundtrip(uuid("12345678-1234-1234-1234-123456789abc"), + '{"!u12345678-1234-1234-1234-123456789abc":"value"}') +check_key_roundtrip("!bang", '{"!!bang":"value"}') +check_key_roundtrip(math.huge, '{"!f1e9999":"value"}') -- Sparse tables: integer keys become !f tagged, avoiding sparse array issues -- (contrast with regular encode which would fail or fill with nulls) local sparse_table = {[1] = "first", [100] = "hundredth"} local sparse_json = lljson.slencode(sparse_table) --- Encoded as object with !f keys, not as array -- Okay, this is a little obnoxious, but we don't really know in which order the -- keys would be serialized. Just accept either/or. assert(sparse_json == '{"!f1":"first","!f100":"hundredth"}' or sparse_json == '{"!f100":"hundredth","!f1":"first"}') @@ -189,48 +210,6 @@ local sparse_decoded = lljson.sldecode(sparse_json) assert(sparse_decoded[1] == "first") assert(sparse_decoded[100] == "hundredth") --- Tagged keys: vector keys -local vec_key_table = {[vector(1, 2, 3)] = "vec"} -local vec_key_json = lljson.slencode(vec_key_table) -assert(vec_key_json == '{"!v<1,2,3>":"vec"}') -local vec_key_decoded = lljson.sldecode(vec_key_json) -assert(vec_key_decoded[vector(1, 2, 3)] == "vec") - --- Tagged keys: quaternion keys (note: table lookup uses reference identity, but == uses value equality) -local quat_key_table = {[quaternion(0, 0, 0, 1)] = "identity"} -local quat_key_json = lljson.slencode(quat_key_table) -assert(quat_key_json == '{"!q<0,0,0,1>":"identity"}') -local quat_key_decoded = lljson.sldecode(quat_key_json) --- Can't lookup by value since table keys use reference identity, need to iterate -local found_quat_key = false -for k, v in quat_key_decoded do - if k == quaternion(0, 0, 0, 1) and v == "identity" then - found_quat_key = true - end -end -assert(found_quat_key) - --- Tagged keys: UUID keys -local uuid_key_table = {[uuid("12345678-1234-1234-1234-123456789abc")] = "some_uuid"} -local uuid_key_json = lljson.slencode(uuid_key_table) -assert(uuid_key_json == '{"!u12345678-1234-1234-1234-123456789abc":"some_uuid"}') -local uuid_key_decoded = lljson.sldecode(uuid_key_json) -assert(uuid_key_decoded[uuid("12345678-1234-1234-1234-123456789abc")] == "some_uuid") - --- Tagged keys: string keys starting with ! get escaped -local bang_key_table = {["!bang"] = "value"} -local bang_key_json = lljson.slencode(bang_key_table) -assert(bang_key_json == '{"!!bang":"value"}') -local bang_key_decoded = lljson.sldecode(bang_key_json) -assert(bang_key_decoded["!bang"] == "value") - --- Infinity in tagged floats -local inf_key_table = {[math.huge] = "infinity"} -local inf_key_json = lljson.slencode(inf_key_table) -assert(inf_key_json == '{"!f1e9999":"infinity"}') -local inf_key_decoded = lljson.sldecode(inf_key_json) -assert(inf_key_decoded[math.huge] == "infinity") - -- NaN can't be used as a table key in Lua ("table index is NaN" error) -- so we can't test NaN keys - but NaN values in vectors still work: @@ -286,46 +265,46 @@ assert(lljson.slencode(empty_buf) == '"!d"') local empty_decoded = lljson.sldecode('"!d"') assert(buffer.len(empty_decoded) == 0) --- Buffer as object key -local buf_key_table = {[buf] = "data"} -local buf_key_json = lljson.slencode(buf_key_table) -assert(buf_key_json == '{"!dAQD/AA==":"data"}') +-- Buffer as object key (no round-trip - buffers use reference identity) +assert(lljson.slencode({[buf] = "value"}) == '{"!dAQD/AA==":"value"}') -- ============================================ --- Tight encoding mode (second arg = true) +-- Tight encoding mode ({tight = true}) -- ============================================ -- Tight vectors: no angle brackets -assert(lljson.slencode(vector(1, 2, 3), true) == '"!v1,2,3"') +assert(lljson.slencode(vector(1, 2, 3), {tight = true}) == '"!v1,2,3"') -- Tight vectors: zeros omitted -assert(lljson.slencode(vector(0, 0, 1), true) == '"!v,,1"') -assert(lljson.slencode(vector(0, 0, 0), true) == '"!v,,"') -assert(lljson.slencode(vector(1, 0, 0), true) == '"!v1,,"') -assert(lljson.slencode(vector(0, 2, 0), true) == '"!v,2,"') +assert(lljson.slencode(vector(0, 0, 1), {tight = true}) == '"!v,,1"') +assert(lljson.slencode(vector(0, 0, 0), {tight = true}) == '"!v"') +assert(lljson.slencode(vector(1, 0, 0), {tight = true}) == '"!v1,,"') +assert(lljson.slencode(vector(0, 2, 0), {tight = true}) == '"!v,2,"') -- Tight quaternions: no angle brackets, zeros omitted -assert(lljson.slencode(quaternion(0, 0, 0, 1), true) == '"!q,,,1"') -assert(lljson.slencode(quaternion(1, 0, 0, 0), true) == '"!q1,,,"') -assert(lljson.slencode(quaternion(0, 0, 0, 0), true) == '"!q,,,"') +assert(lljson.slencode(quaternion(0, 0, 0, 1), {tight = true}) == '"!q"') +assert(lljson.slencode(quaternion(1, 0, 0, 0), {tight = true}) == '"!q1,,,"') +assert(lljson.slencode(quaternion(0, 0, 0, 0), {tight = true}) == '"!q,,,"') -- Tight UUIDs: base64 encoded (22 chars instead of 36) local test_uuid = uuid("12345678-1234-1234-1234-123456789abc") -local tight_uuid_json = lljson.slencode(test_uuid, true) +local tight_uuid_json = lljson.slencode(test_uuid, {tight = true}) -- Should be "!u" + 22 chars of base64 assert(#tight_uuid_json == 26) -- 2 quotes + !u + 22 base64 assert(tight_uuid_json:sub(1, 3) == '"!u') -- Tight null UUID: just "!u" with no payload local null_uuid = uuid("00000000-0000-0000-0000-000000000000") -assert(lljson.slencode(null_uuid, true) == '"!u"') +assert(lljson.slencode(null_uuid, {tight = true}) == '"!u"') assert(lljson.sldecode('"!u"') == null_uuid) -- Decoding tight formats assert(lljson.sldecode('"!v1,2,3"') == vector(1, 2, 3)) assert(lljson.sldecode('"!v,,1"') == vector(0, 0, 1)) assert(lljson.sldecode('"!v,,"') == vector(0, 0, 0)) +assert(lljson.sldecode('"!v"') == vector(0, 0, 0)) assert(lljson.sldecode('"!q,,,1"') == quaternion(0, 0, 0, 1)) +assert(lljson.sldecode('"!q"') == quaternion(0, 0, 0, 1)) assert(lljson.sldecode('"!q,,,"') == quaternion(0, 0, 0, 0)) -- Normal format still works after tight implementation @@ -333,13 +312,13 @@ assert(lljson.sldecode('"!v<1,2,3>"') == vector(1, 2, 3)) assert(lljson.sldecode('"!q<0,0,0,1>"') == quaternion(0, 0, 0, 1)) -- Round-trip with tight encoding -local vec_rt = lljson.sldecode(lljson.slencode(vector(1.5, 0, -2.5), true)) +local vec_rt = lljson.sldecode(lljson.slencode(vector(1.5, 0, -2.5), {tight = true})) assert(vec_rt == vector(1.5, 0, -2.5)) -local quat_rt = lljson.sldecode(lljson.slencode(quaternion(0, 0, 0, 1), true)) +local quat_rt = lljson.sldecode(lljson.slencode(quaternion(0, 0, 0, 1), {tight = true})) assert(quat_rt == quaternion(0, 0, 0, 1)) -local uuid_rt = lljson.sldecode(lljson.slencode(test_uuid, true)) +local uuid_rt = lljson.sldecode(lljson.slencode(test_uuid, {tight = true})) assert(uuid_rt == test_uuid) -- Complex structure with tight encoding @@ -348,7 +327,7 @@ local tight_complex = { rot = quaternion(0, 0, 0, 1), id = uuid("00000000-0000-0000-0000-000000000001"), } -local tight_json = lljson.slencode(tight_complex, true) +local tight_json = lljson.slencode(tight_complex, {tight = true}) local tight_decoded = lljson.sldecode(tight_json) assert(tight_decoded.pos == tight_complex.pos) assert(tight_decoded.rot == tight_complex.rot) @@ -373,6 +352,227 @@ assert(not pcall(lljson.encode, recurse)) -- Non-string input should error, not crash assert(not pcall(lljson.decode, {"1,2,3"})) +-- ============================================ +-- lljson.remove sentinel +-- ============================================ +assert(lljson.remove ~= nil) +assert(lljson.remove ~= lljson.null) +assert(type(lljson.remove) == "userdata") + +-- ============================================ +-- slencode/sldecode !n tag for nil +-- ============================================ + +-- slencode emits "!n" for nil holes in arrays +assert(lljson.slencode({1, nil, 3}) == '[1,"!n",3]') + +-- slencode top-level nil +assert(lljson.slencode(nil) == '"!n"') + +-- sldecode "!n" produces nil (hole in array) +do + local t = lljson.sldecode('[1,"!n",3]') + assert(t[1] == 1) + assert(t[2] == nil) + assert(t[3] == 3) +end + +-- lljson.null still round-trips as null, not !n +do + local t = lljson.sldecode(lljson.slencode({1, lljson.null, 3})) + assert(t[1] == 1) + assert(t[2] == lljson.null) + assert(t[3] == 3) +end + +-- standard encode still uses null for nil (no !n) +assert(lljson.encode({1, nil, 3}) == '[1,null,3]') + +-- ============================================ +-- __tojson context table +-- ============================================ +do + local captured_ctx + local ctx_mt = { __tojson = function(self, ctx) + captured_ctx = ctx + return self.val + end } + + -- encode() passes mode="json", tight=false + lljson.encode(setmetatable({val = 1}, ctx_mt)) + assert(captured_ctx.mode == "json") + assert(captured_ctx.tight == false) + + -- slencode() passes mode="sljson", tight=false + captured_ctx = nil + lljson.slencode(setmetatable({val = 1}, ctx_mt)) + assert(captured_ctx.mode == "sljson") + assert(captured_ctx.tight == false) + + -- slencode(val, {tight = true}) passes tight=true + captured_ctx = nil + lljson.slencode(setmetatable({val = 1}, ctx_mt), {tight = true}) + assert(captured_ctx.mode == "sljson") + assert(captured_ctx.tight == true) + + -- This should definitely be read-only + assert(table.isfrozen(captured_ctx)) +end + + +-- slencode with table arg: tight option +do + local r = lljson.slencode(vector(1, 2, 3), {tight = true}) + assert(r == '"!v1,2,3"') +end + +-- No options: still works +assert(lljson.encode(42) == "42") +assert(lljson.slencode(42) == "42") + +-- Bad arg types error +assert(not pcall(lljson.encode, 1, "string")) +assert(not pcall(lljson.slencode, 1, "string")) +assert(not pcall(lljson.slencode, 1, true)) + +-- sldecode does not set array metatables (slencode ignores them, so attaching would be dishonest) +do + assert(getmetatable(lljson.sldecode("[]")) == nil) + assert(getmetatable(lljson.sldecode("[1,2,3]")) == nil) + -- Standard decode: also no metatables + assert(getmetatable(lljson.decode("[]")) == nil) + assert(getmetatable(lljson.decode("[1,2]")) == nil) + -- Round-trip: non-empty array auto-detected, no metatable needed + local decoded = lljson.sldecode(lljson.slencode({1, 2, 3})) + assert(decoded[1] == 1 and decoded[2] == 2 and decoded[3] == 3) + assert(getmetatable(decoded) == nil) +end + +-- slencode ignores shape metatables - auto-detects from data +do + -- array_mt is ignored by slencode, auto-detects as array anyway + local t = setmetatable({1, 2, 3}, lljson.array_mt) + assert(lljson.slencode(t) == "[1,2,3]") + -- object_mt is ignored by slencode, auto-detects as array + local t2 = setmetatable({10, 20}, lljson.object_mt) + assert(lljson.slencode(t2) == "[10,20]") + -- empty table with object_mt: slencode ignores it, encodes as [] + local t3 = setmetatable({}, lljson.object_mt) + assert(lljson.slencode(t3) == "[]") +end + +-- object_mt forces object encoding +do + -- Sequential integer keys become stringified + local t = setmetatable({10, 20, 30}, lljson.object_mt) + local json = lljson.encode(t) + local decoded = lljson.decode(json) + assert(decoded["1"] == 10) + assert(decoded["2"] == 20) + assert(decoded["3"] == 30) + -- Empty table with object_mt encodes as {} + assert(lljson.encode(setmetatable({}, lljson.object_mt)) == "{}") + -- object_mt is accessible + assert(lljson.object_mt ~= nil) + assert(type(lljson.object_mt) == "table") +end + +-- ============================================ +-- __jsontype metamethod +-- ============================================ +do + -- Custom metatable with __jsontype = "array" + local arr_mt = {__jsontype = "array"} + assert(lljson.encode(setmetatable({}, arr_mt)) == "[]") + assert(lljson.encode(setmetatable({1, 2, 3}, arr_mt)) == "[1,2,3]") + + -- Custom metatable with __jsontype = "object" + local obj_mt = {__jsontype = "object"} + assert(lljson.encode(setmetatable({}, obj_mt)) == "{}") + assert(lljson.encode(setmetatable({1, 2}, obj_mt)) == '{"1":1,"2":2}') + + -- __jsontype + __index: metamethods used for element access + local proxy_mt = { + __jsontype = "array", + __len = function() return 3 end, + __index = function(_, k) return k * 10 end, + } + assert(lljson.encode(setmetatable({}, proxy_mt)) == "[10,20,30]") + + -- __jsontype + __len (custom length) + local len_mt = { + __jsontype = "array", + __len = function() return 2 end, + } + assert(lljson.encode(setmetatable({10, 20, 30}, len_mt)) == "[10,20]") + + -- __tojson provides content, __jsontype provides shape (orthogonal) + -- scalar __tojson result: shape is irrelevant + local scalar_mt = { + __jsontype = "object", + __tojson = function(self) return self.a end, + } + assert(lljson.encode(setmetatable({a = 1}, scalar_mt)) == '1') + + -- table __tojson result + __jsontype = "array": shape applied to result + local arr_tojson_mt = { + __jsontype = "array", + __tojson = function(self) return {self[1] * 10} end, + } + assert(lljson.encode(setmetatable({5}, arr_tojson_mt)) == '[50]') + + -- __tojson converts string-keyed table to array-compatible result + local convert_mt = { + __jsontype = "array", + __tojson = function(self) return {self.x, self.y} end, + } + assert(lljson.encode(setmetatable({x = 1, y = 2}, convert_mt)) == '[1,2]') + + -- table __tojson result + __jsontype = "object": shape applied to result + local obj_tojson_mt = { + __jsontype = "object", + __tojson = function() return {1, 2} end, + } + assert(lljson.encode(setmetatable({}, obj_tojson_mt)) == '{"1":1,"2":2}') + + -- __jsontype = "array" on table with string keys (no __tojson) -> error + assert(not pcall(lljson.encode, setmetatable({x = 1}, {__jsontype = "array"}))) + + -- __jsontype = "array" + __tojson returning string-keyed table -> error + local bad_tojson_mt = { + __jsontype = "array", + __tojson = function() return {x = 1} end, + } + assert(not pcall(lljson.encode, setmetatable({}, bad_tojson_mt))) + + -- Invalid __jsontype value errors + local bad_mt = {__jsontype = "invalid"} + assert(not pcall(lljson.encode, setmetatable({}, bad_mt))) + + -- Non-string __jsontype value errors + assert(not pcall(lljson.encode, setmetatable({}, {__jsontype = 42}))) + assert(not pcall(lljson.encode, setmetatable({}, {__jsontype = true}))) + + -- slencode ignores __jsontype + local slen_mt = {__jsontype = "object"} + assert(lljson.slencode(setmetatable({1, 2}, slen_mt)) == "[1,2]") + assert(lljson.slencode(setmetatable({}, slen_mt)) == "[]") +end + +-- empty_array / empty_object are frozen tables with shape metatables +assert(type(lljson.empty_array) == "table") +assert(type(lljson.empty_object) == "table") +assert(table.isfrozen(lljson.empty_array)) +assert(table.isfrozen(lljson.empty_object)) +assert(lljson.empty_object ~= nil) +assert(lljson.empty_object ~= lljson.null) +assert(lljson.empty_object ~= lljson.empty_array) +-- Metatables are cloned (not shared with array_mt/object_mt) for Ares compatibility +assert(getmetatable(lljson.empty_array) ~= lljson.array_mt) +assert(getmetatable(lljson.empty_object) ~= lljson.object_mt) +assert(getmetatable(lljson.empty_array).__jsontype == "array") +assert(getmetatable(lljson.empty_object).__jsontype == "object") + -- Enable interrupt-driven yields for remaining tests. enable_check_interrupt() @@ -407,6 +607,20 @@ local function consume_nocheck(f, ...) return consume_impl(false, false, f, ...) end +-- Shared metatables for yield tests +local yield_tojson_mt = { __tojson = function(self) + coroutine.yield() + return self.val +end } +local yield_len_mt = { __jsontype = "array", __len = function(self) + coroutine.yield() + return self.n +end, __tojson = function(self) + local t = {} + for i = 1, self.n do t[i] = self[i] end + return t +end } + -- encode flat array: exercises ELEMENT/NEXT_ELEMENT yield path assert(consume(function() return lljson.encode({1, 2, 3, 4, 5}) @@ -446,11 +660,7 @@ end) == "[1,2]") -- multiple __tojson in one encode: two yielding metamethods in the same array assert(consume_nocheck(function() - local mt = { __tojson = function(self) - coroutine.yield() - return self.v - end } - return lljson.encode({setmetatable({v = 10}, mt), setmetatable({v = 20}, mt)}) + return lljson.encode({setmetatable({val = 10}, yield_tojson_mt), setmetatable({val = 20}, yield_tojson_mt)}) end) == "[10,20]") -- encode large array: sustained interrupt-driven yields @@ -478,28 +688,16 @@ end) -- __len that yields: exercises LEN_CHECK/LEN_CALL yield path assert(consume_nocheck(function() - local mt = { __len = function(self) - coroutine.yield() - return self.n - end } - return lljson.encode(setmetatable({10, 20, 30, n = 3}, mt)) + return lljson.encode(setmetatable({10, 20, 30, n = 3}, yield_len_mt)) end) == "[10,20,30]") -- deeply nested encode: arrays of objects of arrays with __tojson and __len at multiple levels consume_nocheck(function() - local tojson_mt = { __tojson = function(self) - coroutine.yield() - return self.val - end } - local len_mt = { __len = function(self) - coroutine.yield() - return self.n - end } local r = lljson.encode({ items = { {name = "a", tags = {1, 2, 3}}, - {name = "b", tags = setmetatable({10, 20, n = 2}, len_mt)}, - {name = "c", custom = setmetatable({val = "hello"}, tojson_mt)}, + {name = "b", tags = setmetatable({10, 20, n = 2}, yield_len_mt)}, + {name = "c", custom = setmetatable({val = "hello"}, yield_tojson_mt)}, }, meta = { nested = { @@ -514,7 +712,7 @@ consume_nocheck(function() assert(t.meta.nested.deep[1][1] == true and t.meta.nested.deep[1][2] == false) end) --- deeply nested decode: exercises recursive json_process_value → parse_object/parse_array at depth +-- deeply nested decode: exercises recursive json_process_value -> parse_object/parse_array at depth consume(function() local src = '{"a":[{"b":[[1,2],[3,4]]},{"c":{"d":[5,6,7],"e":{"f":true}}}],"g":[[[8]]]}' local t = lljson.decode(src) @@ -523,4 +721,43 @@ consume(function() assert(t.g[1][1][1] == 8) end) + +-- __tojson(self, ctx) that yields and uses ctx +assert(consume_nocheck(function() + local mt = { __tojson = function(self, ctx) + coroutine.yield() + if ctx.mode == "json" then + return tostring(self.val) + end + return self.val + end } + return lljson.encode({setmetatable({val = 42}, mt)}) +end) == "[\"42\"]") + +-- slencode with __tojson ctx: mode should be "sljson" +consume_nocheck(function() + local captured_mode + local captured_ctx + local mt = { __tojson = function(self, ctx) + captured_ctx = ctx + captured_mode = type(ctx) == "table" and ctx.mode or nil + return self.val + end } + lljson.slencode(setmetatable({val = 1}, mt)) + assert(captured_mode == "sljson", "expected sljson, got " .. tostring(captured_mode) .. " (ctx type=" .. type(captured_ctx) .. ")") +end) + +-- slencode with __tojson ctx that explicitly yields: exercises Ares round-trip +consume_nocheck(function() + local captured_mode + local mt = { __tojson = function(self, ctx) + coroutine.yield() + captured_mode = ctx.mode + return self.val + end } + lljson.slencode(setmetatable({val = 1}, mt)) + assert(captured_mode == "sljson") +end) + + return 'OK' diff --git a/tests/conformance/lljson_replacer.lua b/tests/conformance/lljson_replacer.lua new file mode 100644 index 00000000..dc418520 --- /dev/null +++ b/tests/conformance/lljson_replacer.lua @@ -0,0 +1,559 @@ +-- ============================================ +-- Reviver for decode() +-- ============================================ + +-- Basic: transform all strings to uppercase +do + local t = lljson.decode('{"a":"hello","b":"world"}', function(key, value) + if type(value) == "string" then return string.upper(value) end + return value + end) + assert(t.a == "HELLO") + assert(t.b == "WORLD") +end + +-- lljson.remove from reviver: omit specific keys from objects +do + local t = lljson.decode('{"keep":"yes","drop":"no","also":"yes"}', function(key, value) + if key == "drop" then return lljson.remove end + return value + end) + assert(t.keep == "yes") + assert(t.also == "yes") + assert(t.drop == nil) +end + +-- lljson.remove from reviver: omit elements from arrays (result is compacted) +do + local t = lljson.decode('[1,2,3,4,5]', function(key, value) + if value == 2 or value == 4 then return lljson.remove end + return value + end) + assert(#t == 3) + assert(t[1] == 1) + assert(t[2] == 3) + assert(t[3] == 5) +end + +-- nil return from reviver encodes as null (not an error) +do + local t = lljson.decode('{"a":1}', function(key, value) + if type(value) == "number" then return nil end + return value + end) + assert(t.a == nil) +end + +-- lljson.null return: stored as JSON null +do + local t = lljson.decode('{"a":"hello"}', function(key, value) + if type(value) == "string" then return lljson.null end + return value + end) + assert(t.a == lljson.null) +end + +-- Root reviver: wrap the root value +do + local result = lljson.decode('42', function(key, value) + if key == nil then return {wrapped = value} end + return value + end) + assert(result.wrapped == 42) +end + +-- Root lljson.remove: returns lljson.null +do + local result = lljson.decode('"hello"', function(key, value) + return lljson.remove + end) + assert(result == lljson.null) +end + +-- Nested objects: verify bottom-up call order +do + local order = {} + lljson.decode('{"a":{"b":1}}', function(key, value) + table.insert(order, key) + return value + end) + -- bottom-up: "b" first (inner), then "a" (outer), then nil (root) + assert(order[1] == "b") + assert(order[2] == "a") + assert(order[3] == nil) +end + +-- Type reconstruction: setmetatable in reviver +do + local Vec2 = {} + Vec2.__index = Vec2 + function Vec2:magnitude() return math.sqrt(self.x * self.x + self.y * self.y) end + + local t = lljson.decode('{"x":3,"y":4}', function(key, value) + if key == nil and type(value) == "table" and value.x and value.y then + return setmetatable(value, Vec2) + end + return value + end) + assert(t:magnitude() == 5) +end + +-- Too many args should error +assert(not pcall(lljson.decode, '"hello"', function() end, "extra")) + +-- Non-function second arg should error if not a table +assert(not pcall(lljson.decode, '"hello"', "not a function")) + +-- ============================================ +-- Reviver for sldecode() +-- ============================================ + +-- Reviver sees vectors/UUIDs (post-tag-parsing), not raw tagged strings +do + local saw_vector = false + local saw_uuid = false + local t = lljson.sldecode('{"v":"!v<1,2,3>","id":"!u12345678-1234-1234-1234-123456789abc"}', function(key, value) + if type(value) == "vector" then saw_vector = true end + if typeof(value) == "uuid" then saw_uuid = true end + return value + end) + assert(saw_vector, "reviver should see parsed vector, not tagged string") + assert(saw_uuid, "reviver should see parsed uuid, not tagged string") + assert(t.v == vector(1, 2, 3)) + assert(t.id == uuid("12345678-1234-1234-1234-123456789abc")) +end + +-- sldecode reviver can transform SL types +do + local t = lljson.sldecode('{"pos":"!v<1,2,3>"}', function(key, value) + if type(value) == "vector" then + return value * 2 + end + return value + end) + assert(t.pos == vector(2, 4, 6)) +end + +-- sldecode: backwards compat without reviver +assert(lljson.sldecode('"!v<1,2,3>"') == vector(1, 2, 3)) + +-- ============================================ +-- Encode replacer +-- ============================================ + +-- Basic object replacer: transform values +do + local r = lljson.encode({a = 1, b = 2}, {replacer = function(key, value) + if type(value) == "number" then return value * 10 end + return value + end}) + local t = lljson.decode(r) + assert(t.a == 10 and t.b == 20) +end + +-- Object replacer with lljson.remove: omit keys +do + local r = lljson.encode({keep = 1, drop = 2, also = 3}, {replacer = function(key, value) + if key == "drop" then return lljson.remove end + return value + end}) + local t = lljson.decode(r) + assert(t.keep == 1 and t.also == 3 and t.drop == nil) +end + +-- Array replacer with lljson.remove: skip elements +do + local r = lljson.encode({1, 2, 3, 4, 5}, {replacer = function(key, value, parent) + if parent == nil then return value end + if value % 2 == 0 then return lljson.remove end + return value + end}) + assert(r == "[1,3,5]") +end + +-- nil return from replacer encodes as null (not an error) +do + local r = lljson.encode({a = 1}, {replacer = function(key, value) + if type(value) == "number" then return nil end + return value + end}) + assert(r == '{"a":null}') +end + +-- Passthrough replacer preserves nil-as-null (no semantic change from adding replacer) +do + local r = lljson.encode({1, nil, 3}, {replacer = function(key, value) + return value + end}) + assert(r == "[1,null,3]", "passthrough replacer should preserve nils as null, got: " .. r) +end + +-- Replacer sees nil array elements and can transform them +do + local r = lljson.encode({1, nil, 3}, {replacer = function(key, value, parent) + if parent ~= nil and value == nil then return 0 end + return value + end}) + assert(r == "[1,0,3]", "replacer should be able to transform nil elements, got: " .. r) +end + +-- slencode: nil return from replacer produces "!n" (preserves nil/null distinction) +do + local r = lljson.slencode({1, nil, 3}, {replacer = function(key, value) + return value + end}) + assert(r == '[1,"!n",3]', "slencode passthrough replacer should preserve !n, got: " .. r) +end + +-- Nested structures: replacer sees leaf values +do + local r = lljson.encode({a = {1, 2}}, {replacer = function(key, value) + if type(value) == "number" then return value + 100 end + return value + end}) + local t = lljson.decode(r) + assert(t.a[1] == 101 and t.a[2] == 102) +end + +-- Replacer + __tojson: __tojson resolves first, replacer sees result (JS compat) +do + local mt = { __tojson = function(self) + return self.v * 10 + end } + local r = lljson.encode({a = setmetatable({v = 3}, mt)}, {replacer = function(key, value) + -- replacer sees 30 (the __tojson-resolved value), not the table + if type(value) == "number" then + return value + 1 + end + return value + end}) + assert(r == '{"a":31}', "expected 31, got " .. r) +end + +-- Replacer + __tojson in arrays: __tojson resolves first (JS compat) +do + local mt = { __tojson = function(self) + return self.v * 10 + end } + local r = lljson.encode({setmetatable({v = 3}, mt)}, {replacer = function(key, value) + if type(value) == "number" then + return value + 1 + end + return value + end}) + assert(r == '[31]', "expected [31], got " .. r) +end + +-- Root __tojson + replacer: __tojson resolves first, replacer sees result (consistent with non-root) +do + local mt = { __tojson = function(self) + return self.v * 10 + end } + local r = lljson.encode(setmetatable({v = 5}, mt), {replacer = function(key, value) + -- replacer should see 50 (the __tojson-resolved value), not the table + if type(value) == "number" then + return value + 1 + end + return value + end}) + assert(r == '51', "root __tojson + replacer: expected 51, got " .. r) +end + +-- Root replacer: replacer receives (nil, value, nil) for root +do + local r = lljson.encode({1, 2, 3}, {replacer = function(key, value, parent) + if key == nil and parent == nil then + -- root call: transform the root value + return {10, 20, 30} + end + return value + end}) + assert(r == '[10,20,30]', "root replacer transform failed: " .. r) +end + +-- Root replacer remove: returning lljson.remove from root -> encodes as null +do + local r = lljson.encode({1, 2}, {replacer = function(key, value, parent) + if key == nil then + return lljson.remove + end + return value + end}) + assert(r == 'null', "root replacer remove failed: " .. r) +end + +-- nil return from root replacer encodes as null +do + local r = lljson.encode({1, 2}, {replacer = function(key, value, parent) + if key == nil then + return nil + end + return value + end}) + assert(r == "null") +end + +-- Replacer parent arg (object): third arg is the containing table +do + local inner = {b = 1} + local seen_parent + local r = lljson.encode({a = inner}, {replacer = function(key, value, parent) + if key == "b" then + seen_parent = parent + end + return value + end}) + assert(seen_parent == inner, "replacer parent should be the inner table") +end + +-- Replacer parent arg (array): third arg is the containing array +do + local inner = {10} + local seen_parent + local r = lljson.encode({inner}, {replacer = function(key, value, parent) + if value == 10 then + seen_parent = parent + end + return value + end}) + assert(seen_parent == inner, "replacer array parent should be the inner array") +end + +-- Reviver parent arg (object): third arg is the containing table +do + local seen_parent + local result = lljson.decode('{"a":{"b":1}}', function(key, value, parent) + if key == "b" then + seen_parent = parent + end + return value + end) + -- parent should be the inner table that contains key "b" + assert(type(seen_parent) == "table", "reviver parent should be a table") + assert(seen_parent.b == 1, "reviver parent should be the inner object") +end + +-- Reviver parent arg (array): third arg is the containing array +do + local seen_parent + local result = lljson.decode('[["hello"]]', function(key, value, parent) + if value == "hello" then + seen_parent = parent + end + return value + end) + assert(type(seen_parent) == "table", "reviver array parent should be a table") + assert(seen_parent[1] == "hello", "reviver array parent should be the inner array") +end + +-- Root reviver: key and parent are both nil +do + local root_key, root_parent, root_called + local result = lljson.decode('42', function(key, value, parent) + root_key = key + root_parent = parent + root_called = true + return value + end) + assert(root_called, "root reviver should be called") + assert(root_key == nil, "root reviver key should be nil") + assert(root_parent == nil, "root reviver parent should be nil") + assert(result == 42, "root reviver should pass through value") +end + +-- slencode with replacer +do + local r = lljson.slencode({a = 1, b = 2}, {replacer = function(key, value) + if key == "b" then return lljson.remove end + return value + end}) + local t = lljson.sldecode(r) + assert(t.a == 1 and t.b == nil) +end + +-- encode with replacer +do + local r = lljson.encode({x = 10}, {replacer = function(key, value) + if type(value) == "number" then return value * 2 end + return value + end}) + assert(lljson.decode(r).x == 20) +end + +-- ============================================ +-- skip_tojson option +-- ============================================ + +-- skip_tojson suppresses __tojson: table encoded as plain object, not __tojson result +do + local mt = { __tojson = function(self) + return self.v * 10 + end } + local obj = setmetatable({v = 3}, mt) + local r = lljson.encode(obj, { + skip_tojson = true, + replacer = function(key, value) return value end, + }) + local t = lljson.decode(r) + assert(t.v == 3, "skip_tojson should encode raw table, got: " .. r) +end + +-- skip_tojson: replacer sees original metatabled table, not __tojson result +do + local mt = { __tojson = function(self) + return self.v * 10 + end } + local obj = setmetatable({v = 7}, mt) + local seen_mt + local r = lljson.encode({a = obj}, { + skip_tojson = true, + replacer = function(key, value) + if key == "a" then + seen_mt = getmetatable(value) + end + return value + end, + }) + assert(seen_mt == mt, "replacer should see original metatable with skip_tojson") +end + +-- skip_tojson: replacer can invoke __tojson manually +do + local mt = { __tojson = function(self) + return self.v * 10 + end } + local obj = setmetatable({v = 5}, mt) + local r = lljson.encode(obj, { + skip_tojson = true, + replacer = function(key, value) + if type(value) == "table" then + local m = getmetatable(value) + if m and m.__tojson then + return m.__tojson(value) + end + end + return value + end, + }) + assert(r == "50", "manual __tojson invocation should work, got: " .. r) +end + +-- skip_tojson = false: __tojson still resolves normally (explicit false) +do + local mt = { __tojson = function(self) + return self.v * 10 + end } + local r = lljson.encode(setmetatable({v = 4}, mt), { + skip_tojson = false, + replacer = function(key, value) + if type(value) == "number" then return value + 1 end + return value + end, + }) + assert(r == "41", "skip_tojson=false should resolve __tojson normally, got: " .. r) +end + +-- skip_tojson in arrays: replacer sees raw table, not __tojson result +do + local mt = { __tojson = function(self) + return self.v * 10 + end } + local r = lljson.encode({setmetatable({v = 2}, mt)}, { + skip_tojson = true, + replacer = function(key, value) return value end, + }) + local t = lljson.decode(r) + assert(t[1].v == 2, "skip_tojson in array should encode raw table, got: " .. r) +end + +-- skip_tojson without replacer: __tojson is still suppressed +do + local mt = { __tojson = function(self) + return self.v * 10 + end } + local r = lljson.encode(setmetatable({v = 6}, mt), { + skip_tojson = true, + }) + local t = lljson.decode(r) + assert(t.v == 6, "skip_tojson without replacer should suppress __tojson, got: " .. r) +end + +-- ============================================ +-- Interrupt tests for replacer/reviver +-- ============================================ + +enable_check_interrupt() + +local function consume_impl(check, expect_yields, f, ...) + clear_check_count() + local co = coroutine.create(f) + local yields = 0 + local ok, a, b, c = coroutine.resume(co, ...) + assert(ok, a) + while coroutine.status(co) ~= "dead" do + yields += 1 + co = ares.unpersist(ares.persist(co)) + collectgarbage() + ok, a, b, c = coroutine.resume(co) + assert(ok, a) + end + if expect_yields then + assert(yields > 0, "no yields occurred") + end + if check then + assert(yields == get_check_count(), + "yield count mismatch: " .. yields .. " actual vs " .. get_check_count() .. " interrupts") + end + return a, b, c, yields +end + +local function consume(f, ...) + return consume_impl(true, true, f, ...) +end + +local function consume_nocheck(f, ...) + return consume_impl(false, false, f, ...) +end + +-- decode with reviver: exercises REVIVER_CHECK/REVIVER_CALL yield paths with Ares round-trip +consume(function() + local src = '{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6,"g":7,"h":8}' + local t = lljson.decode(src, function(key, value) + if type(value) == "number" then return value * 10 end + return value + end) + assert(t.a == 10 and t.b == 20 and t.h == 80) +end) + +-- decode array with reviver + lljson.remove: exercises compaction across yields +consume(function() + local src = '[1,2,3,4,5,6,7,8,9,10]' + local t = lljson.decode(src, function(key, value) + -- remove even numbers + if type(value) == "number" and value % 2 == 0 then return lljson.remove end + return value + end) + assert(#t == 5) + assert(t[1] == 1 and t[2] == 3 and t[3] == 5 and t[4] == 7 and t[5] == 9) +end) + +-- encode with replacer: exercises REPLACER_CHECK/REPLACER_CALL yield paths +consume_nocheck(function() + local r = lljson.encode({a = 1, b = 2, c = 3, d = 4, e = 5, f = 6, g = 7, h = 8}, {replacer = function(key, value) + if type(value) == "number" then return value * 10 end + return value + end}) + local t = lljson.decode(r) + assert(t.a == 10 and t.b == 20 and t.h == 80) +end) + +-- encode array with replacer + lljson.remove across yields +consume_nocheck(function() + local r = lljson.encode({1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, {replacer = function(key, value) + if type(value) == "number" and value % 2 == 0 then return lljson.remove end + return value + end}) + assert(r == "[1,3,5,7,9]") +end) + +return 'OK' diff --git a/tests/conformance/lljson_typedjson.lua b/tests/conformance/lljson_typedjson.lua new file mode 100644 index 00000000..000fbcc4 --- /dev/null +++ b/tests/conformance/lljson_typedjson.lua @@ -0,0 +1,395 @@ +-- TypedJSON: branded serialization via lljson replacer/reviver +-- +-- Wraps lljson to tag marked objects with a type name during encoding +-- and revive them during decoding. The caller provides an explicit +-- name <-> metatable registry. Sentinel tables ensure that incidental +-- "__type" fields on user data are caught as errors. + +local TypedJSON = {} +TypedJSON.__index = TypedJSON + +function TypedJSON.new(type_map: {[string]: table}, options: {[string]: any}?) + options = options or {} + if options.replacer then + error("Replacer may not be specified on options") + end + + local self = setmetatable({}, TypedJSON) + self.name_to_mt = {} + self.mt_to_name = {} + self.name_to_sentinel = {} + self.sentinel_to_name = {} + + -- Set up the sentinel tables so we can use unique table identities + -- to "brand" serialized data through "__type" + -- "name" is a bit of a misnomer since it needn't be a string, but + -- it usually is. + for name, mt in type_map do + local sentinel = table.freeze({name}) + self.name_to_mt[name] = mt + self.mt_to_name[mt] = name + self.name_to_sentinel[name] = sentinel + self.sentinel_to_name[sentinel] = name + end + + self._replacer = self:_make_replacer() + self._reviver = self:_make_reviver() + + local encode_opts = table.clone(options) + encode_opts.replacer = self._replacer + -- We'll manually call `__tojson()` ourselves where necessary, but we + -- want to see the values before they're unwrapped. + encode_opts.skip_tojson = true + self._encode_opts = encode_opts + + return self +end + +function TypedJSON:_make_replacer() + local sentinel_to_name = self.sentinel_to_name + local mt_to_name = self.mt_to_name + local name_to_sentinel = self.name_to_sentinel + -- Per-type reusable wrappers. __type is fixed (never mutated), only + -- __value changes. This is safe even for same-type nesting: lua_next + -- captures values onto the Lua stack before recursion, so the outer + -- __value is unaffected by the inner write. + -- + -- NOTE: This isn't _strictly_ necessary, but it reduces GC pressure + -- by a lot, because we can avoid a temporary table alloc that we'd + -- otherwise need for each individual object we need to wrap. + local wrappers = {} + for name, sentinel in name_to_sentinel do + wrappers[name] = { __type = sentinel, __value = lljson.null } + end + + return function(key, value, parent) + -- Intercept __type fields if present, try to unwrap the branding table + if key == "__type" then + local name = sentinel_to_name[value] + if name then + return name + end + error("field '__type' is reserved for type branding") + end + + -- Don't re-wrap values inside our wrappers, we might just be revisiting. + -- TODO: Hmm, imagine there's a cheaper way to do this. + if key == "__value" and parent then + if sentinel_to_name[rawget(parent, "__type")] then + return value + end + end + + -- Wrap potentially branded tables + if type(value) == "table" then + local mt = getmetatable(value) + if mt then + -- Call __tojson manually on the value, if we have one. + local tojson = mt.__tojson + value = if tojson then tojson(value) else value + + -- Then try to wrap the value in a branded table, if this is one of our known types + -- otherwise just return the value directly. + local name = mt_to_name[mt] + if name then + + -- Returning nil from __tojson here isn't allowed for complicated reasons. Mostly because + -- it'd change the shape of the wrapper table if we set it which could mess up iteration. + if value == nil then + error("__tojson must not return nil (maybe use lljson.null?)") + end + local w = wrappers[name] + w.__value = value + return w + end + end + end + + return value + end +end + +function TypedJSON:_make_reviver() + local name_to_mt = self.name_to_mt + + return function(key, value) + if type(value) == "table" then + local type_name = rawget(value, "__type") + if type_name ~= nil then + local inner = rawget(value, "__value") + if inner ~= nil then + local mt = name_to_mt[type_name] + if not mt then + error(`unknown branded type: {type_name}`) + end + local fromjson = mt.__fromjson + if fromjson then + return fromjson(inner) + end + return setmetatable(inner, mt) + end + end + end + return value + end +end + +function TypedJSON:encode(value) + return lljson.encode(value, self._encode_opts) +end + +function TypedJSON:decode(json_string) + return lljson.decode(json_string, self._reviver) +end + +function TypedJSON:slencode(value) + return lljson.slencode(value, self._encode_opts) +end + +function TypedJSON:sldecode(json_string) + return lljson.sldecode(json_string, self._reviver) +end + +-- ============================================ +-- Tests +-- ============================================ + +-- Define some example types +local Vec2 = {} +Vec2.__index = Vec2 +function Vec2.new(x, y) + return setmetatable({ x = x, y = y }, Vec2) +end +function Vec2:magnitude() + return math.sqrt(self.x * self.x + self.y * self.y) +end + +local Player = {} +Player.__index = Player +function Player.new(name, pos) + return setmetatable({ name = name, pos = pos }, Player) +end + +-- This could just be a vector, but whatever, it's just for demonstration. +local Color = {} +Color.__index = Color +function Color.new(r, g, b) + return setmetatable({ r = r, g = g, b = b }, Color) +end +function Color:__tojson() + return string.format("#%02x%02x%02x", self.r, self.g, self.b) +end +function Color.__fromjson(s) + return Color.new( + tonumber(string.sub(s, 2, 3), 16), + tonumber(string.sub(s, 4, 5), 16), + tonumber(string.sub(s, 6, 7), 16) + ) +end + +local tj = TypedJSON.new({ Vec2 = Vec2, Player = Player, [3] = Color }) + +-- Basic round-trip +do + local v = Vec2.new(3, 4) + local str = tj:encode(v) + local decoded = lljson.decode(str) + assert(decoded.__type == "Vec2") + assert(decoded.__value.x == 3) + assert(decoded.__value.y == 4) + + local revived = tj:decode(str) + assert(getmetatable(revived) == Vec2) + assert(revived.x == 3) + assert(revived.y == 4) + assert(revived:magnitude() == 5) +end + +-- Source object not mutated +do + local v = Vec2.new(1, 2) + tj:encode(v) + assert(rawget(v, "__type") == nil) + assert(rawget(v, "__value") == nil) + assert(getmetatable(v) == Vec2) +end + +-- Nested branded objects +do + local p = Player.new("Alice", Vec2.new(3, 4)) + local str = tj:encode(p) + local revived = tj:decode(str) + + assert(getmetatable(revived) == Player) + assert(revived.name == "Alice") + assert(getmetatable(revived.pos) == Vec2) + assert(revived.pos.x == 3) + assert(revived.pos.y == 4) + assert(revived.pos:magnitude() == 5) +end + +-- Array of branded objects +do + local points = { + Vec2.new(1, 0), + Vec2.new(0, 1), + Vec2.new(3, 4), + } + local str = tj:encode(points) + local revived = tj:decode(str) + + assert(#revived == 3) + for i, v in revived do + assert(getmetatable(v) == Vec2) + end + assert(revived[1].x == 1) + assert(revived[3]:magnitude() == 5) +end + +-- Error on unbranded __type field in encode +do + local bad = { __type = "Vec2", __value = {1,2} } + local ok, err = pcall(function() + tj:encode(bad) + end) + assert(not ok) + assert(string.find(err, "reserved")) +end + +-- Error on unknown type during decode +do + local json_str = '{"__type":"Unknown","__value":{"a":1}}' + local ok, err = pcall(function() + tj:decode(json_str) + end) + assert(not ok) + assert(string.find(err, "unknown branded type")) +end + +-- Unbranded tables pass through fine +do + local plain = { x = 1, y = 2 } + local str = tj:encode(plain) + local revived = tj:decode(str) + assert(revived.x == 1) + assert(revived.y == 2) + assert(getmetatable(revived) == nil) +end + +-- slencode/sldecode round-trip +do + local v = Vec2.new(3, 4) + local str = tj:slencode(v) + local revived = tj:sldecode(str) + assert(getmetatable(revived) == Vec2) + assert(revived.x == 3) + assert(revived.y == 4) + assert(revived:magnitude() == 5) +end + +-- slencode/sldecode with nested branded objects +do + local p = Player.new("Bob", Vec2.new(10, 20)) + local str = tj:slencode(p) + local revived = tj:sldecode(str) + assert(getmetatable(revived) == Player) + assert(revived.name == "Bob") + assert(getmetatable(revived.pos) == Vec2) + assert(revived.pos.x == 10) + assert(revived.pos.y == 20) +end + +-- Table with __value but no __type passes through (not a branded wrapper) +do + local t = { __value = 42, other = "hi" } + local str = tj:encode(t) + local revived = tj:decode(str) + assert(revived.__value == 42) + assert(revived.other == "hi") +end + +-- Branded object inside unbranded table +do + local data = { + label = "origin", + point = Vec2.new(0, 0), + } + local str = tj:encode(data) + local revived = tj:decode(str) + assert(revived.label == "origin") + assert(getmetatable(revived.point) == Vec2) + assert(revived.point.x == 0) + assert(revived.point.y == 0) +end + +-- Deep nesting: exercises wrapper reuse at multiple depths +do + local Team = {} + Team.__index = Team + function Team.new(name, members) + return setmetatable({ name = name, members = members }, Team) + end + + local tj2 = TypedJSON.new({ Vec2 = Vec2, Player = Player, Team = Team }) + + local team = Team.new("Red", { + Player.new("Alice", Vec2.new(1, 2)), + Player.new("Bob", Vec2.new(3, 4)), + }) + local str = tj2:encode(team) + local revived = tj2:decode(str) + + assert(getmetatable(revived) == Team) + assert(revived.name == "Red") + assert(#revived.members == 2) + assert(getmetatable(revived.members[1]) == Player) + assert(getmetatable(revived.members[1].pos) == Vec2) + assert(revived.members[1].pos.x == 1) + assert(getmetatable(revived.members[2]) == Player) + assert(revived.members[2].pos:magnitude() == 5) +end + +-- ============================================ +-- Color: branded type with __tojson +-- ============================================ + +-- Branding with __tojson/__fromjson: compact wire format +do + local c = Color.new(255, 0, 0) + local str = tj:encode(c) + local decoded = lljson.decode(str) + assert(decoded.__type == 3, `should be branded as 3, got: {str}`) + assert(decoded.__value == "#ff0000", "should use __tojson compact form, got: " .. str) + + local revived = tj:decode(str) + assert(getmetatable(revived) == Color) + assert(revived.r == 255 and revived.g == 0 and revived.b == 0) +end + +-- Without skip_tojson, replacer sees __tojson result (a string), can't brand it +do + local seen_type + local r = lljson.encode(Color.new(255, 0, 0), { + replacer = function(key, value) + if key == nil then + seen_type = type(value) + end + return value + end, + }) + assert(seen_type == "string") + assert(r == '"#ff0000"') +end + +-- Round-trip with Color nested inside other branded types +do + local data = { pos = Vec2.new(1, 2), color = Color.new(0, 128, 255) } + local str = tj:encode(data) + local revived = tj:decode(str) + assert(getmetatable(revived.pos) == Vec2) + assert(revived.pos.x == 1) + assert(getmetatable(revived.color) == Color) + assert(revived.color.r == 0 and revived.color.g == 128 and revived.color.b == 255) +end + +return 'OK' diff --git a/tests/conformance/metamethod_and_callback_interrupts.lua b/tests/conformance/metamethod_and_callback_interrupts.lua index 5d550022..48db0f23 100644 --- a/tests/conformance/metamethod_and_callback_interrupts.lua +++ b/tests/conformance/metamethod_and_callback_interrupts.lua @@ -23,7 +23,7 @@ end -- Create test object once to avoid intermediate function calls local obj = create_test_object() local obj_len = {} -setmetatable(obj_len, {__len = test_callback}) +setmetatable(obj_len, {__jsontype = "array", __len = test_callback}) -- Test __tostring (via luaL_callmeta path) reset_interrupt_test() diff --git a/tests/conformance/sl_ares.lua b/tests/conformance/sl_ares.lua index febabaf3..cb1f3921 100644 --- a/tests/conformance/sl_ares.lua +++ b/tests/conformance/sl_ares.lua @@ -1,5 +1,4 @@ local array = setmetatable({}, lljson.array_mt) -local empty_array = setmetatable({}, lljson.empty_array_mt) function roundtrip_persist(val) @@ -11,7 +10,6 @@ function unpersisted_metatable(val) end assert(unpersisted_metatable(array) == lljson.array_mt) -assert(unpersisted_metatable(empty_array) == lljson.empty_array_mt) local vec_mul = getmetatable(vector(1,2,3)).__mul local quat_mul = getmetatable(quaternion(1,2,3,4)).__mul From a52935ad3532dbca52e81da7606b9ee151e4179c Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Mon, 16 Mar 2026 14:51:50 -0700 Subject: [PATCH 02/18] Better handling of sparse tables --- VM/src/cjson/lua_cjson.cpp | 12 ++++++++---- tests/conformance/lljson.lua | 5 +++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/VM/src/cjson/lua_cjson.cpp b/VM/src/cjson/lua_cjson.cpp index c05e192c..9661bae8 100644 --- a/VM/src/cjson/lua_cjson.cpp +++ b/VM/src/cjson/lua_cjson.cpp @@ -230,6 +230,7 @@ static void json_init_lookup_tables() struct json_config_t { int encode_sparse_convert = 0; + bool allow_sparse = false; bool sl_tagged_types = false; bool sl_tight_encoding = false; bool has_replacer = false; @@ -368,7 +369,7 @@ static void json_append_tostring(lua_State *l, strbuf_t *json, int lindex) * -1 object (not a pure array) * >=0 elements in array */ -static int lua_array_length(lua_State *l, json_config_t *cfg, strbuf_t *json) +static int lua_array_length(lua_State *l, json_config_t *cfg, strbuf_t *json, bool force = false) { double k; int max; @@ -399,7 +400,7 @@ static int lua_array_length(lua_State *l, json_config_t *cfg, strbuf_t *json) } /* Encode excessively sparse arrays as objects (if enabled) */ - if (max > items * JSON_SPARSE_RATIO && + if (!force && max > items * JSON_SPARSE_RATIO && max > JSON_SPARSE_SAFE) { if (!cfg->encode_sparse_convert) json_encode_exception(l, cfg, json, -1, "excessively sparse array"); @@ -1055,7 +1056,7 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, if (as_array) { // Validate: __jsontype="array" requires all keys to be positive integers - len = lua_array_length(l, cfg, json); + len = lua_array_length(l, cfg, json, true); if (len < 0) luaL_error(l, "cannot encode as array: table has non-integer keys"); @@ -1076,7 +1077,7 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, YIELD_HELPER(l, APPEND_ARRAY, json_append_array(l, slots, cfg, depth, json, array_length, raw)); } else { - len = lua_array_length(l, cfg, json); + len = lua_array_length(l, cfg, json, cfg->allow_sparse); if (len >= 0) { array_length = len; @@ -1224,6 +1225,9 @@ static int json_encode_common(lua_State* l, bool is_init, bool sl_tagged) skip_tojson = lua_toboolean(l, -1); cfg.skip_tojson = skip_tojson; lua_pop(l, 1); + lua_rawgetfield(l, 3, "allow_sparse"); + cfg.allow_sparse = lua_toboolean(l, -1); + lua_pop(l, 1); lua_rawgetfield(l, 3, "replacer"); if (!lua_isfunction(l, -1)) { lua_pop(l, 1); diff --git a/tests/conformance/lljson.lua b/tests/conformance/lljson.lua index 1023629e..64fc6d4b 100644 --- a/tests/conformance/lljson.lua +++ b/tests/conformance/lljson.lua @@ -40,6 +40,11 @@ assert(not pcall(function() lljson.encode({[vector.one]=1}) end)) assert(lljson.encode({[4]=1}) == '[null,null,null,1]') -- But not _really_ sparse arrays assert(not pcall(function() lljson.encode({[200]=1}) end)) +-- Unless it's specifically an array +local sparse_result = `[{string.rep("null,", 199)}1]` +assert(lljson.encode(setmetatable({[200]=1}, lljson.array_mt)) == sparse_result) +-- Or via the allow_sparse option +assert(lljson.encode({[200]=1}, {allow_sparse=true}) == sparse_result) -- Vector is allowed to have NaN because it turns into a string. local nan_vec = vector(math.huge, -math.huge, 0/0) From 8b49a3de18bcd21a6a03fb7c91fd6e53b0c19289 Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Wed, 18 Mar 2026 13:15:13 -0700 Subject: [PATCH 03/18] Add explicit tests for replacer / reviver order --- tests/conformance/lljson_replacer.lua | 37 +++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/conformance/lljson_replacer.lua b/tests/conformance/lljson_replacer.lua index dc418520..fcd88965 100644 --- a/tests/conformance/lljson_replacer.lua +++ b/tests/conformance/lljson_replacer.lua @@ -380,6 +380,43 @@ do assert(lljson.decode(r).x == 20) end +-- ============================================ +-- Reviver visitation order +-- ============================================ + +-- Helpers: collect visited keys, encode to JSON for easy comparison. +-- nil (root) is recorded as lljson.null so it survives in the array. +local function check_reviver_order(json_str, expected) + local order = {} + lljson.decode(json_str, function(key, value) + table.insert(order, if key == nil then lljson.null else key) + return value + end) + local got = lljson.encode(order) + assert(got == expected, `expected {expected}, got {got}`) +end + +local function check_replacer_order(value, expected) + local order = {} + lljson.encode(value, {replacer = function(key, value) + table.insert(order, if key == nil then lljson.null else key) + return value + end}) + local got = lljson.encode(order) + assert(got == expected, `expected {expected}, got {got}`) +end + +-- Revivers do depth-first, leaf-first visitation +check_reviver_order('{"a":{"b":{"c":1}}}', '["c","b","a",null]') +check_reviver_order('[{"a":1},{"b":2}]', '["a",1,"b",2,null]') +check_reviver_order('{"x":[1,2],"y":[3,4]}', '[1,2,"x",1,2,"y",null]') +check_reviver_order('{"a":1,"b":2,"c":3}', '["a","b","c",null]') + +-- Replacers do depth-first, container-first visitation +check_replacer_order({a = {b = 1, c = 2}}, '[null,"a","c","b"]') +check_replacer_order({{a = 1}, {b = 2}}, '[null,1,"a",2,"b"]') +check_replacer_order({a = 1, b = 2, c = 3}, '[null,"a","c","b"]') + -- ============================================ -- skip_tojson option -- ============================================ From e863750c8d2bcfa2d5324f09e3074911b97d2ba9 Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Wed, 18 Mar 2026 13:25:55 -0700 Subject: [PATCH 04/18] Unify stringified number -> number logic in lljson --- VM/src/cjson/lua_cjson.cpp | 37 ++++++++++-------------------------- tests/conformance/lljson.lua | 3 +++ 2 files changed, 13 insertions(+), 27 deletions(-) diff --git a/VM/src/cjson/lua_cjson.cpp b/VM/src/cjson/lua_cjson.cpp index 9661bae8..9c6d3202 100644 --- a/VM/src/cjson/lua_cjson.cpp +++ b/VM/src/cjson/lua_cjson.cpp @@ -1816,34 +1816,19 @@ static int json_is_invalid_number(json_parse_t *json) return 0; } +// ServerLua: use fpconv_strtod for all numbers; the T_INTEGER path via strtoll +// silently clamped values exceeding LLONG_MAX (e.g. 1e20 -> 9223372036854776000). +// Luau has no separate integer type, so the distinction was meaningless anyway. static void json_next_number_token(json_parse_t *json, json_token_t *token) { char *endptr; - long long tmpval = strtoll(json->ptr, &endptr, 10); - if (json->ptr == endptr || *endptr == '.' || *endptr == 'e' || - *endptr == 'E' || *endptr == 'x') { - token->type = T_NUMBER; - token->value.number = fpconv_strtod(json->ptr, &endptr); - if (json->ptr == endptr) { - json_set_token_error(token, json, "invalid number"); - return; - } - } else if (tmpval > INT32_MAX || tmpval < INT32_MIN) { - /* Typical Lua builds typedef ptrdiff_t to lua_Integer. If tmpval is - * outside the range of that type, we need to use T_NUMBER to avoid - * truncation. - */ - // ServerLua: In our case, it's actually `typedef`'d to `int`, - // but similar logic applies. - token->type = T_NUMBER; - token->value.number = (double)tmpval; - } else { - token->type = T_INTEGER; - token->value.integer = (int)tmpval; + token->type = T_NUMBER; + token->value.number = fpconv_strtod(json->ptr, &endptr); + if (json->ptr == endptr) { + json_set_token_error(token, json, "invalid number"); + return; } - json->ptr = endptr; /* Skip the processed number */ - - return; + json->ptr = endptr; } /* Fills in the token struct. @@ -2249,10 +2234,8 @@ static void json_process_value(lua_State* l, SlotManager& parent_slots, } break; case T_NUMBER: - lua_pushnumber(l, token->value.number); - break; case T_INTEGER: - lua_pushinteger(l, token->value.integer); + lua_pushnumber(l, token->value.number); break; case T_BOOLEAN: lua_pushboolean(l, token->value.boolean); diff --git a/tests/conformance/lljson.lua b/tests/conformance/lljson.lua index 64fc6d4b..6fa9bbe3 100644 --- a/tests/conformance/lljson.lua +++ b/tests/conformance/lljson.lua @@ -764,5 +764,8 @@ consume_nocheck(function() assert(captured_mode == "sljson") end) +-- Numbers exceeding int64 range should parse correctly, not clamp to LLONG_MAX +assert(lljson.decode("100000000000000000000") == 1e20) +assert(lljson.decode("-100000000000000000000") == -1e20) return 'OK' From 38124a5bb05ac9fc133557ac2c51a1ece0a8157c Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Wed, 18 Mar 2026 15:36:13 -0700 Subject: [PATCH 05/18] Allow using UUIDs as object keys in lljson.encode() --- VM/src/cjson/lua_cjson.cpp | 4 ++++ tests/conformance/lljson.lua | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/VM/src/cjson/lua_cjson.cpp b/VM/src/cjson/lua_cjson.cpp index 9c6d3202..375c9be9 100644 --- a/VM/src/cjson/lua_cjson.cpp +++ b/VM/src/cjson/lua_cjson.cpp @@ -903,6 +903,10 @@ static void json_append_object(lua_State* l, SlotManager& parent_slots, } else if (keytype == LUA_TSTRING) { json_append_string(l, json, -2); strbuf_append_char(json, ':'); + // ServerLua: allow UUID keys, encoded as their string form + } else if (keytype == LUA_TUSERDATA && lua_userdatatag(l, -2) == UTAG_UUID) { + json_append_tostring(l, json, -2); + strbuf_append_char(json, ':'); } else { json_encode_exception(l, cfg, json, -2, "table key must be a number or string"); diff --git a/tests/conformance/lljson.lua b/tests/conformance/lljson.lua index 6fa9bbe3..faf939b4 100644 --- a/tests/conformance/lljson.lua +++ b/tests/conformance/lljson.lua @@ -578,6 +578,12 @@ assert(getmetatable(lljson.empty_object) ~= lljson.object_mt) assert(getmetatable(lljson.empty_array).__jsontype == "array") assert(getmetatable(lljson.empty_object).__jsontype == "object") +-- UUID table keys should encode as their string form +assert( + lljson.encode({[uuid("12345678-1234-1234-1234-123456789abc")]="hello" }) == + '{"12345678-1234-1234-1234-123456789abc":"hello"}' +) + -- Enable interrupt-driven yields for remaining tests. enable_check_interrupt() From c4f8a9262cac5e099766a02ecadb9cc2e09178dc Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Wed, 18 Mar 2026 18:10:01 -0700 Subject: [PATCH 06/18] Tighten up validation for floating point coords in lljson --- VM/src/cjson/lua_cjson.cpp | 72 ++++++++++++++++-------------------- tests/conformance/lljson.lua | 13 +++++++ 2 files changed, 45 insertions(+), 40 deletions(-) diff --git a/VM/src/cjson/lua_cjson.cpp b/VM/src/cjson/lua_cjson.cpp index 375c9be9..83009c78 100644 --- a/VM/src/cjson/lua_cjson.cpp +++ b/VM/src/cjson/lua_cjson.cpp @@ -1380,6 +1380,20 @@ static int decode_hex4(const char *hex) digit[3]; } +// ServerLua: skip whitespace, matching tonumber() tolerance +static inline const char *skip_ws(const char *p) { + while (isspace((unsigned char)*p)) p++; + return p; +} + +// ServerLua: skip whitespace, expect delimiter, advance past it +static const char *expect(const char *p, char c, lua_State *l, const char *tag, const char *str) { + p = skip_ws(p); + if (*p != c) + luaL_error(l, "malformed tagged %s: %s", tag, str); + return p + 1; +} + // Helper to parse a tight component (empty string = 0.0f) static float parse_tight_component(const char **ptr, char delimiter) { const char *p = *ptr; @@ -1389,7 +1403,7 @@ static float parse_tight_component(const char **ptr, char delimiter) { } char *end; float val = strtof(p, &end); - *ptr = end; + *ptr = skip_ws(end); return val; } @@ -1430,31 +1444,22 @@ static bool json_parse_tagged_string(lua_State *l, const char *str, size_t len) if (payload[0] == '<') { // Normal format with brackets - if (payload_len < 5 || payload[payload_len - 1] != '>') - luaL_error(l, "malformed tagged vector: %s", str); - char *end; x = strtof(payload + 1, &end); - if (*end != ',') - luaL_error(l, "malformed tagged vector: %s", str); - y = strtof(end + 1, &end); - if (*end != ',') - luaL_error(l, "malformed tagged vector: %s", str); - z = strtof(end + 1, &end); - if (*end != '>') + y = strtof(expect(end, ',', l, "vector", str), &end); + z = strtof(expect(end, ',', l, "vector", str), &end); + if (*skip_ws(expect(end, '>', l, "vector", str)) != '\0') luaL_error(l, "malformed tagged vector: %s", str); } else { // Tight format: !v1,2,3 or !v,,1 (empty = 0) const char *p = payload; x = parse_tight_component(&p, ','); - if (*p != ',') - luaL_error(l, "malformed tagged vector: %s", str); - p++; + p = expect(p, ',', l, "vector", str); y = parse_tight_component(&p, ','); - if (*p != ',') - luaL_error(l, "malformed tagged vector: %s", str); - p++; + p = expect(p, ',', l, "vector", str); z = parse_tight_component(&p, '\0'); + if (*skip_ws(p) != '\0') + luaL_error(l, "malformed tagged vector: %s", str); } lua_pushvector(l, x, y, z); @@ -1473,38 +1478,25 @@ static bool json_parse_tagged_string(lua_State *l, const char *str, size_t len) if (payload[0] == '<') { // Normal format with brackets - if (payload_len < 7 || payload[payload_len - 1] != '>') - luaL_error(l, "malformed tagged quaternion: %s", str); - char *end; x = strtof(payload + 1, &end); - if (*end != ',') - luaL_error(l, "malformed tagged quaternion: %s", str); - y = strtof(end + 1, &end); - if (*end != ',') - luaL_error(l, "malformed tagged quaternion: %s", str); - z = strtof(end + 1, &end); - if (*end != ',') - luaL_error(l, "malformed tagged quaternion: %s", str); - w = strtof(end + 1, &end); - if (*end != '>') + y = strtof(expect(end, ',', l, "quaternion", str), &end); + z = strtof(expect(end, ',', l, "quaternion", str), &end); + w = strtof(expect(end, ',', l, "quaternion", str), &end); + if (*skip_ws(expect(end, '>', l, "quaternion", str)) != '\0') luaL_error(l, "malformed tagged quaternion: %s", str); } else { // Tight format: !q,,,1 (empty = 0) const char *p = payload; x = parse_tight_component(&p, ','); - if (*p != ',') - luaL_error(l, "malformed tagged quaternion: %s", str); - p++; + p = expect(p, ',', l, "quaternion", str); y = parse_tight_component(&p, ','); - if (*p != ',') - luaL_error(l, "malformed tagged quaternion: %s", str); - p++; + p = expect(p, ',', l, "quaternion", str); z = parse_tight_component(&p, ','); - if (*p != ',') - luaL_error(l, "malformed tagged quaternion: %s", str); - p++; + p = expect(p, ',', l, "quaternion", str); w = parse_tight_component(&p, '\0'); + if (*skip_ws(p) != '\0') + luaL_error(l, "malformed tagged quaternion: %s", str); } luaSL_pushquaternion(l, x, y, z, w); @@ -1547,7 +1539,7 @@ static bool json_parse_tagged_string(lua_State *l, const char *str, size_t len) // Float: !f3.14 or !fNaN or !f1e9999 (infinity) char *end; double num = fpconv_strtod(payload, &end); - if (end == payload) + if (end == payload || *skip_ws(end) != '\0') luaL_error(l, "malformed tagged float: %s", str); lua_pushnumber(l, num); return true; diff --git a/tests/conformance/lljson.lua b/tests/conformance/lljson.lua index faf939b4..3649bbd6 100644 --- a/tests/conformance/lljson.lua +++ b/tests/conformance/lljson.lua @@ -618,6 +618,19 @@ local function consume_nocheck(f, ...) return consume_impl(false, false, f, ...) end +-- Trailing garbage in tagged values should error +assert(not pcall(lljson.sldecode, '"!f3.14$$$$"')) +assert(not pcall(lljson.sldecode, '"!v1,2,3junk"')) +assert(not pcall(lljson.sldecode, '"!q1,2,3,4junk"')) +assert(not pcall(lljson.sldecode, '"!q2e3,,0x16,,xyzzz"')) +assert(not pcall(lljson.sldecode, '"!v<1,2,3>junk"')) +assert(not pcall(lljson.sldecode, '"!q<1,2,3,4>junk"')) +-- Whitespace around components/delimiters is OK (matches tonumber()) +assert(lljson.sldecode('"!f3.14 "') == 3.14) +assert(lljson.sldecode('"!f 3.14"') == 3.14) +assert(lljson.sldecode('"!v< 1 , 2 , 3 >"') == vector(1, 2, 3)) +assert(lljson.sldecode('"!q< 1 , 2 , 3 , 4 >"') == quaternion(1, 2, 3, 4)) + -- Shared metatables for yield tests local yield_tojson_mt = { __tojson = function(self) coroutine.yield() From 44c7639909d27d414ceaeb7229fa6c075a9981b4 Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Thu, 19 Mar 2026 09:26:52 -0700 Subject: [PATCH 07/18] Update VM/src/cjson/lua_cjson.cpp Co-authored-by: Tapple Gao --- VM/src/cjson/lua_cjson.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VM/src/cjson/lua_cjson.cpp b/VM/src/cjson/lua_cjson.cpp index 83009c78..2a9ab4fa 100644 --- a/VM/src/cjson/lua_cjson.cpp +++ b/VM/src/cjson/lua_cjson.cpp @@ -909,7 +909,7 @@ static void json_append_object(lua_State* l, SlotManager& parent_slots, strbuf_append_char(json, ':'); } else { json_encode_exception(l, cfg, json, -2, - "table key must be a number or string"); + "table key must be a number, string, or uuid"); /* never returns */ } } From 86cffc1c2fae69b7711f59b4655a75816c91b926 Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Thu, 19 Mar 2026 15:40:12 -0700 Subject: [PATCH 08/18] Ensure __tojson is not run on replacer result --- VM/src/cjson/lua_cjson.cpp | 16 ++++++++++------ tests/conformance/lljson_replacer.lua | 10 ++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/VM/src/cjson/lua_cjson.cpp b/VM/src/cjson/lua_cjson.cpp index 2a9ab4fa..d4bfaab1 100644 --- a/VM/src/cjson/lua_cjson.cpp +++ b/VM/src/cjson/lua_cjson.cpp @@ -433,7 +433,8 @@ static void json_check_encode_depth(lua_State *l, json_config_t *cfg, // ServerLua: Forward declarations for yieldable encode helpers static int json_append_data(lua_State* l, SlotManager& parent_slots, - json_config_t* cfg, int current_depth, strbuf_t* json); + json_config_t* cfg, int current_depth, strbuf_t* json, + bool skip_tojson_once = false); static void json_append_array(lua_State* l, SlotManager& parent_slots, json_config_t* cfg, int current_depth, strbuf_t* json, int array_length, int raw); @@ -532,7 +533,7 @@ static void json_append_array(lua_State* l, SlotManager& parent_slots, // Not a slot: assigned by YIELD_HELPER before read, no goto crosses this decl. bool skip; YIELD_HELPER(l, ELEMENT, - skip = (bool)json_append_data(l, slots, cfg, current_depth, json)); + skip = (bool)json_append_data(l, slots, cfg, current_depth, json, cfg->has_replacer)); if (skip) { strbuf_set_length(json, json_pos); if (comma == 1) { @@ -918,7 +919,7 @@ static void json_append_object(lua_State* l, SlotManager& parent_slots, // Not a slot: assigned by YIELD_HELPER before read, no goto crosses this decl. bool skip; YIELD_HELPER(l, VALUE, - skip = (bool)json_append_data(l, slots, cfg, current_depth, json)); + skip = (bool)json_append_data(l, slots, cfg, current_depth, json, cfg->has_replacer)); if (skip) { strbuf_set_length(json, json_pos); if (comma == 1) { @@ -939,7 +940,8 @@ static void json_append_object(lua_State* l, SlotManager& parent_slots, /* Serialise Lua data into JSON string. */ // ServerLua: Yieldable version of json_append_data. static int json_append_data(lua_State* l, SlotManager& parent_slots, - json_config_t* cfg, int current_depth, strbuf_t* json) + json_config_t* cfg, int current_depth, strbuf_t* json, + bool skip_tojson_once) { YIELDABLE_RETURNS(0); enum class Phase : uint8_t @@ -1034,7 +1036,9 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, lua_pop(l, 1); // pop metatable // __tojson provides content, __jsontype provides shape - if (!cfg->skip_tojson && luaL_getmetafield(l, -1, "__tojson")) { + // skip_tojson_once suppresses __tojson for direct replacer returns, + // This is for symmetry with JS where `toJSON()` is never executed on replacer returns. + if (!cfg->skip_tojson && !skip_tojson_once && luaL_getmetafield(l, -1, "__tojson")) { lua_pushvalue(l, -2); // self lua_pushvalue(l, (int)EncodeStack::CTX); // ctx table YIELD_CHECK(l, TOJSON_CHECK, LUA_INTERRUPT_LLLIB); @@ -1308,7 +1312,7 @@ static int json_encode_common(lua_State* l, bool is_init, bool sl_tagged) lua_pushvalue(l, (int)EncodeStack::VALUE); lua_hardenstack(l, 1); YIELD_HELPER(l, APPEND_DATA, - json_append_data(l, slots, &cfg, 0, buf)); + json_append_data(l, slots, &cfg, 0, buf, cfg.has_replacer)); lua_settop(l, (int)EncodeStack::STRBUF); strbuf_tostring_inplace((int)EncodeStack::STRBUF, true); diff --git a/tests/conformance/lljson_replacer.lua b/tests/conformance/lljson_replacer.lua index fcd88965..a7e45b09 100644 --- a/tests/conformance/lljson_replacer.lua +++ b/tests/conformance/lljson_replacer.lua @@ -259,6 +259,16 @@ do assert(r == '51', "root __tojson + replacer: expected 51, got " .. r) end +-- __tojson is NOT called on the replacer's return value +do + local bomb = { __tojson = function() error("__tojson should not be called on replacer return") end } + local r = lljson.encode(42, {replacer = function(key, value) + if key == nil then return setmetatable({x = 1}, bomb) end + return value + end}) + assert(r == '{"x":1}', 'expected {"x":1}, got ' .. r) +end + -- Root replacer: replacer receives (nil, value, nil) for root do local r = lljson.encode({1, 2, 3}, {replacer = function(key, value, parent) From 3d2ae9e0a94da91560bda0b36932bea0ec88aa25 Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Thu, 19 Mar 2026 18:13:14 -0700 Subject: [PATCH 09/18] Add support for tracking path in lljson.decode() through context --- VM/src/cjson/lua_cjson.cpp | 99 +++++++++++++++++++++++---- tests/conformance/lljson_replacer.lua | 89 ++++++++++++++++++++++++ 2 files changed, 173 insertions(+), 15 deletions(-) diff --git a/VM/src/cjson/lua_cjson.cpp b/VM/src/cjson/lua_cjson.cpp index d4bfaab1..6474be4f 100644 --- a/VM/src/cjson/lua_cjson.cpp +++ b/VM/src/cjson/lua_cjson.cpp @@ -133,6 +133,8 @@ enum class DecodeStack STRBUF = 2, INPUT = 3, REVIVER = 4, + CTX = 5, + PATH = 6, }; typedef enum { @@ -245,6 +247,7 @@ typedef struct { json_config_t *cfg; int current_depth; bool has_reviver; // When true, a reviver function is on the decode stack + bool has_path; // When true, a path table is on the decode stack } json_parse_t; typedef struct { @@ -1999,8 +2002,8 @@ static void json_parse_object_context(lua_State* l, SlotManager& parent_slots, j slots.finalize(); json_restore_offset(json, ptr_offset); - // ServerLua: 7 slots for table, key, value + reviver call args - json_decode_descend(l, json, 7); + // ServerLua: 8 slots for table, key, value + reviver call args + ctx + json_decode_descend(l, json, 8); if (slots.isInit()) { lua_newtable(l); @@ -2030,6 +2033,13 @@ static void json_parse_object_context(lua_State* l, SlotManager& parent_slots, j YIELD_DISPATCH_END(); while (1) { + // Push key onto path before parsing the value + if (json->has_path) { + int path_len = lua_objlen(l, (int)DecodeStack::PATH); + lua_pushvalue(l, -1); + lua_rawseti(l, (int)DecodeStack::PATH, path_len + 1); + } + /* Fetch value */ /* Stack before: [..., table, key] */ { @@ -2043,13 +2053,14 @@ static void json_parse_object_context(lua_State* l, SlotManager& parent_slots, j ptr_offset = json_get_offset(json); if (json->has_reviver) { - // Call reviver(key, value, parent) + // Call reviver(key, value, parent, ctx) lua_pushvalue(l, (int)DecodeStack::REVIVER); lua_pushvalue(l, -3); // key lua_pushvalue(l, -3); // value lua_pushvalue(l, -6); // parent (the object table) + lua_pushvalue(l, (int)DecodeStack::CTX); YIELD_CHECK(l, REVIVER_CHECK, LUA_INTERRUPT_LLLIB); - YIELD_CALL(l, 3, 1, REVIVER_CALL); + YIELD_CALL(l, 4, 1, REVIVER_CALL); // Stack: [..., table, key, value, result] if (lua_tolightuserdatatagged(l, -1, LU_TAG_JSON_INTERNAL) == JSON_REMOVE) { // Omit this key/value pair entirely @@ -2065,6 +2076,13 @@ static void json_parse_object_context(lua_State* l, SlotManager& parent_slots, j lua_rawset(l, -3); } + // Pop key from path + if (json->has_path) { + int path_len = lua_objlen(l, (int)DecodeStack::PATH); + lua_pushnil(l); + lua_rawseti(l, (int)DecodeStack::PATH, path_len); + } + json_token_t token; json_next_token(json, &token); @@ -2106,8 +2124,8 @@ static void json_parse_array_context(lua_State* l, SlotManager& parent_slots, js slots.finalize(); json_restore_offset(json, ptr_offset); - // ServerLua: 6 slots for table, value + reviver call args - json_decode_descend(l, json, 6); + // ServerLua: 7 slots for table, value + reviver call args + ctx + json_decode_descend(l, json, 7); if (slots.isInit()) { lua_newtable(l); @@ -2136,6 +2154,13 @@ static void json_parse_array_context(lua_State* l, SlotManager& parent_slots, js YIELD_DISPATCH_END(); for (; ; i++) { + // Push index onto path before parsing the element + if (json->has_path) { + int path_len = lua_objlen(l, (int)DecodeStack::PATH); + lua_pushinteger(l, i); + lua_rawseti(l, (int)DecodeStack::PATH, path_len + 1); + } + { json_token_t token; YIELD_HELPER(l, ELEMENT, @@ -2151,8 +2176,9 @@ static void json_parse_array_context(lua_State* l, SlotManager& parent_slots, js lua_pushinteger(l, i); // key (1-based source index) lua_pushvalue(l, -3); // value lua_pushvalue(l, -5); // parent (the array table) + lua_pushvalue(l, (int)DecodeStack::CTX); YIELD_CHECK(l, REVIVER_CHECK, LUA_INTERRUPT_LLLIB); - YIELD_CALL(l, 3, 1, REVIVER_CALL); + YIELD_CALL(l, 4, 1, REVIVER_CALL); // Stack: [..., table, value, result] if (lua_tolightuserdatatagged(l, -1, LU_TAG_JSON_INTERNAL) == JSON_REMOVE) { // Omit this element - don't insert, don't increment insert_idx @@ -2168,6 +2194,13 @@ static void json_parse_array_context(lua_State* l, SlotManager& parent_slots, js lua_rawseti(l, -2, i); /* arr[i] = value */ } + // Pop index from path + if (json->has_path) { + int path_len = lua_objlen(l, (int)DecodeStack::PATH); + lua_pushnil(l); + lua_rawseti(l, (int)DecodeStack::PATH, path_len); + } + json_token_t token; json_next_token(json, &token); @@ -2284,12 +2317,30 @@ static int json_decode_common(lua_State* l, bool is_init, bool sl_tagged) if (is_init) { /* Args already validated by init wrapper. * SlotManager inserted nil at pos 1, original args shifted to pos 2+. - * Stack: [opaque(1), input_string(2), reviver?(3)] */ + * Stack: [opaque(1), input_string(2), arg2?(3)] */ - // Ensure reviver or nil occupies a stack slot (will become pos 4 after strbuf insert) + bool path_enabled = false; + + // Normalize: ensure arg2 slot exists if (lua_gettop(l) < 3) lua_pushnil(l); + // If arg 2 is an options table, extract reviver and path + if (lua_istable(l, 3)) { + lua_rawgetfield(l, 3, "track_path"); + path_enabled = lua_toboolean(l, -1); + lua_pop(l, 1); + + lua_rawgetfield(l, 3, "reviver"); + if (!lua_isfunction(l, -1)) { + lua_pop(l, 1); + lua_pushnil(l); + } + // Stack: [opaque(1), input(2), opts(3), reviver_or_nil(4)] + lua_remove(l, 3); + } + // Stack: [opaque(1), input(2), reviver_or_nil(3)] + // ServerLua: Create decode scratch buffer in memcat 1 - it's an // internal intermediary, not user-visible output. Pre-size to // json_len so _unsafe appends can't overflow (decoded <= input). @@ -2305,6 +2356,19 @@ static int json_decode_common(lua_State* l, bool is_init, bool sl_tagged) lua_insert(l, (int)DecodeStack::STRBUF); /* Stack: [opaque(1), strbuf(2), input_string(3), reviver_or_nil(4)] */ + // Create frozen ctx table and path table (or nil) + if (path_enabled) { + lua_newtable(l); + } else { + lua_pushnil(l); + } + lua_newtable(l); + lua_pushvalue(l, -2); + lua_rawsetfield(l, -2, "path"); + lua_setreadonly(l, -1, true); + lua_insert(l, -2); + /* Stack: [opaque(1), strbuf(2), input(3), reviver(4), ctx(5), path_or_nil(6)] */ + size_t json_len; const char* json_data = lua_tolstring(l, (int)DecodeStack::INPUT, &json_len); @@ -2336,6 +2400,7 @@ static int json_decode_common(lua_State* l, bool is_init, bool sl_tagged) json.current_depth = 0; json.tmp = tmp; json.has_reviver = !lua_isnil(l, (int)DecodeStack::REVIVER); + json.has_path = !lua_isnil(l, (int)DecodeStack::PATH); YIELD_DISPATCH_BEGIN(phase, slots); YIELD_DISPATCH(PROCESS_VALUE); @@ -2362,13 +2427,14 @@ static int json_decode_common(lua_State* l, bool is_init, bool sl_tagged) // Call reviver on root value if present if (json.has_reviver) { // Stack: [..., decoded_value] - lua_checkstack(l, 4); + lua_checkstack(l, 5); lua_pushvalue(l, (int)DecodeStack::REVIVER); // reviver lua_pushnil(l); // key = nil (root) lua_pushvalue(l, -3); // decoded value lua_pushnil(l); // parent = nil (root) + lua_pushvalue(l, (int)DecodeStack::CTX); YIELD_CHECK(l, ROOT_REVIVER_CHECK, LUA_INTERRUPT_LLLIB); - YIELD_CALL(l, 3, 1, ROOT_REVIVER_CALL); + YIELD_CALL(l, 4, 1, ROOT_REVIVER_CALL); // Stack: [..., decoded_value, result] if (lua_tolightuserdatatagged(l, -1, LU_TAG_JSON_INTERNAL) == JSON_REMOVE) { lua_pop(l, 2); @@ -2390,8 +2456,8 @@ static int json_decode_v0(lua_State* l) int nargs = lua_gettop(l); luaL_argcheck(l, nargs >= 1 && nargs <= 2, 1, "expected 1-2 arguments"); luaL_checkstring(l, 1); - if (nargs >= 2) - luaL_checktype(l, 2, LUA_TFUNCTION); + if (nargs >= 2 && !lua_isfunction(l, 2) && !lua_istable(l, 2)) + luaL_argerror(l, 2, "expected function or table"); return json_decode_common(l, true, false); } static int json_decode_v0_k(lua_State* l, int) @@ -2404,8 +2470,8 @@ static int json_decode_sl_v0(lua_State* l) int nargs = lua_gettop(l); luaL_argcheck(l, nargs >= 1 && nargs <= 2, 1, "expected 1-2 arguments"); luaL_checkstring(l, 1); - if (nargs >= 2) - luaL_checktype(l, 2, LUA_TFUNCTION); + if (nargs >= 2 && !lua_isfunction(l, 2) && !lua_istable(l, 2)) + luaL_argerror(l, 2, "expected function or table"); return json_decode_common(l, true, true); } static int json_decode_sl_v0_k(lua_State* l, int) @@ -2431,6 +2497,9 @@ static int lua_cjson_new(lua_State *l) luaS_fix(luaS_newliteral(l, "sljson")); luaS_fix(luaS_newliteral(l, "tight")); luaS_fix(luaS_newliteral(l, "replacer")); + luaS_fix(luaS_newliteral(l, "reviver")); + luaS_fix(luaS_newliteral(l, "track_path")); + luaS_fix(luaS_newliteral(l, "path")); luaS_fix(luaS_newliteral(l, "__tojson")); luaS_fix(luaS_newliteral(l, "__jsontype")); luaS_fix(luaS_newliteral(l, "array")); diff --git a/tests/conformance/lljson_replacer.lua b/tests/conformance/lljson_replacer.lua index a7e45b09..deec4ce5 100644 --- a/tests/conformance/lljson_replacer.lua +++ b/tests/conformance/lljson_replacer.lua @@ -525,6 +525,77 @@ do assert(t.v == 6, "skip_tojson without replacer should suppress __tojson, got: " .. r) end +-- ============================================ +-- Reviver ctx and path tracking +-- ============================================ + +-- ctx is always passed as 4th arg and is frozen +do + local seen_ctx + lljson.decode('{"a":1}', function(key, value, parent, ctx) + if key == "a" then seen_ctx = ctx end + return value + end) + assert(type(seen_ctx) == "table", "ctx should be a table") + assert(table.isfrozen(seen_ctx), "ctx should be frozen") +end + +-- ctx.path is nil by default (function reviver) +do + local seen_path_nil = false + lljson.decode('{"a":1}', function(key, value, parent, ctx) + if key == "a" then seen_path_nil = (ctx.path == nil) end + return value + end) + assert(seen_path_nil, "ctx.path should be nil by default") +end + +-- path tracking: nested object +do + local paths = {} + lljson.decode('{"a":{"b":1}}', { reviver = function(key, value, parent, ctx) + paths[key or "ROOT"] = table.clone(ctx.path) + return value + end, track_path = true }) + assert(lljson.encode(paths["b"]) == '["a","b"]') + assert(lljson.encode(paths["a"]) == '["a"]') + assert(lljson.encode(paths["ROOT"]) == '[]') +end + +-- path tracking: arrays with integer indices +do + local paths = {} + lljson.decode('[1,[2,3]]', { reviver = function(key, value, parent, ctx) + if type(value) == "number" then + paths[value] = table.clone(ctx.path) + end + return value + end, track_path = true }) + assert(lljson.encode(paths) == '[[1],[2,1],[2,2]]') +end + +-- path tracking via options table works with sldecode +do + local paths = {} + lljson.sldecode('{"a":1}', { reviver = function(key, value, parent, ctx) + paths[key or "ROOT"] = table.clone(ctx.path) + return value + end, track_path = true }) + assert(lljson.encode(paths["a"]) == '["a"]') +end + +-- ctx is passed with options table (no path) +do + local seen_ctx + lljson.decode('{"a":1}', { reviver = function(key, value, parent, ctx) + if key == "a" then seen_ctx = ctx end + return value + end }) + assert(type(seen_ctx) == "table", "ctx should be passed with options table too") + assert(table.isfrozen(seen_ctx), "ctx should be frozen with options table") + assert(seen_ctx.path == nil, "ctx.path should be nil without path=true") +end + -- ============================================ -- Interrupt tests for replacer/reviver -- ============================================ @@ -603,4 +674,22 @@ consume_nocheck(function() assert(r == "[1,3,5,7,9]") end) +-- decode with path tracking: exercises path table surviving Ares round-trip +consume(function() + local src = '{"a":{"b":1,"c":2},"d":[3,4,5],"e":{"f":{"g":6}},"h":7,"i":8}' + local paths = {} + local t = lljson.decode(src, { reviver = function(key, value, parent, ctx) + if type(value) == "number" then + paths[value] = table.clone(ctx.path) + return value * 10 + end + return value + end, track_path = true }) + assert(t.a.b == 10 and t.a.c == 20) + assert(t.d[1] == 30 and t.d[2] == 40 and t.d[3] == 50) + assert(t.e.f.g == 60) + assert(t.h == 70 and t.i == 80) + assert(lljson.encode(paths) == '[["a","b"],["a","c"],["d",1],["d",2],["d",3],["e","f","g"],["h"],["i"]]') +end) + return 'OK' From 31f7cae1492b147091a7412a40775090198a3028 Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Thu, 19 Mar 2026 18:35:17 -0700 Subject: [PATCH 10/18] Set `object` and `array` metatables as appropriate in `lljson.decode()` This helps improve round-trippability of JSON payloads from outside SL, preserving the `array`-ness or `object`-ness of empty tables especially. --- VM/src/cjson/lua_cjson.cpp | 10 ++++++++++ tests/conformance/lljson.lua | 26 +++++++++++++++++--------- tests/conformance/lljson_typedjson.lua | 2 +- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/VM/src/cjson/lua_cjson.cpp b/VM/src/cjson/lua_cjson.cpp index 6474be4f..ec3329b8 100644 --- a/VM/src/cjson/lua_cjson.cpp +++ b/VM/src/cjson/lua_cjson.cpp @@ -2007,6 +2007,11 @@ static void json_parse_object_context(lua_State* l, SlotManager& parent_slots, j if (slots.isInit()) { lua_newtable(l); + if (!json->cfg->sl_tagged_types) { + lua_pushlightuserdatatagged(l, JSON_OBJECT, LU_TAG_JSON_INTERNAL); + lua_rawget(l, LUA_REGISTRYINDEX); + lua_setmetatable(l, -2); + } json_token_t token; json_next_token(json, &token); @@ -2129,6 +2134,11 @@ static void json_parse_array_context(lua_State* l, SlotManager& parent_slots, js if (slots.isInit()) { lua_newtable(l); + if (!json->cfg->sl_tagged_types) { + lua_pushlightuserdatatagged(l, JSON_ARRAY, LU_TAG_JSON_INTERNAL); + lua_rawget(l, LUA_REGISTRYINDEX); + lua_setmetatable(l, -2); + } /* Peek at first token - check for empty array */ const char* before = json->ptr; diff --git a/tests/conformance/lljson.lua b/tests/conformance/lljson.lua index 3649bbd6..dd42a471 100644 --- a/tests/conformance/lljson.lua +++ b/tests/conformance/lljson.lua @@ -57,10 +57,14 @@ assert(lljson.decode('null') ~= nil) assert(lljson.decode("true") == true) -- Yay, you can actually use unicode escapes. assert(lljson.decode('"\\u0020"') == " ") --- TODO: should this have the custom array metatable where appropriate? --- could people override it if it did? -assert(getmetatable(lljson.decode('[]')) == nil) -assert(getmetatable(lljson.decode('{}')) == nil) +-- decode() sets array_mt/object_mt for round-trippability +assert(getmetatable(lljson.decode('[]')) == lljson.array_mt) +assert(getmetatable(lljson.decode('{}')) == lljson.object_mt) +assert(getmetatable(lljson.decode('[1,2]')) == lljson.array_mt) +assert(getmetatable(lljson.decode('{"a":1}')) == lljson.object_mt) +-- round-trips preserve shape +assert(lljson.encode(lljson.decode('[]')) == '[]') +assert(lljson.encode(lljson.decode('{}')) == '{}') assert(lljson.decode('[5]')[1] == 5) assert(lljson.decode('{"foo":5}').foo == 5) -- Don't automatically cast these @@ -440,14 +444,18 @@ assert(not pcall(lljson.encode, 1, "string")) assert(not pcall(lljson.slencode, 1, "string")) assert(not pcall(lljson.slencode, 1, true)) --- sldecode does not set array metatables (slencode ignores them, so attaching would be dishonest) +-- sldecode does not set metatables (slencode ignores them, so attaching would be dishonest) do assert(getmetatable(lljson.sldecode("[]")) == nil) assert(getmetatable(lljson.sldecode("[1,2,3]")) == nil) - -- Standard decode: also no metatables - assert(getmetatable(lljson.decode("[]")) == nil) - assert(getmetatable(lljson.decode("[1,2]")) == nil) - -- Round-trip: non-empty array auto-detected, no metatable needed + assert(getmetatable(lljson.sldecode("{}")) == nil) + assert(getmetatable(lljson.sldecode('{"a":1}')) == nil) + -- Standard decode sets metatables + assert(getmetatable(lljson.decode("[]")) == lljson.array_mt) + assert(getmetatable(lljson.decode("[1,2]")) == lljson.array_mt) + assert(getmetatable(lljson.decode("{}")) == lljson.object_mt) + assert(getmetatable(lljson.decode('{"a":1}')) == lljson.object_mt) + -- sldecode round-trip: non-empty array auto-detected, no metatable needed local decoded = lljson.sldecode(lljson.slencode({1, 2, 3})) assert(decoded[1] == 1 and decoded[2] == 2 and decoded[3] == 3) assert(getmetatable(decoded) == nil) diff --git a/tests/conformance/lljson_typedjson.lua b/tests/conformance/lljson_typedjson.lua index 000fbcc4..dada2475 100644 --- a/tests/conformance/lljson_typedjson.lua +++ b/tests/conformance/lljson_typedjson.lua @@ -273,7 +273,7 @@ do local revived = tj:decode(str) assert(revived.x == 1) assert(revived.y == 2) - assert(getmetatable(revived) == nil) + assert(getmetatable(revived) == lljson.object_mt) end -- slencode/sldecode round-trip From 2e141b2e5fbf99c31f5f77a4a93c82940ac2515f Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Thu, 19 Mar 2026 20:33:32 -0700 Subject: [PATCH 11/18] Exercise better hygiene when registering object_mt and array_mt for Ares --- VM/src/ares.cpp | 5 +++++ VM/src/cjson/lua_cjson.cpp | 21 +++++++-------------- tests/conformance/lljson.lua | 6 +++--- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/VM/src/ares.cpp b/VM/src/ares.cpp index 5b36e336..c865269a 100644 --- a/VM/src/ares.cpp +++ b/VM/src/ares.cpp @@ -3034,6 +3034,11 @@ static void scan_metatable(lua_State *L, bool forUnpersist, const std::string& p // Register the metatable as a permanent std::string mt_name = prefix + "/mt"; + // empty_array/empty_object are special, they share array_mt/object_mt, which are + // registered under their canonical names. Skip the alias to avoid duplicates. + if (mt_name == "g/lljson/empty_array/mt" || mt_name == "g/lljson/empty_object/mt") + return; + if (forUnpersist) { lua_pushstring(L, mt_name.c_str()); // ... perms obj_table mt name lua_pushvalue(L, -2); // ... perms obj_table mt name mt diff --git a/VM/src/cjson/lua_cjson.cpp b/VM/src/cjson/lua_cjson.cpp index ec3329b8..cdac8385 100644 --- a/VM/src/cjson/lua_cjson.cpp +++ b/VM/src/cjson/lua_cjson.cpp @@ -2533,7 +2533,6 @@ static int lua_cjson_new(lua_State *l) lua_pushliteral(l, "array"); lua_setfield(l, -2, "__jsontype"); lua_setreadonly(l, -1, true); - lua_fixvalue(l, -1); lua_rawset(l, LUA_REGISTRYINDEX); /* object_mt */ @@ -2542,7 +2541,6 @@ static int lua_cjson_new(lua_State *l) lua_pushliteral(l, "object"); lua_setfield(l, -2, "__jsontype"); lua_setreadonly(l, -1, true); - lua_fixvalue(l, -1); lua_rawset(l, LUA_REGISTRYINDEX); } else { lua_pop(l, 1); @@ -2572,32 +2570,27 @@ static int lua_cjson_new(lua_State *l) /* Set cjson.array_mt */ lua_pushlightuserdatatagged(l, JSON_ARRAY, LU_TAG_JSON_INTERNAL); lua_rawget(l, LUA_REGISTRYINDEX); + lua_fixvalue(l, -1); lua_setfield(l, -2, "array_mt"); /* Set cjson.object_mt */ lua_pushlightuserdatatagged(l, JSON_OBJECT, LU_TAG_JSON_INTERNAL); lua_rawget(l, LUA_REGISTRYINDEX); + lua_fixvalue(l, -1); lua_setfield(l, -2, "object_mt"); - /* Set cjson.empty_array / empty_object - frozen tables with cloned shape metatables. - * Cloned (not shared with array_mt/object_mt) to avoid Ares duplicate-permanent-object errors. */ - lua_newtable(l); + /* Set cjson.empty_array / empty_object - frozen sentinel tables sharing array_mt/object_mt. */ lua_newtable(l); - lua_pushliteral(l, "array"); - lua_setfield(l, -2, "__jsontype"); - lua_setreadonly(l, -1, true); - lua_fixvalue(l, -1); + lua_pushlightuserdatatagged(l, JSON_ARRAY, LU_TAG_JSON_INTERNAL); + lua_rawget(l, LUA_REGISTRYINDEX); lua_setmetatable(l, -2); lua_setreadonly(l, -1, true); lua_fixvalue(l, -1); lua_setfield(l, -2, "empty_array"); lua_newtable(l); - lua_newtable(l); - lua_pushliteral(l, "object"); - lua_setfield(l, -2, "__jsontype"); - lua_setreadonly(l, -1, true); - lua_fixvalue(l, -1); + lua_pushlightuserdatatagged(l, JSON_OBJECT, LU_TAG_JSON_INTERNAL); + lua_rawget(l, LUA_REGISTRYINDEX); lua_setmetatable(l, -2); lua_setreadonly(l, -1, true); lua_fixvalue(l, -1); diff --git a/tests/conformance/lljson.lua b/tests/conformance/lljson.lua index dd42a471..a6714f04 100644 --- a/tests/conformance/lljson.lua +++ b/tests/conformance/lljson.lua @@ -580,9 +580,9 @@ assert(table.isfrozen(lljson.empty_object)) assert(lljson.empty_object ~= nil) assert(lljson.empty_object ~= lljson.null) assert(lljson.empty_object ~= lljson.empty_array) --- Metatables are cloned (not shared with array_mt/object_mt) for Ares compatibility -assert(getmetatable(lljson.empty_array) ~= lljson.array_mt) -assert(getmetatable(lljson.empty_object) ~= lljson.object_mt) +-- Metatables are shared with array_mt/object_mt +assert(getmetatable(lljson.empty_array) == lljson.array_mt) +assert(getmetatable(lljson.empty_object) == lljson.object_mt) assert(getmetatable(lljson.empty_array).__jsontype == "array") assert(getmetatable(lljson.empty_object).__jsontype == "object") From b1bf8e6a38449191861500e3f7bf58f61c251645 Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Thu, 19 Mar 2026 21:33:56 -0700 Subject: [PATCH 12/18] Improve GC validation around fixed vars on graylist --- VM/src/lgcdebug.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/VM/src/lgcdebug.cpp b/VM/src/lgcdebug.cpp index 20d17905..83497a1f 100644 --- a/VM/src/lgcdebug.cpp +++ b/VM/src/lgcdebug.cpp @@ -229,6 +229,8 @@ static void validategraylist(global_State* g, GCObject* o) while (o) { LUAU_ASSERT(isgray(o)); + // ServerLua: fixed non-thread objects must never be on gray lists + LUAU_ASSERT(!isfixed(o) || o->gch.tt == LUA_TTHREAD); switch (o->gch.tt) { From d87673b0f3c6cd10fc2133a6a849f35b8505c470 Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Fri, 20 Mar 2026 00:04:17 -0700 Subject: [PATCH 13/18] Add tests clarifying who 'wins' for __jsontype detection --- tests/conformance/lljson.lua | 13 +++++++++++++ tests/conformance/lljson_replacer.lua | 15 +++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/tests/conformance/lljson.lua b/tests/conformance/lljson.lua index a6714f04..5dce2c00 100644 --- a/tests/conformance/lljson.lua +++ b/tests/conformance/lljson.lua @@ -558,6 +558,19 @@ do } assert(not pcall(lljson.encode, setmetatable({}, bad_tojson_mt))) + -- Original's __jsontype wins over __tojson result's metatable (both directions) + local arr_orig_mt = { + __jsontype = "array", + __tojson = function() return setmetatable({10, 20}, lljson.object_mt) end, + } + assert(lljson.encode(setmetatable({}, arr_orig_mt)) == '[10,20]') + + local obj_orig_mt = { + __jsontype = "object", + __tojson = function() return setmetatable({10, 20}, lljson.array_mt) end, + } + assert(lljson.encode(setmetatable({}, obj_orig_mt)) == '{"1":10,"2":20}') + -- Invalid __jsontype value errors local bad_mt = {__jsontype = "invalid"} assert(not pcall(lljson.encode, setmetatable({}, bad_mt))) diff --git a/tests/conformance/lljson_replacer.lua b/tests/conformance/lljson_replacer.lua index deec4ce5..448373c8 100644 --- a/tests/conformance/lljson_replacer.lua +++ b/tests/conformance/lljson_replacer.lua @@ -269,6 +269,21 @@ do assert(r == '{"x":1}', 'expected {"x":1}, got ' .. r) end +-- Replacer return's __jsontype determines shape, not the original's (both directions) +do + local r = lljson.encode(setmetatable({1, 2, 3}, lljson.object_mt), {replacer = function(key, value) + if key == nil then return setmetatable({10, 20}, lljson.array_mt) end + return value + end}) + assert(r == '[10,20]', `replacer array_mt should override original object_mt: got {r}`) + + r = lljson.encode(setmetatable({10, 20}, lljson.array_mt), {replacer = function(key, value) + if key == nil then return setmetatable({30, 40}, lljson.object_mt) end + return value + end}) + assert(r == '{"1":30,"2":40}', `replacer object_mt should override original array_mt: got {r}`) +end + -- Root replacer: replacer receives (nil, value, nil) for root do local r = lljson.encode({1, 2, 3}, {replacer = function(key, value, parent) From fa4a80b46a51b5f09c240bf119e154fcd49edf45 Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Fri, 20 Mar 2026 15:07:00 -0700 Subject: [PATCH 14/18] Correctly track allow_sparse across yields --- VM/src/cjson/lua_cjson.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/VM/src/cjson/lua_cjson.cpp b/VM/src/cjson/lua_cjson.cpp index cdac8385..9761717d 100644 --- a/VM/src/cjson/lua_cjson.cpp +++ b/VM/src/cjson/lua_cjson.cpp @@ -1208,10 +1208,12 @@ static int json_encode_common(lua_State* l, bool is_init, bool sl_tagged) DEFINE_SLOT(Phase, phase, Phase::DEFAULT); DEFINE_SLOT(bool, tight_encoding, false); DEFINE_SLOT(bool, skip_tojson, false); + DEFINE_SLOT(bool, allow_sparse, false); slots.finalize(); json_config_t cfg; cfg.skip_tojson = skip_tojson; + cfg.allow_sparse = allow_sparse; if (sl_tagged) { cfg.sl_tagged_types = true; cfg.encode_sparse_convert = 1; @@ -1233,11 +1235,10 @@ static int json_encode_common(lua_State* l, bool is_init, bool sl_tagged) lua_pop(l, 1); } lua_rawgetfield(l, 3, "skip_tojson"); - skip_tojson = lua_toboolean(l, -1); - cfg.skip_tojson = skip_tojson; + cfg.skip_tojson = skip_tojson = lua_toboolean(l, -1); lua_pop(l, 1); lua_rawgetfield(l, 3, "allow_sparse"); - cfg.allow_sparse = lua_toboolean(l, -1); + allow_sparse = cfg.allow_sparse = lua_toboolean(l, -1); lua_pop(l, 1); lua_rawgetfield(l, 3, "replacer"); if (!lua_isfunction(l, -1)) { From 6c2d9a63ea5e5962aed0351de800e2390c1387ff Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Fri, 20 Mar 2026 17:30:56 -0700 Subject: [PATCH 15/18] __jsontype -> __jsonhint, allow non-arrays with array hint --- VM/src/cjson/lua_cjson.cpp | 37 +++++----- tests/conformance/lljson.lua | 72 +++++++++---------- tests/conformance/lljson_replacer.lua | 2 +- .../metamethod_and_callback_interrupts.lua | 2 +- 4 files changed, 56 insertions(+), 57 deletions(-) diff --git a/VM/src/cjson/lua_cjson.cpp b/VM/src/cjson/lua_cjson.cpp index 9761717d..c0c0f810 100644 --- a/VM/src/cjson/lua_cjson.cpp +++ b/VM/src/cjson/lua_cjson.cpp @@ -1017,28 +1017,28 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, if (has_metatable) { if (!cfg->sl_tagged_types) { - // ServerLua: Check __jsontype metamethod for shape control - lua_rawgetfield(l, -1, "__jsontype"); + // ServerLua: Check __jsonhint metamethod for shape control + lua_rawgetfield(l, -1, "__jsonhint"); if (!lua_isnil(l, -1)) { if (!lua_isstring(l, -1)) - luaL_error(l, "invalid __jsontype value (expected string)"); - const char* jsontype; - jsontype = lua_tostring(l, -1); - if (strcmp(jsontype, "object") == 0) { + luaL_error(l, "invalid __jsonhint value (expected string)"); + const char* jsonhint; + jsonhint = lua_tostring(l, -1); + if (strcmp(jsonhint, "object") == 0) { force_object = true; - } else if (strcmp(jsontype, "array") == 0) { + } else if (strcmp(jsonhint, "array") == 0) { as_array = true; raw = false; } else { - luaL_error(l, "invalid __jsontype value: '%s' (expected \"array\" or \"object\")", jsontype); + luaL_error(l, "invalid __jsonhint value: '%s' (expected \"array\" or \"object\")", jsonhint); } } - lua_pop(l, 1); // pop __jsontype (or nil) + lua_pop(l, 1); // pop __jsonhint (or nil) } lua_pop(l, 1); // pop metatable - // __tojson provides content, __jsontype provides shape + // __tojson provides content, __jsonhint provides shape // skip_tojson_once suppresses __tojson for direct replacer returns, // This is for symmetry with JS where `toJSON()` is never executed on replacer returns. if (!cfg->skip_tojson && !skip_tojson_once && luaL_getmetafield(l, -1, "__tojson")) { @@ -1066,13 +1066,12 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, } if (as_array) { - // Validate: __jsontype="array" requires all keys to be positive integers + // __jsonhint="array" is a soft hint: if the table has non-integer + // keys, fall through to auto-detection (unambiguously an object). len = lua_array_length(l, cfg, json, true); - if (len < 0) - luaL_error(l, "cannot encode as array: table has non-integer keys"); - - // __len overrides the detected length (validation already passed) - if (luaL_getmetafield(l, -1, "__len")) { + if (len < 0) { + as_array = false; + } else if (luaL_getmetafield(l, -1, "__len")) { lua_pushvalue(l, -2); YIELD_CHECK(l, LEN_CHECK, LUA_INTERRUPT_LLLIB); YIELD_CALL(l, 1, 1, LEN_CALL); @@ -2512,7 +2511,7 @@ static int lua_cjson_new(lua_State *l) luaS_fix(luaS_newliteral(l, "track_path")); luaS_fix(luaS_newliteral(l, "path")); luaS_fix(luaS_newliteral(l, "__tojson")); - luaS_fix(luaS_newliteral(l, "__jsontype")); + luaS_fix(luaS_newliteral(l, "__jsonhint")); luaS_fix(luaS_newliteral(l, "array")); luaS_fix(luaS_newliteral(l, "object")); @@ -2532,7 +2531,7 @@ static int lua_cjson_new(lua_State *l) lua_pushlightuserdatatagged(l, JSON_ARRAY, LU_TAG_JSON_INTERNAL); lua_newtable(l); lua_pushliteral(l, "array"); - lua_setfield(l, -2, "__jsontype"); + lua_setfield(l, -2, "__jsonhint"); lua_setreadonly(l, -1, true); lua_rawset(l, LUA_REGISTRYINDEX); @@ -2540,7 +2539,7 @@ static int lua_cjson_new(lua_State *l) lua_pushlightuserdatatagged(l, JSON_OBJECT, LU_TAG_JSON_INTERNAL); lua_newtable(l); lua_pushliteral(l, "object"); - lua_setfield(l, -2, "__jsontype"); + lua_setfield(l, -2, "__jsonhint"); lua_setreadonly(l, -1, true); lua_rawset(l, LUA_REGISTRYINDEX); } else { diff --git a/tests/conformance/lljson.lua b/tests/conformance/lljson.lua index 5dce2c00..e8df9548 100644 --- a/tests/conformance/lljson.lua +++ b/tests/conformance/lljson.lua @@ -25,7 +25,7 @@ assert(lljson.encode({foo=lljson.null}) == '{"foo":null}') assert(lljson.encode(vector(1, 2.5, 22.0 / 7.0)) == '"<1,2.5,3.142857>"') assert(lljson.encode(quaternion(1, 2.5, 22.0 / 7.0, 4)) == '"<1,2.5,3.142857,4>"') --- metatables without __jsontype/__tojson are ignored +-- metatables without __jsonhint/__tojson are ignored local SomeMT = {} function SomeMT.whatever(...) error("Placeholder function called") @@ -491,96 +491,96 @@ do end -- ============================================ --- __jsontype metamethod +-- __jsonhint metamethod -- ============================================ do - -- Custom metatable with __jsontype = "array" - local arr_mt = {__jsontype = "array"} + -- Custom metatable with __jsonhint = "array" + local arr_mt = {__jsonhint = "array"} assert(lljson.encode(setmetatable({}, arr_mt)) == "[]") assert(lljson.encode(setmetatable({1, 2, 3}, arr_mt)) == "[1,2,3]") - -- Custom metatable with __jsontype = "object" - local obj_mt = {__jsontype = "object"} + -- Custom metatable with __jsonhint = "object" + local obj_mt = {__jsonhint = "object"} assert(lljson.encode(setmetatable({}, obj_mt)) == "{}") assert(lljson.encode(setmetatable({1, 2}, obj_mt)) == '{"1":1,"2":2}') - -- __jsontype + __index: metamethods used for element access + -- __jsonhint + __index: metamethods used for element access local proxy_mt = { - __jsontype = "array", + __jsonhint = "array", __len = function() return 3 end, __index = function(_, k) return k * 10 end, } assert(lljson.encode(setmetatable({}, proxy_mt)) == "[10,20,30]") - -- __jsontype + __len (custom length) + -- __jsonhint + __len (custom length) local len_mt = { - __jsontype = "array", + __jsonhint = "array", __len = function() return 2 end, } assert(lljson.encode(setmetatable({10, 20, 30}, len_mt)) == "[10,20]") - -- __tojson provides content, __jsontype provides shape (orthogonal) + -- __tojson provides content, __jsonhint provides shape (orthogonal) -- scalar __tojson result: shape is irrelevant local scalar_mt = { - __jsontype = "object", + __jsonhint = "object", __tojson = function(self) return self.a end, } assert(lljson.encode(setmetatable({a = 1}, scalar_mt)) == '1') - -- table __tojson result + __jsontype = "array": shape applied to result + -- table __tojson result + __jsonhint = "array": shape applied to result local arr_tojson_mt = { - __jsontype = "array", + __jsonhint = "array", __tojson = function(self) return {self[1] * 10} end, } assert(lljson.encode(setmetatable({5}, arr_tojson_mt)) == '[50]') -- __tojson converts string-keyed table to array-compatible result local convert_mt = { - __jsontype = "array", + __jsonhint = "array", __tojson = function(self) return {self.x, self.y} end, } assert(lljson.encode(setmetatable({x = 1, y = 2}, convert_mt)) == '[1,2]') - -- table __tojson result + __jsontype = "object": shape applied to result + -- table __tojson result + __jsonhint = "object": shape applied to result local obj_tojson_mt = { - __jsontype = "object", + __jsonhint = "object", __tojson = function() return {1, 2} end, } assert(lljson.encode(setmetatable({}, obj_tojson_mt)) == '{"1":1,"2":2}') - -- __jsontype = "array" on table with string keys (no __tojson) -> error - assert(not pcall(lljson.encode, setmetatable({x = 1}, {__jsontype = "array"}))) + -- __jsonhint = "array" on table with string keys: hint ignored, encoded as object + assert(lljson.encode(setmetatable({x = 1}, {__jsonhint = "array"})) == '{"x":1}') - -- __jsontype = "array" + __tojson returning string-keyed table -> error - local bad_tojson_mt = { - __jsontype = "array", + -- __jsonhint = "array" + __tojson returning string-keyed table: hint ignored + local fallback_tojson_mt = { + __jsonhint = "array", __tojson = function() return {x = 1} end, } - assert(not pcall(lljson.encode, setmetatable({}, bad_tojson_mt))) + assert(lljson.encode(setmetatable({}, fallback_tojson_mt)) == '{"x":1}') - -- Original's __jsontype wins over __tojson result's metatable (both directions) + -- Original's __jsonhint wins over __tojson result's metatable (both directions) local arr_orig_mt = { - __jsontype = "array", + __jsonhint = "array", __tojson = function() return setmetatable({10, 20}, lljson.object_mt) end, } assert(lljson.encode(setmetatable({}, arr_orig_mt)) == '[10,20]') local obj_orig_mt = { - __jsontype = "object", + __jsonhint = "object", __tojson = function() return setmetatable({10, 20}, lljson.array_mt) end, } assert(lljson.encode(setmetatable({}, obj_orig_mt)) == '{"1":10,"2":20}') - -- Invalid __jsontype value errors - local bad_mt = {__jsontype = "invalid"} + -- Invalid __jsonhint value errors + local bad_mt = {__jsonhint = "invalid"} assert(not pcall(lljson.encode, setmetatable({}, bad_mt))) - -- Non-string __jsontype value errors - assert(not pcall(lljson.encode, setmetatable({}, {__jsontype = 42}))) - assert(not pcall(lljson.encode, setmetatable({}, {__jsontype = true}))) + -- Non-string __jsonhint value errors + assert(not pcall(lljson.encode, setmetatable({}, {__jsonhint = 42}))) + assert(not pcall(lljson.encode, setmetatable({}, {__jsonhint = true}))) - -- slencode ignores __jsontype - local slen_mt = {__jsontype = "object"} + -- slencode ignores __jsonhint + local slen_mt = {__jsonhint = "object"} assert(lljson.slencode(setmetatable({1, 2}, slen_mt)) == "[1,2]") assert(lljson.slencode(setmetatable({}, slen_mt)) == "[]") end @@ -596,8 +596,8 @@ assert(lljson.empty_object ~= lljson.empty_array) -- Metatables are shared with array_mt/object_mt assert(getmetatable(lljson.empty_array) == lljson.array_mt) assert(getmetatable(lljson.empty_object) == lljson.object_mt) -assert(getmetatable(lljson.empty_array).__jsontype == "array") -assert(getmetatable(lljson.empty_object).__jsontype == "object") +assert(getmetatable(lljson.empty_array).__jsonhint == "array") +assert(getmetatable(lljson.empty_object).__jsonhint == "object") -- UUID table keys should encode as their string form assert( @@ -657,7 +657,7 @@ local yield_tojson_mt = { __tojson = function(self) coroutine.yield() return self.val end } -local yield_len_mt = { __jsontype = "array", __len = function(self) +local yield_len_mt = { __jsonhint = "array", __len = function(self) coroutine.yield() return self.n end, __tojson = function(self) diff --git a/tests/conformance/lljson_replacer.lua b/tests/conformance/lljson_replacer.lua index 448373c8..ab91d21d 100644 --- a/tests/conformance/lljson_replacer.lua +++ b/tests/conformance/lljson_replacer.lua @@ -269,7 +269,7 @@ do assert(r == '{"x":1}', 'expected {"x":1}, got ' .. r) end --- Replacer return's __jsontype determines shape, not the original's (both directions) +-- Replacer return's __jsonhint determines shape, not the original's (both directions) do local r = lljson.encode(setmetatable({1, 2, 3}, lljson.object_mt), {replacer = function(key, value) if key == nil then return setmetatable({10, 20}, lljson.array_mt) end diff --git a/tests/conformance/metamethod_and_callback_interrupts.lua b/tests/conformance/metamethod_and_callback_interrupts.lua index 48db0f23..4065a2cc 100644 --- a/tests/conformance/metamethod_and_callback_interrupts.lua +++ b/tests/conformance/metamethod_and_callback_interrupts.lua @@ -23,7 +23,7 @@ end -- Create test object once to avoid intermediate function calls local obj = create_test_object() local obj_len = {} -setmetatable(obj_len, {__jsontype = "array", __len = test_callback}) +setmetatable(obj_len, {__jsonhint = "array", __len = test_callback}) -- Test __tostring (via luaL_callmeta path) reset_interrupt_test() From 0e4a513704f7e08a878d261f1fb8f97c876b2913 Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Fri, 20 Mar 2026 17:31:38 -0700 Subject: [PATCH 16/18] Remove __len sniffing and __index use in array serialization You can just use __tojson() for doing these kinds of things now, and it's very strange to only consult `__index` in the array case. Barely even works, and was just inherited from cjson. Let's ditch it. --- VM/src/cjson/lua_cjson.cpp | 31 ++++-------------- tests/conformance/lljson.lua | 32 ++++--------------- .../metamethod_and_callback_interrupts.lua | 8 ----- 3 files changed, 13 insertions(+), 58 deletions(-) diff --git a/VM/src/cjson/lua_cjson.cpp b/VM/src/cjson/lua_cjson.cpp index c0c0f810..04a4206d 100644 --- a/VM/src/cjson/lua_cjson.cpp +++ b/VM/src/cjson/lua_cjson.cpp @@ -440,7 +440,7 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, bool skip_tojson_once = false); static void json_append_array(lua_State* l, SlotManager& parent_slots, json_config_t* cfg, int current_depth, - strbuf_t* json, int array_length, int raw); + strbuf_t* json, int array_length); static void json_append_object(lua_State* l, SlotManager& parent_slots, json_config_t* cfg, int current_depth, strbuf_t* json); @@ -453,7 +453,7 @@ static strbuf_t* json_get_strbuf(lua_State* l) static void json_append_array(lua_State* l, SlotManager& parent_slots, json_config_t* cfg, int current_depth, - strbuf_t* json, int array_length, int raw) + strbuf_t* json, int array_length) { YIELDABLE_RETURNS_VOID; enum class Phase : uint8_t @@ -492,12 +492,7 @@ static void json_append_array(lua_State* l, SlotManager& parent_slots, for (; i <= array_length; ++i) { replacer_removed = false; - if (raw) { - lua_rawgeti(l, -1, i); - } else { - lua_pushinteger(l, i); - lua_gettable(l, -2); - } + lua_rawgeti(l, -1, i); /* table, value */ if (cfg->has_replacer) { @@ -957,16 +952,13 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, APPEND_ARRAY_AUTO = 5, APPEND_ARRAY_LUD = 6, TOJSON_CHECK = 7, - LEN_CHECK = 8, - LEN_CALL = 9, - APPEND_OBJECT_MT = 10, + APPEND_OBJECT_MT = 8, }; SlotManager slots(parent_slots); DEFINE_SLOT(Phase, phase, Phase::DEFAULT); DEFINE_SLOT(int32_t, depth, current_depth); DEFINE_SLOT(int32_t, array_length, 0); - DEFINE_SLOT(bool, raw, true); DEFINE_SLOT(uint8_t, type, LUA_TNIL); DEFINE_SLOT(bool, as_array, false); DEFINE_SLOT(bool, force_object, false); @@ -984,8 +976,6 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, YIELD_DISPATCH(TOJSON_RECURSE); YIELD_DISPATCH(APPEND_ARRAY_AUTO); YIELD_DISPATCH(APPEND_ARRAY_LUD); - YIELD_DISPATCH(LEN_CHECK); - YIELD_DISPATCH(LEN_CALL); YIELD_DISPATCH(APPEND_OBJECT_MT); YIELD_DISPATCH_END(); @@ -1028,7 +1018,6 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, force_object = true; } else if (strcmp(jsonhint, "array") == 0) { as_array = true; - raw = false; } else { luaL_error(l, "invalid __jsonhint value: '%s' (expected \"array\" or \"object\")", jsonhint); } @@ -1071,12 +1060,6 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, len = lua_array_length(l, cfg, json, true); if (len < 0) { as_array = false; - } else if (luaL_getmetafield(l, -1, "__len")) { - lua_pushvalue(l, -2); - YIELD_CHECK(l, LEN_CHECK, LUA_INTERRUPT_LLLIB); - YIELD_CALL(l, 1, 1, LEN_CALL); - array_length = lua_tonumber(l, -1); - lua_pop(l, 1); } else { array_length = len; } @@ -1085,14 +1068,14 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, if (as_array) { YIELD_HELPER(l, APPEND_ARRAY, - json_append_array(l, slots, cfg, depth, json, array_length, raw)); + json_append_array(l, slots, cfg, depth, json, array_length)); } else { len = lua_array_length(l, cfg, json, cfg->allow_sparse); if (len >= 0) { array_length = len; YIELD_HELPER(l, APPEND_ARRAY_AUTO, - json_append_array(l, slots, cfg, depth, json, array_length, raw)); + json_append_array(l, slots, cfg, depth, json, array_length)); } else { YIELD_HELPER(l, APPEND_OBJECT, json_append_object(l, slots, cfg, depth, json)); @@ -1115,7 +1098,7 @@ static int json_append_data(lua_State* l, SlotManager& parent_slots, break; } else if (json_internal_val == JSON_ARRAY) { YIELD_HELPER(l, APPEND_ARRAY_LUD, - json_append_array(l, slots, cfg, depth, json, 0, 1)); + json_append_array(l, slots, cfg, depth, json, 0)); break; } } else if (lua_tolightuserdatatagged(l, -1, LU_TAG_LSL_INTEGER) != nullptr) { diff --git a/tests/conformance/lljson.lua b/tests/conformance/lljson.lua index e8df9548..2e050ec9 100644 --- a/tests/conformance/lljson.lua +++ b/tests/conformance/lljson.lua @@ -504,20 +504,13 @@ do assert(lljson.encode(setmetatable({}, obj_mt)) == "{}") assert(lljson.encode(setmetatable({1, 2}, obj_mt)) == '{"1":1,"2":2}') - -- __jsonhint + __index: metamethods used for element access + -- __index and __len are NOT consulted during encode: raw values and length are used local proxy_mt = { __jsonhint = "array", - __len = function() return 3 end, - __index = function(_, k) return k * 10 end, + __index = function(_, k) return k * 100 end, + __len = function() return 1 end, } - assert(lljson.encode(setmetatable({}, proxy_mt)) == "[10,20,30]") - - -- __jsonhint + __len (custom length) - local len_mt = { - __jsonhint = "array", - __len = function() return 2 end, - } - assert(lljson.encode(setmetatable({10, 20, 30}, len_mt)) == "[10,20]") + assert(lljson.encode(setmetatable({1, 2, 3}, proxy_mt)) == '[1,2,3]') -- __tojson provides content, __jsonhint provides shape (orthogonal) -- scalar __tojson result: shape is irrelevant @@ -657,14 +650,6 @@ local yield_tojson_mt = { __tojson = function(self) coroutine.yield() return self.val end } -local yield_len_mt = { __jsonhint = "array", __len = function(self) - coroutine.yield() - return self.n -end, __tojson = function(self) - local t = {} - for i = 1, self.n do t[i] = self[i] end - return t -end } -- encode flat array: exercises ELEMENT/NEXT_ELEMENT yield path assert(consume(function() @@ -731,17 +716,12 @@ consume(function() assert(t.pos == vector(1, 2, 3) and t.id == uuid("12345678-1234-1234-1234-123456789abc")) end) --- __len that yields: exercises LEN_CHECK/LEN_CALL yield path -assert(consume_nocheck(function() - return lljson.encode(setmetatable({10, 20, 30, n = 3}, yield_len_mt)) -end) == "[10,20,30]") - --- deeply nested encode: arrays of objects of arrays with __tojson and __len at multiple levels +-- deeply nested encode: arrays of objects of arrays with __tojson at multiple levels consume_nocheck(function() local r = lljson.encode({ items = { {name = "a", tags = {1, 2, 3}}, - {name = "b", tags = setmetatable({10, 20, n = 2}, yield_len_mt)}, + {name = "b", tags = {10, 20}}, {name = "c", custom = setmetatable({val = "hello"}, yield_tojson_mt)}, }, meta = { diff --git a/tests/conformance/metamethod_and_callback_interrupts.lua b/tests/conformance/metamethod_and_callback_interrupts.lua index 4065a2cc..f618480d 100644 --- a/tests/conformance/metamethod_and_callback_interrupts.lua +++ b/tests/conformance/metamethod_and_callback_interrupts.lua @@ -22,9 +22,6 @@ end -- Create test object once to avoid intermediate function calls local obj = create_test_object() -local obj_len = {} -setmetatable(obj_len, {__jsonhint = "array", __len = test_callback}) - -- Test __tostring (via luaL_callmeta path) reset_interrupt_test() local result = tostring(obj) @@ -37,11 +34,6 @@ result = lljson.encode(obj) assert(result == '"ok"', "__tojson should return '\"ok\"'") assert(check_callback_ran(), "__tojson metamethod should have run") --- Test __len (via JSON encoder when no __tojson) -reset_interrupt_test() -result = lljson.encode(obj_len) -assert(check_callback_ran(), "__len metamethod should have run") - -- Test arithmetic metamethods (__add, __sub, __mul, __div, __unm) -- These go through callTMres or luaV_callTM paths reset_interrupt_test() From eec525027321bd3249fb11b306a6ad2f8f24c3c9 Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Fri, 20 Mar 2026 17:36:54 -0700 Subject: [PATCH 17/18] Add yield step for allow_sparse preservation --- tests/conformance/lljson.lua | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/conformance/lljson.lua b/tests/conformance/lljson.lua index 2e050ec9..f1994d9a 100644 --- a/tests/conformance/lljson.lua +++ b/tests/conformance/lljson.lua @@ -716,6 +716,14 @@ consume(function() assert(t.pos == vector(1, 2, 3) and t.id == uuid("12345678-1234-1234-1234-123456789abc")) end) +-- allow_sparse preserved across yields +consume_nocheck(function() + local sparse = {} + sparse[200] = 1 + local r = lljson.encode(sparse, {allow_sparse = true}) + assert(r == `[{string.rep("null,", 199)}1]`) +end) + -- deeply nested encode: arrays of objects of arrays with __tojson at multiple levels consume_nocheck(function() local r = lljson.encode({ From 9dfb4e065ddbf6a5600ff4ea769dd23f31d376cf Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Fri, 20 Mar 2026 17:41:17 -0700 Subject: [PATCH 18/18] Error on non-function `replacer` and `reviver` --- VM/src/cjson/lua_cjson.cpp | 4 ++++ tests/conformance/lljson.lua | 2 ++ 2 files changed, 6 insertions(+) diff --git a/VM/src/cjson/lua_cjson.cpp b/VM/src/cjson/lua_cjson.cpp index 04a4206d..ee3f5408 100644 --- a/VM/src/cjson/lua_cjson.cpp +++ b/VM/src/cjson/lua_cjson.cpp @@ -1224,6 +1224,8 @@ static int json_encode_common(lua_State* l, bool is_init, bool sl_tagged) lua_pop(l, 1); lua_rawgetfield(l, 3, "replacer"); if (!lua_isfunction(l, -1)) { + if (!lua_isnil(l, -1)) + luaL_error(l, "'replacer' must be a function"); lua_pop(l, 1); lua_pushnil(l); } @@ -2326,6 +2328,8 @@ static int json_decode_common(lua_State* l, bool is_init, bool sl_tagged) lua_rawgetfield(l, 3, "reviver"); if (!lua_isfunction(l, -1)) { + if (!lua_isnil(l, -1)) + luaL_error(l, "'reviver' must be a function"); lua_pop(l, 1); lua_pushnil(l); } diff --git a/tests/conformance/lljson.lua b/tests/conformance/lljson.lua index f1994d9a..9cf18e34 100644 --- a/tests/conformance/lljson.lua +++ b/tests/conformance/lljson.lua @@ -443,6 +443,8 @@ assert(lljson.slencode(42) == "42") assert(not pcall(lljson.encode, 1, "string")) assert(not pcall(lljson.slencode, 1, "string")) assert(not pcall(lljson.slencode, 1, true)) +assert(not pcall(lljson.encode, {}, {replacer = "oops"})) +assert(not pcall(lljson.decode, "[]", {reviver = "oops"})) -- sldecode does not set metatables (slencode ignores them, so attaching would be dishonest) do