feat: add native SQLite function mode and deprecate legacy overrides

Makefile

@@ -23,6 +23,7 @@ SQLITE_COMPILATION_FLAGS = \
 	-DSQLITE_DISABLE_LFS \
 	-DSQLITE_ENABLE_FTS3 \
 	-DSQLITE_ENABLE_FTS3_PARENTHESIS \
+	-DSQLITE_ENABLE_MATH_FUNCTIONS \
 	-DSQLITE_THREADSAFE=0 \
 	-DSQLITE_ENABLE_NORMALIZE
 
@@ -183,16 +184,16 @@ dist/worker.sql-wasm-inline-debug.js: dist/sql-wasm-inline-debug.js src/worker.j
 # 	#mv out/sql-wasm-debug.wasm dist/sql-wasm-debug.wasm
 # 	rm out/tmp-raw.js
 
-out/sqlite3.o: sqlite-src/$(SQLITE_AMALGAMATION)
+out/sqlite3.o: sqlite-src/$(SQLITE_AMALGAMATION) Makefile
 	mkdir -p out
 	# Generate llvm bitcode
 	$(EMCC) $(SQLITE_COMPILATION_FLAGS) -c sqlite-src/$(SQLITE_AMALGAMATION)/sqlite3.c -o $@
 
 # Since the extension-functions.c includes other headers in the sqlite_amalgamation, we declare that this depends on more than just extension-functions.c
-out/extension-functions.o: sqlite-src/$(SQLITE_AMALGAMATION)
+out/extension-functions.o: sqlite-src/$(SQLITE_AMALGAMATION) src/legacy-functions.c Makefile
 	mkdir -p out
 	# Generate llvm bitcode
-	$(EMCC) $(SQLITE_COMPILATION_FLAGS) -c sqlite-src/$(SQLITE_AMALGAMATION)/extension-functions.c -o $@
+	$(EMCC) $(SQLITE_COMPILATION_FLAGS) -I sqlite-src/$(SQLITE_AMALGAMATION) -c src/legacy-functions.c -o $@
 
 # TODO: This target appears to be unused. If we re-instatate it, we'll need to add more files inside of the JS folder
 # module.tar.gz: test package.json AUTHORS README.md dist/sql-asm.js

README.md

@@ -8,7 +8,7 @@
 
 *sql.js* is a javascript SQL database. It allows you to create a relational database and query it entirely in the browser. You can try it in [this online demo](https://sql.js.org/examples/GUI/). It uses a [virtual database file stored in memory](https://emscripten.org/docs/porting/files/file_systems_overview.html), and thus **doesn't persist the changes** made to the database. However, it allows you to **import** any existing sqlite file, and to **export** the created database as a [JavaScript typed array](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Typed_arrays).
 
-*sql.js* uses [emscripten](https://emscripten.org/docs/introducing_emscripten/about_emscripten.html) to compile [SQLite](http://sqlite.org/about.html) to WebAssembly. It includes [contributed math and string extension functions](https://www.sqlite.org/contrib?orderby=date).
+*sql.js* uses [emscripten](https://emscripten.org/docs/introducing_emscripten/about_emscripten.html) to compile [SQLite](http://sqlite.org/about.html) to WebAssembly. It enables SQLite's native math functions and, by default, the [legacy math and string extensions](https://www.sqlite.org/contrib?orderby=date). Calling a legacy function prints a deprecation warning; initialize with `useOldExtensionFunctions: false` to opt into [the next major version's behavior](https://github.com/sql-js/sql.js/pull/619).
 
 sql.js can be used like any traditional JavaScript library. If you are building a native application in JavaScript (using Electron for instance), or are working in node.js, you will likely prefer to use [a native binding of SQLite to JavaScript](https://www.npmjs.com/package/sqlite3). A native binding will not only be faster because it will run native code, but it will also be able to work on database files directly instead of having to load the entire database in memory, avoiding out of memory errors and further improving performances.
 

src/api.js

@@ -26,9 +26,14 @@
  */
 
 /**
- * @typedef {{locateFile:function(string):string}} SqlJsConfig
- * @property {function(string):string} locateFile
+ * @typedef {Object} SqlJsConfig
+ * @property {function(string):string} [locateFile]
  * a function that returns the full path to a resource given its file name
+ * @property {boolean} [useOldExtensionFunctions=true]
+ * Register deprecated extension functions for backwards compatibility.
+ * Calling a legacy function prints a deprecation warning.
+ * Set to false to use SQLite's native functions without legacy overrides.
+ * The first initialization determines this option for all databases.
  * @see https://emscripten.org/docs/api_reference/module.html
  */
 
@@ -52,6 +57,7 @@
  */
 // Wait for preRun to run, and then finish our initialization
 Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
+    var useOldExtensionFunctions = Module["useOldExtensionFunctions"] !== false;
     // Declare toplevel variables
     // register, used for temporary stack values
     var apiTemp = stackAlloc(4);
@@ -855,7 +861,9 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
         }
         this.handleError(sqlite3_open(this.filename, apiTemp));
         this.db = getValue(apiTemp, "i32");
-        registerExtensionFunctions(this.db);
+        if (useOldExtensionFunctions) {
+            registerExtensionFunctions(this.db);
+        }
         // A list of all prepared statements of the database
         this.statements = {};
         // Callback pointers for each user function, keyed by name and arity.

src/legacy-functions.c

@@ -0,0 +1,52 @@
+#include <assert.h>
+#include <emscripten.h>
+#include "sqlite3.h"
+
+typedef void (*ScalarFunction)(sqlite3_context *, int, sqlite3_value **);
+typedef void (*FinalFunction)(sqlite3_context *);
+
+typedef struct {
+  const char *name;
+  ScalarFunction scalar;
+  FinalFunction final;
+} LegacyFunction;
+
+EM_JS(void, warnLegacyFunction, (const char *name), {
+  console.warn("sql.js: legacy function " + UTF8ToString(name) +
+    "() is deprecated and will be disabled by default in the next major " +
+    "version. Initialize with useOldExtensionFunctions: false to use " +
+    "native SQLite functions. See https://github.com/sql-js/sql.js/pull/619");
+});
+
+static void legacyScalar(sqlite3_context *context, int argc,
+                         sqlite3_value **argv) {
+  LegacyFunction *function = sqlite3_user_data(context);
+  warnLegacyFunction(function->name);
+  function->scalar(context, argc, argv);
+}
+
+static void legacyFinal(sqlite3_context *context) {
+  LegacyFunction *function = sqlite3_user_data(context);
+  warnLegacyFunction(function->name);
+  function->final(context);
+}
+
+static int registerLegacyFunction(sqlite3 *db, const char *name, int argc,
+                                  int encoding, void *userData,
+                                  ScalarFunction scalar, ScalarFunction step,
+                                  FinalFunction final) {
+  /* The pinned legacy extension never uses sqlite3_user_data(). */
+  assert(userData == 0);
+  LegacyFunction *function = sqlite3_malloc(sizeof(*function));
+  if (!function) return SQLITE_NOMEM;
+  function->name = name;
+  function->scalar = scalar;
+  function->final = final;
+  return sqlite3_create_function_v2(db, name, argc, encoding, function,
+    scalar ? legacyScalar : 0, step, final ? legacyFinal : 0, sqlite3_free);
+}
+
+/* Wrap registrations without modifying or duplicating the upstream source. */
+#define sqlite3_create_function registerLegacyFunction
+#include "extension-functions.c"
+#undef sqlite3_create_function

src/worker.js

@@ -89,9 +89,15 @@ function onError(err) {
 }
 
 db = null;
-var sqlModuleReady = initSqlJs();
+var sqlModuleReady;
 
 function global_sqljs_message_handler(event) {
+    if (!sqlModuleReady) {
+        var config = event["data"]["config"] || {};
+        sqlModuleReady = initSqlJs({
+            useOldExtensionFunctions: config["useOldExtensionFunctions"]
+        });
+    }
     return sqlModuleReady
         .then(onModuleReady.bind(event))
         .catch(onError.bind(event));

test/builds.js

@@ -51,7 +51,12 @@ async function testBrowserBuild(file) {
         await context.self.onmessage({ data: { id: 3, action: "close" } });
     }
 
-    assert.equal(warnings.length, 0, "WebAssembly builds do not warn");
+    assert.equal(warnings.length, 0, "Initialization and ordinary SQL do not warn");
+    const legacyDb = new SQL.Database();
+    assert.equal(legacyDb.exec("SELECT reverse('abc')")[0].values[0][0], "cba");
+    legacyDb.close();
+    assert.equal(warnings.length, 1, "Calling a legacy function warns");
+    assert.match(warnings[0], /reverse\(\).*https:\/\/github.com\/sql-js\/sql.js\/pull\/619/);
     console.log("OK   browser distribution " + file);
 }
 

test/test_native_math_functions.js

@@ -0,0 +1,106 @@
+const fs = require("node:fs");
+const path = require("node:path");
+
+async function initialize(config) {
+  const target = process.argv[2] || "wasm";
+  const file = require.resolve("../dist/sql-" + target + ".js");
+  const cached = require.cache[file];
+  delete require.cache[file];
+  const initSqlJs = require(file);
+  require.cache[file] = cached;
+  if (target.startsWith("wasm") && !target.includes("inline")) {
+    config.wasmBinary = fs.readFileSync(path.join(
+      __dirname, "../dist/sql-" + target + ".wasm"
+    ));
+  }
+  return initSqlJs(config);
+}
+
+exports.test = async function(SQL, assert) {
+  const warnings = [];
+  const warn = console.warn;
+  console.warn = message => warnings.push(message);
+  try {
+    await testFunctions(assert, warnings);
+  } finally {
+    console.warn = warn;
+  }
+};
+
+async function testFunctions(assert, warnings) {
+  for (const config of [{}, { useOldExtensionFunctions: true }]) {
+    warnings.length = 0;
+    const SQL = await initialize(config);
+    const db = new SQL.Database();
+    try {
+      db.exec("SELECT 42");
+      assert.deepEqual(db.exec(`SELECT ln(1), log2(8), mod(5.5, 2),
+        trunc(-1.9), pow(2, 3), ceiling(1.2), log(2, 8)`)[0].values,
+      [[0, 3, 1.5, -1, 8, 2, 3]],
+      "Native-only names and arities are available in legacy mode");
+      assert.deepEqual(warnings, [],
+        "Initialization, opening databases and native functions do not warn");
+      assert.deepEqual(db.exec(`SELECT log(100), sqrt('abc'),
+        sign('abc'), typeof(sign(2.5)), reverse('abc')`)[0].values,
+      [[Math.log(100), 0, 0, "real", "cba"]]);
+      assert.equal(warnings.length, 5, "Each legacy scalar call warns");
+      assert.ok(warnings.every(message =>
+        message.includes("deprecated") &&
+        message.includes("next major version") &&
+        message.includes("https://github.com/sql-js/sql.js/pull/619")
+      ), "Usage warnings explain deprecation and link to the discussion");
+      assert.ok(warnings[0].includes("log()"), "Warnings name the function");
+      warnings.length = 0;
+      db.exec("SELECT reverse(x) FROM (SELECT 'a' AS x UNION ALL SELECT 'b')");
+      assert.equal(warnings.length, 2, "Legacy calls on separate rows each warn");
+      warnings.length = 0;
+      assert.deepEqual(db.exec(`SELECT stdev(x), variance(x), mode(x),
+        median(x), lower_quartile(x), upper_quartile(x)
+        FROM (SELECT 2 AS x UNION ALL SELECT 2)`)[0].values,
+      [[0, 0, 2, 2, 2, 2]], "Legacy aggregates keep their results");
+      assert.equal(warnings.length, 6, "Each legacy aggregate result warns");
+      warnings.length = 0;
+      db.exec("SELECT median(x) FROM (SELECT 1 AS x WHERE 0)");
+      assert.equal(warnings.length, 1, "Empty aggregates also warn");
+    } finally {
+      db.close();
+    }
+  }
+
+  warnings.length = 0;
+  const native = await initialize({ useOldExtensionFunctions: false });
+  const db = new native.Database();
+  try {
+    assert.deepEqual(db.exec(`SELECT log(100), ln(1), log(2, 8),
+      sqrt('abc'), sign('abc'), sign(x'01'), sign(''),
+      typeof(sign(2.5))`)[0].values,
+    [[2, 0, 3, null, null, null, null, "integer"]]);
+    db.run("CREATE TABLE t(a); CREATE INDEX ti ON t(sign(a));" +
+      "CREATE TABLE g(a, b GENERATED ALWAYS AS (sign(a)));");
+    db.run("INSERT INTO g(a) VALUES (-2.5), ('abc')");
+    assert.deepEqual(db.exec("SELECT b FROM g")[0].values, [[-1], [null]],
+      "Native sign works in generated columns");
+    const reopened = new native.Database(db.export());
+    try {
+      assert.deepEqual(reopened.exec("SELECT b FROM g")[0].values,
+        [[-1], [null]], "Schemas using native sign survive export and reopen");
+    } finally {
+      reopened.close();
+    }
+    for (const expression of [
+      "atn2(1, 1)", "square(2)", "cot(1)", "coth(1)",
+      "replicate('a', 2)", "difference('a', 'b')",
+      "charindex('a', 'abc')", "charindex('a', 'abc', 1)",
+      "leftstr('abc', 1)", "rightstr('abc', 1)", "reverse('abc')",
+      "proper('abc')", "padl('a', 2)", "padr('a', 2)", "padc('a', 2)",
+      "strfilter('abc', 'ab')", "stdev(1)", "variance(1)", "mode(1)",
+      "median(1)", "lower_quartile(1)", "upper_quartile(1)"
+    ]) {
+      assert.throws(() => db.exec("SELECT " + expression), /no such function/,
+        expression + " is not registered in native mode");
+    }
+  } finally {
+    db.close();
+  }
+  assert.deepEqual(warnings, [], "Native mode does not warn");
+}

test/test_worker.js

@@ -153,6 +153,25 @@ exports.test = async function test(SQL, assert) {
   await worker.postMessage({ action: "open" });
   data = await worker.postMessage({ action: "exec", sql: "SELECT 1" });
   assert.deepEqual(data.results, [{"columns":["1"],"values":[[1]]}]);
+  data = await worker.postMessage({ action: "exec", sql: "SELECT log(100), log(2, 8)" });
+  assert.deepEqual(data.results[0].values, [[Math.log(100), 3]],
+    'Workers default to legacy overrides with native-only arities');
+  await worker.worker.terminate();
+
+  const nativeWorker = await SQLWorker.fromFile(path.join(__dirname, filename));
+  try {
+    await nativeWorker.postMessage({
+      action: 'open', config: { useOldExtensionFunctions: false }
+    });
+    data = await nativeWorker.postMessage({
+      action: 'exec',
+      sql: "SELECT log(100), sign('abc'), typeof(sign(2.5))"
+    });
+    assert.deepEqual(data.results[0].values, [[2, null, 'integer']],
+      'The first worker message can select native SQLite behavior');
+  } finally {
+    await nativeWorker.worker.terminate();
+  }
 }
 
 function obj2array(obj) {