diff --git a/.gitignore b/.gitignore index dad3e94e..08c7825d 100644 --- a/.gitignore +++ b/.gitignore @@ -84,12 +84,10 @@ android/gradle/ !.yarn/sdks !.yarn/versions -# Generated by the build pipelines from generate_tokenizers_header_file.js +# Copied out of the app's own c_sources by android/build.gradle android/c_sources -# c_sources/ is also generated (copied out of the app's own c_sources by the -# podspec/Package.swift) but stays tracked: SwiftPM evaluates Package.swift in a -# sandbox that can deny that copy, so an SPM build off a fresh checkout needs the -# files to already be there. Android no longer depends on it. +# c_sources/ is also a copy (made by the podspec and react-native.config.js). It is +# tracked, but no build depends on that: they recreate it. # c_sources/ scripts/sqlite-vec-* diff --git a/Package.swift b/Package.swift index dd89ddc5..257f5ed3 100644 --- a/Package.swift +++ b/Package.swift @@ -4,12 +4,11 @@ import PackageDescription // Hand-authored SPM manifest for React Native's experimental SwiftPM support // (RN 0.87, "Preview" — see scripts/spm/__doc__/spm-scripts.md in react-native). -// This is a direct translation of the logic in `op-sqlite.podspec` + -// `generate_tokenizers_header_file.rb`: CocoaPods evaluates that podspec (and -// its dynamic per-consumer-app configuration) as Ruby during `pod install`. +// This is a direct translation of the logic in `op-sqlite.podspec`: +// CocoaPods evaluates that podspec (and its dynamic per-consumer-app configuration) as Ruby during `pod install`. // SwiftPM evaluates *this* file as Swift on every resolve, so the same -// dynamic logic — reading the consumer app's package.json, generating -// cpp/tokenizers.h, choosing which SQLite backend & xcframeworks to link — +// dynamic logic — reading the consumer app's package.json, finding its +// tokenizers, choosing which SQLite backend & xcframeworks to link — // is reproduced here with Foundation instead of Ruby's FileUtils/JSON. // // Per RN's "hand-authored community library contract" @@ -25,8 +24,12 @@ import PackageDescription let packageRoot = URL(fileURLWithPath: #filePath).deletingLastPathComponent() +// #filePath keeps the autolinker's symlink path (see the header), which is not +// inside node_modules, so this has to look at the real directory. +let resolvedPackageRoot = packageRoot.resolvingSymlinksInPath() + // In the sample app the dir is not inside of node_modules. -let isUserApp = packageRoot.path.contains("node_modules") +let isUserApp = resolvedPackageRoot.path.contains("node_modules") func readJSONObject(atPath path: String) -> [String: Any] { guard let data = FileManager.default.contents(atPath: path), @@ -38,15 +41,27 @@ func readJSONObject(atPath path: String) -> [String: Any] { return dict } -// When installed into node_modules, find the consumer's package.json by -// searching up through parent directories (starting one level above this -// package, i.e. outside node_modules/@op-engineering/op-sqlite). In the -// monorepo/dev checkout it's simply example/package.json. let appPackageJSONPath: String = { + // Prefer the app, the same package.json Android reads. It is the folder above + // the one holding build/generated/autolinking; the search further down can't + // find it under pnpm or when hoisted, where it ends at the workspace root. + // Without the autolinker's symlink the app is unknown. + if let marker = packageRoot.path.range(of: "/build/generated/autolinking/libs/") { + let candidate = URL(fileURLWithPath: String(packageRoot.path[.. = ["ios", "cpp", "c_sources"] +// c_sources stays out when tokenizers are off, in case an earlier run left a copy. +let neededTopLevelDirs: Set = tokenizers.isEmpty ? ["ios", "cpp"] : ["ios", "cpp", "c_sources"] let topLevelExcludes: [String] = (try? FileManager.default.contentsOfDirectory(atPath: packageRoot.path))? .filter { !neededTopLevelDirs.contains($0) && $0 != "Package.swift" && $0 != "op-sqlite-spm-prefix.h" } diff --git a/README.md b/README.md index 4e7796e1..8c83d83a 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,14 @@ Some of the big supported features: It also contains a simple [Key-Value store](https://op-engineering.github.io/op-sqlite/docs/key_value_storage) you can use without adding one more dependency to your app. +# Custom tokenizers + +1. Add `"tokenizers": ["your_tokenizer"]` to the `"op-sqlite"` config in your app's `package.json`. +2. Run `npx react-native config` from your app's folder. It creates `c_sources/tokenizers.h`. +3. Write your implementation in `c_sources/tokenizers.cpp` and commit the `c_sources` folder. + +Run the command again whenever you change `"tokenizers"`. See the [tokenizers docs](https://op-engineering.github.io/op-sqlite/docs/tokenizers) for the details. + # License MIT License. diff --git a/android/build.gradle b/android/build.gradle index ec4967b2..001bc4b3 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -143,38 +143,6 @@ if(!tokenizers.isEmpty()) { println "[OP-SQLITE] Tokenizers enabled. Detected tokenizers: " + tokenizers } -// Generates the tokenizers.h that cpp/OPBridge.cpp includes (through -// TOKENIZERS_HEADER_PATH) and that the copied tokenizers.cpp includes by name. -// -// The generation itself lives in generate_tokenizers_header_file.js, shared with -// the CocoaPods (generate_tokenizers_header_file.rb) and SwiftPM (Package.swift) -// pipelines, so all three emit the same header. Before this, Android had no -// generator of its own and could only build if an iOS pod install (or a header -// committed to the repo) had produced the file first. -// -// Uses a plain ProcessBuilder rather than project.exec: this runs at -// configuration time, where that API is deprecated/removed in recent Gradle. -def generateTokenizersHeaderFile(List names, File headerFile) { - def packageDir = buildscript.sourceFile.parentFile.parentFile - def script = new File(packageDir, "generate_tokenizers_header_file.js") - // NODE_BINARY is React Native's own convention for pointing at a specific node. - def node = System.getenv("NODE_BINARY") - if (!node) { - node = "node" - } - - def command = [node, script.absolutePath, headerFile.absolutePath] + names - def process = new ProcessBuilder(command).redirectErrorStream(true).start() - def output = process.inputStream.text - def status = process.waitFor() - - if (status != 0) { - throw new GradleException( - "[OP-SQLITE] Could not generate ${headerFile}. Command: ${command.join(' ')}\n" + - "Make sure node is on your PATH or set NODE_BINARY to its location.\n${output}") - } -} - // Build the list of library-default SQLite compile flags driven by package.json // toggles. These are emitted via CMake add_definitions() BEFORE the user-supplied // sqliteFlags, so that user flags take precedence when both define the same macro @@ -231,11 +199,13 @@ android { if (!tokenizers.isEmpty()) { def sourceDir = isUserApp ? file("$rootDir/../../../c_sources") : file("$rootDir/../c_sources") def destDir = file("$buildscript.sourceFile.parentFile/c_sources") - // Generated into the app's own c_sources, exactly where the podspec and - // Package.swift put it, so the header the user's tokenizers.cpp includes - // is there no matter which platform was built last. The copy below then - // carries it over with the sources. - generateTokenizersHeaderFile(tokenizers, new File(sourceDir, "tokenizers.h")) + // tokenizers.h is generated by react-native.config.js. Autolinking loads + // it when the app's package.json changes, before this runs. The copy + // below carries the header over with the sources. + def tokenizersHeader = new File(sourceDir, "tokenizers.h") + if (!tokenizersHeader.exists()) { + throw new GradleException("[OP-SQLITE] Tokenizers ${tokenizers} are enabled, but ${tokenizersHeader} is missing. Run `npx react-native config` from your app's folder to create it.") + } copy { from sourceDir into destDir diff --git a/docs/docs/installation.md b/docs/docs/installation.md index 3ec9cb8e..30439c51 100644 --- a/docs/docs/installation.md +++ b/docs/docs/installation.md @@ -49,10 +49,12 @@ Required response headers (for worker/OPFS setup): SQLite is very customizable on compilation level. op-sqlite also allows you add extensions or even change the base implementation. You can do this by adding the following to your `package.json`: > [!IMPORTANT] -> When using a monorepo, be sure to add the "op-sqlite" config to the monorepo root `package.json` file. -> This is necessary because the `node_modules/@op-engineering/op-sqlite/op-sqlite.podspec` will search for first the `package.json` file in a parent directory. -> Alternative, you may be able to solve this by blocking this package from being hoisted to the root `node_modules`. -> Check your ios/Podfile.lock to see where it's being installed. +> In a monorepo, put the "op-sqlite" config in your app's `package.json` (the folder above `ios/` and `android/`). Android reads only that file. +> On iOS (CocoaPods and SwiftPM), if the app's `package.json` has no "op-sqlite" key, the build falls back to the first `package.json` above the installed package, usually the monorepo root. A config kept only in the root therefore applies to iOS but not to Android. If both have one, the app's is used and the two are not merged. +> `pod install` prints which file it used (`[OP-SQLITE] Configuration found at ...`). SwiftPM prints `[OP-SQLITE] SPM configuration found at ...`, but only when it evaluates the manifest rather than reusing its cache. + +> [!WARNING] +> SwiftPM caches the evaluated manifest and doesn't notice changes to `package.json`, so with SwiftPM a config change can silently have no effect, even after a clean build. After changing the "op-sqlite" config, run `rm -rf ~/Library/Caches/org.swift.swiftpm/manifests` and do a clean build. ```json { diff --git a/docs/docs/tokenizers.md b/docs/docs/tokenizers.md index fe009cd2..8033aaf1 100644 --- a/docs/docs/tokenizers.md +++ b/docs/docs/tokenizers.md @@ -8,6 +8,13 @@ Tokenizers are custom C functions that allow you to turn a stream of characters op-sqlite has a novel way for you to create your tokenizers. +> [!NOTE] +> With SwiftPM on iOS, run `npx react-native spm update` where these steps say `pod install`, including after editing your files in `c_sources`: the build compiles a copy of them. +> +> After changing "tokenizers", also clear SwiftPM's manifest cache as described in [the configuration section](./installation.md#configuration). +> +> `c_sources` doesn't exist until you generate it. Once you have filled it in, commit it. + 1. Declare which tokenizers you want on the `package.json`: ```json @@ -18,7 +25,7 @@ op-sqlite has a novel way for you to create your tokenizers. } ``` -2. Run `pod install` (iOS) or a Gradle build (Android). Both contain a code generation step. It will create a `c_sources` folder at the root of your project. It will create a `tokenizers.h` file. DON’T TOUCH THIS FILE. It will be overwritten every time. You need to create a `c_sources/tokenizers.cpp` file. Here you need to provide your tokenizer implementation. The `tokenizer.h` file contains the function declaration that will be executed when registering your tokenizer. In this case here is a sample `tokenizers.cpp` implementation +2. Run `npx react-native config` from your app's folder (the one holding `ios/` and `android/`). You can ignore what it prints. It will create a `c_sources` folder at the root of your project (on iOS, next to the `package.json` holding your "op-sqlite" config). It will create a `tokenizers.h` file. DON’T TOUCH THIS FILE. It will be overwritten every time. Run the command again whenever you change "tokenizers". You need to create a `c_sources/tokenizers.cpp` file. Here you need to provide your tokenizer implementation. The `tokenizer.h` file contains the function declaration that will be executed when registering your tokenizer. In this case here is a sample `tokenizers.cpp` implementation ```cpp #include "tokenizers.h" @@ -102,7 +109,7 @@ op-sqlite has a novel way for you to create your tokenizers. You need to keep the namespace and the function signature intact. For now the `sqlite3_api_routines` parameter will always be a null pointer. 3. Once you are done. You need to run `pod install` again. It will then copy the files you created to the pod sources in order to compile `op-sqlite` together with your new C++ code in one go. -4. The `tokenizers.h` generation runs in every pipeline: CocoaPods, SwiftPM and Gradle all call the same `generate_tokenizers_header_file.js` from the package, so an Android build no longer needs an iOS `pod install` to have happened first. Adding or removing a **file** inside `c_sources` still needs a `pod install` on iOS, since the pod's file list is fixed at install time; Android picks up new files on the next Gradle build. On your CI, you will also need to do a pod install even if your pods are cached, in order to copy the sources. +4. Commit `c_sources`. The build needs `tokenizers.h` to be there, and stops with an error telling you to run the command above if it isn't. An Android build doesn't need an iOS `pod install` to have happened first, or the other way around. Adding or removing a **file** inside `c_sources` still needs a `pod install` on iOS, since the pod's file list is fixed at install time; Android picks up new files on the next Gradle build. On your CI, you will also need to do a pod install even if your pods are cached, in order to copy the sources. 5. You can then create a FTS5 virtual table with your tokenizer: ```tsx diff --git a/generate_tokenizers_header_file.js b/generate_tokenizers_header_file.js index 6f7b459e..ce64d47b 100644 --- a/generate_tokenizers_header_file.js +++ b/generate_tokenizers_header_file.js @@ -1,14 +1,6 @@ -#!/usr/bin/env node - -// Single source of truth for the generated `tokenizers.h`. -// -// Every build pipeline needs this file: CocoaPods (via -// generate_tokenizers_header_file.rb), SwiftPM (Package.swift) and Gradle -// (android/build.gradle). They all shell out to this script instead of -// reimplementing the generation, so the header cannot drift between platforms -// and TOKENIZER_LIST only has to be kept in sync with cpp/OPBridge.cpp here. -// -// Usage: node generate_tokenizers_header_file.js ... +// Single source of truth for the generated `tokenizers.h`, used by +// react-native.config.js. TOKENIZER_LIST only has to be kept in sync with +// cpp/OPBridge.cpp here. const fs = require("node:fs"); const path = require("node:path"); @@ -48,8 +40,8 @@ function generateTokenizersHeader(names) { * Returns true when the file was written, false when it was already up to date. * * Rewriting an identical file would bump its mtime and force a recompile of - * everything that includes it. Gradle in particular runs this on every - * configuration. + * everything that includes it. The React Native CLI loads the config file that + * calls this for most of its commands. */ function writeTokenizersHeader(names, filePath) { const contents = generateTokenizersHeader(names); @@ -64,17 +56,4 @@ function writeTokenizersHeader(names, filePath) { return true; } -module.exports = { generateTokenizersHeader, writeTokenizersHeader }; - -if (require.main === module) { - const [filePath, ...names] = process.argv.slice(2); - - if (!filePath) { - console.error( - "[OP-SQLITE] usage: node generate_tokenizers_header_file.js ...", - ); - process.exit(1); - } - - writeTokenizersHeader(names, filePath); -} +module.exports = { writeTokenizersHeader }; diff --git a/generate_tokenizers_header_file.rb b/generate_tokenizers_header_file.rb deleted file mode 100644 index fd3fcd13..00000000 --- a/generate_tokenizers_header_file.rb +++ /dev/null @@ -1,19 +0,0 @@ -require 'shellwords' - -# Thin wrapper around generate_tokenizers_header_file.js, which holds the actual -# generation logic so CocoaPods, SwiftPM and Gradle all emit the exact same -# header. Node is already a hard requirement for building a React Native app. -def generate_tokenizers_header_file(names, file_path) - script_path = File.join(__dir__, "generate_tokenizers_header_file.js") - # NODE_BINARY is React Native's own convention for pointing at a specific - # node (see the .xcode.env files it generates). - node = ENV["NODE_BINARY"] - node = "node" if node.nil? || node.empty? - - command = [node, script_path, file_path, *names] - - unless system(*command) - raise "[OP-SQLITE] Could not generate the tokenizers header. Command: #{command.shelljoin}. " \ - "Make sure node is on your PATH or set NODE_BINARY to its location." - end -end diff --git a/op-sqlite.podspec b/op-sqlite.podspec index 905d8e95..d47983e7 100644 --- a/op-sqlite.podspec +++ b/op-sqlite.podspec @@ -1,5 +1,4 @@ require "json" -require_relative "./generate_tokenizers_header_file" log_message = lambda do |message| puts "\e[34m#{message}\e[0m" @@ -14,24 +13,41 @@ package_json_path = nil # When installed on user node_modules lives inside node_modules/@op-engineering/op-sqlite # Find the users package.json by searching up through parent directories if is_user_app - current_dir = File.expand_path(__dir__) - # Move one level up to the parent directory - current_dir = File.dirname(current_dir) - - # Find the package.json by searching up through parent directories - loop do - package_path = File.join(current_dir, "package.json") - if File.exist?(package_path) - package_json_path = package_path - break + # Prefer the app, the same package.json Android reads. It is the folder above + # the Podfile; the search further down can't find it under pnpm or when + # hoisted, where it ends at the workspace root. `pod install` finds the Podfile + # before evaluating any podspec; when none was found, podfile_path is nil. + podfile_path = defined?(Pod::Config) ? Pod::Config.instance.podfile_path : nil + if podfile_path + package_path = File.join(File.dirname(File.dirname(podfile_path.to_s)), "package.json") + candidate = begin + JSON.parse(File.read(package_path)) + rescue Errno::ENOENT, JSON::ParserError + nil end - - parent_dir = File.dirname(current_dir) - break if parent_dir == current_dir # reached filesystem root - current_dir = parent_dir + package_json_path = package_path if candidate.is_a?(Hash) && candidate.key?("op-sqlite") end - raise "package.json not found" if package_json_path.nil? + if package_json_path.nil? + current_dir = File.expand_path(__dir__) + # Move one level up to the parent directory + current_dir = File.dirname(current_dir) + + # Find the package.json by searching up through parent directories + loop do + package_path = File.join(current_dir, "package.json") + if File.exist?(package_path) + package_json_path = package_path + break + end + + parent_dir = File.dirname(current_dir) + break if parent_dir == current_dir # reached filesystem root + current_dir = parent_dir + end + + raise "package.json not found" if package_json_path.nil? + end else package_json_path = File.join(__dir__, "example", "package.json") end @@ -113,19 +129,22 @@ Pod::Spec.new do |s| # Strictly blocks all headers from being public s.public_header_files = ["ios/OPSQLite.h", "cpp/sqlite3.h", "cpp/OPBridge.hpp"] - # Set the path to the `c_sources` directory based on environment - if is_user_app - c_sources_dir = File.join("..", "..", "..", "c_sources") - else - c_sources_dir = File.join("example", "c_sources") - end + # `c_sources` lives next to the package.json holding the config. A path relative + # to this podspec would land inside the pnpm store. + c_sources_dir = File.join(File.dirname(package_json_path), "c_sources") if tokenizers.any? if use_turso then raise "Tokenizers are not supported with turso backend. Please disable tokenizers or do not enable turso." end - generate_tokenizers_header_file(tokenizers, File.join(c_sources_dir, "tokenizers.h")) + # Generated by react-native.config.js. `use_native_modules!` loads it before any + # podspec is evaluated, but a Podfile without it won't have. + tokenizers_header = File.join(c_sources_dir, "tokenizers.h") + unless File.exist?(tokenizers_header) + raise "[OP-SQLITE] Tokenizers #{tokenizers} are enabled, but #{tokenizers_header} is missing. Run `npx react-native config` from your app's folder to create it." + end + FileUtils.cp_r(c_sources_dir, __dir__) # # Add all .h and .c files from the `c_sources` directory source_files += Dir.glob(File.join("c_sources", "**/*.{h,cpp}")) diff --git a/package.json b/package.json index 5bb127ab..c9f9d5bf 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,6 @@ "node/dist", "node/package.json", "*.podspec", - "*.rb", "generate_tokenizers_header_file.js", "react-native.config.js", "ios/**.xcframework", diff --git a/react-native.config.js b/react-native.config.js new file mode 100644 index 00000000..884ef1ae --- /dev/null +++ b/react-native.config.js @@ -0,0 +1,72 @@ +// The only place that generates c_sources/tokenizers.h. Also copies the app's +// c_sources into this package so SwiftPM can compile them. +// +// The React Native CLI loads this file, so `npx react-native config` runs it, and +// so do `use_native_modules!` in a Podfile and Gradle's autolinking when the app's +// package.json changes. React Native's SwiftPM scripts load it on every `spm add`, +// `update` and `sync`. +// +// Package.swift can't do this because SwiftPM sandboxes manifest evaluation, and +// spm.autolinkingPlugin isn't an option because a dependency declaring one loses +// its autolinked target. +const fs = require("node:fs"); +const path = require("node:path"); +const { writeTokenizersHeader } = require("./generate_tokenizers_header_file.js"); + +function readJSON(file) { + try { + return JSON.parse(fs.readFileSync(file, "utf8")); + } catch { + return null; + } +} + +function findUp(startDir) { + let dir = startDir; + while (true) { + const candidate = path.join(dir, "package.json"); + if (fs.existsSync(candidate)) { + return candidate; + } + const parent = path.dirname(dir); + if (parent === dir) { + return null; + } + dir = parent; + } +} + +// The same lookup as op-sqlite.podspec and Package.swift: the app first, which +// these scripts run from or from its ios/ folder, then the old search. +function findConfigPackageJSON() { + const app = findUp(process.cwd()); + if (app && readJSON(app)?.["op-sqlite"] != null) { + return app; + } + const packageRoot = fs.realpathSync(__dirname); + if (!packageRoot.includes("node_modules")) { + return path.join(packageRoot, "example", "package.json"); + } + return findUp(path.dirname(packageRoot)); +} + +try { + const packageJSON = findConfigPackageJSON(); + const tokenizers = packageJSON && readJSON(packageJSON)?.["op-sqlite"]?.tokenizers; + if (Array.isArray(tokenizers) && tokenizers.length > 0) { + const cSources = path.join(path.dirname(packageJSON), "c_sources"); + writeTokenizersHeader(tokenizers, path.join(cSources, "tokenizers.h")); + + // Replaced, so a file removed from the app doesn't keep compiling. Keeping the + // timestamps means an unchanged file doesn't get recompiled. + const copy = path.join(__dirname, "c_sources"); + fs.rmSync(copy, { recursive: true, force: true }); + fs.cpSync(cSources, copy, { recursive: true, preserveTimestamps: true }); + } +} catch (error) { + // Never break the config load: the CLI also reads this file for Android, Metro + // and everything else. + console.warn(`[OP-SQLITE] could not prepare c_sources: ${error.message}`); +} + +module.exports = {};