From fdc4b6daa1c4df2d663f4c3c3a2973f0fa2b2641 Mon Sep 17 00:00:00 2001 From: uw4 Date: Sat, 14 Nov 2026 16:41:22 +0100 Subject: [PATCH] Serve the config database from SQL When dashjoin.configdb.url is set, the config database is stored in one SQL table (dj_config) instead of the model folder, so several instances can share it. Layers: app (model folder of a git commit, read-only), local (runtime changes, deletion markers) and effective (the merged result, maintained on every write in the same transaction). Reads and queries are served from the merged records; on PostgreSQL, filters, sorting and paging run in SQL. - SqlConfigStore: tables, transactions and record locks (advisory locks on PostgreSQL), import of the git checkout, app sync, merged records, rebuild when the platform defaults change - JSONSqlDatabase: one layer; SqlConfigView: the injected config database (reads merged records, writes the local layer) - $appSync(dryRun?) updates the app layer; $gitPull() does the same in this mode - UnionDatabase, Services, PojoDatabase: small hooks Co-Authored-By: Claude Opus 5.5 --- .../java/org/dashjoin/function/AppSync.java | 61 ++ .../java/org/dashjoin/function/GitPull.java | 6 + .../service/JSONClassloaderDatabase.java | 15 + .../org/dashjoin/service/JSONSqlDatabase.java | 244 +++++ .../org/dashjoin/service/PojoDatabase.java | 3 +- .../java/org/dashjoin/service/Services.java | 14 + .../org/dashjoin/service/SqlConfigStore.java | 887 ++++++++++++++++++ .../org/dashjoin/service/SqlConfigView.java | 81 ++ .../org/dashjoin/service/UnionDatabase.java | 56 +- .../services/org.dashjoin.function.Function | 1 + .../dashjoin/service/SqlConfigStoreTest.java | 323 +++++++ dashjoin-docs/docs/administration.md | 47 + dashjoin-docs/docs/appendix-jsonata.md | 18 + dashjoin-docs/docs/jsonata-reference.md | 1 + dashjoin-docs/llms/jsonata.json | 25 + 15 files changed, 1780 insertions(+), 2 deletions(-) create mode 100644 dashjoin-core/src/main/java/org/dashjoin/function/AppSync.java create mode 100644 dashjoin-core/src/main/java/org/dashjoin/service/JSONSqlDatabase.java create mode 100644 dashjoin-core/src/main/java/org/dashjoin/service/SqlConfigStore.java create mode 100644 dashjoin-core/src/main/java/org/dashjoin/service/SqlConfigView.java create mode 100644 dashjoin-core/src/test/java/org/dashjoin/service/SqlConfigStoreTest.java diff --git a/dashjoin-core/src/main/java/org/dashjoin/function/AppSync.java b/dashjoin-core/src/main/java/org/dashjoin/function/AppSync.java new file mode 100644 index 000000000..9684dff39 --- /dev/null +++ b/dashjoin-core/src/main/java/org/dashjoin/function/AppSync.java @@ -0,0 +1,61 @@ +package org.dashjoin.function; + +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; +import org.dashjoin.service.PojoDatabase; +import org.dashjoin.service.Services; +import org.dashjoin.service.SqlConfigStore; + +/** + * Updates the app in the SQL config database: git pull, then the model folder of the new HEAD + * replaces the app layer. Changes made at runtime (local layer) stay. $appSync(true) only reports + * what would change. + */ +public class AppSync extends AbstractFunction> { + + private final static Logger logger = Logger.getLogger(AppSync.class.getName()); + + @Override + public Map run(Boolean dryRun) throws Exception { + if (!sc.isUserInRole("admin")) + throw new Exception("must be admin to sync the app"); + return sync(services, Boolean.TRUE.equals(dryRun)); + } + + /** + * pull and sync the app; reconnects the databases whose definition changed + */ + @SuppressWarnings("unchecked") + public static Map sync(Services services, boolean dryRun) throws Exception { + SqlConfigStore store = SqlConfigStore.get(); + if (store == null) + throw new Exception( + "The config database is not stored in SQL (dashjoin.configdb.url is not set)"); + Map report = store.sync(services.getTenantHome(), true, dryRun); + if (!dryRun) + for (String id : (List) report.get("changedDatabases")) + try { + ((PojoDatabase) services.getConfig()).metadataCollection(id); + } catch (Exception e) { + // e.g. the database was removed + logger.info("Not reconnecting database " + id + ": " + e); + } + return report; + } + + @Override + public Class getArgumentClass() { + return Boolean.class; + } + + @Override + public String getID() { + return "appSync"; + } + + @Override + public String getType() { + return "write"; + } +} diff --git a/dashjoin-core/src/main/java/org/dashjoin/function/GitPull.java b/dashjoin-core/src/main/java/org/dashjoin/function/GitPull.java index 517039b05..e154dfc29 100644 --- a/dashjoin-core/src/main/java/org/dashjoin/function/GitPull.java +++ b/dashjoin-core/src/main/java/org/dashjoin/function/GitPull.java @@ -1,5 +1,6 @@ package org.dashjoin.function; +import org.dashjoin.service.SqlConfigStore; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.internal.storage.file.FileRepository; @@ -9,6 +10,11 @@ public class GitPull extends AbstractFunction { public Void run(Void arg) throws Exception { if (!sc.isUserInRole("admin")) throw new Exception("must be admin to perform Git operations"); + if (SqlConfigStore.enabled()) { + // the config is served from SQL: pull and update the app layer + AppSync.sync(services, false); + return null; + } try (Git git = new Git(new FileRepository(services.getTenantHome() + "/.git"))) { git.pull().call(); return null; diff --git a/dashjoin-core/src/main/java/org/dashjoin/service/JSONClassloaderDatabase.java b/dashjoin-core/src/main/java/org/dashjoin/service/JSONClassloaderDatabase.java index 713fe08bb..de0d08551 100644 --- a/dashjoin-core/src/main/java/org/dashjoin/service/JSONClassloaderDatabase.java +++ b/dashjoin-core/src/main/java/org/dashjoin/service/JSONClassloaderDatabase.java @@ -6,6 +6,7 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.regex.Pattern; import jakarta.enterprise.context.ApplicationScoped; import org.dashjoin.model.QueryMeta; @@ -30,6 +31,20 @@ Set scan(String part) { return cache.get(part); } + /** + * the tables that have records on the classpath + */ + public Set tables() { + Set res = new TreeSet<>(); + for (String s : new Reflections("model", new ResourcesScanner()) + .getResources(Pattern.compile(".*\\.json"))) { + String[] parts = s.split("/"); + if (parts.length == 3 && parts[0].equals("model")) + res.add(parts[1]); + } + return res; + } + @Override public Map> queryMap(QueryMeta info, Map arguments) throws Exception { diff --git a/dashjoin-core/src/main/java/org/dashjoin/service/JSONSqlDatabase.java b/dashjoin-core/src/main/java/org/dashjoin/service/JSONSqlDatabase.java new file mode 100644 index 000000000..308e81159 --- /dev/null +++ b/dashjoin-core/src/main/java/org/dashjoin/service/JSONSqlDatabase.java @@ -0,0 +1,244 @@ +package org.dashjoin.service; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.Callable; +import org.dashjoin.model.QueryMeta; +import org.dashjoin.model.Table; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * One layer of the config database stored in SQL (see SqlConfigStore). The "app" layer holds the + * app definition from git and is read-only; the "local" layer holds everything written at runtime; + * the "effective" layer holds the merged result of all layers (read-only, maintained by the store). + * + * Records are stored as JSON text, which keeps the field order. Deleting a record that comes from a + * lower layer leaves a deletion marker in the local layer (see UnionDatabase), which read and + * queryMap skip. + */ +public class JSONSqlDatabase extends JSONDatabase { + + final SqlConfigStore store; + final String layer; + final boolean readOnly; + + JSONSqlDatabase(SqlConfigStore store, String layer, boolean readOnly) { + this.store = store; + this.layer = layer; + this.readOnly = readOnly; + } + + static String id(Object id) { + return "" + id; + } + + @Override + public Map read(Table s, Map search) throws Exception { + try (SqlConfigStore.Con c = store.con(); + PreparedStatement stmt = c.con.prepareStatement( + "select doc from dj_config where layer = ? and tbl = ? and id = ? and deleted = ?")) { + stmt.setString(1, layer); + stmt.setString(2, s.name); + stmt.setString(3, id(search.get("ID"))); + stmt.setBoolean(4, false); + try (ResultSet res = stmt.executeQuery()) { + if (res.next()) + return fromJsonString(res.getString(1)); + return null; + } + } + } + + @Override + public Map> queryMap(QueryMeta info, Map arguments) + throws Exception { + Map> res = new LinkedHashMap<>(); + String[] parts = info.query.split("/", 2); + if (parts.length == 1 && arguments != null && arguments.get("ID") != null) + parts = new String[] {parts[0], id(arguments.get("ID"))}; + if (parts.length == 2) { + Map i = read(Table.ofName(parts[0]), Map.of("ID", parts[1])); + if (i != null) + res.put(id(i.get("ID")), i); + return res; + } + try (SqlConfigStore.Con c = store.con(); + PreparedStatement stmt = c.con.prepareStatement( + "select id, doc from dj_config where layer = ? and tbl = ? and deleted = ? order by id" + + store.binaryCollation())) { + stmt.setString(1, layer); + stmt.setString(2, parts[0]); + stmt.setBoolean(3, false); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) + res.put(rs.getString(1), fromJsonString(rs.getString(2))); + } + } + return res; + } + + void checkWritable() throws Exception { + if (readOnly) + throw new Exception("The " + layer + " layer of the config database is read only"); + } + + /** + * like the JSON files, create overwrites an existing record (and a deletion marker) + */ + @Override + public void create(Table s, Map object) throws Exception { + checkWritable(); + if (object.get("ID") == null) + throw new Exception("Object must contain an ID"); + try (SqlConfigStore.Con c = store.con()) { + store.upsert(c.con, layer, s.name, id(object.get("ID")), toJsonString(object), false); + } + changed(s, object.get("ID")); + } + + /** the effective record must be recomputed */ + void changed(Table s, Object id) throws Exception { + if (layer.equals(SqlConfigStore.LOCAL)) + store.changed(s.name, id(id)); + } + + @Override + public boolean delete(Table s, Map search) throws Exception { + checkWritable(); + try (SqlConfigStore.Con c = store.con(); + PreparedStatement stmt = c.con.prepareStatement( + "delete from dj_config where layer = ? and tbl = ? and id = ? and deleted = ?")) { + stmt.setString(1, layer); + stmt.setString(2, s.name); + stmt.setString(3, id(search.get("ID"))); + stmt.setBoolean(4, false); + boolean res = stmt.executeUpdate() > 0; + if (inLowerLayer(s, search.get("ID"))) { + // hide the record of the lower layer + markDeleted(s, search.get("ID")); + return true; + } + changed(s, search.get("ID")); + return res; + } + } + + /** the local layer: does a layer below it (app, platform defaults) have the record? */ + boolean inLowerLayer(Table s, Object id) throws Exception { + if (!layer.equals(SqlConfigStore.LOCAL) || store.base == null) + return false; + for (JSONDatabase d : store.base) + if (d.read(s, Map.of("ID", id(id))) != null) + return true; + return false; + } + + /** + * hide the record of a lower layer + */ + public void markDeleted(Table s, Object id) throws Exception { + checkWritable(); + try (SqlConfigStore.Con c = store.con()) { + store.upsert(c.con, layer, s.name, id(id), null, true); + } + changed(s, id); + } + + public boolean isDeleted(Table s, Object id) throws Exception { + try (SqlConfigStore.Con c = store.con(); + PreparedStatement stmt = c.con.prepareStatement( + "select 1 from dj_config where layer = ? and tbl = ? and id = ? and deleted = ?")) { + stmt.setString(1, layer); + stmt.setString(2, s.name); + stmt.setString(3, id(id)); + stmt.setBoolean(4, true); + try (ResultSet res = stmt.executeQuery()) { + return res.next(); + } + } + } + + public Set deletedIds(String table) throws Exception { + Set res = new HashSet<>(); + try (SqlConfigStore.Con c = store.con(); + PreparedStatement stmt = c.con.prepareStatement( + "select id from dj_config where layer = ? and tbl = ? and deleted = ?")) { + stmt.setString(1, layer); + stmt.setString(2, table); + stmt.setBoolean(3, true); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) + res.add(rs.getString(1)); + } + } + return res; + } + + /** + * all() of the union database on the effective layer, run in SQL on PostgreSQL: the arguments + * filter by field value (null matches a missing field), sort orders by a field (records without + * the field come last, or first when descending) + */ + public List> allEffective(Table s, Integer offset, Integer limit, String sort, + boolean descending, Map arguments) throws Exception { + StringBuilder sql = + new StringBuilder("select doc from dj_config where layer = ? and tbl = ? and deleted = ?"); + List params = new ArrayList<>(List.of(layer, s.name, false)); + if (arguments != null) + for (Entry e : arguments.entrySet()) + if (e.getValue() == null) { + sql.append(" and (docb -> ? is null or docb -> ? = 'null'::jsonb)"); + params.add(e.getKey()); + params.add(e.getKey()); + } else { + // containment uses the index, equality makes it exact (for arrays and objects) + String value = om.writeValueAsString(e.getValue()); + sql.append(" and docb @> jsonb_build_object(?::text, ?::jsonb) and docb -> ? = ?::jsonb"); + params.addAll(List.of(e.getKey(), value, e.getKey(), value)); + } + if (sort != null) { + sql.append(" order by docb -> ? " + (descending ? "desc" : "asc") + ", id" + + store.binaryCollation()); + params.add(sort); + } else + sql.append(" order by id" + store.binaryCollation()); + if (limit != null) { + sql.append(" limit ?"); + params.add(limit); + } + if (offset != null) { + sql.append(" offset ?"); + params.add(offset); + } + List> res = new ArrayList<>(); + try (SqlConfigStore.Con c = store.con(); + PreparedStatement stmt = c.con.prepareStatement(sql.toString())) { + for (int i = 0; i < params.size(); i++) + stmt.setObject(i + 1, params.get(i)); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) + res.add(fromJsonString(rs.getString(1))); + } + } + return res; + } + + static final ObjectMapper om = new ObjectMapper(); + + @Override + public T locked(Table s, Object id, Callable body) throws Exception { + return store.locked(s.name + "/" + id(id), body); + } + + @Override + public String toString() { + return "JSONSqlDatabase(" + layer + ")"; + } +} diff --git a/dashjoin-core/src/main/java/org/dashjoin/service/PojoDatabase.java b/dashjoin-core/src/main/java/org/dashjoin/service/PojoDatabase.java index b3a1f9582..fbd235987 100644 --- a/dashjoin-core/src/main/java/org/dashjoin/service/PojoDatabase.java +++ b/dashjoin-core/src/main/java/org/dashjoin/service/PojoDatabase.java @@ -583,7 +583,8 @@ public void create(Table m, Map object) throws Exception { throw new Exception("Database name must not contain special characters"); String djClassName = (String) object.get("djClassName"); String url = (String) object.get("url"); - if (!(services.persistantDB instanceof JSONFileDatabase)) + if (!(services.persistantDB instanceof JSONFileDatabase + || services.persistantDB instanceof SqlConfigView)) if ("org.dashjoin.service.SQLDatabase".equals(djClassName)) if (url != null) if (url.startsWith("jdbc:sqlite:")) diff --git a/dashjoin-core/src/main/java/org/dashjoin/service/Services.java b/dashjoin-core/src/main/java/org/dashjoin/service/Services.java index 558f1ad8b..a0457b369 100644 --- a/dashjoin-core/src/main/java/org/dashjoin/service/Services.java +++ b/dashjoin-core/src/main/java/org/dashjoin/service/Services.java @@ -125,7 +125,21 @@ PojoDatabase pojoDatabase() { config._dbs = new CopyOnWriteArrayList(); JSONDatabase cl = readonlyDB; config._dbs.add(cl); + // SQL config store: the app layer sits between the platform defaults and the local layer + SqlConfigStore sql = + persistantDB instanceof SqlConfigView ? ((SqlConfigView) persistantDB).store : null; + if (sql != null) { + config._user = sql.local(); + config._dbs.add(sql.app()); + } config._dbs.add(new PolymorphismDatabase()); + if (sql != null) + try { + // imports / syncs the app on the first call and keeps the merged records up to date + sql.attach(config._dbs); + } catch (Exception e) { + throw new RuntimeException("Cannot initialize the SQL config database", e); + } return config; } diff --git a/dashjoin-core/src/main/java/org/dashjoin/service/SqlConfigStore.java b/dashjoin-core/src/main/java/org/dashjoin/service/SqlConfigStore.java new file mode 100644 index 000000000..823fee555 --- /dev/null +++ b/dashjoin-core/src/main/java/org/dashjoin/service/SqlConfigStore.java @@ -0,0 +1,887 @@ +package org.dashjoin.service; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.security.MessageDigest; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantLock; +import java.util.logging.Logger; +import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.io.FileUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.dashjoin.model.QueryMeta; +import org.dashjoin.model.Table; +import org.dashjoin.util.Home; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.internal.storage.file.FileRepository; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.eclipse.microprofile.config.ConfigProvider; + +/** + * Serves the config database from SQL instead of the model folder (JSON files). + * + * Enabled by the property dashjoin.configdb.url (with dashjoin.configdb.username and + * dashjoin.configdb.password). The config has these layers (higher wins, see UnionDatabase): + * + * - local (SQL, writable): everything written at runtime (UI, API, logins), records that only exist + * locally and deletion markers + * + * - app (SQL, read only): the model folder of one git commit of the app. A sync replaces it as a + * whole, the local layer stays + * + * - platform defaults (classpath and the metadata of the installed function and database classes) + * + * The merged result of all layers is stored as well (layer "effective"), so that reads and queries + * do not merge the layers in memory. Every write to the local layer recomputes the effective record + * in the same transaction; an app sync recomputes the records it changed; a change of the platform + * defaults (detected by a hash at startup) rebuilds all effective records. + * + * On first start, the model folder of the git checkout is imported: the HEAD commit becomes the app + * layer, local changes in the working tree become the local layer. Later, $appSync() pulls the app + * and replaces the app layer. The checkout is still used for all other files of the app. + */ +public class SqlConfigStore { + + private final static Logger logger = Logger.getLogger(SqlConfigStore.class.getName()); + + public static final String APP = "app"; + public static final String LOCAL = "local"; + public static final String EFFECTIVE = "effective"; + + /** increase when the way effective records are computed changes: rebuilds them on startup */ + static final String EFFECTIVE_VERSION = "1"; + + /** sync replaces the app records of these tables; comma separated */ + static final String EXCLUDE_PROPERTY = "dashjoin.configdb.sync.exclude"; + + static volatile SqlConfigStore instance; + + /** + * the store if dashjoin.configdb.url is set, null otherwise. The first call creates the tables. + * The import of the model folder (if the store is empty) runs when the config database attaches + * its other layers (see attach) + */ + public static SqlConfigStore get() { + if (instance == null) + synchronized (SqlConfigStore.class) { + if (instance == null) { + String url = property("dashjoin.configdb.url"); + if (url == null || url.isBlank()) + return null; + SqlConfigStore store = new SqlConfigStore(url, property("dashjoin.configdb.username"), + property("dashjoin.configdb.password")); + store.home = Home.get().getHome(); + store.hasAppUrl = property("dashjoin.appurl") != null; + try { + store.createTables(); + } catch (Exception e) { + throw new RuntimeException("Cannot initialize the SQL config database", e); + } + instance = store; + } + } + return instance; + } + + public static boolean enabled() { + String url = property("dashjoin.configdb.url"); + return url != null && !url.isBlank(); + } + + static String property(String name) { + try { + return ConfigProvider.getConfig().getOptionalValue(name, String.class).orElse(null); + } catch (Exception e) { + return null; + } + } + + final BasicDataSource ds; + final boolean postgres; + final JSONSqlDatabase app; + final JSONSqlDatabase local; + final JSONSqlDatabase effective; + final SqlConfigView view; + + /** app checkout to import / sync on attach (null: none) */ + String home; + boolean hasAppUrl; + + /** + * the layers below the local layer, lowest first (platform defaults, app, polymorphism). null + * until attach: effective records cannot be computed before + */ + volatile List base; + + boolean attached; + + /** tables never written to the app layer */ + Set exclude = new HashSet<>(); + + SqlConfigStore(String url, String username, String password) { + ds = new BasicDataSource(); + ds.setUrl(url); + ds.setUsername(username); + ds.setPassword(password); + postgres = url.startsWith("jdbc:postgresql:"); + app = new JSONSqlDatabase(this, APP, true); + local = new JSONSqlDatabase(this, LOCAL, false); + effective = new JSONSqlDatabase(this, EFFECTIVE, true); + view = new SqlConfigView(this); + String ex = property(EXCLUDE_PROPERTY); + if (ex != null) + for (String t : ex.split(",")) + if (!t.isBlank()) + exclude.add(t.trim()); + } + + public JSONSqlDatabase app() { + return app; + } + + public JSONSqlDatabase local() { + return local; + } + + public JSONSqlDatabase effective() { + return effective; + } + + /** + * the config database to inject as JSONDatabase: reads merged records, writes the local layer + */ + public SqlConfigView view() { + return view; + } + + /** + * called when the config database is set up: base are the layers below the local layer (lowest + * first). The first call imports or syncs the app and brings the effective records up to date + */ + public void attach(List base) throws Exception { + synchronized (this) { + this.base = List.copyOf(base); + if (attached) + return; + attached = true; + } + syncLocked(() -> { + if (home != null) + bootstrap(home, hasAppUrl); + maintainEffective(); + return null; + }); + } + + public void close() throws Exception { + ds.close(); + } + + // ---------------------------------------------------------------- connections and locks + + /** connection of the transaction of locked() on this thread */ + final ThreadLocal tx = new ThreadLocal<>(); + + final Map jvmLocks = new ConcurrentHashMap<>(); + + /** a connection that is closed unless it belongs to the running transaction */ + static class Con implements AutoCloseable { + final Connection con; + final boolean own; + + Con(Connection con, boolean own) { + this.con = con; + this.own = own; + } + + @Override + public void close() throws Exception { + if (own) + con.close(); + } + } + + Con con() throws Exception { + Connection c = tx.get(); + if (c != null) + return new Con(c, false); + return new Con(ds.getConnection(), true); + } + + /** + * runs body in a transaction that holds the lock of key: a JVM lock and, on PostgreSQL, an + * advisory lock, so read-modify-write sequences of other instances cannot interleave. Nested + * calls join the transaction + */ + T locked(String key, Callable body) throws Exception { + ReentrantLock jvm = jvmLocks.computeIfAbsent(key, k -> new ReentrantLock()); + jvm.lock(); + try { + return transaction(c -> { + advisoryLock(c, key); + return body.call(); + }); + } finally { + jvm.unlock(); + } + } + + interface TxBody { + T run(Connection c) throws Exception; + } + + /** local records written in the transaction of this thread: their effective records are stale */ + final ThreadLocal>> dirty = ThreadLocal.withInitial(LinkedHashSet::new); + + /** + * runs body in a transaction whose connection con() returns on this thread. Before the commit, + * the effective records of the local records written in the transaction are recomputed. Nested + * calls join the transaction + */ + T transaction(TxBody body) throws Exception { + Connection cur = tx.get(); + if (cur != null) + return body.run(cur); + try (Connection c = ds.getConnection()) { + c.setAutoCommit(false); + tx.set(c); + try { + T res = body.run(c); + flush(c); + c.commit(); + return res; + } catch (Throwable t) { + c.rollback(); + throw t; + } finally { + tx.remove(); + dirty.remove(); + c.setAutoCommit(true); + } + } + } + + /** IDs are ordered like Java strings (and SQLite), not by the locale of the database */ + String binaryCollation() { + return postgres ? " collate \"C\"" : ""; + } + + static String key(String tbl, String id) { + return tbl + "/" + id; + } + + void advisoryLock(Connection c, String key) throws Exception { + if (!postgres) + return; + try (PreparedStatement stmt = c.prepareStatement("select pg_advisory_xact_lock(hashtext(?))")) { + stmt.setString(1, key); + stmt.execute(); + } + } + + /** key of the cluster wide lock for import and sync */ + static final long SYNC_LOCK = 0x646a636f6e666967L; + + final ReentrantLock syncLock = new ReentrantLock(); + + /** runs body while holding the cluster wide import / sync lock */ + T syncLocked(Callable body) throws Exception { + syncLock.lock(); + try { + // nested (e.g. sync during bootstrap): this thread already holds the lock + if (!postgres || syncLock.getHoldCount() > 1) + return body.call(); + try (Connection c = ds.getConnection()) { + try (Statement stmt = c.createStatement()) { + stmt.execute("select pg_advisory_lock(" + SYNC_LOCK + ")"); + } + try { + return body.call(); + } finally { + try (Statement stmt = c.createStatement()) { + stmt.execute("select pg_advisory_unlock(" + SYNC_LOCK + ")"); + } + } + } + } finally { + syncLock.unlock(); + } + } + + static final String UPSERT = "insert into dj_config (layer, tbl, id, doc, deleted)" + + " values (?, ?, ?, ?, ?) on conflict (layer, tbl, id) do update set doc = excluded.doc," + + " deleted = excluded.deleted"; + + void upsert(Connection c, String layer, String tbl, String id, String doc, boolean deleted) + throws Exception { + try (PreparedStatement stmt = c.prepareStatement(UPSERT)) { + stmt.setString(1, layer); + stmt.setString(2, tbl); + stmt.setString(3, id); + stmt.setString(4, doc); + stmt.setBoolean(5, deleted); + stmt.executeUpdate(); + } + } + + void createTables() throws Exception { + try (Connection c = ds.getConnection(); Statement stmt = c.createStatement()) { + stmt.execute("create table if not exists dj_config (layer varchar(16) not null," + + " tbl varchar(255) not null, id varchar(1024) not null, doc text," + + " deleted boolean not null default false, primary key (layer, tbl, id))"); + stmt.execute("create table if not exists dj_app_sync (synced varchar(64) not null," + + " app_commit varchar(64), report text)"); + stmt.execute("create table if not exists dj_config_meta (k varchar(64) not null primary key," + + " v text)"); + } + if (postgres) + // queries of the effective records run in SQL (see JSONSqlDatabase.allEffective) + for (String ddl : new String[] { + "alter table dj_config add column if not exists docb jsonb" + + " generated always as (doc::jsonb) stored", + "create index if not exists dj_config_docb on dj_config using gin (docb jsonb_path_ops)" + + " where layer = 'effective'"}) + try (Connection c = ds.getConnection(); Statement stmt = c.createStatement()) { + stmt.execute(ddl); + } catch (Exception e) { + // another instance did it at the same time + logger.info("Not executed (" + e.getMessage() + "): " + ddl); + } + } + + String getMeta(String k) throws Exception { + try (Con c = con(); + PreparedStatement stmt = c.con.prepareStatement("select v from dj_config_meta where k = ?")) { + stmt.setString(1, k); + try (ResultSet rs = stmt.executeQuery()) { + return rs.next() ? rs.getString(1) : null; + } + } + } + + void setMeta(String k, String v) throws Exception { + try (Con c = con(); PreparedStatement stmt = c.con.prepareStatement( + "insert into dj_config_meta (k, v) values (?, ?) on conflict (k) do update set v = excluded.v")) { + stmt.setString(1, k); + stmt.setString(2, v); + stmt.executeUpdate(); + } + } + + // ---------------------------------------------------------------- effective records + + /** + * a record of the local or app layer was written. Its effective record is recomputed before the + * transaction commits (or right away outside of a transaction) + */ + void changed(String tbl, String id) throws Exception { + if (base == null) { + // before attach: all effective records are rebuilt then + setMeta("rebuild", "true"); + return; + } + if (tx.get() != null) + dirty.get().add(List.of(tbl, id)); + else + locked(key(tbl, id), () -> { + dirty.get().add(List.of(tbl, id)); + return null; + }); + } + + void flush(Connection c) throws Exception { + Set> d = dirty.get(); + while (!d.isEmpty()) { + List k = d.iterator().next(); + d.remove(k); + advisoryLock(c, key(k.get(0), k.get(1))); + computeEffective(c, k.get(0), k.get(1)); + } + } + + /** + * the local layer as the union database reads it: passwords are masked (PasswordDatabase). The + * encrypted password stays in the local layer only + */ + static Map masked(Map local) { + if (local != null && local.get("password") instanceof String) { + local = new LinkedHashMap<>(local); + local.put("password", "********"); + } + return local; + } + + /** merge all layers of one record, like UnionDatabase.read */ + void computeEffective(Connection c, String tbl, String id) throws Exception { + Table t = Table.ofName(tbl); + Map search = Map.of("ID", id); + Map res = null; + if (!local.isDeleted(t, id)) { + for (JSONDatabase d : base) + res = UnionDatabase.merge(res, d.read(t, search)); + res = UnionDatabase.merge(res, masked(local.read(t, search))); + } + if (res == null || res.get("ID") == null) + try (PreparedStatement stmt = + c.prepareStatement("delete from dj_config where layer = ? and tbl = ? and id = ?")) { + stmt.setString(1, EFFECTIVE); + stmt.setString(2, tbl); + stmt.setString(3, id); + stmt.executeUpdate(); + } + else + upsert(c, EFFECTIVE, tbl, id, JSONDatabase.toJsonString(res), false); + } + + /** tables a layer that is not stored in SQL has records in */ + static Set layerTables(JSONDatabase d) { + if (d instanceof JSONClassloaderDatabase) + return ((JSONClassloaderDatabase) d).tables(); + if (d instanceof PolymorphismDatabase) + return Set.of("dj-database"); + return Set.of(); + } + + static final ObjectMapper om = new ObjectMapper(); + + /** hash of the layers that are not stored in SQL (platform defaults, polymorphism) */ + String platformHash() throws Exception { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + md.update(EFFECTIVE_VERSION.getBytes()); + for (JSONDatabase d : base) + if (!(d instanceof JSONSqlDatabase)) { + md.update(d.getClass().getName().getBytes()); + for (String table : new TreeSet<>(layerTables(d))) { + QueryMeta q = new QueryMeta(); + q.query = table; + md.update(table.getBytes()); + md.update(om.writeValueAsBytes(new TreeMap<>(d.queryMap(q, null)))); + } + } + return HexFormat.of().formatHex(md.digest()); + } + + /** + * on attach: rebuild the effective records if the platform defaults changed, if they were never + * built or if records were written before attach + */ + void maintainEffective() throws Exception { + if (platformHash().equals(getMeta("platform")) && getMeta("rebuild") == null) + return; + transaction(c -> { + rebuildEffective(c); + return null; + }); + } + + /** recompute all effective records (in the transaction of c) */ + void rebuildEffective(Connection c) throws Exception { + long start = System.currentTimeMillis(); + try (PreparedStatement stmt = c.prepareStatement("delete from dj_config where layer = ?")) { + stmt.setString(1, EFFECTIVE); + stmt.executeUpdate(); + } + Set tables = new TreeSet<>(); + try (PreparedStatement stmt = + c.prepareStatement("select distinct tbl from dj_config where layer in (?, ?)")) { + stmt.setString(1, APP); + stmt.setString(2, LOCAL); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) + tables.add(rs.getString(1)); + } + } + for (JSONDatabase d : base) + tables.addAll(layerTables(d)); + + int n = 0; + try (PreparedStatement stmt = c.prepareStatement(UPSERT)) { + for (String table : tables) { + QueryMeta q = new QueryMeta(); + q.query = table; + Map> res = new LinkedHashMap<>(); + for (JSONDatabase d : base) + for (Entry> e : d.queryMap(q, null).entrySet()) + res.put(e.getKey(), UnionDatabase.merge(res.get(e.getKey()), e.getValue())); + for (Entry> e : local.queryMap(q, null).entrySet()) + res.put(e.getKey(), UnionDatabase.merge(res.get(e.getKey()), masked(e.getValue()))); + res.keySet().removeAll(local.deletedIds(table)); + for (Entry> e : res.entrySet()) + if (e.getValue() != null && e.getValue().get("ID") != null) { + stmt.setString(1, EFFECTIVE); + stmt.setString(2, table); + stmt.setString(3, e.getKey()); + stmt.setString(4, JSONDatabase.toJsonString(e.getValue())); + stmt.setBoolean(5, false); + stmt.addBatch(); + n++; + } + stmt.executeBatch(); + } + } + setMeta("platform", platformHash()); + try (PreparedStatement stmt = c.prepareStatement("delete from dj_config_meta where k = ?")) { + stmt.setString(1, "rebuild"); + stmt.executeUpdate(); + } + logger.info("Rebuilt " + n + " effective records of the SQL config database in " + + (System.currentTimeMillis() - start) + " ms"); + } + + // ---------------------------------------------------------------- reading a model folder + + /** + * all records of a model folder: table -> ID -> record. Externalized strings are inlined + */ + static Map>> readModel(File root, Set skip) + throws Exception { + Map>> res = new TreeMap<>(); + File[] dirs = new File(root, "model").listFiles(File::isDirectory); + if (dirs == null) + return res; + Arrays.sort(dirs); + JSONFileDatabase db = new JSONFileDatabase() { + @Override + protected File getFile(String path) { + return new File(root, path); + } + }; + for (File dir : dirs) { + String table = dir.getName(); + if (table.startsWith(".") || skip.contains(table)) + continue; + QueryMeta q = new QueryMeta(); + q.query = table; + Map> records = new LinkedHashMap<>(); + for (Map record : db.queryMap(q, null).values()) { + if (record.get("ID") == null) + continue; + stripPointers(record); + records.put(JSONSqlDatabase.id(record.get("ID")), record); + } + if (!records.isEmpty()) + res.put(table, records); + } + return res; + } + + /** the SQL store holds externalized strings inline; drop the X-pointer keys */ + @SuppressWarnings("unchecked") + static void stripPointers(Object o) { + if (o instanceof Map) { + ((Map) o).keySet().removeIf(k -> k.endsWith("-pointer")); + for (Object v : ((Map) o).values()) + stripPointers(v); + } + if (o instanceof List) + for (Object v : (List) o) + stripPointers(v); + } + + /** + * exports the model folder of the HEAD commit of the checkout in home to a temp folder + */ + static Path exportHead(String home, String[] commit) throws Exception { + Path tmp = Files.createTempDirectory("dj-app-"); + try (Repository repo = new FileRepository(new File(home, ".git"))) { + ObjectId head = repo.resolve("HEAD"); + if (head == null) + throw new Exception("The app checkout in " + home + " has no commits"); + commit[0] = head.getName(); + try (RevWalk rw = new RevWalk(repo); TreeWalk tw = new TreeWalk(repo)) { + RevCommit rc = rw.parseCommit(head); + tw.addTree(rc.getTree()); + tw.setRecursive(true); + tw.setFilter(PathFilter.create("model")); + while (tw.next()) { + File f = tmp.resolve(tw.getPathString()).toFile(); + f.getParentFile().mkdirs(); + try (OutputStream out = new FileOutputStream(f)) { + repo.open(tw.getObjectId(0)).copyTo(out); + } + } + } + } + return tmp; + } + + static boolean isGit(String home) { + return new File(home, ".git").exists(); + } + + // ---------------------------------------------------------------- bootstrap, import, sync + + void bootstrap(String home, boolean hasAppUrl) throws Exception { + createTables(); + syncLocked(() -> { + if (lastSync() == null) + importFromHome(home); + else if (hasAppUrl && isGit(home)) { + // Home pulled the app at startup: bring the app layer to the new HEAD + Map report = sync(home, false, false); + logger.info("App sync at startup: " + summary(report)); + } + return null; + }); + } + + /** commit and time of the last import / sync, null if the store is empty */ + Map lastSync() throws Exception { + try (Connection c = ds.getConnection(); + Statement stmt = c.createStatement(); + ResultSet rs = stmt.executeQuery( + "select synced, app_commit from dj_app_sync order by synced desc")) { + if (!rs.next()) + return null; + Map res = new LinkedHashMap<>(); + res.put("synced", rs.getString(1)); + res.put("commit", rs.getString(2)); + return res; + } + } + + /** + * first start: the HEAD commit of the checkout becomes the app layer; records that differ in the + * working tree become the local layer (only the changed fields), records deleted in the working + * tree become deletion markers. Without git, all records go to the local layer + */ + Map importFromHome(String home) throws Exception { + Map>> working = readModel(new File(home), Set.of()); + + String[] commit = new String[1]; + Map>> appRecords = new TreeMap<>(); + if (isGit(home)) { + Path tmp = exportHead(home, commit); + try { + appRecords = readModel(tmp.toFile(), exclude); + } finally { + FileUtils.deleteQuietly(tmp.toFile()); + } + } + + List lostFields = new ArrayList<>(); + Map>> app = appRecords; + return transaction(c -> { + int nApp = 0, nLocal = 0, nDeleted = 0; + for (Entry>> t : app.entrySet()) + for (Entry> r : t.getValue().entrySet()) { + upsert(c, APP, t.getKey(), r.getKey(), JSONDatabase.toJsonString(r.getValue()), false); + nApp++; + } + for (Entry>> t : working.entrySet()) + for (Entry> r : t.getValue().entrySet()) { + Map appRecord = app.getOrDefault(t.getKey(), Map.of()).get(r.getKey()); + Map diff = diff(appRecord, r.getValue()); + if (appRecord != null) + for (String k : appRecord.keySet()) + if (!r.getValue().containsKey(k)) + lostFields.add(t.getKey() + "/" + r.getKey() + "." + k); + if (diff != null) { + upsert(c, LOCAL, t.getKey(), r.getKey(), JSONDatabase.toJsonString(diff), false); + nLocal++; + } + } + for (Entry>> t : app.entrySet()) + for (String id : t.getValue().keySet()) + if (!working.getOrDefault(t.getKey(), Map.of()).containsKey(id)) { + upsert(c, LOCAL, t.getKey(), id, null, true); + nDeleted++; + } + Map report = new LinkedHashMap<>(); + report.put("import", true); + report.put("commit", commit[0]); + report.put("appRecords", nApp); + report.put("localRecords", nLocal); + report.put("deletedRecords", nDeleted); + if (!lostFields.isEmpty()) + report.put("fieldsRemovedLocallyNotImported", lostFields); + recordSync(c, commit[0], report); + if (base != null) + rebuildEffective(c); + else + setMeta("rebuild", "true"); + logger.info("Imported the model folder into the SQL config database: " + summary(report)); + if (!lostFields.isEmpty()) + logger.warning("Fields removed in the working tree cannot be represented in the local" + + " layer and are visible again: " + lostFields); + return report; + }); + } + + /** + * fields of local that differ from base (with ID), the whole record if there is no base, null if + * there is no difference + */ + static Map diff(Map base, Map local) { + if (base == null) + return local; + Map res = new LinkedHashMap<>(); + for (Entry e : local.entrySet()) + if (!e.getKey().equals("ID") && !Objects.equals(e.getValue(), base.get(e.getKey()))) + res.put(e.getKey(), e.getValue()); + if (res.isEmpty()) + return null; + Map withId = new LinkedHashMap<>(); + withId.put("ID", local.get("ID")); + withId.putAll(res); + return withId; + } + + void recordSync(Connection c, String commit, Map report) throws Exception { + try (PreparedStatement stmt = + c.prepareStatement("insert into dj_app_sync (synced, app_commit, report) values (?, ?, ?)")) { + stmt.setString(1, Instant.now().toString()); + stmt.setString(2, commit); + stmt.setString(3, JSONDatabase.toJsonString(report)); + stmt.executeUpdate(); + } + } + + /** + * replaces the app layer with the model folder of the HEAD commit (after a git pull if pull is + * set). The local layer stays. Returns a report: added, changed and removed app records and the + * local overrides of fields the app changed + */ + public Map sync(String home, boolean pull, boolean dryRun) throws Exception { + return syncLocked(() -> { + if (!isGit(home)) + throw new Exception("The app folder " + home + " is not a git checkout"); + if (pull && !dryRun) + try (Git git = new Git(new FileRepository(new File(home, ".git")))) { + git.pull().call(); + } + + String[] commit = new String[1]; + Map>> newApp; + Path tmp = exportHead(home, commit); + try { + newApp = readModel(tmp.toFile(), exclude); + } finally { + FileUtils.deleteQuietly(tmp.toFile()); + } + + Map>> oldApp = readLayer(APP); + Map>> localLayer = readLayer(LOCAL); + Map last = lastSync(); + + List added = new ArrayList<>(), changed = new ArrayList<>(), + removed = new ArrayList<>(), overrides = new ArrayList<>(); + Set changedDatabases = new HashSet<>(); + for (Entry>> t : newApp.entrySet()) + for (Entry> r : t.getValue().entrySet()) { + Map old = oldApp.getOrDefault(t.getKey(), Map.of()).get(r.getKey()); + String key = t.getKey() + "/" + r.getKey(); + if (old == null) + added.add(key); + else if (!old.equals(r.getValue())) { + changed.add(key); + Map loc = + localLayer.getOrDefault(t.getKey(), Map.of()).get(r.getKey()); + if (loc != null) { + List fields = new ArrayList<>(); + for (String f : loc.keySet()) + if (!f.equals("ID") && !Objects.equals(old.get(f), r.getValue().get(f))) + fields.add(f); + if (!fields.isEmpty()) + overrides.add(key + ": " + String.join(", ", fields)); + } + } else + continue; + if (t.getKey().equals("dj-database")) + changedDatabases.add(r.getKey()); + } + for (Entry>> t : oldApp.entrySet()) + for (String id : t.getValue().keySet()) + if (!newApp.getOrDefault(t.getKey(), Map.of()).containsKey(id)) { + removed.add(t.getKey() + "/" + id); + if (t.getKey().equals("dj-database")) + changedDatabases.add(id); + } + + Map report = new LinkedHashMap<>(); + report.put("commit", commit[0]); + report.put("previousCommit", last == null ? null : last.get("commit")); + report.put("dryRun", dryRun); + report.put("added", added); + report.put("changed", changed); + report.put("removed", removed); + report.put("localOverridesOfChangedFields", overrides); + report.put("changedDatabases", new ArrayList<>(changedDatabases)); + if (dryRun) + return report; + + transaction(c -> { + try (PreparedStatement stmt = c.prepareStatement("delete from dj_config where layer = ?")) { + stmt.setString(1, APP); + stmt.executeUpdate(); + } + for (Entry>> t : newApp.entrySet()) + for (Entry> r : t.getValue().entrySet()) { + upsert(c, APP, t.getKey(), r.getKey(), JSONDatabase.toJsonString(r.getValue()), false); + if (!r.getValue().equals(oldApp.getOrDefault(t.getKey(), Map.of()).get(r.getKey()))) + changed(t.getKey(), r.getKey()); + } + for (Entry>> t : oldApp.entrySet()) + for (String id : t.getValue().keySet()) + if (!newApp.getOrDefault(t.getKey(), Map.of()).containsKey(id)) + changed(t.getKey(), id); + recordSync(c, commit[0], report); + // the effective records of the changed app records are recomputed before the commit + return null; + }); + return report; + }); + } + + /** all records of a layer (without deletion markers): table -> ID -> record */ + Map>> readLayer(String layer) throws Exception { + Map>> res = new TreeMap<>(); + try (Connection c = ds.getConnection(); + PreparedStatement stmt = c.prepareStatement( + "select tbl, id, doc from dj_config where layer = ? and deleted = ?")) { + stmt.setString(1, layer); + stmt.setBoolean(2, false); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) + res.computeIfAbsent(rs.getString(1), k -> new LinkedHashMap<>()).put(rs.getString(2), + JSONDatabase.fromJsonString(rs.getString(3))); + } + } + return res; + } + + static String summary(Map report) { + Map res = new LinkedHashMap<>(); + for (Entry e : report.entrySet()) + res.put(e.getKey(), e.getValue() instanceof List ? ((List) e.getValue()).size() + : e.getValue()); + return res.toString(); + } +} diff --git a/dashjoin-core/src/main/java/org/dashjoin/service/SqlConfigView.java b/dashjoin-core/src/main/java/org/dashjoin/service/SqlConfigView.java new file mode 100644 index 000000000..3bfee1ce8 --- /dev/null +++ b/dashjoin-core/src/main/java/org/dashjoin/service/SqlConfigView.java @@ -0,0 +1,81 @@ +package org.dashjoin.service; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.Callable; +import org.dashjoin.model.QueryMeta; +import org.dashjoin.model.Table; + +/** + * The config database as code outside the config database API sees it in SQL mode, e.g. the + * session and tenant managers that use the injected JSONDatabase: reads return the merged records + * of all layers, writes go to the local layer. In file mode, the injected database is the model + * folder, which holds all of these records as well. + */ +public class SqlConfigView extends JSONDatabase { + + final SqlConfigStore store; + + SqlConfigView(SqlConfigStore store) { + this.store = store; + } + + /** the merged records once the layers are attached, the local layer before */ + JSONSqlDatabase reads() { + return store.base == null ? store.local() : store.effective(); + } + + @Override + public Map read(Table s, Map search) throws Exception { + return reads().read(s, search); + } + + @Override + public Map> queryMap(QueryMeta info, Map arguments) + throws Exception { + return reads().queryMap(info, arguments); + } + + @Override + public void create(Table s, Map object) throws Exception { + store.local().create(s, object); + } + + /** + * a record that only a lower layer has (e.g. @EVERYONE from the app) gets a local record with the + * given fields, like an update through the config database API + */ + @Override + public boolean update(Table s, Map search, Map object) + throws Exception { + return locked(s, search.get("ID"), () -> { + if (store.local().read(s, search) != null) + return store.local().update(s, search, object); + if (read(s, search) == null) + return false; + Map local = new LinkedHashMap<>(); + local.put("ID", search.get("ID")); + for (Entry e : object.entrySet()) + if (e.getValue() != null && !e.getKey().equals("ID")) + local.put(e.getKey(), e.getValue()); + store.local().create(s, local); + return true; + }); + } + + @Override + public boolean delete(Table s, Map search) throws Exception { + return store.local().delete(s, search); + } + + @Override + public T locked(Table s, Object id, Callable body) throws Exception { + return store.local().locked(s, id, body); + } + + @Override + public String toString() { + return "SqlConfigView"; + } +} diff --git a/dashjoin-core/src/main/java/org/dashjoin/service/UnionDatabase.java b/dashjoin-core/src/main/java/org/dashjoin/service/UnionDatabase.java index 8d1e2e297..d334935d8 100644 --- a/dashjoin-core/src/main/java/org/dashjoin/service/UnionDatabase.java +++ b/dashjoin-core/src/main/java/org/dashjoin/service/UnionDatabase.java @@ -61,6 +61,34 @@ PasswordDatabase user() { return new PasswordDatabase(((PojoDatabase) services.getConfig())._user); } + /** + * the writable layer if it is the SQL config store, which supports deletion markers for records + * of lower layers. null in file mode + */ + JSONSqlDatabase sqlUser() { + JSONDatabase u = _user != null ? _user : ((PojoDatabase) services.getConfig())._user; + return u instanceof JSONSqlDatabase ? (JSONSqlDatabase) u : null; + } + + /** + * the merged records of the SQL config store if records of table can be read from them: the + * store is attached and no layer that is not stored in SQL (e.g. the metadata a connected + * database provides) has records of the table. null otherwise: the layers are merged in memory + */ + JSONSqlDatabase effective(String table) { + JSONSqlDatabase sql = sqlUser(); + if (sql == null || sql.store.base == null) + return null; + for (JSONDatabase d : dbs()) + if (d instanceof ProviderDatabase) { + if (((ProviderDatabase) d).data.containsKey(table)) + return null; + } else if (!(d instanceof JSONSqlDatabase) + && sql.store.base.stream().noneMatch(b -> b.getClass().equals(d.getClass()))) + return null; + return sql.store.effective(); + } + List dbs() { if (_user != null) return _dbs; @@ -79,6 +107,10 @@ List all() { @Override public List> query(QueryMeta info, Map arguments) throws Exception { + JSONSqlDatabase eff = effective(info.query.split("/", 2)[0]); + if (eff != null) + return new ArrayList<>(eff.queryMap(info, arguments).values()); + Map> res = new LinkedHashMap<>(); for (JSONDatabase d : all()) { for (Entry> e : d.queryMap(info, arguments).entrySet()) { @@ -98,6 +130,11 @@ public List> query(QueryMeta info, Map argum } } + // records of lower layers deleted in the SQL config store + JSONSqlDatabase sql = sqlUser(); + if (sql != null) + res.keySet().removeAll(sql.deletedIds(info.query.split("/", 2)[0])); + // if files are deleted on the FS, there might be stale providers which cause // zombie objects without ID, remove these List> list = new ArrayList<>(res.values()); @@ -170,6 +207,13 @@ public void create(Table m, Map object) throws Exception { @Override public Map read(Table s, Map search) throws Exception { + JSONSqlDatabase eff = effective(s.name); + if (eff != null) + return eff.read(s, search); + JSONSqlDatabase sql = sqlUser(); + if (sql != null && search.get("ID") != null && sql.isDeleted(s, search.get("ID"))) + return null; + // merge all DBs (user DB is last) Map res = null; for (JSONDatabase d : all()) @@ -197,7 +241,13 @@ public Map readDBs(Table s, Map search) throws E public boolean update(Table schema, Map search, Map object) throws Exception { // the steps below read and write the user DB several times; hold the record lock throughout - return user().locked(schema, search.get("ID"), () -> updateLocked(schema, search, object)); + return user().locked(schema, search.get("ID"), () -> { + // a record deleted in the SQL config store cannot be updated + JSONSqlDatabase sql = sqlUser(); + if (sql != null && sql.isDeleted(schema, search.get("ID"))) + return false; + return updateLocked(schema, search, object); + }); } boolean updateLocked(Table schema, Map search, Map object) @@ -382,6 +432,10 @@ public QueryResponse getInitialQuery(InitialQueryRequest ac) throws Exception { @Override public List> all(Table s, Integer offset, Integer limit, String sort, boolean descending, Map arguments) throws Exception { + JSONSqlDatabase eff = effective(s.name); + if (eff != null && eff.store.postgres) + return eff.allEffective(s, offset, limit, sort, descending, arguments); + QueryMeta info = new QueryMeta(); info.query = s.name; List> res = new ArrayList<>(); diff --git a/dashjoin-core/src/main/resources/META-INF/services/org.dashjoin.function.Function b/dashjoin-core/src/main/resources/META-INF/services/org.dashjoin.function.Function index 5cac984c4..035405dbc 100644 --- a/dashjoin-core/src/main/resources/META-INF/services/org.dashjoin.function.Function +++ b/dashjoin-core/src/main/resources/META-INF/services/org.dashjoin.function.Function @@ -45,6 +45,7 @@ org.dashjoin.function.SaveApi org.dashjoin.function.ERDiagram org.dashjoin.function.Stats org.dashjoin.function.GitPull +org.dashjoin.function.AppSync org.dashjoin.function.GitCheckout org.dashjoin.function.GitRestore org.dashjoin.function.GitCommit diff --git a/dashjoin-core/src/test/java/org/dashjoin/service/SqlConfigStoreTest.java b/dashjoin-core/src/test/java/org/dashjoin/service/SqlConfigStoreTest.java new file mode 100644 index 000000000..b44bc8537 --- /dev/null +++ b/dashjoin-core/src/test/java/org/dashjoin/service/SqlConfigStoreTest.java @@ -0,0 +1,323 @@ +package org.dashjoin.service; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import org.dashjoin.model.QueryMeta; +import org.dashjoin.model.Table; +import org.eclipse.jgit.api.Git; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * config served from SQL: import of a git checkout, layers, deletion markers, app sync, locking. + * + * The PostgreSQL variant runs when DJ_TEST_PG_URL (and DJ_TEST_PG_USER, DJ_TEST_PG_PASSWORD) are + * set. It works in a schema of its own, which is dropped afterwards. + */ +public class SqlConfigStoreTest { + + @TempDir + Path tmp; + + static final Table PAGE = Table.ofName("page"); + static final Table FUNCTION = Table.ofName("dj-function"); + static final Table TENANTUSERS = Table.ofName("tenantusers"); + + Path home; + Git git; + + void write(String path, String content) throws Exception { + Path f = home.resolve(path); + Files.createDirectories(f.getParent()); + Files.writeString(f, content); + } + + void commit(String message) throws Exception { + git.add().addFilepattern(".").call(); + git.add().addFilepattern(".").setUpdate(true).call(); + git.commit().setMessage(message).setAuthor("test", "test@example.org") + .setCommitter("test", "test@example.org").call(); + } + + /** an app checkout: one commit plus local changes in the working tree */ + void createApp() throws Exception { + home = Files.createDirectories(tmp.resolve("app")); + git = Git.init().setDirectory(home.toFile()).call(); + write("model/page/a.json", "{\"ID\": \"a\", \"title\": \"A\", \"color\": \"red\"}"); + write("model/dj-function/f.json", + "{\"ID\": \"f\", \"djClassName\": \"x\", \"expression-pointer\": \"expression.jsonata\"}"); + write("model/dj-function/f.expression.jsonata", "1 + 1"); + write("model/tenantusers/@EVERYONE.json", + "{\"ID\": \"@EVERYONE\", \"active\": true, \"roles\": [\"user\"]}"); + write("model/tenantusers/@gone.json", "{\"ID\": \"@gone\", \"active\": true}"); + write("upload/file.txt", "not config"); + write(".gitignore", "model/dj-function/secret.json\n"); + commit("app v1"); + + // local changes, not committed + write("model/page/a.json", "{\"ID\": \"a\", \"title\": \"A local\", \"color\": \"red\"}"); + write("model/dj-function/secret.json", "{\"ID\": \"secret\", \"djClassName\": \"y\"}"); + Files.delete(home.resolve("model/tenantusers/@gone.json")); + } + + /** the union database as Services sets it up: attaching brings the effective records up to date */ + UnionDatabase union(SqlConfigStore store) throws Exception { + store.attach(List.of(store.app())); + UnionDatabase u = new UnionDatabase(); + u._user = store.local(); + u._dbs = new ArrayList<>(List.of(store.app())); + return u; + } + + static Map map(Object... kv) { + Map res = new HashMap<>(); + for (int i = 0; i < kv.length; i += 2) + res.put((String) kv[i], kv[i + 1]); + return res; + } + + static List ids(List> records) { + return records.stream().map(r -> r.get("ID")).sorted().map(Object.class::cast).toList(); + } + + /** + * the effective records maintained incrementally are the same as the ones of a full rebuild, and + * the same as merging the layers in memory + */ + void assertEffectiveConsistent(SqlConfigStore store) throws Exception { + Map>> incremental = + store.readLayer(SqlConfigStore.EFFECTIVE); + store.transaction(c -> { + store.rebuildEffective(c); + return null; + }); + Assertions.assertEquals(store.readLayer(SqlConfigStore.EFFECTIVE), incremental); + + UnionDatabase inMemory = new UnionDatabase() { + @Override + JSONSqlDatabase effective(String table) { + return null; + } + }; + inMemory._user = store.local(); + inMemory._dbs = List.of(store.app()); + for (String table : incremental.keySet()) + for (Map r : incremental.get(table).values()) + Assertions.assertEquals(inMemory.read(Table.ofName(table), Map.of("ID", r.get("ID"))), r); + } + + List> all(UnionDatabase u, String table) throws Exception { + QueryMeta q = new QueryMeta(); + q.query = table; + return u.query(q, null); + } + + @SuppressWarnings("unchecked") + void scenario(SqlConfigStore store) throws Exception { + createApp(); + store.bootstrap(home.toString(), false); + UnionDatabase u = union(store); + + // import: HEAD is the app layer, the working tree changes are the local layer + Assertions.assertEquals(map("ID", "a", "title", "A local", "color", "red"), + u.read(PAGE, Map.of("ID", "a"))); + Assertions.assertEquals(map("ID", "a", "title", "A local"), + store.local().read(PAGE, Map.of("ID", "a")), "only the changed field is local"); + Assertions.assertEquals(map("ID", "f", "djClassName", "x", "expression", "1 + 1"), + u.read(FUNCTION, Map.of("ID", "f")), "externalized string inlined, pointer dropped"); + Assertions.assertEquals(map("ID", "secret", "djClassName", "y"), + store.local().read(FUNCTION, Map.of("ID", "secret")), "local only record"); + Assertions.assertNull(u.read(TENANTUSERS, Map.of("ID", "@gone")), "deleted locally"); + Assertions.assertEquals(List.of("@EVERYONE"), ids(all(u, "tenantusers"))); + + // runtime writes go to the local layer + u.create(TENANTUSERS, map("ID", "joe@x.org", "active", false)); + u.update(TENANTUSERS, Map.of("ID", "@EVERYONE"), map("roles", List.of("user", "admin"))); + Assertions.assertEquals(map("ID", "@EVERYONE", "roles", List.of("user", "admin")), + store.local().read(TENANTUSERS, Map.of("ID", "@EVERYONE"))); + Assertions.assertEquals(List.of("@EVERYONE", "joe@x.org"), ids(all(u, "tenantusers"))); + + // deleting an app record leaves a marker; creating it again removes the marker + Assertions.assertTrue(u.delete(PAGE, Map.of("ID", "a"))); + Assertions.assertNull(u.read(PAGE, Map.of("ID", "a"))); + Assertions.assertTrue(all(u, "page").isEmpty()); + Assertions.assertFalse(u.update(PAGE, Map.of("ID", "a"), map("title", "x")), + "a deleted record cannot be updated"); + u.create(PAGE, map("ID", "a", "title", "A again")); + Assertions.assertEquals("A again", u.read(PAGE, Map.of("ID", "a")).get("title")); + u.delete(PAGE, Map.of("ID", "a")); + u.update(TENANTUSERS, Map.of("ID", "@EVERYONE"), map("roles", List.of("user", "admin"))); + assertEffectiveConsistent(store); + + // queries: filter, sort, paging (in SQL on PostgreSQL) + u.create(TENANTUSERS, map("ID", "ann@x.org", "active", true, "n", 2, "tags", List.of("a"))); + u.create(TENANTUSERS, map("ID", "bob@x.org", "active", true, "n", 1)); + Assertions.assertEquals(List.of("@EVERYONE", "ann@x.org", "bob@x.org"), + u.all(TENANTUSERS, null, null, null, false, map("active", true)).stream() + .map(r -> r.get("ID")).toList()); + Assertions.assertEquals(List.of("bob@x.org", "ann@x.org", "@EVERYONE", "joe@x.org"), + u.all(TENANTUSERS, null, null, "n", false, null).stream().map(r -> r.get("ID")).toList(), + "records without the field come last"); + Assertions.assertEquals(List.of("ann@x.org"), + u.all(TENANTUSERS, 2, 1, "n", true, null).stream().map(r -> r.get("ID")).toList()); + Assertions.assertEquals(List.of("ann@x.org"), u + .all(TENANTUSERS, null, null, null, false, map("tags", List.of("a"))).stream() + .map(r -> r.get("ID")).toList()); + Assertions.assertEquals(List.of("@EVERYONE", "joe@x.org"), + u.all(TENANTUSERS, null, null, null, false, map("n", null)).stream() + .map(r -> r.get("ID")).toList()); + u.delete(TENANTUSERS, Map.of("ID", "ann@x.org")); + u.delete(TENANTUSERS, Map.of("ID", "bob@x.org")); + + // new app version: the app layer is replaced, the local layer stays + git.checkout().setAllPaths(true).call(); // discard working tree changes of the checkout + write("model/page/a.json", "{\"ID\": \"a\", \"title\": \"A v2\", \"color\": \"blue\"}"); + write("model/page/b.json", "{\"ID\": \"b\", \"title\": \"B\"}"); + write("model/tenantusers/@EVERYONE.json", + "{\"ID\": \"@EVERYONE\", \"active\": true, \"roles\": [\"guest\"], \"comment\": \"v2\"}"); + Files.delete(home.resolve("model/dj-function/f.json")); + Files.delete(home.resolve("model/dj-function/f.expression.jsonata")); + commit("app v2"); + + Map dry = store.sync(home.toString(), false, true); + Assertions.assertEquals(List.of("page/b"), dry.get("added")); + Assertions.assertNotNull(u.read(FUNCTION, Map.of("ID", "f")), "dry run changes nothing"); + + Map report = store.sync(home.toString(), false, false); + Assertions.assertEquals(List.of("page/b"), report.get("added")); + Assertions.assertEquals(List.of("page/a", "tenantusers/@EVERYONE"), + ((List) report.get("changed")).stream().sorted().toList()); + Assertions.assertEquals(List.of("dj-function/f"), report.get("removed")); + Assertions.assertEquals(List.of("tenantusers/@EVERYONE: roles"), + report.get("localOverridesOfChangedFields")); + + Assertions.assertNull(u.read(PAGE, Map.of("ID", "a")), "still deleted locally"); + Assertions.assertEquals(map("ID", "b", "title", "B"), u.read(PAGE, Map.of("ID", "b"))); + Assertions.assertNull(u.read(FUNCTION, Map.of("ID", "f"))); + Assertions.assertNotNull(u.read(FUNCTION, Map.of("ID", "secret"))); + Assertions.assertEquals( + map("ID", "@EVERYONE", "active", true, "roles", List.of("user", "admin"), "comment", "v2"), + u.read(TENANTUSERS, Map.of("ID", "@EVERYONE")), "local override wins, app change shows"); + Assertions.assertNotNull(u.read(TENANTUSERS, Map.of("ID", "joe@x.org"))); + assertEffectiveConsistent(store); + + // a second start does not import again + store.bootstrap(home.toString(), false); + Assertions.assertNull(u.read(PAGE, Map.of("ID", "a"))); + + // concurrent updates of one record are not lost + int threads = 8, perThread = 10; + ExecutorService pool = Executors.newFixedThreadPool(threads); + List> fs = new ArrayList<>(); + for (int t = 0; t < threads; t++) { + int tt = t; + fs.add(pool.submit(() -> { + for (int i = 0; i < perThread; i++) + u.update(TENANTUSERS, Map.of("ID", "joe@x.org"), map("f" + tt + "_" + i, i)); + return null; + })); + } + for (Future f : fs) + f.get(); + pool.shutdown(); + Assertions.assertEquals(threads * perThread + 2, + u.read(TENANTUSERS, Map.of("ID", "joe@x.org")).size()); + assertEffectiveConsistent(store); + + // a changed platform (here: an unknown hash) rebuilds the effective records on startup + store.setMeta("platform", "other"); + store.transaction(c -> { + try (java.sql.Statement stmt = c.createStatement()) { + stmt.execute("delete from dj_config where layer = 'effective'"); + } + return null; + }); + store.maintainEffective(); + Assertions.assertNotNull(u.read(TENANTUSERS, Map.of("ID", "joe@x.org"))); + } + + /** + * the injected config database (e.g. for the session and tenant managers) reads merged records, + * including records from the app, and writes the local layer + */ + @Test + public void view() throws Exception { + SqlConfigStore store = + new SqlConfigStore("jdbc:sqlite:" + tmp.resolve("config.db"), null, null); + try { + createApp(); + store.bootstrap(home.toString(), false); + UnionDatabase u = union(store); + SqlConfigView view = store.view(); + + Assertions.assertEquals(map("ID", "@EVERYONE", "active", true, "roles", List.of("user")), + view.read(TENANTUSERS, Map.of("ID", "@EVERYONE")), "app record"); + Assertions.assertTrue( + view.update(TENANTUSERS, Map.of("ID", "@EVERYONE"), map("roles", List.of("admin")))); + Assertions.assertEquals(map("ID", "@EVERYONE", "roles", List.of("admin")), + store.local().read(TENANTUSERS, Map.of("ID", "@EVERYONE")), "only the change is local"); + Assertions.assertEquals(List.of("admin"), + u.read(TENANTUSERS, Map.of("ID", "@EVERYONE")).get("roles")); + Assertions.assertFalse(view.update(TENANTUSERS, Map.of("ID", "x"), map("active", true))); + + view.create(TENANTUSERS, map("ID", "new@x.org", "active", true)); + Assertions.assertEquals(map("ID", "new@x.org", "active", true), + u.read(TENANTUSERS, Map.of("ID", "new@x.org"))); + Assertions.assertTrue(view.delete(TENANTUSERS, Map.of("ID", "@EVERYONE"))); + Assertions.assertNull(view.read(TENANTUSERS, Map.of("ID", "@EVERYONE")), "marker written"); + assertEffectiveConsistent(store); + } finally { + store.close(); + git.close(); + } + } + + @Test + public void sqlite() throws Exception { + SqlConfigStore store = + new SqlConfigStore("jdbc:sqlite:" + tmp.resolve("config.db"), null, null); + try { + scenario(store); + } finally { + store.close(); + git.close(); + } + } + + @Test + public void postgres() throws Exception { + String url = System.getenv("DJ_TEST_PG_URL"); + Assumptions.assumeTrue(url != null, "DJ_TEST_PG_URL not set"); + String user = System.getenv("DJ_TEST_PG_USER"); + String password = System.getenv("DJ_TEST_PG_PASSWORD"); + String schema = "dj_config_test_" + Long.toHexString(System.nanoTime()); + try (Connection con = DriverManager.getConnection(url, user, password); + Statement stmt = con.createStatement()) { + stmt.execute("create schema " + schema); + } + SqlConfigStore store = new SqlConfigStore( + url + (url.contains("?") ? "&" : "?") + "currentSchema=" + schema, user, password); + try { + scenario(store); + } finally { + store.close(); + git.close(); + try (Connection con = DriverManager.getConnection(url, user, password); + Statement stmt = con.createStatement()) { + stmt.execute("drop schema " + schema + " cascade"); + } + } + } +} diff --git a/dashjoin-docs/docs/administration.md b/dashjoin-docs/docs/administration.md index e71a00596..4e97e7b28 100644 --- a/dashjoin-docs/docs/administration.md +++ b/dashjoin-docs/docs/administration.md @@ -371,3 +371,50 @@ The sessions are tracked in the table user within the config database (field ses Only a hash of each session ID is stored, along with the creation time and the browser's user agent. Users that logged in before the feature was enabled are not affected until they log in again. The local admin user is never restricted. To turn the feature off, remove the map from the single-session entry. + +## Serving the Config Database from SQL + +When several instances of the platform run the same app, the config database can be served from a SQL database +(currently PostgreSQL) instead of the model folder. Set these properties (or the environment variables +DASHJOIN_CONFIGDB_URL, DASHJOIN_CONFIGDB_USERNAME, DASHJOIN_CONFIGDB_PASSWORD): + +```text +dashjoin.configdb.url=jdbc:postgresql://host:5432/db?currentSchema=djconfig +dashjoin.configdb.username=postgres +dashjoin.configdb.password=... +``` + +All instances must use the same encryption key for stored passwords: set the environment variable DJ_SID +(for an existing installation, use the content of model/.secrets.id). + +The config then consists of three layers (the higher one wins, field by field): + +* local: everything written at runtime (UI, API, logins), records that only exist on this installation + (e.g. git-ignored credentials) and deleted records +* app: the model folder of one git commit of the app +* the platform defaults + +The merged result of the layers is stored as well, so reads and queries (filter, sort, paging) run in +SQL without merging the layers in memory. Every write recomputes the merged record in the same +transaction, and a sync recomputes the records it changes. When the platform defaults change (e.g. +after an upgrade or when plugins are added), all merged records are rebuilt on startup. Therefore, all +instances should run the same platform version. Tables that connected databases add metadata to at +runtime (dj-database) are still merged in memory. + +Users (user) and tenant users (tenantusers) are stored like all other records. Records checked into git, +such as @EVERYONE, come from the app layer; logins and invitations write to the local layer. + +On the first start, the model folder of the app checkout (see DASHJOIN_APP_URL) is imported: +the HEAD commit becomes the app layer, records changed in the working tree become the local layer +(only the changed fields), and records deleted in the working tree are marked as deleted. +Without a git checkout, all records go to the local layer. + +To update the app, run $appSync() (or $gitPull(), which does the same in this mode). It pulls the app, +replaces the app layer with the new version and keeps the local layer. When DASHJOIN_APP_URL is set, +this also happens on every start. The result lists the records added, changed and removed, and the local +changes of fields the new version changed; $appSync(true) only computes this report. +Tables listed in dashjoin.configdb.sync.exclude are never taken from the app. + +The app checkout is still used for all other files (uploads, assets, scripts). +Changes made in the UI are no longer written to the model folder; develop apps on an instance without +dashjoin.configdb.url and commit them to git as before. diff --git a/dashjoin-docs/docs/appendix-jsonata.md b/dashjoin-docs/docs/appendix-jsonata.md index b0ed65c2f..9e0e5f26c 100644 --- a/dashjoin-docs/docs/appendix-jsonata.md +++ b/dashjoin-docs/docs/appendix-jsonata.md @@ -528,6 +528,24 @@ Sample output: ```json null ``` +### appSync +Pull the dashjoin app and update the app records of the SQL config database +``` +$appSync() +``` +Sample output: +```json +{ + "commit" : "3f2a9c1e5b7d4f6a8c0e2b4d6f8a0c2e4b6d8f0a", + "previousCommit" : "9d1c3e5a7b9f1d3c5e7a9b1d3f5c7e9a1b3d5f7c", + "dryRun" : false, + "added" : [ "page/b" ], + "changed" : [ "page/a", "tenantusers/@EVERYONE" ], + "removed" : [ "dj-function/f" ], + "localOverridesOfChangedFields" : [ "tenantusers/@EVERYONE: roles" ], + "changedDatabases" : [ ] +} +``` ### gitRestore git restore 'dashjoin-demo.db' ``` diff --git a/dashjoin-docs/docs/jsonata-reference.md b/dashjoin-docs/docs/jsonata-reference.md index d0e1a14ba..ed80d0b6b 100644 --- a/dashjoin-docs/docs/jsonata-reference.md +++ b/dashjoin-docs/docs/jsonata-reference.md @@ -43,6 +43,7 @@ Function | Syntax | Returns adHocQuery| $adHocQuery(database, query, limit?) | Runs as ad hoc select / read query all| $all(database, table) | array of all table records all| $all(database, table, offset, limit, sort, descending, filter) | array of all table records whose columns match the filter key's values +appSync| $appSync(dryRun?) | If the config database is stored in SQL: git pull the app and replace the app records with the new version. Changes made at runtime are kept. Returns a report of the changes; with dryRun=true, only the report is computed call| $call(function, argument) | Dashjoin function result chat | $chat(url, query, messages, tools?, options?, headers?) | Encapsulates API calls to LLMs running at url. Query is the next user message. messages is an array of role / content containing the previous chat interactions. Tools is an array of strings referencing Invoke functions in the function catalog that should be used as tools during the LLM call. Options contains additional LLM parameters like "model": "gpt-4o-mini", etc. Finally, headers works like with the curl function. classifyEntities | $classifyEntities([entities], entity-language?, limit?, subclass-depth?) | Reconciles entities and finds common classifications that all entities are an instance of. The parameters are similar to the reconcileEntity function. The subclass depth (default 1) describes the number of superclasses that are included in the results (see [AI & ML Integration](ai-ml-integration.md)) diff --git a/dashjoin-docs/llms/jsonata.json b/dashjoin-docs/llms/jsonata.json index ac8d676b1..ef4e5b255 100644 --- a/dashjoin-docs/llms/jsonata.json +++ b/dashjoin-docs/llms/jsonata.json @@ -577,6 +577,31 @@ "code": "$gitPull()" } ], + "appSync": [ + { + "description": "Pull the dashjoin app and update the app records of the SQL config database", + "code": "$appSync()", + "output": { + "commit": "3f2a9c1e5b7d4f6a8c0e2b4d6f8a0c2e4b6d8f0a", + "previousCommit": "9d1c3e5a7b9f1d3c5e7a9b1d3f5c7e9a1b3d5f7c", + "dryRun": false, + "added": [ + "page/b" + ], + "changed": [ + "page/a", + "tenantusers/@EVERYONE" + ], + "removed": [ + "dj-function/f" + ], + "localOverridesOfChangedFields": [ + "tenantusers/@EVERYONE: roles" + ], + "changedDatabases": [] + } + } + ], "gitCheckout": [ { "description": "Checks out and switches the dashjoin app branch",