Make the database portable and encryptable (#3848)

.gitattributes

@@ -0,0 +1,8 @@
+# Vendored third-party sources. Marking them keeps them out of language
+# statistics and collapses them in diffs, so a routine engine update does not
+# read as a 400,000 line change and does not push review tooling past its
+# size limits.
+vm/ByteCodeTranslator/src/cn1_sqlite3_amalgamation.h linguist-vendored linguist-generated -diff
+vm/ByteCodeTranslator/src/cn1_sqlite3.h             linguist-vendored linguist-generated -diff
+Ports/JavaScriptPort/src/main/webapp/js/sqlite3mc.js              linguist-vendored -diff
+Ports/JavaScriptPort/src/main/webapp/js/sqlite3-opfs-async-proxy.js linguist-vendored -diff

.github/codeql/codeql-config.yml

@@ -9,3 +9,10 @@ name: "Codename One CodeQL configuration"
 # String sinks across the rest of the codebase).
 paths-ignore:
   - 'scripts/cn1playground/**/bsh/cn1/gen/**'
+  # Vendored SQLite engine (SQLite3 Multiple Ciphers), 13MB of upstream C carried
+  # verbatim. Analysing it reports upstream's findings as ours, and its sheer size
+  # pushes CodeQL off incremental analysis so it re-reports pre-existing alerts
+  # across unrelated files. Provenance and update instructions are in
+  # vm/ByteCodeTranslator/src/cn1_sqlite3_README.md.
+  - 'vm/ByteCodeTranslator/src/cn1_sqlite3_amalgamation.h'
+  - 'vm/ByteCodeTranslator/src/cn1_sqlite3.h'

.github/workflows/pr.yml

@@ -181,6 +181,15 @@ jobs:
             -Dcodename1.platform=javase \
             -Dcn1.binaries="${CN1_BINARIES}" \
             -pl javase -Plocal-dev-javase test
+      - name: Verify encrypted databases interoperate with reference SQLCipher
+        if: ${{ matrix.java-version == 8 }}
+        # The portability claim is that every port writes one on-disk format. A cipher
+        # misconfiguration produces files each platform reads happily and nothing else can
+        # touch, which no single-platform test would catch, so check ours against the stock
+        # sqlcipher client in both directions.
+        run: |
+          apt-get update -qq && apt-get install -y -qq sqlcipher
+          scripts/ci/db-cipher-interop.sh
       - name: Determine AI/ad cn1libs to build
         id: cn1libs
         if: ${{ matrix.java-version == 8 }}
@@ -344,6 +353,15 @@ jobs:
             -Dcn1.binaries="${CN1_BINARIES}" \
             -Pcompile-android \
             -pl android,ios,codenameone-maven-plugin -am verify
+      - name: Type-check the SQLCipher-backed database package
+        if: ${{ matrix.java-version == 8 }}
+        env:
+          CN1_BINARIES: ${{ github.workspace }}/maven/target/cn1-binaries
+        # That package is excluded from the port build on purpose, so nothing above compiles it
+        # and a member of com.codename1.impl.android it reaches can stop being public without
+        # anything noticing until an Android app build forty minutes later. Runs after the port
+        # is compiled, which is what it type-checks against.
+        run: scripts/ci/check-android-cipher-package-compiles.sh
       - name: Run SpotBugs for ByteCodeTranslator
         if: ${{ matrix.java-version == 8 }}
         run: mvn -B -DskipTests=true -f vm/ByteCodeTranslator/pom.xml verify

CodenameOne/src/com/codename1/db/Cursor.java

@@ -24,124 +24,117 @@
 
 import java.io.IOException;
 
-/// The Cursor interface is used to iterate over the results returned from a database query.
-/// **IMPORTANT:** Notice that some methods might not be supported on all platforms!
+/// Iterates over the results returned from a database query.
 ///
-/// There is more thorough coverage of the `Database API here`.
-///
-/// The sample code below presents a Database Explorer tool that allows executing arbitrary SQL and
-/// viewing the tabular results:
+/// Positions are counted from zero and a new cursor sits before the first row, so the usual loop
+/// is simply:
 ///
 /// ```java
-/// Toolbar.setGlobalToolbar(true);
-/// Style s = UIManager.getInstance().getComponentStyle("TitleCommand");
-/// FontImage icon = FontImage.createMaterial(FontImage.MATERIAL_QUERY_BUILDER, s);
-/// Form hi = new Form("SQL Explorer", new BorderLayout());
-/// hi.getToolbar().addCommandToRightBar("", icon, (e) -> {
-///     TextArea query = new TextArea(3, 80);
-///     Command ok = new Command("Execute");
-///     Command cancel = new Command("Cancel");
-///     if(Dialog.show("Query", query, ok, cancel) == ok) {
-///         Database db = null;
-///         Cursor cur = null;
-///         try {
-///             db = Display.getInstance().openOrCreate("MyDB.db");
-///             if(query.getText().startsWith("select")) {
-///                 cur = db.executeQuery(query.getText());
-///                 int columns = cur.getColumnCount();
-///                 hi.removeAll();
-///                 if(columns > 0) {
-///                     boolean next = cur.next();
-///                     if(next) {
-///                         ArrayList data = new ArrayList<>();
-///                         String[] columnNames = new String[columns];
-///                         for(int iter = 0 ; iter
+/// Cursor cur = db.executeQuery("SELECT id, body FROM notes ORDER BY id");
+/// try {
+///     while (cur.next()) {
+///         Row row = cur.getRow();
+///         System.out.println(row.getInteger(0) + ": " + row.getString(1));
+///     }
+/// } finally {
+///     cur.close();
+/// }
+/// ```
+///
+/// Every navigation method works on every platform. Only the cost varies: `#next()` is uniformly
+/// cheap, while `#last()`, `#prev()` and `#position(int)` may have to rewind and re-step the
+/// underlying statement, which costs time proportional to the distance from the start. For a large
+/// result set, prefer iterating forward with `#next()`.
+///
+/// Because a backward seek re-runs the statement, a cursor is a repeatable read only inside a
+/// transaction. See the `com.codename1.db` package documentation for the full contract.
 ///
 /// @author Chen
 public interface Cursor {
 
-    /// Move the cursor to the first row.
-    ///
-    /// If cursor provides forward-only navigation and is positioned after the
-    /// first row then calling first() method would throw a IOException.
+    /// Moves the cursor onto the first row.
     ///
     /// #### Returns
     ///
-    /// true if succeeded
+    /// true if there is a first row, false for an empty result set
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: if the cursor is closed
     // PMD Fix (UnnecessaryModifier): Interface methods are implicitly public; remove redundant modifiers.
     boolean first() throws IOException;
 
-    /// Move the cursor to the last row.
+    /// Moves the cursor onto the last row.
+    ///
+    /// Costs a full pass over the result set the first time it is called.
     ///
     /// #### Returns
     ///
-    /// true if succeeded
+    /// true if there is a last row, false for an empty result set
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: if the cursor is closed
     boolean last() throws IOException;
 
-    /// Moves the cursor to the next row.
-    /// Calling next() method the first time will position cursor on the first.
+    /// Advances the cursor one row.
+    ///
+    /// A new cursor sits before the first row, so the first call lands on it.
     ///
     /// #### Returns
     ///
-    /// true if succeeded
+    /// true if a row was reached, false at the end of the result set
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: if the cursor is closed
     boolean next() throws IOException;
 
-    /// Moves the cursor to the previous row.
-    /// If cursor is forward type then calling prev() would throw a IOException.
+    /// Moves the cursor back one row.
     ///
     /// #### Returns
     ///
-    /// true if succeeded
+    /// true if a row was reached, false when already at or before the first row
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: if the cursor is closed
     boolean prev() throws IOException;
 
-    /// Returns the zero-based index for a given column name.
-    /// Note that columns meta information is available only after navigation to
-    /// the first row
+    /// Returns the zero-based index of a column, or -1 if there is no such column.
+    ///
+    /// The comparison is case-insensitive and matches the result set label, so a column selected
+    /// as `SELECT a AS b` is found under `b`. Available as soon as the query returns, before the
+    /// first `#next()`.
     ///
     /// #### Parameters
     ///
-    /// - `columnName`: the name of the column.
+    /// - `columnName`: the name of the column
     ///
     /// #### Returns
     ///
-    /// the index of the column
+    /// the zero-based index, or -1 when the column is not in the result set
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: if the cursor is closed
     int getColumnIndex(String columnName) throws IOException;
 
-    /// Returns the column name at a given zero-based column index.
-    /// Note that columns meta information is available only after navigation to
-    /// the first row
+    /// Returns the label of the column at a zero-based index.
+    ///
+    /// Available as soon as the query returns, before the first `#next()`.
     ///
     /// #### Parameters
     ///
-    /// - `columnIndex`: the index of the column
+    /// - `columnIndex`: the zero-based index of the column
     ///
     /// #### Returns
     ///
-    /// the name of the column
+    /// the column label
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: if the cursor is closed
     String getColumnName(int columnIndex) throws IOException;
 
     /// Returns the column count
@@ -155,48 +148,57 @@ public interface Cursor {
     /// - `IOException`
     int getColumnCount() throws IOException;
 
-    /// Returns the current Cursor position.
+    /// Returns the zero-based position of the cursor.
+    ///
+    /// Reports -1 before any successful move, and the row count once the result set is exhausted.
     ///
     /// #### Returns
     ///
     /// the cursor position
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: if the cursor is closed
     int getPosition() throws IOException;
 
-    /// Move the cursor to an absolute row position
+    /// Moves the cursor to an absolute zero-based row.
+    ///
+    /// Passing -1 rewinds to before the first row and returns false.
     ///
     /// #### Parameters
     ///
-    /// - `row`: position to move to
+    /// - `row`: the zero-based row to move to
     ///
     /// #### Returns
     ///
-    /// true if succeeded
+    /// true if the row exists, false if it is out of range
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: if the cursor is closed
     boolean position(int row) throws IOException;
 
-    /// Close the cursor and release its resources
+    /// Closes the cursor and releases its resources.
+    ///
+    /// Calling this more than once is harmless.
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: if the underlying statement cannot be released
     void close() throws IOException;
 
-    /// Get the Row data Object.
+    /// Returns the current row.
+    ///
+    /// Valid only while the cursor is on a row.
     ///
     /// #### Returns
     ///
-    /// a Row Object
+    /// the current row
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: @throws IOException if the cursor is closed, or is before the first row or
+    ///                   past the last one
     Row getRow() throws IOException;
 
 }

CodenameOne/src/com/codename1/db/CursorExt.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Codename One designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Codename One through http://www.codenameone.com/ if you
+ * need additional information or have any questions.
+ */
+package com.codename1.db;
+
+import java.io.IOException;
+
+/// Optional cursor capabilities, in the same spirit as `RowExt`.
+///
+/// These are not on `Cursor` itself because that interface is public and is
+/// implemented outside this repository; adding a method to it would break every
+/// third party library that provides one. Reach these through the static helpers
+/// on `Database`, which degrade gracefully on a cursor that does not implement
+/// this interface:
+///
+/// ```java
+/// Database.beforeFirst(cursor);
+/// int rows = Database.count(cursor); // -1 when the port cannot say cheaply
+/// ```
+///
+/// Every cursor returned by a Codename One port implements this.
+public interface CursorExt extends Cursor {
+
+    /// Rewinds to before the first row, without landing on a row.
+    ///
+    /// After this call `Cursor#getPosition()` reports -1 and `Cursor#getRow()`
+    /// throws, exactly as on a freshly returned cursor. This is the operation
+    /// `Cursor#position(int)` performs when given -1.
+    ///
+    /// #### Throws
+    ///
+    /// - `IOException`: if the cursor is closed or the rewind fails
+    void beforeFirst() throws IOException;
+
+    /// Returns the number of rows in the result set, or -1 when the port cannot
+    /// determine it without walking the whole set.
+    ///
+    /// Ports that already track the count report it directly. Ports backed by a
+    /// forward-only statement have to step to the end and rewind, so they report
+    /// the count once it is known and -1 before that. Treat -1 as "unknown", not
+    /// as "empty".
+    ///
+    /// #### Returns
+    ///
+    /// the row count, or -1 when it is not cheaply available
+    ///
+    /// #### Throws
+    ///
+    /// - `IOException`: if the cursor is closed
+    int getCount() throws IOException;
+}