diff --git a/.github/workflows/compile_sqlite.yml b/.github/workflows/compile_sqlite.yml index 34f2fbd4..22b00bd0 100644 --- a/.github/workflows/compile_sqlite.yml +++ b/.github/workflows/compile_sqlite.yml @@ -101,6 +101,7 @@ jobs: build_sqlite: needs: [download_sqlite, build_openssl] strategy: + fail-fast: true matrix: os: [ubuntu-latest, macos-latest, windows-latest] @@ -123,6 +124,13 @@ jobs: name: sqlite-src path: sqlite-src + - name: Download compiled OpenSSL + if: runner.os == 'Linux' && steps.cache_build.outputs.cache-hit != 'true' + uses: actions/download-artifact@v8 + with: + name: openssl-libs-${{ runner.os }} + path: openssl-compiled + - uses: dart-lang/setup-dart@v1 if: steps.cache_build.outputs.cache-hit != 'true' diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 64c1fcef..622963c5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -157,6 +157,15 @@ jobs: dart test --test-randomize-ordering-seed "random" -P ci working-directory: sqlite3/ + - name: Enable sqlcipher + run: | + dart run tool/hook_overrides.dart compiled-sqlcipher + - name: Test sqlite3 package with sqlcipher + run: | + dart test --test-randomize-ordering-seed "random" -P ci + if: ${{ runner.os != 'windows' }} # TODO: SQLCipher Windows not supported yet + working-directory: sqlite3/ + - name: Prepare for tests with system SQLite run: | dart run tool/hook_overrides.dart system-os-specific @@ -432,3 +441,29 @@ jobs: emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none script: flutter test integration_test -Dsqlite3.multipleciphers=true working-directory: "examples/flutter_integration_tests" + + - name: Enable sqlcipher + run: | + dart run tool/hook_overrides.dart compiled-sqlcipher + + - name: Flutter sqlcipher tests on macOS + if: runner.os == 'macos' + working-directory: examples/flutter_integration_tests + run: | + flutter config --enable-swift-package-manager + flutter test integration_test -Dsqlite3.sqlcipher=true + flutter test integration_test -Dsqlite3.sqlcipher=true -d macos + + - name: sqlcipher Android emulator tests + uses: reactivecircus/android-emulator-runner@e89f39f1abbbd05b1113a29cf4db69e7540cae5a # v2.37.0 + if: runner.os == 'linux' + with: + api-level: 34 + force-avd-creation: false + target: google_apis + arch: x86_64 + disable-animations: false + avd-name: $AVD_NAME + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + script: flutter test integration_test -Dsqlite3.sqlcipher=true + working-directory: "examples/flutter_integration_tests" diff --git a/examples/flutter_integration_tests/integration_test/integration_test.dart b/examples/flutter_integration_tests/integration_test/integration_test.dart index 563e4f4a..bf141bfc 100644 --- a/examples/flutter_integration_tests/integration_test/integration_test.dart +++ b/examples/flutter_integration_tests/integration_test/integration_test.dart @@ -1,7 +1,10 @@ // ignore_for_file: avoid_print +import 'dart:io'; + import 'package:integration_test/integration_test.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart'; import 'package:sqlite3/sqlite3.dart'; import 'package:sqlite3_connection_pool/sqlite3_connection_pool.dart'; @@ -118,6 +121,55 @@ void main() { print(db.select('select sqlite3mc_config(?)', ['cipher'])); }); } + + const sqlcipher = bool.fromEnvironment('sqlite3.sqlcipher'); + if (sqlcipher) { + test('cipher_version', () { + final db = sqlite3.openInMemory()..closeWhenDone(); + final cipherVersionRows = db.select('PRAGMA cipher_version'); + print(cipherVersionRows); + expect(cipherVersionRows, isNotEmpty); + }); + } + + if (ciphers || sqlcipher) { + test('encryption', () { + final dir = Directory.systemTemp.createTempSync(); + final path = join(dir.path, 'test.db'); + final db = sqlite3.open(path); + addTearDown(() { + db.close(); + }); + + final key = 'my_secret'; + db.execute("PRAGMA key = '$key'"); + db.execute("CREATE TABLE users (id INTEGER, username TEXT)"); + db.close(); + + final dbAfterEnc = sqlite3.open(path); + addTearDown(() => dbAfterEnc.close()); + expect( + () => dbAfterEnc.select('SELECT * FROM sqlite_master'), + throwsSqlError(SqlError.SQLITE_NOTADB, SqlError.SQLITE_NOTADB), + ); + dbAfterEnc.execute("PRAGMA key = '$key'"); + + // Reads the db after setting the key + expect(dbAfterEnc.select('SELECT * FROM sqlite_master'), isNotEmpty); + }); + } +} + +Matcher throwsSqlError(int resultCode, int extendedResultCode) { + return throwsA( + isA() + .having( + (e) => e.extendedResultCode, + 'extendedResultCode', + extendedResultCode, + ) + .having((e) => e.resultCode, 'resultCode', resultCode), + ); } extension on Database { diff --git a/examples/flutter_integration_tests/pubspec.yaml b/examples/flutter_integration_tests/pubspec.yaml index 7890bce9..4b6b049f 100644 --- a/examples/flutter_integration_tests/pubspec.yaml +++ b/examples/flutter_integration_tests/pubspec.yaml @@ -11,6 +11,7 @@ dependencies: sdk: flutter sqlite3: sqlite3_connection_pool: + path: ^1.8.3 dev_dependencies: flutter_test: diff --git a/sqlite3/hook/build.dart b/sqlite3/hook/build.dart index ddaa0d60..d0af4fbb 100644 --- a/sqlite3/hook/build.dart +++ b/sqlite3/hook/build.dart @@ -60,15 +60,70 @@ ${usedSqliteSymbols.map((symbol) => ' $symbol;').join('\n')} } final isSqlcipher = input.userDefines['is_sqlcipher'] as bool? ?? false; + final targetOS = input.config.code.targetOS; + final targetArchitecture = input.config.code.targetArchitecture; final isAppleTarget = targetOS == OS.iOS || targetOS == OS.macOS; + // Directory where the architecture compiled OpenSSL is located. Null if OpenSSL is not used + Directory? openSslCompileDir; + File? openSslStaticLib; + if (isSqlcipher) { + final linksWithOpenSSL = + targetOS == OS.android || + targetOS == OS.linux || + targetOS == OS.windows; + if (linksWithOpenSSL) { + const openSSLCompiledRootKey = 'openssl_compiled_root'; + final Uri? opensslCompiledRoot = input.userDefines.path( + openSSLCompiledRootKey, + ); + + if (opensslCompiledRoot == null) { + throw StateError( + 'Target $targetOS needs OpenSSL compiled dir root with \'openSSLCompiledRootKey\'', + ); + } + + openSslCompileDir = Directory( + p.join( + opensslCompiledRoot.toFilePath(), + "${targetOS.name}-${targetArchitecture.name}", + ), + ); + + if (!await openSslCompileDir.exists()) { + throw StateError( + 'Expected OpenSSL compiled directory at ${openSslCompileDir.path}', + ); + } + + openSslStaticLib = File( + p.join( + openSslCompileDir.path, + _getOpenSslLibFolderName(targetOS, targetArchitecture), + targetOS.staticlibFileName('crypto'), + ), + ); + + if (!await openSslStaticLib.exists()) { + throw StateError( + 'Expected OpenSSL static library at ${openSslStaticLib.path}', + ); + } + } + } + final library = CBuilder.library( name: 'sqlite3', packageName: 'sqlite3', assetName: name, sources: [sourceFile], - includes: [p.dirname(sourceFile)], + includes: [ + p.dirname(sourceFile), + if (openSslCompileDir != null) + p.join(openSslCompileDir.path, 'include'), + ], defines: defines, flags: [ if (input.config.code.targetOS == OS.linux) ...[ @@ -78,12 +133,11 @@ ${usedSqliteSymbols.map((symbol) => ' $symbol;').join('\n')} // export, we might as well strip the rest. // TODO: Port this to other targets too. '-Wl,--version-script=$linkerScript', + // Strip symbols + '-s', '-ffunction-sections', '-fdata-sections', '-Wl,--gc-sections', - if (isSqlcipher) - // TODO: Link OpenSSL statically? - '-lcrypto', ], if (isAppleTarget) ...[ '-headerpad_max_install_names', @@ -101,10 +155,17 @@ ${usedSqliteSymbols.map((symbol) => ' $symbol;').join('\n')} ], ], ], + libraryDirectories: [ + if (openSslStaticLib != null) openSslStaticLib.parent.path, + ], libraries: [ - if (input.config.code.targetOS == OS.android) + if (targetOS == OS.android) ...[ // We need to link the math library on Android. 'm', + if (isSqlcipher) 'log', + ], + // Link with OpenSSL (SQLCipher builds) + if (openSslCompileDir != null) 'crypto', ], ); @@ -121,5 +182,12 @@ ${usedSqliteSymbols.map((symbol) => ' $symbol;').join('\n')} }); } +String _getOpenSslLibFolderName(OS os, Architecture architecture) { + return switch ((os, architecture)) { + (OS.linux, Architecture.x64) => 'lib64', + _ => 'lib', + }; +} + const package = 'sqlite3'; const name = 'src/ffi/libsqlite3.g.dart'; diff --git a/sqlite3/lib/src/hook/description.dart b/sqlite3/lib/src/hook/description.dart index c1d0affc..69db2cef 100644 --- a/sqlite3/lib/src/hook/description.dart +++ b/sqlite3/lib/src/hook/description.dart @@ -22,10 +22,14 @@ sealed class SqliteBinary { return PrecompiledFromGithubAssets(LibraryType.sqlite3); case 'sqlite3mc': return PrecompiledFromGithubAssets(LibraryType.sqlite3mc); + case 'sqlcipher': + return PrecompiledFromGithubAssets(LibraryType.sqlcipher); case 'test-sqlite3': return PrecompiledForTesting(LibraryType.sqlite3); case 'test-sqlite3mc': return PrecompiledForTesting(LibraryType.sqlite3mc); + case 'test-sqlcipher': + return PrecompiledForTesting(LibraryType.sqlcipher); case 'system': final osSpecificNameKey = 'name_${input.config.code.targetOS.name}'; @@ -38,11 +42,13 @@ sealed class SqliteBinary { case 'executable': return SimpleBinary.fromExecutable; case 'source': + final isSqlcipher = userDefines['is_sqlcipher'] as bool? ?? false; return CompileSqlite( sourceFile: userDefines.path('path')!.toFilePath(), defines: CompilerDefines.parse( userDefines, - input.config.code.targetOS, + targetOS: input.config.code.targetOS, + isSqlcipher: isSqlcipher, ), ); default: @@ -339,7 +345,11 @@ extension type const CompilerDefines(Map flags) return CompilerDefines({...flags, ...other.flags}); } - static CompilerDefines parse(HookInputUserDefines defines, OS targetOS) { + static CompilerDefines parse( + HookInputUserDefines defines, { + required OS targetOS, + required bool isSqlcipher, + }) { final obj = defines['defines']; // Include default options when not explicitly disabled. @@ -357,7 +367,7 @@ extension type const CompilerDefines(Map flags) }; final start = includeDefaults - ? CompilerDefines.defaults(targetOS == OS.windows) + ? CompilerDefines.defaults(targetOS: targetOS, isSqlcipher: isSqlcipher) : const CompilerDefines({}); return switch (additionalDefines) { @@ -394,11 +404,44 @@ extension type const CompilerDefines(Map flags) return CompilerDefines(entries); } - static CompilerDefines defaults(bool windows) { + static CompilerDefines defaults({ + required OS targetOS, + required bool isSqlcipher, + }) { final defines = _parseLines(const LineSplitter().convert(_defaultDefines)); - if (windows) { + if (targetOS == OS.windows) { defines['SQLITE_API'] = '__declspec(dllexport)'; } + + // Minimum extra flags to build SQLCipher + if (isSqlcipher) { + defines.addAll({ + 'SQLITE_HAS_CODEC': null, + 'SQLITE_TEMP_STORE': "2", + 'SQLITE_EXTRA_INIT': 'sqlcipher_extra_init', + 'SQLITE_EXTRA_SHUTDOWN': 'sqlcipher_extra_shutdown', + + // Default flags from SQLCipher community builds to keep compatibility with the old sqlcipher_flutter_libs + // which was using SQLCipher Community binaries under the hood + // https://github.com/sqlcipher/sqlcipher-android/blob/7fab57af75039e5004b087086142b11a9d2a2380/sqlcipher/src/main/jni/sqlcipher/Android.mk#L9 + ...{ + // Most modern unix systems support nanosleep, but if it wouldn't be available + // we want to fallback to usleep (microseconds) instead of sleep (seconds) + 'HAVE_USLEEP': null, + + // URI support + 'SQLITE_USE_URI ': null, + + // Not clear if it has an impact in all applications + 'SQLITE_ENABLE_MEMORY_MANAGEMENT': null, + }, + + // Link with CommonCrypto on Apple platforms + if (targetOS == OS.macOS || targetOS == OS.iOS) + 'SQLCIPHER_CRYPTO_CC': null, + }); + } + return defines; } } diff --git a/sqlite3/test/ffi/encryption_test.dart b/sqlite3/test/ffi/encryption_test.dart new file mode 100644 index 00000000..26ef44f6 --- /dev/null +++ b/sqlite3/test/ffi/encryption_test.dart @@ -0,0 +1,66 @@ +@Tags(['ffi']) +library; + +import 'package:sqlite3/sqlite3.dart'; +import 'package:sqlite3/src/hook/assets.dart'; +import 'package:test/test.dart'; +import 'package:test_descriptor/test_descriptor.dart' as d; + +import '../common/utils.dart'; + +void main() { + test('encryption', () { + final LibraryType libraryType = _inferLibraryType(); + + if (libraryType == LibraryType.sqlcipher || + libraryType == LibraryType.sqlite3mc) { + testEncryptAndOpenEncrypted(); + } + }); +} + +void testEncryptAndOpenEncrypted() { + final path = d.path('test.db'); + final db = sqlite3.open(path); + addTearDown(() { + db.close(); + }); + + final key = 'my_secret'; + db.execute("PRAGMA key = '$key'"); + db.execute("CREATE TABLE users (id INTEGER, username TEXT)"); + db.close(); + + final dbAfterEnc = sqlite3.open(path); + addTearDown(() => dbAfterEnc.close()); + expect( + () => dbAfterEnc.select('SELECT * FROM sqlite_master'), + throwsSqlError(SqlError.SQLITE_NOTADB, SqlError.SQLITE_NOTADB), + ); + dbAfterEnc.execute("PRAGMA key = '$key'"); + + // Reads the db after setting the key + expect(dbAfterEnc.select('SELECT * FROM sqlite_master'), isNotEmpty); +} + +LibraryType _inferLibraryType() { + final db = sqlite3.openInMemory(); + + try { + // Sqlcipher can check PRAGMA cipher_version + final cipherVersionRows = db.select('PRAGMA cipher_version'); + if (cipherVersionRows.isNotEmpty) { + return LibraryType.sqlcipher; + } + + // PRAGMA cipher is available in sqlite3mc + final cipherRows = db.select('PRAGMA cipher'); + if (cipherRows.isNotEmpty) { + return LibraryType.sqlite3mc; + } + + return LibraryType.sqlite3; + } finally { + db.close(); + } +} diff --git a/sqlite3_connection_pool/lib/src/raw.dart b/sqlite3_connection_pool/lib/src/raw.dart index ad571d41..0c330274 100644 --- a/sqlite3_connection_pool/lib/src/raw.dart +++ b/sqlite3_connection_pool/lib/src/raw.dart @@ -243,8 +243,7 @@ final class PoolConnections { extension type PoolConnectionRef( /// The pool connection, used to manage cached prepared statements. - Pointer - connection + Pointer connection ) { /// The `sqlite3*` connection pointer. Pointer get rawDatabase => connection.ref.raw; diff --git a/tool/build_openssl.dart b/tool/build_openssl.dart index e8a901b9..c5387ce5 100644 --- a/tool/build_openssl.dart +++ b/tool/build_openssl.dart @@ -24,30 +24,43 @@ void main(List args) async { var hadFailure = false; for (final platform in args) { + final OS targetOS; + final List targetArchs; + switch (platform) { case 'linux': - for (final arch in _linuxArchitectures) { - try { - await _buildOpenSSL( - targetOS: OS.linux, - targetArchitecture: arch, - sharedOutputDirectory: target, - openSslSrcDir: src, - ); - } catch (e, s) { - hadFailure = true; - print('Build failed for $arch: $e'); - print(s); - } - } - case 'windows': + targetArchs = _linuxArchitectures; + targetOS = OS.linux; break; case 'android': + targetArchs = _androidArchitectures; + targetOS = OS.android; + break; + case 'windows': + targetArchs = [ + // TODO: Windows + ]; + targetOS = OS.windows; break; default: throw UnsupportedError( 'Unsupported target OS, expected linux, windows or android.'); } + + for (final arch in targetArchs) { + try { + await _buildOpenSSL( + targetOS: targetOS, + targetArchitecture: arch, + sharedOutputDirectory: target, + openSslSrcDir: src, + ); + } catch (e, s) { + hadFailure = true; + print('Build failed for $platform-$arch: $e'); + print(s); + } + } } if (hadFailure) exit(1); @@ -77,8 +90,18 @@ Future _buildOpenSSL({ targetOS, targetArchitecture, ); + + final Map extraEnv = {}; if (targetOS == OS.android) { - throw 'TODO: Android'; + final String? ndkRoot = Platform.environment['ANDROID_NDK_ROOT']; + + if (ndkRoot == null) { + throw Exception('Android NDK not found. Set ANDROID_NDK_ROOT'); + } + + final existingPath = Platform.environment['PATH'] ?? ''; + extraEnv['PATH'] = + '$ndkRoot/toolchains/llvm/prebuilt/linux-x86_64/bin:$existingPath'; } final extraConfigureArgs = [ @@ -98,31 +121,37 @@ Future _buildOpenSSL({ case OS.linux: // run ./Configure with the target OS and architecture await _run( - 'perl', - [ - configureProgramPath, - configName, - ..._configArgs, - ...extraConfigureArgs, - ], - workingDirectory: openSslBuildDirPath); + 'perl', + [ + configureProgramPath, + configName, + ..._configArgs, + ...extraConfigureArgs, + ], + workingDirectory: openSslBuildDirPath, + environment: extraEnv, + ); // Build static libraries await _run( - 'make', - [ - '-j', - '${Platform.numberOfProcessors}', - ], - workingDirectory: openSslBuildDirPath); + 'make', + [ + '-j', + '${Platform.numberOfProcessors}', + ], + workingDirectory: openSslBuildDirPath, + environment: extraEnv, + ); // Copy compiled libraries into output directory await _run( - 'make', - [ - 'install', - ], - workingDirectory: openSslBuildDirPath); + 'make', + [ + 'install', + ], + workingDirectory: openSslBuildDirPath, + environment: extraEnv, + ); break; } @@ -130,13 +159,18 @@ Future _buildOpenSSL({ await tmp.delete(recursive: true); } -Future _run(String executable, List args, - {String? workingDirectory}) async { +Future _run( + String executable, + List args, { + String? workingDirectory, + Map? environment, +}) async { final proc = await Process.start( executable, args, mode: ProcessStartMode.inheritStdio, workingDirectory: workingDirectory, + environment: environment, ); final exitCode = await proc.exitCode; @@ -224,3 +258,10 @@ const _linuxArchitectures = [ Architecture.x64, //Architecture.riscv64, ]; + +const _androidArchitectures = [ + Architecture.arm, + Architecture.arm64, + Architecture.ia32, + Architecture.x64, +]; diff --git a/tool/build_sqlite.dart b/tool/build_sqlite.dart index a286a21e..40ae1804 100644 --- a/tool/build_sqlite.dart +++ b/tool/build_sqlite.dart @@ -20,9 +20,15 @@ void main(List args) async { var operatingSystems = args.map(OS.fromString).toList(); if (operatingSystems.isEmpty) { if (Platform.isLinux) { - operatingSystems = [OS.linux, OS.android]; + operatingSystems = [ + OS.linux, + OS.android, + ]; } else if (Platform.isMacOS) { - operatingSystems = [OS.macOS, OS.iOS]; + operatingSystems = [ + OS.macOS, + OS.iOS, + ]; } else if (Platform.isWindows) { operatingSystems = [OS.windows]; } @@ -107,22 +113,14 @@ void main(List args) async { 'source': 'source', 'path': p.relative(sourcePath, from: fs.currentDirectory.path), if (mode == SqliteFork.sqlcipher) ...{ - 'defines': { - 'default_options': true, - 'defines': [ - 'SQLITE_HAS_CODEC=1', - 'SQLITE_EXTRA_INIT=sqlcipher_extra_init', - 'SQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown', - // SQLCipher uses it in their Community builds. - // Not clear if it has an impact in all applications - // https://github.com/sqlcipher/sqlcipher-android/blob/7fab57af75039e5004b087086142b11a9d2a2380/sqlcipher/src/main/jni/sqlcipher/Android.mk#L9 - 'SQLITE_ENABLE_MEMORY_MANAGEMENT=1', - if (os case OS.iOS || OS.macOS) - // Link with CommonCrypto on Apple platforms - 'SQLCIPHER_CRYPTO_CC=1', - ] - }, 'is_sqlcipher': true, + // This is the folder where all the openssl compiled archs are located + 'openssl_compiled_root': p.relative( + fs.currentDirectory.parent + .childDirectory('openssl-compiled') + .path, + from: fs.currentDirectory.path, + ), } }, basePath: fs.currentDirectory.uri, @@ -170,8 +168,6 @@ bool _skipBuild(OS targetOS, Architecture targetArch, SqliteFork type) { case SqliteFork.sqlcipher: // TODO: Build for Windows if (targetOS == OS.windows) return true; - // TODO: Build for Android - if (targetOS == OS.android) return true; // TODO: Other Linux architectures if (targetOS == OS.linux) { return targetArch != Architecture.x64; diff --git a/tool/build_with_sanitizers.dart b/tool/build_with_sanitizers.dart index f6d475b3..56f04c8f 100644 --- a/tool/build_with_sanitizers.dart +++ b/tool/build_with_sanitizers.dart @@ -58,7 +58,10 @@ void main() async { includes: [p.dirname(sourceFile)], defines: { 'SQLITE_ENABLE_API_ARMOR': '1', - ...CompilerDefines.defaults(false), + ...CompilerDefines.defaults( + targetOS: input.config.code.targetOS, + isSqlcipher: false, + ), }, flags: [ '-fsanitize=$sanitizer', diff --git a/tool/download_sqlite.dart b/tool/download_sqlite.dart index 533158d1..dd87e980 100644 --- a/tool/download_sqlite.dart +++ b/tool/download_sqlite.dart @@ -17,13 +17,20 @@ const tmpDir = 'tmp'; /// really supposed to be used outside of that, but can be used to reproduce /// what hooks are doing in the CI. void main(List args) async { + if (await Directory(tmpDir).exists()) { + await Directory(tmpDir).delete(recursive: true); + } await Directory(tmpDir).create(); await _downloadAndExtract(sqliteSource, 'sqlite3'); await _downloadAndExtract(sqliteMultipleCiphersSource, 'sqlite3mc'); await _downloadAndExtract(sqlcipherSource, 'sqlcipher'); + if (await Directory('sqlite-src').exists()) { + await Directory('sqlite-src').delete(recursive: true); + } await Directory('sqlite-src').create(); + await Directory('sqlite-src/sqlite3mc').create(); await File('$tmpDir/sqlite3mc_amalgamation.h') .copy('sqlite-src/sqlite3mc/sqlite3mc_amalgamation.h'); diff --git a/tool/hook_overrides.dart b/tool/hook_overrides.dart index a155c20b..f660ce79 100644 --- a/tool/hook_overrides.dart +++ b/tool/hook_overrides.dart @@ -48,6 +48,16 @@ hooks: sqlite3: source: test-sqlite3mc directory: $outPath/ +'''); + case 'compiled-sqlcipher': + final outPath = p.relative('sqlite-compiled', from: p.dirname(path)); + + out.write(''' +hooks: + user_defines: + sqlite3: + source: test-sqlcipher + directory: $outPath/ '''); default: throw 'Unsupported mode, can use system, system-os-specific, '