[Issue #1393] support derby metadata storage

docs/INSTALL.md

@@ -78,15 +78,12 @@ source ~/.bashrc
 ./install.sh
 ```
 
-But you still need to:
-- Put the [MySQL JDBC connector](https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/mysql-connector-j-8.0.33.jar) into `PIXELS_HOME/lib`.
-- Modify `PIXELS_HOME/etc/pixels.properties` to ensure the following properties are valid:
+But you still need to modify `PIXELS_HOME/etc/pixels.properties` to ensure the following properties are valid:
 ```properties
 pixels.var.dir=/home/pixels/opt/pixels/var/
-metadata.db.driver=com.mysql.jdbc.Driver
 metadata.db.user=pixels
 metadata.db.password=password
-metadata.db.url=jdbc:mysql://localhost:3306/pixels_metadata?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull
+metadata.db.url=jdbc:derby:/home/pixels/opt/pixels/var/pixels_metadata;create=true
 metadata.server.port=18888
 metadata.server.host=localhost
 trans.server.port=18889
@@ -145,9 +142,10 @@ Leave the other config parameters as default.
 
 Set `cache.enabled` to `false` in `PIXELS_HOME/etc/pixels.properties` if you don't use pixels-cache.
 
-## Install MySQL
-MySQL and etcd are used to store the metadata and states of Pixels. MySQL/MariaDB 5.5 or later has been tested. Other forks or variants may also work.
-You only need to install one etcd and one MySQL instance, even in a cluster.
+## Install MySQL*
+Mysql is optional. Pixels uses the embedded database Derby to store the metadata by default. 
+However, we also support MySQL as the metadata database.
+MySQL/MariaDB 5.5 or later has been tested. Other forks or variants may also work.
 
 To install MySQL:
 ```bash
@@ -177,8 +175,14 @@ binds the server to localhost thus declines remote connections.
 
 Use `scripts/sql/metadata_schema.sql` to create tables in `pixels_metadata`.
 
+Then, put the [MySQL JDBC connector](https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/mysql-connector-j-8.0.33.jar) into `PIXELS_HOME/lib` and set `metadata.db.url=jdbc:mysql://localhost:3306/pixels_metadata?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull`
+in `PIXELS_HOME/etc/pixels.properties` to enable MySQL as the metadata storage in Pixels.
+Change `localhost` in the URL to the hostname of the MySQL server if it is not running on the same node as Pixels coordinator.
+
 ## Install etcd
 
+Etcd is used to store the states of Pixels.
+
 First install go-lang:
 ```bash
 sudo apt install golang

pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/MetaDBUtil.java

@@ -55,12 +55,10 @@ private MetaDBUtil()
         try
         {
             ConfigFactory config = ConfigFactory.Instance();
-            String driver = config.getProperty("metadata.db.driver");
             url = config.getProperty("metadata.db.url");
             user = config.getProperty("metadata.db.user");
             pass = config.getProperty("metadata.db.password");
 
-            Class.forName(driver);
             this.connection = DriverManager.getConnection(url, user, pass);
         }
         catch (Exception e)

pixels-common/src/main/resources/pixels.properties

@@ -2,10 +2,10 @@
 # pixels.var.dir is where the lock files are created
 pixels.var.dir=/home/pixels/opt/pixels/var/
 # metadata database connection properties
-metadata.db.driver=com.mysql.cj.jdbc.Driver
 metadata.db.user=pixels
 metadata.db.password=password
-metadata.db.url=jdbc:mysql://localhost:3306/pixels_metadata?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull
+#metadata.db.url=jdbc:mysql://localhost:3306/pixels_metadata?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull
+metadata.db.url=jdbc:derby:/home/pixels/opt/pixels/var/pixels_metadata;create=true
 # metadata server host and port
 metadata.server.port=18888
 metadata.server.host=localhost

pixels-common/src/test/java/io/pixelsdb/pixels/common/TestMetadataService.java → pixels-common/src/test/java/io/pixelsdb/pixels/common/metadata/TestMetadataService.java

@@ -17,11 +17,10 @@
  * License along with Pixels.  If not, see
  * <https://www.gnu.org/licenses/>.
  */
-package io.pixelsdb.pixels.common;
+package io.pixelsdb.pixels.common.metadata;
 
 import com.alibaba.fastjson.JSON;
 import io.pixelsdb.pixels.common.exception.MetadataException;
-import io.pixelsdb.pixels.common.metadata.MetadataService;
 import io.pixelsdb.pixels.common.metadata.domain.*;
 import org.junit.Before;
 import org.junit.Test;

pixels-daemon/pom.xml

@@ -87,6 +87,11 @@
             <artifactId>jetcd-core</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.derby</groupId>
+            <artifactId>derby</artifactId>
+        </dependency>
+
         <!-- trino-jdbc -->
         <dependency>
             <groupId>io.trino</groupId>

pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/TestRdbDaos.java

@@ -21,16 +21,34 @@
 
 import io.pixelsdb.pixels.common.metadata.domain.Layout;
 import io.pixelsdb.pixels.common.metadata.domain.Ordered;
+import io.pixelsdb.pixels.common.utils.MetaDBUtil;
 import io.pixelsdb.pixels.daemon.MetadataProto;
 import org.junit.Test;
 
 import java.io.*;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
 public class TestRdbDaos
 {
+    @Test
+    public void testRdbConnection() throws SQLException
+    {
+        MetaDBUtil db = MetaDBUtil.Instance();
+        Connection conn = db.getConnection();
+        Statement st = conn.createStatement();
+        ResultSet rs = st.executeQuery("VALUES 1"); // VALUES 1 for derby, SELCT 1 for MySQL
+        assert rs != null && rs.next();
+        assert rs.getInt(1) == 1;
+        assert !rs.next();
+        conn.close();
+    }
+
     @Test
     public void testSchema ()
     {

pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java

@@ -19,62 +19,39 @@
  */
 package io.pixelsdb.pixels.retina;
 
+import com.google.common.collect.ImmutableList;
 import io.pixelsdb.pixels.common.index.service.LocalIndexService;
 import io.pixelsdb.pixels.common.metadata.MetadataService;
-import io.pixelsdb.pixels.common.utils.CheckpointFileIO;
-import io.pixelsdb.pixels.common.utils.ConfigFactory;
-import io.pixelsdb.pixels.common.utils.MetaDBUtil;
-import io.pixelsdb.pixels.common.utils.PixelsFileNameUtils;
-import io.pixelsdb.pixels.common.utils.RetinaUtils;
 import io.pixelsdb.pixels.common.metadata.domain.Column;
 import io.pixelsdb.pixels.common.metadata.domain.File;
 import io.pixelsdb.pixels.common.metadata.domain.Layout;
 import io.pixelsdb.pixels.common.physical.Storage;
 import io.pixelsdb.pixels.common.physical.StorageFactory;
-import io.pixelsdb.pixels.core.PixelsFooterCache;
-import io.pixelsdb.pixels.core.PixelsReader;
-import io.pixelsdb.pixels.core.PixelsReaderImpl;
-import io.pixelsdb.pixels.core.PixelsWriter;
-import io.pixelsdb.pixels.core.PixelsWriterImpl;
-import io.pixelsdb.pixels.core.TypeDescription;
+import io.pixelsdb.pixels.common.utils.ConfigFactory;
+import io.pixelsdb.pixels.common.utils.PixelsFileNameUtils;
+import io.pixelsdb.pixels.common.utils.RetinaUtils;
+import io.pixelsdb.pixels.core.*;
 import io.pixelsdb.pixels.core.encoding.EncodingLevel;
 import io.pixelsdb.pixels.core.reader.PixelsReaderOption;
 import io.pixelsdb.pixels.core.reader.PixelsRecordReader;
 import io.pixelsdb.pixels.core.vector.BinaryColumnVector;
 import io.pixelsdb.pixels.core.vector.DoubleColumnVector;
 import io.pixelsdb.pixels.core.vector.LongColumnVector;
 import io.pixelsdb.pixels.core.vector.VectorizedRowBatch;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
+import org.junit.*;
 
 import java.io.IOException;
 import java.lang.reflect.Field;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.sql.PreparedStatement;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.*;
 
 /**
  * Tests for {@link StorageGarbageCollector}, covering scan/grouping, data rewrite,
@@ -3313,8 +3290,7 @@ private Map<String, RGVisibility> getRgVisibilityMap()
     // Helpers: catalog registration
     // =======================================================================
 
-    private long registerTestFile(String name, File.Type type,
-                                  int numRg, long minRow, long maxRow)
+    private long registerTestFile(String name, File.Type type, int numRg, long minRow, long maxRow)
             throws Exception
     {
         File f = new File();
@@ -3330,29 +3306,22 @@ private long registerTestFile(String name, File.Type type,
         return id;
     }
 
-    private long insertRawFileWithType(String name, int fileType,
-                                       int numRg, long minRow, long maxRow)
-            throws Exception
+    private long insertRawFileWithType(String name, int fileType, int numRg, long minRow, long maxRow) throws Exception
     {
-        String sql = "INSERT INTO FILES(FILE_NAME, FILE_TYPE, FILE_NUM_RG, FILE_MIN_ROW_ID, FILE_MAX_ROW_ID, PATHS_PATH_ID) " +
-                "VALUES (?, ?, ?, ?, ?, ?)";
-        try (PreparedStatement pst = MetaDBUtil.Instance().getConnection().prepareStatement(sql))
-        {
-            pst.setString(1, name);
-            pst.setInt(2, fileType);
-            pst.setInt(3, numRg);
-            pst.setLong(4, minRow);
-            pst.setLong(5, maxRow);
-            pst.setLong(6, testPathId);
-            assertEquals("raw test file insert should affect one row", 1, pst.executeUpdate());
-        }
+        File file = new File();
+        file.setName(name);
+        file.setType(File.Type.valueOf(fileType));
+        file.setNumRowGroup(numRg);
+        file.setMinRowId(minRow);
+        file.setMaxRowId(maxRow);
+        file.setPathId(testPathId);
+        metadataService.addFiles(ImmutableList.of(file));
         long id = metadataService.getFileId(testOrderedPathUri + "/" + name);
         assertTrue(name + " must have valid id", id > 0);
         return id;
     }
 
-    private long[] registerTestFiles(String[] names, File.Type[] types,
-                                     int[] numRgs, long[] minRows, long[] maxRows)
+    private long[] registerTestFiles(String[] names, File.Type[] types, int[] numRgs, long[] minRows, long[] maxRows)
             throws Exception
     {
         List<File> files = new ArrayList<>();
@@ -3428,8 +3397,7 @@ private static void assertNoIndexSwitchingTask(List<StorageGcWal.Task> tasks)
     // Helpers: GC factory for grouping tests
     // =======================================================================
 
-    private static StorageGarbageCollector newGcForGrouping(
-            long targetFileSize, int maxFilesPerGroup, int maxGroups)
+    private static StorageGarbageCollector newGcForGrouping(long targetFileSize, int maxFilesPerGroup, int maxGroups)
     {
         return new StorageGarbageCollector(
                 null, null, null, 0.5, targetFileSize, maxFilesPerGroup, maxGroups,
@@ -3448,8 +3416,7 @@ private static StorageGarbageCollector newGcForGrouping(
      *   <li>The sentinel entry equals the expected total surviving rows</li>
      * </ul>
      */
-    private static void assertRewriteResultConsistency(
-            StorageGarbageCollector.RewriteResult result, int expectedTotalRows)
+    private static void assertRewriteResultConsistency(StorageGarbageCollector.RewriteResult result, int expectedTotalRows)
     {
         assertTrue("newFileRgCount must be at least 1", result.newFileRgCount >= 1);
         assertEquals(result.newFileRgCount, result.newFileRgActualRecordNums.length);
@@ -3579,8 +3546,7 @@ private static String writeTestFile(String fileName, TypeDescription schema,
      * the Pixels writer flushes one RG per call.  Row values are sequential integers
      * starting from 0.
      */
-    private static String writeTestFileMultiRg(String fileName, TypeDescription schema,
-                                               int numRgs, int rowsPerRg) throws Exception
+    private static String writeTestFileMultiRg(String fileName, TypeDescription schema, int numRgs, int rowsPerRg) throws Exception
     {
         return writeTestFileMultiRg(fileName, schema, numRgs, rowsPerRg, 10_000);
     }
@@ -3671,8 +3637,7 @@ private static long[][] readAllRows(String path, TypeDescription schema,
      * {@code pathId} is set to {@link #testPathId} so that {@code addFiles} satisfies
      * the foreign key constraint against the PATHS table.
      */
-    private static StorageGarbageCollector.FileGroup makeGroup(
-            long fileId, String filePath, TypeDescription schema) throws Exception
+    private static StorageGarbageCollector.FileGroup makeGroup(long fileId, String filePath, TypeDescription schema) throws Exception
     {
         int rgCount;
         try (PixelsReader r = PixelsReaderImpl.newBuilder()
@@ -3701,10 +3666,8 @@ private static StorageGarbageCollector.FileGroup makeGroup(
      * {@link StorageGarbageCollector.FileCandidate} objects backed by distinct files.
      * Both files share the same {@code (tableId=1, virtualNodeId=0)}.
      */
-    private static StorageGarbageCollector.FileGroup makeMultiFileGroup(
-            TypeDescription schema,
-            long fileIdA, String pathA,
-            long fileIdB, String pathB) throws Exception
+    private static StorageGarbageCollector.FileGroup makeMultiFileGroup(TypeDescription schema,
+            long fileIdA, String pathA, long fileIdB, String pathB) throws Exception
     {
         List<StorageGarbageCollector.FileCandidate> candidates = new ArrayList<>();
         for (long[] pair : new long[][]{{fileIdA, 0}, {fileIdB, 0}})

pom.xml

@@ -118,6 +118,8 @@
         <!-- jetcd-core-0.7.7 uses grpc-1.60.0 and netty-4.1.100-Final,
         however it is tested to be compatible with grpc-1.74.0 and netty-4.1.118-Final -->
         <dep.jetcd.version>0.7.7</dep.jetcd.version>
+        <!-- 10.14.2.0 is the latest version derby compatible with JDK 8 -->
+        <dep.derby.version>10.14.2.0</dep.derby.version>
         <dep.jna.version>5.13.0</dep.jna.version>
         <dep.javax.annotation.version>1.3.2</dep.javax.annotation.version>
         <dep.prometheus.client.version>0.16.0</dep.prometheus.client.version>
@@ -351,6 +353,13 @@
                 <version>${dep.trino.version}</version>
             </dependency>
 
+            <!-- derby database and embedded jdbc -->
+            <dependency>
+                <groupId>org.apache.derby</groupId>
+                <artifactId>derby</artifactId>
+                <version>${dep.derby.version}</version>
+            </dependency>
+
             <!-- prometheus -->
             <dependency>
                 <groupId>io.prometheus</groupId>