diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 2df9d89ea4..a4a70750a3 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -78,15 +78,12 @@ source ~/.bashrc ./install.sh ``` -But you still need to: -- Put the [MySQL JDBC connector](https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/mysql-connector-j-8.0.33.jar) into `PIXELS_HOME/lib`. -- Modify `PIXELS_HOME/etc/pixels.properties` to ensure the following properties are valid: +But you still need to modify `PIXELS_HOME/etc/pixels.properties` to ensure the following properties are valid: ```properties pixels.var.dir=/home/pixels/opt/pixels/var/ -metadata.db.driver=com.mysql.jdbc.Driver metadata.db.user=pixels metadata.db.password=password -metadata.db.url=jdbc:mysql://localhost:3306/pixels_metadata?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull +metadata.db.url=jdbc:derby:/home/pixels/opt/pixels/var/pixels_metadata;create=true metadata.server.port=18888 metadata.server.host=localhost trans.server.port=18889 @@ -145,9 +142,10 @@ Leave the other config parameters as default. Set `cache.enabled` to `false` in `PIXELS_HOME/etc/pixels.properties` if you don't use pixels-cache. -## Install MySQL -MySQL and etcd are used to store the metadata and states of Pixels. MySQL/MariaDB 5.5 or later has been tested. Other forks or variants may also work. -You only need to install one etcd and one MySQL instance, even in a cluster. +## Install MySQL* +Mysql is optional. Pixels uses the embedded database Derby to store the metadata by default. +However, we also support MySQL as the metadata database. +MySQL/MariaDB 5.5 or later has been tested. Other forks or variants may also work. To install MySQL: ```bash @@ -177,8 +175,14 @@ binds the server to localhost thus declines remote connections. Use `scripts/sql/metadata_schema.sql` to create tables in `pixels_metadata`. +Then, put the [MySQL JDBC connector](https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/mysql-connector-j-8.0.33.jar) into `PIXELS_HOME/lib` and set `metadata.db.url=jdbc:mysql://localhost:3306/pixels_metadata?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull` +in `PIXELS_HOME/etc/pixels.properties` to enable MySQL as the metadata storage in Pixels. +Change `localhost` in the URL to the hostname of the MySQL server if it is not running on the same node as Pixels coordinator. + ## Install etcd +Etcd is used to store the states of Pixels. + First install go-lang: ```bash sudo apt install golang diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/MetaDBUtil.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/MetaDBUtil.java index 9fe5e4f0c3..bb2319c388 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/MetaDBUtil.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/utils/MetaDBUtil.java @@ -55,12 +55,10 @@ private MetaDBUtil() try { ConfigFactory config = ConfigFactory.Instance(); - String driver = config.getProperty("metadata.db.driver"); url = config.getProperty("metadata.db.url"); user = config.getProperty("metadata.db.user"); pass = config.getProperty("metadata.db.password"); - Class.forName(driver); this.connection = DriverManager.getConnection(url, user, pass); } catch (Exception e) diff --git a/pixels-common/src/main/resources/pixels.properties b/pixels-common/src/main/resources/pixels.properties index 74915a9882..a9934f399b 100644 --- a/pixels-common/src/main/resources/pixels.properties +++ b/pixels-common/src/main/resources/pixels.properties @@ -2,10 +2,10 @@ # pixels.var.dir is where the lock files are created pixels.var.dir=/home/pixels/opt/pixels/var/ # metadata database connection properties -metadata.db.driver=com.mysql.cj.jdbc.Driver metadata.db.user=pixels metadata.db.password=password -metadata.db.url=jdbc:mysql://localhost:3306/pixels_metadata?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull +#metadata.db.url=jdbc:mysql://localhost:3306/pixels_metadata?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull +metadata.db.url=jdbc:derby:/home/pixels/opt/pixels/var/pixels_metadata;create=true # metadata server host and port metadata.server.port=18888 metadata.server.host=localhost diff --git a/pixels-common/src/test/java/io/pixelsdb/pixels/common/TestMetadataService.java b/pixels-common/src/test/java/io/pixelsdb/pixels/common/metadata/TestMetadataService.java similarity index 97% rename from pixels-common/src/test/java/io/pixelsdb/pixels/common/TestMetadataService.java rename to pixels-common/src/test/java/io/pixelsdb/pixels/common/metadata/TestMetadataService.java index 6f4ec327c9..56aebb4624 100644 --- a/pixels-common/src/test/java/io/pixelsdb/pixels/common/TestMetadataService.java +++ b/pixels-common/src/test/java/io/pixelsdb/pixels/common/metadata/TestMetadataService.java @@ -17,11 +17,10 @@ * License along with Pixels. If not, see * . */ -package io.pixelsdb.pixels.common; +package io.pixelsdb.pixels.common.metadata; import com.alibaba.fastjson.JSON; import io.pixelsdb.pixels.common.exception.MetadataException; -import io.pixelsdb.pixels.common.metadata.MetadataService; import io.pixelsdb.pixels.common.metadata.domain.*; import org.junit.Before; import org.junit.Test; diff --git a/pixels-daemon/pom.xml b/pixels-daemon/pom.xml index 987b089e80..4dcac2147e 100644 --- a/pixels-daemon/pom.xml +++ b/pixels-daemon/pom.xml @@ -87,6 +87,11 @@ jetcd-core + + org.apache.derby + derby + + io.trino diff --git a/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/TestRdbDaos.java b/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/TestRdbDaos.java index f8dcf3367b..673953ae14 100644 --- a/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/TestRdbDaos.java +++ b/pixels-daemon/src/test/java/io/pixelsdb/pixels/daemon/metadata/dao/TestRdbDaos.java @@ -21,16 +21,34 @@ import io.pixelsdb.pixels.common.metadata.domain.Layout; import io.pixelsdb.pixels.common.metadata.domain.Ordered; +import io.pixelsdb.pixels.common.utils.MetaDBUtil; import io.pixelsdb.pixels.daemon.MetadataProto; import org.junit.Test; import java.io.*; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.util.HashSet; import java.util.List; import java.util.Set; public class TestRdbDaos { + @Test + public void testRdbConnection() throws SQLException + { + MetaDBUtil db = MetaDBUtil.Instance(); + Connection conn = db.getConnection(); + Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery("VALUES 1"); // VALUES 1 for derby, SELCT 1 for MySQL + assert rs != null && rs.next(); + assert rs.getInt(1) == 1; + assert !rs.next(); + conn.close(); + } + @Test public void testSchema () { diff --git a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java index 9f55fd1b47..da9365d11f 100644 --- a/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java +++ b/pixels-retina/src/test/java/io/pixelsdb/pixels/retina/TestStorageGarbageCollector.java @@ -19,24 +19,18 @@ */ package io.pixelsdb.pixels.retina; +import com.google.common.collect.ImmutableList; import io.pixelsdb.pixels.common.index.service.LocalIndexService; import io.pixelsdb.pixels.common.metadata.MetadataService; -import io.pixelsdb.pixels.common.utils.CheckpointFileIO; -import io.pixelsdb.pixels.common.utils.ConfigFactory; -import io.pixelsdb.pixels.common.utils.MetaDBUtil; -import io.pixelsdb.pixels.common.utils.PixelsFileNameUtils; -import io.pixelsdb.pixels.common.utils.RetinaUtils; import io.pixelsdb.pixels.common.metadata.domain.Column; import io.pixelsdb.pixels.common.metadata.domain.File; import io.pixelsdb.pixels.common.metadata.domain.Layout; import io.pixelsdb.pixels.common.physical.Storage; import io.pixelsdb.pixels.common.physical.StorageFactory; -import io.pixelsdb.pixels.core.PixelsFooterCache; -import io.pixelsdb.pixels.core.PixelsReader; -import io.pixelsdb.pixels.core.PixelsReaderImpl; -import io.pixelsdb.pixels.core.PixelsWriter; -import io.pixelsdb.pixels.core.PixelsWriterImpl; -import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.common.utils.ConfigFactory; +import io.pixelsdb.pixels.common.utils.PixelsFileNameUtils; +import io.pixelsdb.pixels.common.utils.RetinaUtils; +import io.pixelsdb.pixels.core.*; import io.pixelsdb.pixels.core.encoding.EncodingLevel; import io.pixelsdb.pixels.core.reader.PixelsReaderOption; import io.pixelsdb.pixels.core.reader.PixelsRecordReader; @@ -44,37 +38,20 @@ import io.pixelsdb.pixels.core.vector.DoubleColumnVector; import io.pixelsdb.pixels.core.vector.LongColumnVector; import io.pixelsdb.pixels.core.vector.VectorizedRowBatch; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; +import org.junit.*; import java.io.IOException; import java.lang.reflect.Field; import java.nio.file.Files; import java.nio.file.Path; -import java.sql.PreparedStatement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; /** * Tests for {@link StorageGarbageCollector}, covering scan/grouping, data rewrite, @@ -3313,8 +3290,7 @@ private Map getRgVisibilityMap() // Helpers: catalog registration // ======================================================================= - private long registerTestFile(String name, File.Type type, - int numRg, long minRow, long maxRow) + private long registerTestFile(String name, File.Type type, int numRg, long minRow, long maxRow) throws Exception { File f = new File(); @@ -3330,29 +3306,22 @@ private long registerTestFile(String name, File.Type type, return id; } - private long insertRawFileWithType(String name, int fileType, - int numRg, long minRow, long maxRow) - throws Exception + private long insertRawFileWithType(String name, int fileType, int numRg, long minRow, long maxRow) throws Exception { - String sql = "INSERT INTO FILES(FILE_NAME, FILE_TYPE, FILE_NUM_RG, FILE_MIN_ROW_ID, FILE_MAX_ROW_ID, PATHS_PATH_ID) " + - "VALUES (?, ?, ?, ?, ?, ?)"; - try (PreparedStatement pst = MetaDBUtil.Instance().getConnection().prepareStatement(sql)) - { - pst.setString(1, name); - pst.setInt(2, fileType); - pst.setInt(3, numRg); - pst.setLong(4, minRow); - pst.setLong(5, maxRow); - pst.setLong(6, testPathId); - assertEquals("raw test file insert should affect one row", 1, pst.executeUpdate()); - } + File file = new File(); + file.setName(name); + file.setType(File.Type.valueOf(fileType)); + file.setNumRowGroup(numRg); + file.setMinRowId(minRow); + file.setMaxRowId(maxRow); + file.setPathId(testPathId); + metadataService.addFiles(ImmutableList.of(file)); long id = metadataService.getFileId(testOrderedPathUri + "/" + name); assertTrue(name + " must have valid id", id > 0); return id; } - private long[] registerTestFiles(String[] names, File.Type[] types, - int[] numRgs, long[] minRows, long[] maxRows) + private long[] registerTestFiles(String[] names, File.Type[] types, int[] numRgs, long[] minRows, long[] maxRows) throws Exception { List files = new ArrayList<>(); @@ -3428,8 +3397,7 @@ private static void assertNoIndexSwitchingTask(List tasks) // Helpers: GC factory for grouping tests // ======================================================================= - private static StorageGarbageCollector newGcForGrouping( - long targetFileSize, int maxFilesPerGroup, int maxGroups) + private static StorageGarbageCollector newGcForGrouping(long targetFileSize, int maxFilesPerGroup, int maxGroups) { return new StorageGarbageCollector( null, null, null, 0.5, targetFileSize, maxFilesPerGroup, maxGroups, @@ -3448,8 +3416,7 @@ private static StorageGarbageCollector newGcForGrouping( *
  • The sentinel entry equals the expected total surviving rows
    • * */ - private static void assertRewriteResultConsistency( - StorageGarbageCollector.RewriteResult result, int expectedTotalRows) + private static void assertRewriteResultConsistency(StorageGarbageCollector.RewriteResult result, int expectedTotalRows) { assertTrue("newFileRgCount must be at least 1", result.newFileRgCount >= 1); assertEquals(result.newFileRgCount, result.newFileRgActualRecordNums.length); @@ -3579,8 +3546,7 @@ private static String writeTestFile(String fileName, TypeDescription schema, * the Pixels writer flushes one RG per call. Row values are sequential integers * starting from 0. */ - private static String writeTestFileMultiRg(String fileName, TypeDescription schema, - int numRgs, int rowsPerRg) throws Exception + private static String writeTestFileMultiRg(String fileName, TypeDescription schema, int numRgs, int rowsPerRg) throws Exception { return writeTestFileMultiRg(fileName, schema, numRgs, rowsPerRg, 10_000); } @@ -3671,8 +3637,7 @@ private static long[][] readAllRows(String path, TypeDescription schema, * {@code pathId} is set to {@link #testPathId} so that {@code addFiles} satisfies * the foreign key constraint against the PATHS table. */ - private static StorageGarbageCollector.FileGroup makeGroup( - long fileId, String filePath, TypeDescription schema) throws Exception + private static StorageGarbageCollector.FileGroup makeGroup(long fileId, String filePath, TypeDescription schema) throws Exception { int rgCount; try (PixelsReader r = PixelsReaderImpl.newBuilder() @@ -3701,10 +3666,8 @@ private static StorageGarbageCollector.FileGroup makeGroup( * {@link StorageGarbageCollector.FileCandidate} objects backed by distinct files. * Both files share the same {@code (tableId=1, virtualNodeId=0)}. */ - private static StorageGarbageCollector.FileGroup makeMultiFileGroup( - TypeDescription schema, - long fileIdA, String pathA, - long fileIdB, String pathB) throws Exception + private static StorageGarbageCollector.FileGroup makeMultiFileGroup(TypeDescription schema, + long fileIdA, String pathA, long fileIdB, String pathB) throws Exception { List candidates = new ArrayList<>(); for (long[] pair : new long[][]{{fileIdA, 0}, {fileIdB, 0}}) diff --git a/pom.xml b/pom.xml index 92e8d7c814..8cc964974b 100644 --- a/pom.xml +++ b/pom.xml @@ -118,6 +118,8 @@ 0.7.7 + + 10.14.2.0 5.13.0 1.3.2 0.16.0 @@ -351,6 +353,13 @@ ${dep.trino.version}     + + + org.apache.derby + derby + ${dep.derby.version} + + io.prometheus