[Issue pixelsdb#1394] add a init-metadata command to create the metadata tables

docker/Dockerfile

@@ -20,8 +20,8 @@ RUN sed -i 's|http://mirrors.cloud.aliyuncs.com|http://mirrors.aliyun.com|g' /et
     git maven less g++ cmake make netcat-openbsd\
     python-is-python3 \
     wget tar unzip openssh-server vim \
-    mysql-server golang \
-    ca-certificates libmysqlclient-dev \
+    golang \
+    ca-certificates \
     # Additional dependency packages
     xz-utils \
     libx11-6 libxau6 libxcb1 libxdmcp6 \
@@ -50,10 +50,6 @@ RUN sed -i 's|http://mirrors.cloud.aliyuncs.com|http://mirrors.aliyun.com|g' /et
 && cd $PIXELS_HOME && mvn install \
 && bash $PIXELS_HOME/install.sh \
 
-# Prepare MySQL Connector/J
-&& cd $PIXELS_HOME/lib/ \
-&& wget https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/mysql-connector-j-8.0.33.jar \
-
 # Clone and compile pixels-trino
 && cd $HOME/opt/ \
 && git clone --depth 1 https://github.com/pixelsdb/pixels-trino.git \
@@ -127,34 +123,11 @@ RUN sed -i 's|http://mirrors.cloud.aliyuncs.com|http://mirrors.aliyun.com|g' /et
 && echo "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED" >> $HOME/opt/trino-server/etc/jvm.config \
 && echo "--add-opens=java.base/java.nio=ALL-UNNAMED" >> $HOME/opt/trino-server/etc/jvm.config \
 
-# Configure MySQL
-&& cd $HOME \
-# Initialize root user credentials (username: root, password: password)
-&& echo "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'password';" >> $HOME/root.sql \
-# Create pixels user and database
-&& echo "CREATE USER 'pixels'@'%' IDENTIFIED BY 'password';" >> $HOME/user.sql \
-&& echo "CREATE DATABASE pixels_metadata;" >> $HOME/user.sql \
-&& echo "GRANT ALL PRIVILEGES ON pixels_metadata.* to 'pixels'@'%';" >> $HOME/user.sql \
-&& echo "FLUSH PRIVILEGES;" >> $HOME/user.sql \
-
-# Initialize MySQL service
-&& usermod -d /var/lib/mysql/ mysql \
-&& service mysql start \
-&& while ! mysqladmin ping -hlocalhost --silent; do sleep 1; done  \
-# Initialize root user
-&& mysql < $HOME/root.sql \
-# Create root credential file
-&& echo "[client]" >> $HOME/root.cnf \
-&& echo "user=root" >> $HOME/root.cnf \
-&& echo "password=password" >> $HOME/root.cnf \
-# Create pixels user credential file
-&& echo "[client]" >> $HOME/pixels.cnf \
-&& echo "user=pixels" >> $HOME/pixels.cnf \
-&& echo "password=password" >> $HOME/pixels.cnf \
-# Initialize metadata schema
-&& mysql --defaults-file=root.cnf < $HOME/user.sql \
-&& mysql --defaults-file=pixels.cnf < $HOME/opt/pixels/scripts/sql/metadata_schema.sql \
-&& service mysql stop \
+# Configure Derby as the metadata database and create tables with INIT-META
+&& sed -i 's|^metadata.db.driver=.*|metadata.db.driver=org.apache.derby.jdbc.EmbeddedDriver|' $PIXELS_HOME/etc/pixels.properties \
+&& sed -i "s|^metadata.db.url=.*|metadata.db.url=jdbc:derby:${PIXELS_HOME}/var/pixels_metadata;create=true|" $PIXELS_HOME/etc/pixels.properties \
+&& mkdir -p $PIXELS_HOME/var \
+&& printf 'INIT-META\nexit\n' | java -jar $PIXELS_HOME/sbin/pixels-cli-*-full.jar \
 
 # Install ETCD
 && cp -v $HOME/opt/pixels/scripts/tars/etcd-v3.3.4-linux-amd64.tar.xz $HOME/opt/ \

docs/INSTALL.md

@@ -98,6 +98,10 @@ presto.pixels.jdbc.url=jdbc:trino://localhost:8080/pixels/tpch
 ```
 The hostnames, ports, paths, usernames, and passwords in these properties are to be configured in the following steps of installation.
 
+The default metadata store is the embedded Derby database under `PIXELS_HOME/var`.
+No extra database server or JDBC connector is required. After the properties are valid,
+create the metadata tables with pixels-cli `INIT-META` (see [Initialize Metadata](#initialize-metadata)).
+
 > **Note:** optionally, you can also set the `PIXEL_CONFIG` system environment variable
 > to specify a different location of `pixels.properties`. This can be a http or https URL
 > to a remote location.
@@ -134,16 +138,38 @@ mkdir var
 ```
 Put the sh scripts in `scripts/bin` and `scripts/sbin` into `PIXELS_HOME/bin` and `PIXELS_HOME/sbin`, respectively.
 Put `pixels-daemon-*-full.jar` into `PIXELS_HOME/bin`.
-Put `pixels-cli-*-full.jar` into `PIXELS_HOME/sbin`
-Put the jdbc connector of MySQL into `PIXELS_HOME/lib`.
+Put `pixels-cli-*-full.jar` into `PIXELS_HOME/sbin`.
 Put `pixels-common/src/main/resources/pixels.properties` into `PIXELS_HOME/etc`.
 Modify `pixels.properties` to ensure that the URLs, ports, paths, usernames, and passwords are valid.
-Leave the other config parameters as default.
+Leave the other config parameters as default. Derby is bundled in pixels-cli and pixels-daemon,
+so you do not need a MySQL JDBC connector unless you switch the metadata database to MySQL.
 
 Set `cache.enabled` to `false` in `PIXELS_HOME/etc/pixels.properties` if you don't use pixels-cache.
 
+Then run `INIT-META` as described in [Initialize Metadata](#initialize-metadata).
+
+## Initialize Metadata
+
+Pixels stores table/layout metadata in the database configured by `metadata.db.driver`
+and `metadata.db.url`. Create those tables with the `INIT-META` command in pixels-cli.
+No need to source the SQL scripts by hand; `INIT-META` selects the matching schema
+(`pixels_metadata_derby.sql` or `pixels_metadata_mysql.sql`) from the configured JDBC URL.
+
+```bash
+mkdir -p $PIXELS_HOME/var
+java -jar $PIXELS_HOME/sbin/pixels-cli-*-full.jar
+# then in the pixels-cli prompt:
+INIT-META
+```
+
+It should print `Initializing metadata tables in DERBY database...` (or `MYSQL` if configured) and
+`INIT-META finished`.
+
+Run `INIT-META` once on the node that hosts the metadata database (the coordinator when
+using the default Derby URL). Workers do not need their own metadata schema.
+
 ## Install MySQL*
-Mysql is optional. Pixels uses the embedded database Derby to store the metadata by default. 
+MySQL is optional. Pixels uses the embedded database Derby to store the metadata by default.
 However, we also support MySQL as the metadata database.
 MySQL/MariaDB 5.5 or later has been tested. Other forks or variants may also work.
 
@@ -173,11 +199,18 @@ FLUSH PRIVILEGES;
 Ensure that MySQL server can be accessed remotely. Sometimes the default MySQL configuration
 binds the server to localhost thus declines remote connections.
 
-Use `scripts/sql/metadata_schema.sql` to create tables in `pixels_metadata`.
+Then put the [MySQL JDBC connector](https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/mysql-connector-j-8.0.33.jar)
+into `PIXELS_HOME/lib` and switch the metadata properties in `PIXELS_HOME/etc/pixels.properties`:
+```properties
+metadata.db.user=pixels
+metadata.db.password=password
+metadata.db.url=jdbc:mysql://localhost:3306/pixels_metadata?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull
+```
+Change `localhost` in the URL to the hostname of the MySQL server if it is not running on the same node as the Pixels coordinator.
 
-Then, put the [MySQL JDBC connector](https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/mysql-connector-j-8.0.33.jar) into `PIXELS_HOME/lib` and set `metadata.db.url=jdbc:mysql://localhost:3306/pixels_metadata?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull`
-in `PIXELS_HOME/etc/pixels.properties` to enable MySQL as the metadata storage in Pixels.
-Change `localhost` in the URL to the hostname of the MySQL server if it is not running on the same node as Pixels coordinator.
+After the properties point at MySQL, create the tables with the same `INIT-META` command
+described in [Initialize Metadata](#initialize-metadata). It should report
+`Initializing metadata tables in MYSQL database...`.
 
 ## Install etcd
 

pixels-cli/pom.xml

@@ -97,6 +97,15 @@
             <version>${dep.jline.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.derby</groupId>
+            <artifactId>derby</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.mysql</groupId>
+            <artifactId>mysql-connector-j</artifactId>
+        </dependency>
+
         <dependency>
             <!-- this is only for third-party libs that use slf4j -->
             <groupId>org.apache.logging.log4j</groupId>
@@ -110,6 +119,13 @@
             <resource>
                 <directory>src/main/resources</directory>
             </resource>
+            <resource>
+                <directory>${project.basedir}/../pixels-daemon/src/main/resources</directory>
+                <includes>
+                    <include>pixels_metadata_mysql.sql</include>
+                    <include>pixels_metadata_derby.sql</include>
+                </includes>
+            </resource>
         </resources>
 
         <plugins>

pixels-cli/src/main/java/io/pixelsdb/pixels/cli/Main.java

@@ -72,6 +72,9 @@
  * <p>
  * STAT -s tpch -t region
  * </p>
+ * <p>
+ * INIT-META
+ * </p>
  */
 public class Main
 {
@@ -140,7 +143,8 @@ public static void main(String[] args)
                             "STAT\n" +
                             "QUERY\n" +
                             "COPY\n" +
-                            "FILE_META");
+                            "FILE_META\n" +
+                            "INIT-META");
                     System.out.println("{command} -h to show the usage of a command.\nexit / quit / -q to exit.\n");
                     continue;
                 }
@@ -366,6 +370,33 @@ public static void main(String[] args)
                     }
                 }
 
+                if (command.equals("INIT-META"))
+                {
+                    ArgumentParser argumentParser = ArgumentParsers.newArgumentParser("Pixels INIT-META")
+                            .defaultHelp(true);
+
+                    Namespace ns;
+                    try
+                    {
+                        String argsLine = inputStr.substring(command.length()).trim();
+                        ns = argumentParser.parseArgs(argsLine.isEmpty() ? new String[0] : argsLine.split("\\s+"));
+                    } catch (ArgumentParserException e)
+                    {
+                        argumentParser.handleError(e);
+                        continue;
+                    }
+
+                    try
+                    {
+                        InitMetaExecutor initMetaExecutor = new InitMetaExecutor();
+                        initMetaExecutor.execute(ns, command);
+                    }
+                    catch (Exception e)
+                    {
+                        e.printStackTrace();
+                    }
+                }
+
                 if (command.equals("FILE_META"))
                 {
                     ArgumentParser argumentParser = ArgumentParsers.newArgumentParser("Pixels File Metadata Explorer")
@@ -401,7 +432,8 @@ public static void main(String[] args)
                         !command.equals("COMPACT") &&
                         !command.equals("STAT") &&
                         !command.equals("IMPORT") &&
-                        !command.equals("FILE_META"))
+                        !command.equals("FILE_META") &&
+                        !command.equals("INIT-META"))
                 {
                     System.out.println("Command '" + command + "' not found");
                 }

pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/InitMetaExecutor.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2026 PixelsDB.
+ *
+ * This file is part of Pixels.
+ *
+ * Pixels is free software: you can redistribute it and/or modify
+ * it under the terms of the Affero GNU General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * Pixels is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * Affero GNU General Public License for more details.
+ *
+ * You should have received a copy of the Affero GNU General Public
+ * License along with Pixels.  If not, see
+ * <https://www.gnu.org/licenses/>.
+ */
+package io.pixelsdb.pixels.cli.executor;
+
+import io.pixelsdb.pixels.cli.initmeta.MetadataSchemaInitializer;
+import io.pixelsdb.pixels.common.metadata.MetadataDbType;
+import io.pixelsdb.pixels.common.utils.ConfigFactory;
+import net.sourceforge.argparse4j.inf.Namespace;
+
+/**
+ * Create metadata tables in the configured metadata database.
+ *
+ * @author hank
+ * @create 2026-09-02
+ */
+public class InitMetaExecutor implements CommandExecutor
+{
+    @Override
+    public void execute(Namespace ns, String command) throws Exception
+    {
+        ConfigFactory config = ConfigFactory.Instance();
+        String driver = config.getProperty("metadata.db.driver");
+        String url = config.getProperty("metadata.db.url");
+        MetadataDbType dbType = MetadataDbType.from(driver, url);
+        System.out.println("Initializing metadata tables in " + dbType + " database...");
+        System.out.println("JDBC URL: " + url);
+        int executed = MetadataSchemaInitializer.initialize();
+        System.out.println("INIT-META finished, executed " + executed +
+                " statement(s) from " + dbType.getSchemaResource());
+    }
+}