From ef4dafe8e79d9d1a14618dc619e06cf1d302b9c0 Mon Sep 17 00:00:00 2001 From: haoyueli Date: Wed, 2 Sep 2026 20:15:44 +0800 Subject: [PATCH 1/4] [Issue pixelsdb#1394] add a init-metadata command to create the metadata tables --- docker/Dockerfile | 2 +- docs/INSTALL.md | 9 +- pixels-cli/pom.xml | 21 + .../java/io/pixelsdb/pixels/cli/Main.java | 36 +- .../pixels/cli/executor/InitMetaExecutor.java | 48 +++ .../src/main/resources/pixels.properties | 1 + pixels-daemon/pom.xml | 6 + .../daemon/metadata/MetadataDbType.java | 78 ++++ .../metadata/MetadataSchemaInitializer.java | 180 +++++++++ .../daemon/metadata/dao/DaoFactory.java | 14 + scripts/sql/metadata_schema.sql | 372 ------------------ skills/pixels-install/claude/agent.md | 2 +- skills/pixels-install/codex/SKILL.md | 2 +- skills/pixels-install/cursor/SKILL.md | 2 +- .../pixels-install/scripts/install_mysql.sh | 2 +- 15 files changed, 395 insertions(+), 380 deletions(-) create mode 100644 pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/InitMetaExecutor.java create mode 100644 pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataDbType.java create mode 100644 pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataSchemaInitializer.java delete mode 100644 scripts/sql/metadata_schema.sql diff --git a/docker/Dockerfile b/docker/Dockerfile index 50e88009f1..6d4ba08a0d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -153,7 +153,7 @@ RUN sed -i 's|http://mirrors.cloud.aliyuncs.com|http://mirrors.aliyun.com|g' /et && echo "password=password" >> $HOME/pixels.cnf \ # Initialize metadata schema && mysql --defaults-file=root.cnf < $HOME/user.sql \ -&& mysql --defaults-file=pixels.cnf < $HOME/opt/pixels/scripts/sql/metadata_schema.sql \ +&& mysql --defaults-file=pixels.cnf < $HOME/opt/pixels/pixels-daemon/src/main/resources/pixels_metadata_mysql.sql \ && service mysql stop \ # Install ETCD diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 2df9d89ea4..8009e51064 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -175,7 +175,14 @@ FLUSH PRIVILEGES; Ensure that MySQL server can be accessed remotely. Sometimes the default MySQL configuration binds the server to localhost thus declines remote connections. -Use `scripts/sql/metadata_schema.sql` to create tables in `pixels_metadata`. +Use the `INIT-META` command in pixels-cli to create tables in the configured metadata database +(`pixels-daemon/src/main/resources/pixels_metadata_mysql.sql` for MySQL, or +`pixels-daemon/src/main/resources/pixels_metadata_derby.sql` for Derby): +```bash +java -jar $PIXELS_HOME/sbin/pixels-cli-*-full.jar +# then in the pixels-cli prompt: +INIT-META +``` ## Install etcd diff --git a/pixels-cli/pom.xml b/pixels-cli/pom.xml index f68d843bfa..a254751088 100644 --- a/pixels-cli/pom.xml +++ b/pixels-cli/pom.xml @@ -51,6 +51,27 @@ io.pixelsdb pixels-storage-s3 + + io.pixelsdb + pixels-daemon + ${project.version} + + + * + * + + + + + com.mysql + mysql-connector-j + 8.0.33 + + + org.apache.derby + derby + 10.14.2.0 + com.facebook.presto diff --git a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/Main.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/Main.java index e138325037..376b82a862 100644 --- a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/Main.java +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/Main.java @@ -72,6 +72,9 @@ *

* STAT -s tpch -t region *

+ *

+ * INIT-META + *

*/ public class Main { @@ -140,7 +143,8 @@ public static void main(String[] args) "STAT\n" + "QUERY\n" + "COPY\n" + - "FILE_META"); + "FILE_META\n" + + "INIT-META"); System.out.println("{command} -h to show the usage of a command.\nexit / quit / -q to exit.\n"); continue; } @@ -366,6 +370,33 @@ public static void main(String[] args) } } + if (command.equals("INIT-META")) + { + ArgumentParser argumentParser = ArgumentParsers.newArgumentParser("Pixels INIT-META") + .defaultHelp(true); + + Namespace ns; + try + { + String argsLine = inputStr.substring(command.length()).trim(); + ns = argumentParser.parseArgs(argsLine.isEmpty() ? new String[0] : argsLine.split("\\s+")); + } catch (ArgumentParserException e) + { + argumentParser.handleError(e); + continue; + } + + try + { + InitMetaExecutor initMetaExecutor = new InitMetaExecutor(); + initMetaExecutor.execute(ns, command); + } + catch (Exception e) + { + e.printStackTrace(); + } + } + if (command.equals("FILE_META")) { ArgumentParser argumentParser = ArgumentParsers.newArgumentParser("Pixels File Metadata Explorer") @@ -401,7 +432,8 @@ public static void main(String[] args) !command.equals("COMPACT") && !command.equals("STAT") && !command.equals("IMPORT") && - !command.equals("FILE_META")) + !command.equals("FILE_META") && + !command.equals("INIT-META")) { System.out.println("Command '" + command + "' not found"); } diff --git a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/InitMetaExecutor.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/InitMetaExecutor.java new file mode 100644 index 0000000000..f0c93b9b23 --- /dev/null +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/InitMetaExecutor.java @@ -0,0 +1,48 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.cli.executor; + +import io.pixelsdb.pixels.common.utils.ConfigFactory; +import io.pixelsdb.pixels.daemon.metadata.MetadataDbType; +import io.pixelsdb.pixels.daemon.metadata.MetadataSchemaInitializer; +import net.sourceforge.argparse4j.inf.Namespace; + +/** + * Create metadata tables in the configured metadata database. + * + * @author hank + * @create 2026-09-02 + */ +public class InitMetaExecutor implements CommandExecutor +{ + @Override + public void execute(Namespace ns, String command) throws Exception + { + ConfigFactory config = ConfigFactory.Instance(); + String driver = config.getProperty("metadata.db.driver"); + String url = config.getProperty("metadata.db.url"); + MetadataDbType dbType = MetadataDbType.from(driver, url); + System.out.println("Initializing metadata tables in " + dbType + " database..."); + System.out.println("JDBC URL: " + url); + int executed = MetadataSchemaInitializer.initialize(); + System.out.println("INIT-META finished, executed " + executed + + " statement(s) from " + dbType.getSchemaResource()); + } +} diff --git a/pixels-common/src/main/resources/pixels.properties b/pixels-common/src/main/resources/pixels.properties index 74915a9882..ac97235870 100644 --- a/pixels-common/src/main/resources/pixels.properties +++ b/pixels-common/src/main/resources/pixels.properties @@ -2,6 +2,7 @@ # pixels.var.dir is where the lock files are created pixels.var.dir=/home/pixels/opt/pixels/var/ # metadata database connection properties +# The metadata database type is detected from the driver and url (MySQL or Derby). metadata.db.driver=com.mysql.cj.jdbc.Driver metadata.db.user=pixels metadata.db.password=password diff --git a/pixels-daemon/pom.xml b/pixels-daemon/pom.xml index 987b089e80..958ada7120 100644 --- a/pixels-daemon/pom.xml +++ b/pixels-daemon/pom.xml @@ -154,6 +154,12 @@ ec2 + + org.apache.derby + derby + 10.14.2.0 + + org.apache.logging.log4j diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataDbType.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataDbType.java new file mode 100644 index 0000000000..c874581f24 --- /dev/null +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataDbType.java @@ -0,0 +1,78 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.daemon.metadata; + +import io.pixelsdb.pixels.common.utils.ConfigFactory; + +/** + * Supported metadata database backends. The type is detected from + * {@code metadata.db.driver} and {@code metadata.db.url} in pixels.properties. + * + * @author hank + * @create 2026-09-02 + */ +public enum MetadataDbType +{ + MYSQL("pixels_metadata_mysql.sql"), + DERBY("pixels_metadata_derby.sql"); + + private final String schemaResource; + + MetadataDbType(String schemaResource) + { + this.schemaResource = schemaResource; + } + + /** + * @return the classpath resource that contains CREATE TABLE statements for this backend + */ + public String getSchemaResource() + { + return this.schemaResource; + } + + /** + * Detect the metadata database type from {@code pixels.properties}. + */ + public static MetadataDbType fromConfig() + { + ConfigFactory config = ConfigFactory.Instance(); + return from(config.getProperty("metadata.db.driver"), config.getProperty("metadata.db.url")); + } + + /** + * Detect the metadata database type from the JDBC driver class and URL. + */ + public static MetadataDbType from(String driver, String url) + { + String driverLower = driver == null ? "" : driver.toLowerCase(); + String urlLower = url == null ? "" : url.toLowerCase(); + if (driverLower.contains("derby") || urlLower.contains("jdbc:derby")) + { + return DERBY; + } + if (driverLower.contains("mysql") || urlLower.contains("jdbc:mysql")) + { + return MYSQL; + } + throw new IllegalArgumentException("unsupported metadata database, driver=" + + driver + ", url=" + url); + } +} diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataSchemaInitializer.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataSchemaInitializer.java new file mode 100644 index 0000000000..46f4c34be2 --- /dev/null +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataSchemaInitializer.java @@ -0,0 +1,180 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ +package io.pixelsdb.pixels.daemon.metadata; + +import io.pixelsdb.pixels.common.utils.ConfigFactory; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; + +/** + * Creates metadata tables in the configured metadata database. + * + * @author hank + * @create 2026-09-02 + */ +public class MetadataSchemaInitializer +{ + private static final Logger log = LogManager.getLogger(MetadataSchemaInitializer.class); + + private MetadataSchemaInitializer() { } + + /** + * Detect the metadata database type from {@code pixels.properties} and execute + * the matching CREATE TABLE script. + * + * @return the number of SQL statements that were executed successfully + */ + public static int initialize() throws Exception + { + ConfigFactory config = ConfigFactory.Instance(); + String driver = config.getProperty("metadata.db.driver"); + String url = config.getProperty("metadata.db.url"); + String user = config.getProperty("metadata.db.user"); + String pass = config.getProperty("metadata.db.password"); + MetadataDbType dbType = MetadataDbType.from(driver, url); + return initialize(dbType, driver, url, user, pass); + } + + /** + * Execute the CREATE TABLE script of the given metadata database type. + * + * @return the number of SQL statements that were executed successfully + */ + public static int initialize(MetadataDbType dbType, String driver, String url, + String user, String pass) throws Exception + { + if (driver == null || driver.isEmpty()) + { + throw new IllegalArgumentException("metadata.db.driver is not set"); + } + if (url == null || url.isEmpty()) + { + throw new IllegalArgumentException("metadata.db.url is not set"); + } + + Class.forName(driver); + List statements = loadStatements(dbType.getSchemaResource()); + int executed = 0; + try (Connection conn = DriverManager.getConnection(url, user, pass); + Statement stmt = conn.createStatement()) + { + for (String sql : statements) + { + try + { + stmt.execute(sql); + executed++; + } + catch (SQLException e) + { + if (isAlreadyExists(e)) + { + log.warn("skip existing metadata object: {}", summarize(sql)); + } + else + { + throw new SQLException("failed to execute: " + summarize(sql), e); + } + } + } + } + return executed; + } + + private static List loadStatements(String resource) throws IOException + { + InputStream in = MetadataSchemaInitializer.class.getClassLoader().getResourceAsStream(resource); + if (in == null) + { + throw new IOException("metadata schema resource not found: " + resource); + } + + StringBuilder current = new StringBuilder(); + List statements = new ArrayList<>(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) + { + String line; + while ((line = reader.readLine()) != null) + { + String trimmed = stripLineComment(line).trim(); + if (trimmed.isEmpty()) + { + continue; + } + current.append(trimmed).append(' '); + if (trimmed.endsWith(";")) + { + String sql = current.toString().trim(); + sql = sql.substring(0, sql.length() - 1).trim(); + if (!sql.isEmpty()) + { + statements.add(sql); + } + current.setLength(0); + } + } + } + String remaining = current.toString().trim(); + if (!remaining.isEmpty()) + { + statements.add(remaining); + } + return statements; + } + + private static String stripLineComment(String line) + { + int comment = line.indexOf("--"); + if (comment < 0) + { + return line; + } + return line.substring(0, comment); + } + + private static boolean isAlreadyExists(SQLException e) + { + String state = e.getSQLState(); + if ("42S01".equals(state) || "42S11".equals(state) || "X0Y32".equals(state)) + { + return true; + } + String message = e.getMessage(); + return message != null && message.toLowerCase().contains("already exists"); + } + + private static String summarize(String sql) + { + String compact = sql.replaceAll("\\s+", " "); + return compact.length() > 120 ? compact.substring(0, 117) + "..." : compact; + } +} diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/DaoFactory.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/DaoFactory.java index 3d6a35d4fc..560a5315aa 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/DaoFactory.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/DaoFactory.java @@ -1,6 +1,9 @@ package io.pixelsdb.pixels.daemon.metadata.dao; +import io.pixelsdb.pixels.daemon.metadata.MetadataDbType; import io.pixelsdb.pixels.daemon.metadata.dao.impl.*; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /** * @author hank @@ -8,6 +11,8 @@ */ public class DaoFactory { + private static final Logger log = LogManager.getLogger(DaoFactory.class); + private static final class InstanceHolder { private static final DaoFactory instance = new DaoFactory(); @@ -18,6 +23,8 @@ public static DaoFactory Instance () return InstanceHolder.instance; } + private final MetadataDbType metadataDbType; + private final ColumnDao columnDao; private final LayoutDao layoutDao; private final SchemaDao schemaDao; @@ -34,6 +41,8 @@ public static DaoFactory Instance () private DaoFactory () { + this.metadataDbType = MetadataDbType.fromConfig(); + log.info("detected metadata database type: {}", this.metadataDbType); this.columnDao = new RdbColumnDao(); this.layoutDao = new RdbLayoutDao(); this.schemaDao = new RdbSchemaDao(); @@ -49,6 +58,11 @@ private DaoFactory () this.singlePointIndexDao = new RdbSinglePointIndexDao(); } + public MetadataDbType getMetadataDbType() + { + return this.metadataDbType; + } + public ColumnDao getColumnDao () { return this.columnDao; diff --git a/scripts/sql/metadata_schema.sql b/scripts/sql/metadata_schema.sql deleted file mode 100644 index 2558d2d1af..0000000000 --- a/scripts/sql/metadata_schema.sql +++ /dev/null @@ -1,372 +0,0 @@ --- Create the MySQL metadata database for Pixels. - -SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0; -SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0; -SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'; - --- ----------------------------------------------------- --- Schema pixels_metadata --- ----------------------------------------------------- -CREATE SCHEMA IF NOT EXISTS `pixels_metadata` ; -USE `pixels_metadata` ; - --- ----------------------------------------------------- --- Table `pixels_metadata`.`DBS` --- ----------------------------------------------------- -CREATE TABLE IF NOT EXISTS `pixels_metadata`.`DBS` ( - `DB_ID` BIGINT NOT NULL AUTO_INCREMENT, - `DB_NAME` VARCHAR(128) NOT NULL, - `DB_DESC` VARCHAR(4000) NULL, - PRIMARY KEY (`DB_ID`), - UNIQUE INDEX `DB_NAME_UNIQUE` (`DB_NAME` ASC)) - ENGINE = InnoDB - DEFAULT CHARACTER SET = utf8mb4 - COLLATE = utf8mb4_bin; - - --- ----------------------------------------------------- --- Table `pixels_metadata`.`TBLS` --- ----------------------------------------------------- -CREATE TABLE IF NOT EXISTS `pixels_metadata`.`TBLS` ( - `TBL_ID` BIGINT NOT NULL AUTO_INCREMENT, - `TBL_NAME` VARCHAR(128) NOT NULL, - `TBL_TYPE` VARCHAR(128) NULL, - `TBL_STORAGE_SCHEME` VARCHAR(32) NOT NULL DEFAULT 'file' COMMENT 'The name of the storage scheme of the files stored in all the paths of this table.', - `TBL_ROW_COUNT` BIGINT NOT NULL DEFAULT 0 COMMENT 'The number of rows in this table.', - `DBS_DB_ID` BIGINT NOT NULL, - PRIMARY KEY (`TBL_ID`), - INDEX `fk_TBLS_DBS_idx` (`DBS_DB_ID` ASC), - UNIQUE INDEX `TBL_NAME_DB_ID_UNIQUE` (`TBL_NAME` ASC, `DBS_DB_ID` ASC), - CONSTRAINT `fk_TBLS_DBS` - FOREIGN KEY (`DBS_DB_ID`) - REFERENCES `pixels_metadata`.`DBS` (`DB_ID`) - ON DELETE CASCADE - ON UPDATE CASCADE) - ENGINE = InnoDB - DEFAULT CHARACTER SET = utf8mb4 - COLLATE = utf8mb4_bin; - - --- ----------------------------------------------------- --- Table `pixels_metadata`.`COLS` --- ----------------------------------------------------- -CREATE TABLE IF NOT EXISTS `pixels_metadata`.`COLS` ( - `COL_ID` BIGINT NOT NULL AUTO_INCREMENT, - `COL_NAME` VARCHAR(128) NOT NULL, - `COL_TYPE` VARCHAR(128) NOT NULL, - `COL_CHUNK_SIZE` DOUBLE NOT NULL DEFAULT 0, - `COL_SIZE` DOUBLE NOT NULL DEFAULT 0, - `COL_NULL_FRACTION` DOUBLE NOT NULL DEFAULT 0, - `COL_CARDINALITY` BIGINT NOT NULL DEFAULT 0, - `COL_RECORD_STATS` BLOB NULL DEFAULT NULL, - `TBLS_TBL_ID` BIGINT NOT NULL, - PRIMARY KEY (`COL_ID`), - INDEX `fk_COLS_TBLS_idx` (`TBLS_TBL_ID` ASC), - UNIQUE INDEX `COL_NAME_TBL_ID_UNIQUE` (`COL_NAME` ASC, `TBLS_TBL_ID` ASC), - CONSTRAINT `fk_COLS_TBLS` - FOREIGN KEY (`TBLS_TBL_ID`) - REFERENCES `pixels_metadata`.`TBLS` (`TBL_ID`) - ON DELETE CASCADE - ON UPDATE CASCADE) - ENGINE = InnoDB - DEFAULT CHARACTER SET = utf8mb4 - COLLATE = utf8mb4_bin; - - --- ----------------------------------------------------- --- Table `pixels_metadata`.`SCHEMA_VERSIONS` --- ----------------------------------------------------- -CREATE TABLE IF NOT EXISTS `pixels_metadata`.`SCHEMA_VERSIONS` ( - `SV_ID` BIGINT NOT NULL AUTO_INCREMENT, - `SV_COLUMNS` MEDIUMTEXT NOT NULL COMMENT 'The json string that contains the ids of the columns owned by this schema version.', - `SV_TRANS_TS` BIGINT NOT NULL COMMENT 'The transaction timestamp of this schema version.', - `TBLS_TBL_ID` BIGINT NOT NULL, - PRIMARY KEY (`SV_ID`), - INDEX `fk_SCHEMA_VERSIONS_TBLS_idx` (`TBLS_TBL_ID` ASC), - CONSTRAINT `fk_SCHEMA_VERSIONS_TBLS` - FOREIGN KEY (`TBLS_TBL_ID`) - REFERENCES `pixels_metadata`.`TBLS` (`TBL_ID`) - ON DELETE CASCADE - ON UPDATE CASCADE) - ENGINE = InnoDB - DEFAULT CHARACTER SET = utf8mb4 - COLLATE = utf8mb4_bin; - - --- ----------------------------------------------------- --- Table `pixels_metadata`.`LAYOUTS` --- ----------------------------------------------------- -CREATE TABLE IF NOT EXISTS `pixels_metadata`.`LAYOUTS` ( - `LAYOUT_ID` BIGINT NOT NULL AUTO_INCREMENT, - `LAYOUT_VERSION` BIGINT NOT NULL COMMENT 'The version of this layout.', - `LAYOUT_CREATE_AT` BIGINT NOT NULL COMMENT 'The milliseconds of moment since the unix epoch that this layout is created.', - `LAYOUT_PERMISSION` TINYINT NOT NULL COMMENT '<0 for not readable and writable, 0 for readable only, >0 for readable and writable.', - `LAYOUT_ORDERED` MEDIUMTEXT NOT NULL COMMENT 'The default order of this layout. It is used to determine the column order in a single-row-group blocks.', - `LAYOUT_COMPACT` LONGTEXT NOT NULL COMMENT 'the layout strategy, stored as json. It is used to determine how row groups are compacted into a big block.', - `LAYOUT_SPLITS` LONGTEXT NOT NULL COMMENT 'The suggested split size for access patterns, stored as json.', - `LAYOUT_PROJECTIONS` LONGTEXT NOT NULL COMMENT 'The projections each maps a set of columns to a different set of paths.', - `TBLS_TBL_ID` BIGINT NOT NULL, - `SCHEMA_VERSIONS_SV_ID` BIGINT NOT NULL, - PRIMARY KEY (`LAYOUT_ID`), - INDEX `fk_LAYOUTS_TBLS_idx` (`TBLS_TBL_ID` ASC), - INDEX `fk_LAYOUTS_SCHEMA_VERSIONS_idx` (`SCHEMA_VERSIONS_SV_ID` ASC), - CONSTRAINT `fk_LAYOUTS_TBLS` - FOREIGN KEY (`TBLS_TBL_ID`) - REFERENCES `pixels_metadata`.`TBLS` (`TBL_ID`) - ON DELETE CASCADE - ON UPDATE CASCADE, - CONSTRAINT `fk_LAYOUTS_SCHEMA_VERSIONS` - FOREIGN KEY (`SCHEMA_VERSIONS_SV_ID`) - REFERENCES `pixels_metadata`.`SCHEMA_VERSIONS` (`SV_ID`) - ON DELETE CASCADE - ON UPDATE CASCADE) - ENGINE = InnoDB - DEFAULT CHARACTER SET = utf8mb4 - COLLATE = utf8mb4_bin; - - --- ----------------------------------------------------- --- Table `pixels_metadata`.`VIEWS` --- ----------------------------------------------------- -CREATE TABLE IF NOT EXISTS `pixels_metadata`.`VIEWS` ( - `VIEW_ID` BIGINT NOT NULL AUTO_INCREMENT, - `VIEW_NAME` VARCHAR(128) NOT NULL, - `VIEW_TYPE` VARCHAR(128) NULL, - `VIEW_DATA` LONGTEXT NOT NULL, - `DBS_DB_ID` BIGINT NOT NULL, - PRIMARY KEY (`VIEW_ID`), - INDEX `fk_VIEWS_DBS_idx` (`DBS_DB_ID` ASC), - CONSTRAINT `fk_VIEWS_DBS` - FOREIGN KEY (`DBS_DB_ID`) - REFERENCES `pixels_metadata`.`DBS` (`DB_ID`) - ON DELETE CASCADE - ON UPDATE CASCADE) - ENGINE = InnoDB - DEFAULT CHARACTER SET = utf8mb4 - COLLATE = utf8mb4_bin; - - --- ----------------------------------------------------- --- Table `pixels_metadata`.`RANGE_INDICES` --- ----------------------------------------------------- -CREATE TABLE IF NOT EXISTS `pixels_metadata`.`RANGE_INDICES` ( - `RI_ID` BIGINT NOT NULL AUTO_INCREMENT, - `RI_KEY_COLUMNS` TEXT NOT NULL COMMENT 'The ids of the key columns, stored in csv format.', - `TBLS_TBL_ID` BIGINT NOT NULL, - `SCHEMA_VERSIONS_SV_ID` BIGINT NOT NULL, - PRIMARY KEY (`RI_ID`), - INDEX `fk_RANGE_INDICES_TBLS_idx` (`TBLS_TBL_ID` ASC), - INDEX `fk_RANGE_INDICES_SCHEMA_VERSIONS_idx` (`SCHEMA_VERSIONS_SV_ID` ASC), - UNIQUE INDEX `TBL_ID_SV_ID_UNIQUE` (`TBLS_TBL_ID` ASC, `SCHEMA_VERSIONS_SV_ID` ASC) COMMENT 'We ensure every (table, schema_version) has only one range index.', - CONSTRAINT `fk_RANGE_INDICES_TBLS` - FOREIGN KEY (`TBLS_TBL_ID`) - REFERENCES `pixels_metadata`.`TBLS` (`TBL_ID`) - ON DELETE CASCADE - ON UPDATE CASCADE, - CONSTRAINT `fk_RANGE_INDICES_SCHEMA_VERSIONS` - FOREIGN KEY (`SCHEMA_VERSIONS_SV_ID`) - REFERENCES `pixels_metadata`.`SCHEMA_VERSIONS` (`SV_ID`) - ON DELETE CASCADE - ON UPDATE CASCADE) - ENGINE = InnoDB - DEFAULT CHARACTER SET = utf8mb4 - COLLATE = utf8mb4_bin; - - --- ----------------------------------------------------- --- Table `pixels_metadata`.`RANGES` --- ----------------------------------------------------- -CREATE TABLE IF NOT EXISTS `pixels_metadata`.`RANGES` ( - `RANGE_ID` BIGINT NOT NULL AUTO_INCREMENT, - `RANGE_MIN` BLOB NOT NULL COMMENT 'The min value of the key column(s).', - `RANGE_MAX` BLOB NOT NULL COMMENT 'The max value of the key column(s).', - `RANGE_PARENT_ID` BIGINT NULL, - `RANGE_INDICES_RI_ID` BIGINT NOT NULL, - PRIMARY KEY (`RANGE_ID`), - INDEX `fk_RANGES_RANGE_INDICES_idx` (`RANGE_INDICES_RI_ID` ASC), - INDEX `fk_RANGES_RANGES_idx` (`RANGE_PARENT_ID` ASC), - CONSTRAINT `fk_RANGES_RANGE_INDICES` - FOREIGN KEY (`RANGE_INDICES_RI_ID`) - REFERENCES `pixels_metadata`.`RANGE_INDICES` (`RI_ID`) - ON DELETE CASCADE - ON UPDATE CASCADE, - CONSTRAINT `fk_RANGES_RANGES` - FOREIGN KEY (`RANGE_PARENT_ID`) - REFERENCES `pixels_metadata`.`RANGES` (`RANGE_ID`) - ON DELETE SET NULL - ON UPDATE CASCADE) - ENGINE = InnoDB - DEFAULT CHARACTER SET = utf8mb4 - COLLATE = utf8mb4_bin; - - --- ----------------------------------------------------- --- Table `pixels_metadata`.`PATHS` --- ----------------------------------------------------- -CREATE TABLE IF NOT EXISTS `pixels_metadata`.`PATHS` ( - `PATH_ID` BIGINT NOT NULL AUTO_INCREMENT, - `PATH_URI` VARCHAR(512) NOT NULL COMMENT 'The storage path uri containing the storage scheme prefix.', - `PATH_TYPE` TINYINT NOT NULL COMMENT 'Valid value can be 0 (ordered), 1 (compact), or 2 (projection).', - `LAYOUTS_LAYOUT_ID` BIGINT NOT NULL, - `RANGES_RANGE_ID` BIGINT NULL DEFAULT NULL, - PRIMARY KEY (`PATH_ID`), - INDEX `fk_PATHS_RANGES_idx` (`RANGES_RANGE_ID` ASC), - INDEX `fk_PATHS_LAYOUTS_idx` (`LAYOUTS_LAYOUT_ID` ASC), - UNIQUE INDEX `PATH_URI_UNIQUE` (`PATH_URI` ASC), - CONSTRAINT `fk_PATHS_RANGES` - FOREIGN KEY (`RANGES_RANGE_ID`) - REFERENCES `pixels_metadata`.`RANGES` (`RANGE_ID`) - ON DELETE SET NULL - ON UPDATE CASCADE, - CONSTRAINT `fk_PATHS_LAYOUTS` - FOREIGN KEY (`LAYOUTS_LAYOUT_ID`) - REFERENCES `pixels_metadata`.`LAYOUTS` (`LAYOUT_ID`) - ON DELETE CASCADE - ON UPDATE CASCADE) - ENGINE = InnoDB - DEFAULT CHARACTER SET = utf8mb4 - COLLATE = utf8mb4_bin; - - --- ----------------------------------------------------- --- Table `pixels_metadata`.`USERS` --- ----------------------------------------------------- -CREATE TABLE IF NOT EXISTS `pixels_metadata`.`USERS` ( - `USER_ID` BIGINT NOT NULL AUTO_INCREMENT, - `USER_NAME` VARCHAR(128) NOT NULL, - `USER_PASSWORD` VARCHAR(128) NOT NULL, - `USER_EMAIL` VARCHAR(128) NOT NULL, - PRIMARY KEY (`USER_ID`), - UNIQUE INDEX `USER_NAME_UNIQUE` (`USER_NAME` ASC)) - ENGINE = InnoDB - DEFAULT CHARACTER SET = utf8mb4 - COLLATE = utf8mb4_bin; - - --- ----------------------------------------------------- --- Table `pixels_metadata`.`USER_HAS_DB` --- ----------------------------------------------------- -CREATE TABLE IF NOT EXISTS `pixels_metadata`.`USER_HAS_DB` ( - `USERS_USER_ID` BIGINT NOT NULL, - `DBS_DB_ID` BIGINT NOT NULL, - `USER_DB_PERMITION` TINYINT NOT NULL, - PRIMARY KEY (`USERS_USER_ID`, `DBS_DB_ID`), - INDEX `fk_USERS_has_DBS_DBS_idx` (`DBS_DB_ID` ASC), - INDEX `fk_USERS_has_DBS_USERS_idx` (`USERS_USER_ID` ASC), - CONSTRAINT `fk_USERS_has_DBS_USERS` - FOREIGN KEY (`USERS_USER_ID`) - REFERENCES `pixels_metadata`.`USERS` (`USER_ID`) - ON DELETE CASCADE - ON UPDATE CASCADE, - CONSTRAINT `fk_USERS_has_DBS_DBS` - FOREIGN KEY (`DBS_DB_ID`) - REFERENCES `pixels_metadata`.`DBS` (`DB_ID`) - ON DELETE CASCADE - ON UPDATE CASCADE) - ENGINE = InnoDB - DEFAULT CHARACTER SET = utf8mb4 - COLLATE = utf8mb4_bin; - - --- ----------------------------------------------------- --- Table `pixels_metadata`.`PEERS` --- ----------------------------------------------------- -CREATE TABLE IF NOT EXISTS `pixels_metadata`.`PEERS` ( - `PEER_ID` BIGINT NOT NULL AUTO_INCREMENT, - `PEER_NAME` VARCHAR(128) NOT NULL COMMENT 'The name of the peer, must be unique.', - `PEER_LOCATION` VARCHAR(1024) NOT NULL COMMENT 'The geographic location of the peer.', - `PEER_HOST` VARCHAR(128) NOT NULL COMMENT 'The registered host name or ip address of the peer.', - `PEER_PORT` INT NOT NULL COMMENT 'The registered port of this peer.', - `PEER_STORAGE_SCHEME` VARCHAR(32) NOT NULL, - PRIMARY KEY (`PEER_ID`), - UNIQUE INDEX `PEER_NAME_UNIQUE` (`PEER_NAME` ASC)) - ENGINE = InnoDB - DEFAULT CHARACTER SET = utf8mb4 - COLLATE = utf8mb4_bin; - - --- ----------------------------------------------------- --- Table `pixels_metadata`.`PEER_PATHS` --- ----------------------------------------------------- -CREATE TABLE IF NOT EXISTS `pixels_metadata`.`PEER_PATHS` ( - `PEER_PATH_ID` BIGINT NOT NULL AUTO_INCREMENT, - `PEER_PATH_URI` VARCHAR(32) NOT NULL, - `PEER_PATH_COLUMNS` MEDIUMTEXT NOT NULL COMMENT 'The json string that contains the ids of the columns stored in this peer path.', - `PATHS_PATH_ID` BIGINT NOT NULL, - `PEERS_PEER_ID` BIGINT NOT NULL, - PRIMARY KEY (`PEER_PATH_ID`), - INDEX `fk_PEER_PATHS_PATHS_idx` (`PATHS_PATH_ID` ASC), - INDEX `fk_PEER_PATHS_PEERS_idx` (`PEERS_PEER_ID` ASC), - CONSTRAINT `fk_PEER_PATHS_PATHS` - FOREIGN KEY (`PATHS_PATH_ID`) - REFERENCES `pixels_metadata`.`PATHS` (`PATH_ID`) - ON DELETE CASCADE - ON UPDATE CASCADE, - CONSTRAINT `fk_PEER_PATHS_PEERS` - FOREIGN KEY (`PEERS_PEER_ID`) - REFERENCES `pixels_metadata`.`PEERS` (`PEER_ID`) - ON DELETE CASCADE - ON UPDATE CASCADE) - ENGINE = InnoDB - DEFAULT CHARACTER SET = utf8mb4 - COLLATE = utf8mb4_bin; - - --- ----------------------------------------------------- --- Table `pixels_metadata`.`FILES` --- ----------------------------------------------------- -CREATE TABLE IF NOT EXISTS `pixels_metadata`.`FILES` ( - `FILE_ID` BIGINT NOT NULL AUTO_INCREMENT, - `FILE_NAME` VARCHAR(128) NOT NULL, - `FILE_TYPE` TINYINT NOT NULL COMMENT "Valid value can be 0 (temporary ingest), 1 (regular), 2 (temporary gc), or 3 (retired).", - `FILE_NUM_RG` INT NOT NULL, - `FILE_MIN_ROW_ID` BIGINT NOT NULL, - `FILE_MAX_ROW_ID` BIGINT NOT NULL, - `FILE_CLEANUP_AT` BIGINT NULL COMMENT "Earliest cleanup deadline in epoch milliseconds; meaningful only when FILE_TYPE = 3 (retired).", - `PATHS_PATH_ID` BIGINT NOT NULL, - PRIMARY KEY (`FILE_ID`), - INDEX `fk_FILES_PATHS_idx` (`PATHS_PATH_ID` ASC), - UNIQUE INDEX `PATH_ID_FILE_NAME_UNIQUE` (`PATHS_PATH_ID` ASC, `FILE_NAME` ASC), - INDEX `FILE_ROW_ID_INDEX` USING BTREE (`FILE_MIN_ROW_ID`, `FILE_MAX_ROW_ID`), - CONSTRAINT `fk_FILES_PATHS` - FOREIGN KEY (`PATHS_PATH_ID`) - REFERENCES `pixels_metadata`.`PATHS` (`PATH_ID`) - ON DELETE CASCADE - ON UPDATE CASCADE) - ENGINE = InnoDB - DEFAULT CHARACTER SET = utf8mb4 - COLLATE = utf8mb4_bin; - - --- ----------------------------------------------------- --- Table `pixels_metadata`.`SINGLE_POINT_INDICES` --- ----------------------------------------------------- -CREATE TABLE IF NOT EXISTS `pixels_metadata`.`SINGLE_POINT_INDICES` ( - `SPI_ID` BIGINT NOT NULL AUTO_INCREMENT, - `SPI_KEY_COLUMNS` TEXT NOT NULL COMMENT 'The ids of the key columns of this index, stored in json format.', - `SPI_PRIMARY` TINYINT NOT NULL COMMENT 'True (1) if this single point index is the primary index. There can be only one primary index on a table.', - `SPI_UNIQUE` TINYINT NOT NULL COMMENT 'True (1) if this single point index is an unique index.', - `SPI_INDEX_SCHEME` VARCHAR(32) NOT NULL COMMENT 'The index scheme, e.g., rocksdb or rockset, of this single pint index.', - `TBLS_TBL_ID` BIGINT NOT NULL, - `SCHEMA_VERSIONS_SV_ID` BIGINT NOT NULL, - PRIMARY KEY (`SPI_ID`), - INDEX `fk_SINGLE_POINT_INDICES_TBLS_idx` (`TBLS_TBL_ID` ASC), - INDEX `fk_SINGLE_POINT_INDICES_SCHEMA_VERSIONS_idx` (`SCHEMA_VERSIONS_SV_ID` ASC), - CONSTRAINT `fk_SINGLE_POINT_INDICES_TBLS` - FOREIGN KEY (`TBLS_TBL_ID`) - REFERENCES `pixels_metadata`.`TBLS` (`TBL_ID`) - ON DELETE CASCADE - ON UPDATE CASCADE, - CONSTRAINT `fk_SINGLE_POINT_INDICES_SCHEMA_VERSIONS` - FOREIGN KEY (`SCHEMA_VERSIONS_SV_ID`) - REFERENCES `pixels_metadata`.`SCHEMA_VERSIONS` (`SV_ID`) - ON DELETE CASCADE - ON UPDATE CASCADE) - ENGINE = InnoDB - DEFAULT CHARACTER SET = utf8mb4 - COLLATE = utf8mb4_bin; - - -SET SQL_MODE=@OLD_SQL_MODE; -SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS; -SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS; diff --git a/skills/pixels-install/claude/agent.md b/skills/pixels-install/claude/agent.md index 6633f9423b..1694d68eb2 100644 --- a/skills/pixels-install/claude/agent.md +++ b/skills/pixels-install/claude/agent.md @@ -17,7 +17,7 @@ Use these helper scripts when they directly fit the current environment and the - `check_prerequisites.sh`: validate OS, architecture, memory, disk, ports, host resolution, privilege, and optional SSH reachability. Runs every check and exits with a structured `=== check_prerequisites result ===` summary (one `ok|warn|fail|skip : ` line per check, plus a final `summary: ok=N warn=N fail=N skip=N status=pass|fail` line) instead of stopping at the first failure, so a single run shows every problem at once. - `install_jdk.sh`: install a Zulu OpenJDK build (default JDK 23) matching the server's CPU architecture (x86_64 or aarch64) under `~/opt`, and persist `JAVA_HOME` only into the current user's shell profile. Looks for a JDK that already satisfies `JDK_VERSION` first — checking `JAVA_HOME` if set, otherwise resolving whatever `java` is on `PATH` (e.g. an `apt`-installed JDK that never exported `JAVA_HOME`) — and if one is found, skips the download entirely and only fixes up the environment variable to point at it. Only downloads/installs a fresh Zulu build when nothing on the system satisfies the version requirement. Falls back to pointing at the manual `.deb` method in `docs/INSTALL.md` if the Azul metadata API lookup fails. - `install_maven.sh`: install or validate Maven 3.8+ (default pinned version `3.9.8`; override with `MAVEN_VERSION` only if the user explicitly asks for a different one) when the current Maven is missing or incompatible with the selected JDK. Same "skip if already satisfied" logic as `install_jdk.sh`: if an existing `mvn` (from `apt`, a prior manual install, etc.) already meets the minimum version, it reuses that installation's home directory instead of downloading anything. Fresh installs go into `~/opt/apache-maven-` with a `~/opt/maven` symlink, never into `/opt`. -- `install_mysql.sh`: install MySQL, then interactively confirm the root password and the `pixels` metadata-DB-user password (default `password` for both, never applied silently), create the `pixels_metadata` database/user, and load `scripts/sql/metadata_schema.sql`. It validates DB/user identifiers, quotes SQL values, avoids putting the root password in command-line arguments, and writes shell-quoted `STATE_DIR/deployment.secrets.env` (mode 600, untracked) so the confirmed credentials flow into `configure_pixels.sh` automatically. +- `install_mysql.sh`: install MySQL, then interactively confirm the root password and the `pixels` metadata-DB-user password (default `password` for both, never applied silently), create the `pixels_metadata` database/user, and load `pixels-daemon/src/main/resources/pixels_metadata_mysql.sql`. It validates DB/user identifiers, quotes SQL values, avoids putting the root password in command-line arguments, and writes shell-quoted `STATE_DIR/deployment.secrets.env` (mode 600, untracked) so the confirmed credentials flow into `configure_pixels.sh` automatically. - `install_etcd.sh`: install and optionally start the bundled etcd 3.3.4 package under `~/opt`. Always fixes the shipped `conf.yml`'s hardcoded `/home/ubuntu/...` `data-dir` to the real install path. By default (`ETCD_ALLOW_REMOTE=false`) it keeps etcd localhost-only. For a cluster, set `ETCD_ALLOW_REMOTE=true` only after confirming private networking/security-group rules; the script then requires `CONFIRM_ETCD_REMOTE_ACCESS=true`, `ASSUME_YES=true`, or an interactive confirmation before binding the client/peer listeners for remote access. Can also install and enable a `systemd` unit so etcd survives reboots and restarts on failure, but only after asking — it never installs this silently. Leave `INSTALL_ETCD_SYSTEMD_SERVICE` unset to be prompted interactively (`[y/N]`, defaults to no); set it to `true`/`false` to answer non-interactively once the user has actually agreed, or `ASSUME_YES=true` to answer yes to this and other yes/no prompts. Declining (or running non-interactively with no explicit answer) just skips the unit — etcd still starts, as a manually backgrounded process instead. Falls back the same way when `systemctl` isn't available at all. Restarts/re-enables the installed service automatically when the config actually changed. - `build_install_pixels.sh`: build and install Pixels into the confirmed `PIXELS_HOME` from `deployment.env`, then add the MySQL JDBC connector. It prints the repo, deployment file, and install target before running; in non-interactive mode, set `CONFIRM_PIXELS_INSTALL=true` only after the user confirms the target. If you must bypass the prepared deployment file, pass `USE_DEPLOYMENT_FILE=false PIXELS_HOME=` explicitly. `AUTO_CONFIRM_INSTALL` (default `true`) auto-answers `install.sh`'s prompts only when that is safe; if `PIXELS_HOME/etc/pixels.properties` or `pixels-cpp.properties` already exist, it forces the interactive path instead, because `install.sh` would otherwise prompt to add/remove config options or overwrite `pixels-cpp.properties` outright — exactly what `Claude agent`/`SKILL.md` say not to do without the user's say-so. - `configure_pixels.sh`: update `PIXELS_HOME/etc/pixels.properties` after database, etcd, host, port, topology, and path values are known. Automatically sources `deployment.env` and `deployment.secrets.env` (if present), so `PIXELS_HOME`, coordinator service hosts, worker names, and MySQL credentials stay in sync with what the earlier scripts confirmed. It writes `$PIXELS_HOME/etc/workers` from `PIXELS_WORKERS` by default. It fails instead of silently writing the documented default DB password unless `METADATA_DB_PASSWORD` is explicit or `ALLOW_DEFAULT_METADATA_PASSWORD=true` is set. diff --git a/skills/pixels-install/codex/SKILL.md b/skills/pixels-install/codex/SKILL.md index 602913e028..592215bf73 100644 --- a/skills/pixels-install/codex/SKILL.md +++ b/skills/pixels-install/codex/SKILL.md @@ -25,7 +25,7 @@ Use these helpers when they directly fit the current environment and the user-ap - `check_prerequisites.sh`: validate OS, architecture, memory, disk, ports, host resolution, privilege, and optional SSH reachability. Runs every check and exits with a structured `=== check_prerequisites result ===` summary (one `ok|warn|fail|skip : ` line per check, plus a final `summary: ok=N warn=N fail=N skip=N status=pass|fail` line) instead of stopping at the first failure, so a single run shows every problem at once. - `install_jdk.sh`: install a Zulu OpenJDK build (default JDK 23) matching the server's CPU architecture (x86_64 or aarch64) under `~/opt`, and persist `JAVA_HOME` only into the current user's shell profile. Looks for a JDK that already satisfies `JDK_VERSION` first — checking `JAVA_HOME` if set, otherwise resolving whatever `java` is on `PATH` (e.g. an `apt`-installed JDK that never exported `JAVA_HOME`) — and if one is found, skips the download entirely and only fixes up the environment variable to point at it. Only downloads/installs a fresh Zulu build when nothing on the system satisfies the version requirement. Falls back to pointing at the manual `.deb` method in `docs/INSTALL.md` if the Azul metadata API lookup fails. - `install_maven.sh`: install or validate Maven 3.8+ (default pinned version `3.9.8`; override with `MAVEN_VERSION` only if the user explicitly asks for a different one) when the current Maven is missing or incompatible with the selected JDK. Same "skip if already satisfied" logic as `install_jdk.sh`: if an existing `mvn` (from `apt`, a prior manual install, etc.) already meets the minimum version, it reuses that installation's home directory instead of downloading anything. Fresh installs go into `~/opt/apache-maven-` with a `~/opt/maven` symlink, never into `/opt`. -- `install_mysql.sh`: install MySQL, then interactively confirm the root password and the `pixels` metadata-DB-user password (default `password` for both, never applied silently), create the `pixels_metadata` database/user, and load `scripts/sql/metadata_schema.sql` from the Pixels source tree. It validates DB/user identifiers, quotes SQL values, avoids putting the root password in command-line arguments, and writes shell-quoted `STATE_DIR/deployment.secrets.env` (mode 600) so the confirmed credentials flow into `configure_pixels.sh` automatically. +- `install_mysql.sh`: install MySQL, then interactively confirm the root password and the `pixels` metadata-DB-user password (default `password` for both, never applied silently), create the `pixels_metadata` database/user, and load `pixels-daemon/src/main/resources/pixels_metadata_mysql.sql` from the Pixels source tree. It validates DB/user identifiers, quotes SQL values, avoids putting the root password in command-line arguments, and writes shell-quoted `STATE_DIR/deployment.secrets.env` (mode 600) so the confirmed credentials flow into `configure_pixels.sh` automatically. - `install_etcd.sh`: install and optionally start the bundled etcd 3.3.4 package under `~/opt`. Always fixes the shipped `conf.yml`'s hardcoded `/home/ubuntu/...` `data-dir` to the real install path. By default (`ETCD_ALLOW_REMOTE=false`) it keeps etcd localhost-only. For a cluster, set `ETCD_ALLOW_REMOTE=true` only after confirming private networking/security-group rules; the script then requires `CONFIRM_ETCD_REMOTE_ACCESS=true`, `ASSUME_YES=true`, or an interactive confirmation before binding the client/peer listeners for remote access. Can also install and enable a `systemd` unit so etcd survives reboots and restarts on failure, but only after asking — it never installs this silently. Leave `INSTALL_ETCD_SYSTEMD_SERVICE` unset to be prompted interactively (`[y/N]`, defaults to no); set it to `true`/`false` to answer non-interactively once the user has actually agreed, or `ASSUME_YES=true` to answer yes to this and other yes/no prompts. Declining (or running non-interactively with no explicit answer) just skips the unit — etcd still starts, as a manually backgrounded process instead. Falls back the same way when `systemctl` isn't available at all. Restarts/re-enables the installed service automatically when the config actually changed. - `build_install_pixels.sh`: build and install Pixels into the confirmed `PIXELS_HOME` from `deployment.env`, then add the MySQL JDBC connector. It prints the repo, deployment file, and install target before running; in non-interactive mode, set `CONFIRM_PIXELS_INSTALL=true` only after the user confirms the target. If you must bypass the prepared deployment file, pass `USE_DEPLOYMENT_FILE=false PIXELS_HOME=` explicitly. `AUTO_CONFIRM_INSTALL` (default `true`) auto-answers `install.sh`'s prompts only when that is safe; if `PIXELS_HOME/etc/pixels.properties` or `pixels-cpp.properties` already exist, it forces the interactive path instead, because `install.sh` would otherwise prompt to add/remove config options or overwrite `pixels-cpp.properties` outright — exactly what `Claude agent`/`SKILL.md` say not to do without the user's say-so. - `configure_pixels.sh`: update `PIXELS_HOME/etc/pixels.properties` after database, etcd, host, port, topology, and path values are known. Automatically sources `deployment.env` and `deployment.secrets.env` (if present), so `PIXELS_HOME`, coordinator service hosts, worker names, and MySQL credentials stay in sync with what the earlier scripts confirmed. It writes `$PIXELS_HOME/etc/workers` from `PIXELS_WORKERS` by default. It fails instead of silently writing the documented default DB password unless `METADATA_DB_PASSWORD` is explicit or `ALLOW_DEFAULT_METADATA_PASSWORD=true` is set. diff --git a/skills/pixels-install/cursor/SKILL.md b/skills/pixels-install/cursor/SKILL.md index fcb4de4c35..4708908877 100644 --- a/skills/pixels-install/cursor/SKILL.md +++ b/skills/pixels-install/cursor/SKILL.md @@ -25,7 +25,7 @@ Use these helpers when they directly fit the current environment and the user-ap - `check_prerequisites.sh`: validate OS, architecture, memory, disk, ports, host resolution, privilege, and optional SSH reachability. Runs every check and exits with a structured `=== check_prerequisites result ===` summary (one `ok|warn|fail|skip : ` line per check, plus a final `summary: ok=N warn=N fail=N skip=N status=pass|fail` line) instead of stopping at the first failure, so a single run shows every problem at once. - `install_jdk.sh`: install a Zulu OpenJDK build (default JDK 23) matching the server's CPU architecture (x86_64 or aarch64) under `~/opt`, and persist `JAVA_HOME` only into the current user's shell profile. Looks for a JDK that already satisfies `JDK_VERSION` first — checking `JAVA_HOME` if set, otherwise resolving whatever `java` is on `PATH` (e.g. an `apt`-installed JDK that never exported `JAVA_HOME`) — and if one is found, skips the download entirely and only fixes up the environment variable to point at it. Only downloads/installs a fresh Zulu build when nothing on the system satisfies the version requirement. Falls back to pointing at the manual `.deb` method in `docs/INSTALL.md` if the Azul metadata API lookup fails. - `install_maven.sh`: install or validate Maven 3.8+ (default pinned version `3.9.8`; override with `MAVEN_VERSION` only if the user explicitly asks for a different one) when the current Maven is missing or incompatible with the selected JDK. Same "skip if already satisfied" logic as `install_jdk.sh`: if an existing `mvn` (from `apt`, a prior manual install, etc.) already meets the minimum version, it reuses that installation's home directory instead of downloading anything. Fresh installs go into `~/opt/apache-maven-` with a `~/opt/maven` symlink, never into `/opt`. -- `install_mysql.sh`: install MySQL, then interactively confirm the root password and the `pixels` metadata-DB-user password (default `password` for both, never applied silently), create the `pixels_metadata` database/user, and load `scripts/sql/metadata_schema.sql` from the Pixels source tree. It validates DB/user identifiers, quotes SQL values, avoids putting the root password in command-line arguments, and writes shell-quoted `STATE_DIR/deployment.secrets.env` (mode 600) so the confirmed credentials flow into `configure_pixels.sh` automatically. +- `install_mysql.sh`: install MySQL, then interactively confirm the root password and the `pixels` metadata-DB-user password (default `password` for both, never applied silently), create the `pixels_metadata` database/user, and load `pixels-daemon/src/main/resources/pixels_metadata_mysql.sql` from the Pixels source tree. It validates DB/user identifiers, quotes SQL values, avoids putting the root password in command-line arguments, and writes shell-quoted `STATE_DIR/deployment.secrets.env` (mode 600) so the confirmed credentials flow into `configure_pixels.sh` automatically. - `install_etcd.sh`: install and optionally start the bundled etcd 3.3.4 package under `~/opt`. Always fixes the shipped `conf.yml`'s hardcoded `/home/ubuntu/...` `data-dir` to the real install path. By default (`ETCD_ALLOW_REMOTE=false`) it keeps etcd localhost-only. For a cluster, set `ETCD_ALLOW_REMOTE=true` only after confirming private networking/security-group rules; the script then requires `CONFIRM_ETCD_REMOTE_ACCESS=true`, `ASSUME_YES=true`, or an interactive confirmation before binding the client/peer listeners for remote access. Can also install and enable a `systemd` unit so etcd survives reboots and restarts on failure, but only after asking — it never installs this silently. Leave `INSTALL_ETCD_SYSTEMD_SERVICE` unset to be prompted interactively (`[y/N]`, defaults to no); set it to `true`/`false` to answer non-interactively once the user has actually agreed, or `ASSUME_YES=true` to answer yes to this and other yes/no prompts. Declining (or running non-interactively with no explicit answer) just skips the unit — etcd still starts, as a manually backgrounded process instead. Falls back the same way when `systemctl` isn't available at all. Restarts/re-enables the installed service automatically when the config actually changed. - `build_install_pixels.sh`: build and install Pixels into the confirmed `PIXELS_HOME` from `deployment.env`, then add the MySQL JDBC connector. It prints the repo, deployment file, and install target before running; in non-interactive mode, set `CONFIRM_PIXELS_INSTALL=true` only after the user confirms the target. If you must bypass the prepared deployment file, pass `USE_DEPLOYMENT_FILE=false PIXELS_HOME=` explicitly. `AUTO_CONFIRM_INSTALL` (default `true`) auto-answers `install.sh`'s prompts only when that is safe; if `PIXELS_HOME/etc/pixels.properties` or `pixels-cpp.properties` already exist, it forces the interactive path instead, because `install.sh` would otherwise prompt to add/remove config options or overwrite `pixels-cpp.properties` outright — exactly what `Claude agent`/`SKILL.md` say not to do without the user's say-so. - `configure_pixels.sh`: update `PIXELS_HOME/etc/pixels.properties` after database, etcd, host, port, topology, and path values are known. Automatically sources `deployment.env` and `deployment.secrets.env` (if present), so `PIXELS_HOME`, coordinator service hosts, worker names, and MySQL credentials stay in sync with what the earlier scripts confirmed. It writes `$PIXELS_HOME/etc/workers` from `PIXELS_WORKERS` by default. It fails instead of silently writing the documented default DB password unless `METADATA_DB_PASSWORD` is explicit or `ALLOW_DEFAULT_METADATA_PASSWORD=true` is set. diff --git a/skills/pixels-install/scripts/install_mysql.sh b/skills/pixels-install/scripts/install_mysql.sh index 3a55e494eb..13cb03a884 100755 --- a/skills/pixels-install/scripts/install_mysql.sh +++ b/skills/pixels-install/scripts/install_mysql.sh @@ -39,7 +39,7 @@ METADATA_DB_NAME="${METADATA_DB_NAME:-pixels_metadata}" # '%' allows the pixels DB user to connect from any host; set to 'localhost' # for a single-node deployment with no remote metadata access. METADATA_DB_USER_HOST="${METADATA_DB_USER_HOST:-%}" -SCHEMA_FILE="${SCHEMA_FILE:-$REPO_ROOT/scripts/sql/metadata_schema.sql}" +SCHEMA_FILE="${SCHEMA_FILE:-$REPO_ROOT/pixels-daemon/src/main/resources/pixels_metadata_mysql.sql}" SECRETS_FILE="${SECRETS_FILE:-$STATE_DIR/deployment.secrets.env}" ASSUME_YES="${ASSUME_YES:-false}" DEFAULT_PASSWORD="password" From d926bc70865db52a454b172981d79031a7bb6c68 Mon Sep 17 00:00:00 2001 From: haoyueli Date: Thu, 3 Sep 2026 14:20:53 +0800 Subject: [PATCH 2/4] fix: add .sql file --- .../main/resources/pixels_metadata_derby.sql | 294 ++++++++++++++ .../main/resources/pixels_metadata_mysql.sql | 372 ++++++++++++++++++ 2 files changed, 666 insertions(+) create mode 100644 pixels-daemon/src/main/resources/pixels_metadata_derby.sql create mode 100644 pixels-daemon/src/main/resources/pixels_metadata_mysql.sql diff --git a/pixels-daemon/src/main/resources/pixels_metadata_derby.sql b/pixels-daemon/src/main/resources/pixels_metadata_derby.sql new file mode 100644 index 0000000000..7d06b20324 --- /dev/null +++ b/pixels-daemon/src/main/resources/pixels_metadata_derby.sql @@ -0,0 +1,294 @@ +-- Create the Derby metadata tables for Pixels. + +-- ----------------------------------------------------- +-- Table DBS +-- ----------------------------------------------------- +CREATE TABLE DBS ( + DB_ID BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + DB_NAME VARCHAR(128) NOT NULL, + DB_DESC VARCHAR(4000), + PRIMARY KEY (DB_ID) +); +CREATE UNIQUE INDEX DB_NAME_UNIQUE ON DBS (DB_NAME); + +-- ----------------------------------------------------- +-- Table TBLS +-- ----------------------------------------------------- +CREATE TABLE TBLS ( + TBL_ID BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + TBL_NAME VARCHAR(128) NOT NULL, + TBL_TYPE VARCHAR(128), + TBL_STORAGE_SCHEME VARCHAR(32) NOT NULL DEFAULT 'file', + TBL_ROW_COUNT BIGINT NOT NULL DEFAULT 0, + DBS_DB_ID BIGINT NOT NULL, + PRIMARY KEY (TBL_ID), + CONSTRAINT fk_TBLS_DBS + FOREIGN KEY (DBS_DB_ID) + REFERENCES DBS (DB_ID) + ON DELETE CASCADE +); +CREATE INDEX fk_TBLS_DBS_idx ON TBLS (DBS_DB_ID); +CREATE UNIQUE INDEX TBL_NAME_DB_ID_UNIQUE ON TBLS (TBL_NAME, DBS_DB_ID); + +-- ----------------------------------------------------- +-- Table COLS +-- ----------------------------------------------------- +CREATE TABLE COLS ( + COL_ID BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + COL_NAME VARCHAR(128) NOT NULL, + COL_TYPE VARCHAR(128) NOT NULL, + COL_CHUNK_SIZE DOUBLE NOT NULL DEFAULT 0, + COL_SIZE DOUBLE NOT NULL DEFAULT 0, + COL_NULL_FRACTION DOUBLE NOT NULL DEFAULT 0, + COL_CARDINALITY BIGINT NOT NULL DEFAULT 0, + COL_RECORD_STATS BLOB, + TBLS_TBL_ID BIGINT NOT NULL, + PRIMARY KEY (COL_ID), + CONSTRAINT fk_COLS_TBLS + FOREIGN KEY (TBLS_TBL_ID) + REFERENCES TBLS (TBL_ID) + ON DELETE CASCADE +); +CREATE INDEX fk_COLS_TBLS_idx ON COLS (TBLS_TBL_ID); +CREATE UNIQUE INDEX COL_NAME_TBL_ID_UNIQUE ON COLS (COL_NAME, TBLS_TBL_ID); + +-- ----------------------------------------------------- +-- Table SCHEMA_VERSIONS +-- ----------------------------------------------------- +CREATE TABLE SCHEMA_VERSIONS ( + SV_ID BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + SV_COLUMNS CLOB NOT NULL, + SV_TRANS_TS BIGINT NOT NULL, + TBLS_TBL_ID BIGINT NOT NULL, + PRIMARY KEY (SV_ID), + CONSTRAINT fk_SCHEMA_VERSIONS_TBLS + FOREIGN KEY (TBLS_TBL_ID) + REFERENCES TBLS (TBL_ID) + ON DELETE CASCADE +); +CREATE INDEX fk_SCHEMA_VERSIONS_TBLS_idx ON SCHEMA_VERSIONS (TBLS_TBL_ID); + +-- ----------------------------------------------------- +-- Table LAYOUTS +-- ----------------------------------------------------- +CREATE TABLE LAYOUTS ( + LAYOUT_ID BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + LAYOUT_VERSION BIGINT NOT NULL, + LAYOUT_CREATE_AT BIGINT NOT NULL, + LAYOUT_PERMISSION SMALLINT NOT NULL, + LAYOUT_ORDERED CLOB NOT NULL, + LAYOUT_COMPACT CLOB NOT NULL, + LAYOUT_SPLITS CLOB NOT NULL, + LAYOUT_PROJECTIONS CLOB NOT NULL, + TBLS_TBL_ID BIGINT NOT NULL, + SCHEMA_VERSIONS_SV_ID BIGINT NOT NULL, + PRIMARY KEY (LAYOUT_ID), + CONSTRAINT fk_LAYOUTS_TBLS + FOREIGN KEY (TBLS_TBL_ID) + REFERENCES TBLS (TBL_ID) + ON DELETE CASCADE, + CONSTRAINT fk_LAYOUTS_SCHEMA_VERSIONS + FOREIGN KEY (SCHEMA_VERSIONS_SV_ID) + REFERENCES SCHEMA_VERSIONS (SV_ID) + ON DELETE CASCADE +); +CREATE INDEX fk_LAYOUTS_TBLS_idx ON LAYOUTS (TBLS_TBL_ID); +CREATE INDEX fk_LAYOUTS_SCHEMA_VERSIONS_idx ON LAYOUTS (SCHEMA_VERSIONS_SV_ID); + +-- ----------------------------------------------------- +-- Table VIEWS +-- ----------------------------------------------------- +CREATE TABLE VIEWS ( + VIEW_ID BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + VIEW_NAME VARCHAR(128) NOT NULL, + VIEW_TYPE VARCHAR(128), + VIEW_DATA CLOB NOT NULL, + DBS_DB_ID BIGINT NOT NULL, + PRIMARY KEY (VIEW_ID), + CONSTRAINT fk_VIEWS_DBS + FOREIGN KEY (DBS_DB_ID) + REFERENCES DBS (DB_ID) + ON DELETE CASCADE +); +CREATE INDEX fk_VIEWS_DBS_idx ON VIEWS (DBS_DB_ID); + +-- ----------------------------------------------------- +-- Table RANGE_INDICES +-- ----------------------------------------------------- +CREATE TABLE RANGE_INDICES ( + RI_ID BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + RI_KEY_COLUMNS CLOB NOT NULL, + TBLS_TBL_ID BIGINT NOT NULL, + SCHEMA_VERSIONS_SV_ID BIGINT NOT NULL, + PRIMARY KEY (RI_ID), + CONSTRAINT fk_RANGE_INDICES_TBLS + FOREIGN KEY (TBLS_TBL_ID) + REFERENCES TBLS (TBL_ID) + ON DELETE CASCADE, + CONSTRAINT fk_RANGE_INDICES_SCHEMA_VERSIONS + FOREIGN KEY (SCHEMA_VERSIONS_SV_ID) + REFERENCES SCHEMA_VERSIONS (SV_ID) + ON DELETE CASCADE +); +CREATE INDEX fk_RANGE_INDICES_TBLS_idx ON RANGE_INDICES (TBLS_TBL_ID); +CREATE INDEX fk_RANGE_INDICES_SCHEMA_VERSIONS_idx ON RANGE_INDICES (SCHEMA_VERSIONS_SV_ID); +CREATE UNIQUE INDEX TBL_ID_SV_ID_UNIQUE ON RANGE_INDICES (TBLS_TBL_ID, SCHEMA_VERSIONS_SV_ID); + +-- ----------------------------------------------------- +-- Table RANGES +-- ----------------------------------------------------- +CREATE TABLE RANGES ( + RANGE_ID BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + RANGE_MIN BLOB NOT NULL, + RANGE_MAX BLOB NOT NULL, + RANGE_PARENT_ID BIGINT, + RANGE_INDICES_RI_ID BIGINT NOT NULL, + PRIMARY KEY (RANGE_ID), + CONSTRAINT fk_RANGES_RANGE_INDICES + FOREIGN KEY (RANGE_INDICES_RI_ID) + REFERENCES RANGE_INDICES (RI_ID) + ON DELETE CASCADE, + CONSTRAINT fk_RANGES_RANGES + FOREIGN KEY (RANGE_PARENT_ID) + REFERENCES RANGES (RANGE_ID) + ON DELETE CASCADE +); +CREATE INDEX fk_RANGES_RANGE_INDICES_idx ON RANGES (RANGE_INDICES_RI_ID); +CREATE INDEX fk_RANGES_RANGES_idx ON RANGES (RANGE_PARENT_ID); + +-- ----------------------------------------------------- +-- Table PATHS +-- ----------------------------------------------------- +CREATE TABLE PATHS ( + PATH_ID BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + PATH_URI VARCHAR(512) NOT NULL, + PATH_TYPE SMALLINT NOT NULL, + LAYOUTS_LAYOUT_ID BIGINT NOT NULL, + RANGES_RANGE_ID BIGINT, + PRIMARY KEY (PATH_ID), + CONSTRAINT fk_PATHS_RANGES + FOREIGN KEY (RANGES_RANGE_ID) + REFERENCES RANGES (RANGE_ID) + ON DELETE CASCADE, + CONSTRAINT fk_PATHS_LAYOUTS + FOREIGN KEY (LAYOUTS_LAYOUT_ID) + REFERENCES LAYOUTS (LAYOUT_ID) + ON DELETE CASCADE +); +CREATE INDEX fk_PATHS_RANGES_idx ON PATHS (RANGES_RANGE_ID); +CREATE INDEX fk_PATHS_LAYOUTS_idx ON PATHS (LAYOUTS_LAYOUT_ID); +CREATE UNIQUE INDEX PATH_URI_UNIQUE ON PATHS (PATH_URI); + +-- ----------------------------------------------------- +-- Table USERS +-- ----------------------------------------------------- +CREATE TABLE USERS ( + USER_ID BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + USER_NAME VARCHAR(128) NOT NULL, + USER_PASSWORD VARCHAR(128) NOT NULL, + USER_EMAIL VARCHAR(128) NOT NULL, + PRIMARY KEY (USER_ID) +); +CREATE UNIQUE INDEX USER_NAME_UNIQUE ON USERS (USER_NAME); + +-- ----------------------------------------------------- +-- Table USER_HAS_DB +-- ----------------------------------------------------- +CREATE TABLE USER_HAS_DB ( + USERS_USER_ID BIGINT NOT NULL, + DBS_DB_ID BIGINT NOT NULL, + USER_DB_PERMITION SMALLINT NOT NULL, + PRIMARY KEY (USERS_USER_ID, DBS_DB_ID), + CONSTRAINT fk_USERS_has_DBS_USERS + FOREIGN KEY (USERS_USER_ID) + REFERENCES USERS (USER_ID) + ON DELETE CASCADE, + CONSTRAINT fk_USERS_has_DBS_DBS + FOREIGN KEY (DBS_DB_ID) + REFERENCES DBS (DB_ID) + ON DELETE CASCADE +); +CREATE INDEX fk_USERS_has_DBS_DBS_idx ON USER_HAS_DB (DBS_DB_ID); +CREATE INDEX fk_USERS_has_DBS_USERS_idx ON USER_HAS_DB (USERS_USER_ID); + +-- ----------------------------------------------------- +-- Table PEERS +-- ----------------------------------------------------- +CREATE TABLE PEERS ( + PEER_ID BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + PEER_NAME VARCHAR(128) NOT NULL, + PEER_LOCATION VARCHAR(1024) NOT NULL, + PEER_HOST VARCHAR(128) NOT NULL, + PEER_PORT INT NOT NULL, + PEER_STORAGE_SCHEME VARCHAR(32) NOT NULL, + PRIMARY KEY (PEER_ID) +); +CREATE UNIQUE INDEX PEER_NAME_UNIQUE ON PEERS (PEER_NAME); + +-- ----------------------------------------------------- +-- Table PEER_PATHS +-- ----------------------------------------------------- +CREATE TABLE PEER_PATHS ( + PEER_PATH_ID BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + PEER_PATH_URI VARCHAR(32) NOT NULL, + PEER_PATH_COLUMNS CLOB NOT NULL, + PATHS_PATH_ID BIGINT NOT NULL, + PEERS_PEER_ID BIGINT NOT NULL, + PRIMARY KEY (PEER_PATH_ID), + CONSTRAINT fk_PEER_PATHS_PATHS + FOREIGN KEY (PATHS_PATH_ID) + REFERENCES PATHS (PATH_ID) + ON DELETE CASCADE, + CONSTRAINT fk_PEER_PATHS_PEERS + FOREIGN KEY (PEERS_PEER_ID) + REFERENCES PEERS (PEER_ID) + ON DELETE CASCADE +); +CREATE INDEX fk_PEER_PATHS_PATHS_idx ON PEER_PATHS (PATHS_PATH_ID); +CREATE INDEX fk_PEER_PATHS_PEERS_idx ON PEER_PATHS (PEERS_PEER_ID); + +-- ----------------------------------------------------- +-- Table FILES +-- ----------------------------------------------------- +CREATE TABLE FILES ( + FILE_ID BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + FILE_NAME VARCHAR(128) NOT NULL, + FILE_TYPE SMALLINT NOT NULL, + FILE_NUM_RG INT NOT NULL, + FILE_MIN_ROW_ID BIGINT NOT NULL, + FILE_MAX_ROW_ID BIGINT NOT NULL, + FILE_CLEANUP_AT BIGINT, + PATHS_PATH_ID BIGINT NOT NULL, + PRIMARY KEY (FILE_ID), + CONSTRAINT fk_FILES_PATHS + FOREIGN KEY (PATHS_PATH_ID) + REFERENCES PATHS (PATH_ID) + ON DELETE CASCADE +); +CREATE INDEX fk_FILES_PATHS_idx ON FILES (PATHS_PATH_ID); +CREATE UNIQUE INDEX PATH_ID_FILE_NAME_UNIQUE ON FILES (PATHS_PATH_ID, FILE_NAME); +CREATE INDEX FILE_ROW_ID_INDEX ON FILES (FILE_MIN_ROW_ID, FILE_MAX_ROW_ID); + +-- ----------------------------------------------------- +-- Table SINGLE_POINT_INDICES +-- ----------------------------------------------------- +CREATE TABLE SINGLE_POINT_INDICES ( + SPI_ID BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + SPI_KEY_COLUMNS CLOB NOT NULL, + SPI_PRIMARY SMALLINT NOT NULL, + SPI_UNIQUE SMALLINT NOT NULL, + SPI_INDEX_SCHEME VARCHAR(32) NOT NULL, + TBLS_TBL_ID BIGINT NOT NULL, + SCHEMA_VERSIONS_SV_ID BIGINT NOT NULL, + PRIMARY KEY (SPI_ID), + CONSTRAINT fk_SINGLE_POINT_INDICES_TBLS + FOREIGN KEY (TBLS_TBL_ID) + REFERENCES TBLS (TBL_ID) + ON DELETE CASCADE, + CONSTRAINT fk_SINGLE_POINT_INDICES_SCHEMA_VERSIONS + FOREIGN KEY (SCHEMA_VERSIONS_SV_ID) + REFERENCES SCHEMA_VERSIONS (SV_ID) + ON DELETE CASCADE +); +CREATE INDEX fk_SINGLE_POINT_INDICES_TBLS_idx ON SINGLE_POINT_INDICES (TBLS_TBL_ID); +CREATE INDEX fk_SINGLE_POINT_INDICES_SCHEMA_VERSIONS_idx ON SINGLE_POINT_INDICES (SCHEMA_VERSIONS_SV_ID); diff --git a/pixels-daemon/src/main/resources/pixels_metadata_mysql.sql b/pixels-daemon/src/main/resources/pixels_metadata_mysql.sql new file mode 100644 index 0000000000..2558d2d1af --- /dev/null +++ b/pixels-daemon/src/main/resources/pixels_metadata_mysql.sql @@ -0,0 +1,372 @@ +-- Create the MySQL metadata database for Pixels. + +SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0; +SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0; +SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'; + +-- ----------------------------------------------------- +-- Schema pixels_metadata +-- ----------------------------------------------------- +CREATE SCHEMA IF NOT EXISTS `pixels_metadata` ; +USE `pixels_metadata` ; + +-- ----------------------------------------------------- +-- Table `pixels_metadata`.`DBS` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `pixels_metadata`.`DBS` ( + `DB_ID` BIGINT NOT NULL AUTO_INCREMENT, + `DB_NAME` VARCHAR(128) NOT NULL, + `DB_DESC` VARCHAR(4000) NULL, + PRIMARY KEY (`DB_ID`), + UNIQUE INDEX `DB_NAME_UNIQUE` (`DB_NAME` ASC)) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8mb4 + COLLATE = utf8mb4_bin; + + +-- ----------------------------------------------------- +-- Table `pixels_metadata`.`TBLS` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `pixels_metadata`.`TBLS` ( + `TBL_ID` BIGINT NOT NULL AUTO_INCREMENT, + `TBL_NAME` VARCHAR(128) NOT NULL, + `TBL_TYPE` VARCHAR(128) NULL, + `TBL_STORAGE_SCHEME` VARCHAR(32) NOT NULL DEFAULT 'file' COMMENT 'The name of the storage scheme of the files stored in all the paths of this table.', + `TBL_ROW_COUNT` BIGINT NOT NULL DEFAULT 0 COMMENT 'The number of rows in this table.', + `DBS_DB_ID` BIGINT NOT NULL, + PRIMARY KEY (`TBL_ID`), + INDEX `fk_TBLS_DBS_idx` (`DBS_DB_ID` ASC), + UNIQUE INDEX `TBL_NAME_DB_ID_UNIQUE` (`TBL_NAME` ASC, `DBS_DB_ID` ASC), + CONSTRAINT `fk_TBLS_DBS` + FOREIGN KEY (`DBS_DB_ID`) + REFERENCES `pixels_metadata`.`DBS` (`DB_ID`) + ON DELETE CASCADE + ON UPDATE CASCADE) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8mb4 + COLLATE = utf8mb4_bin; + + +-- ----------------------------------------------------- +-- Table `pixels_metadata`.`COLS` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `pixels_metadata`.`COLS` ( + `COL_ID` BIGINT NOT NULL AUTO_INCREMENT, + `COL_NAME` VARCHAR(128) NOT NULL, + `COL_TYPE` VARCHAR(128) NOT NULL, + `COL_CHUNK_SIZE` DOUBLE NOT NULL DEFAULT 0, + `COL_SIZE` DOUBLE NOT NULL DEFAULT 0, + `COL_NULL_FRACTION` DOUBLE NOT NULL DEFAULT 0, + `COL_CARDINALITY` BIGINT NOT NULL DEFAULT 0, + `COL_RECORD_STATS` BLOB NULL DEFAULT NULL, + `TBLS_TBL_ID` BIGINT NOT NULL, + PRIMARY KEY (`COL_ID`), + INDEX `fk_COLS_TBLS_idx` (`TBLS_TBL_ID` ASC), + UNIQUE INDEX `COL_NAME_TBL_ID_UNIQUE` (`COL_NAME` ASC, `TBLS_TBL_ID` ASC), + CONSTRAINT `fk_COLS_TBLS` + FOREIGN KEY (`TBLS_TBL_ID`) + REFERENCES `pixels_metadata`.`TBLS` (`TBL_ID`) + ON DELETE CASCADE + ON UPDATE CASCADE) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8mb4 + COLLATE = utf8mb4_bin; + + +-- ----------------------------------------------------- +-- Table `pixels_metadata`.`SCHEMA_VERSIONS` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `pixels_metadata`.`SCHEMA_VERSIONS` ( + `SV_ID` BIGINT NOT NULL AUTO_INCREMENT, + `SV_COLUMNS` MEDIUMTEXT NOT NULL COMMENT 'The json string that contains the ids of the columns owned by this schema version.', + `SV_TRANS_TS` BIGINT NOT NULL COMMENT 'The transaction timestamp of this schema version.', + `TBLS_TBL_ID` BIGINT NOT NULL, + PRIMARY KEY (`SV_ID`), + INDEX `fk_SCHEMA_VERSIONS_TBLS_idx` (`TBLS_TBL_ID` ASC), + CONSTRAINT `fk_SCHEMA_VERSIONS_TBLS` + FOREIGN KEY (`TBLS_TBL_ID`) + REFERENCES `pixels_metadata`.`TBLS` (`TBL_ID`) + ON DELETE CASCADE + ON UPDATE CASCADE) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8mb4 + COLLATE = utf8mb4_bin; + + +-- ----------------------------------------------------- +-- Table `pixels_metadata`.`LAYOUTS` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `pixels_metadata`.`LAYOUTS` ( + `LAYOUT_ID` BIGINT NOT NULL AUTO_INCREMENT, + `LAYOUT_VERSION` BIGINT NOT NULL COMMENT 'The version of this layout.', + `LAYOUT_CREATE_AT` BIGINT NOT NULL COMMENT 'The milliseconds of moment since the unix epoch that this layout is created.', + `LAYOUT_PERMISSION` TINYINT NOT NULL COMMENT '<0 for not readable and writable, 0 for readable only, >0 for readable and writable.', + `LAYOUT_ORDERED` MEDIUMTEXT NOT NULL COMMENT 'The default order of this layout. It is used to determine the column order in a single-row-group blocks.', + `LAYOUT_COMPACT` LONGTEXT NOT NULL COMMENT 'the layout strategy, stored as json. It is used to determine how row groups are compacted into a big block.', + `LAYOUT_SPLITS` LONGTEXT NOT NULL COMMENT 'The suggested split size for access patterns, stored as json.', + `LAYOUT_PROJECTIONS` LONGTEXT NOT NULL COMMENT 'The projections each maps a set of columns to a different set of paths.', + `TBLS_TBL_ID` BIGINT NOT NULL, + `SCHEMA_VERSIONS_SV_ID` BIGINT NOT NULL, + PRIMARY KEY (`LAYOUT_ID`), + INDEX `fk_LAYOUTS_TBLS_idx` (`TBLS_TBL_ID` ASC), + INDEX `fk_LAYOUTS_SCHEMA_VERSIONS_idx` (`SCHEMA_VERSIONS_SV_ID` ASC), + CONSTRAINT `fk_LAYOUTS_TBLS` + FOREIGN KEY (`TBLS_TBL_ID`) + REFERENCES `pixels_metadata`.`TBLS` (`TBL_ID`) + ON DELETE CASCADE + ON UPDATE CASCADE, + CONSTRAINT `fk_LAYOUTS_SCHEMA_VERSIONS` + FOREIGN KEY (`SCHEMA_VERSIONS_SV_ID`) + REFERENCES `pixels_metadata`.`SCHEMA_VERSIONS` (`SV_ID`) + ON DELETE CASCADE + ON UPDATE CASCADE) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8mb4 + COLLATE = utf8mb4_bin; + + +-- ----------------------------------------------------- +-- Table `pixels_metadata`.`VIEWS` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `pixels_metadata`.`VIEWS` ( + `VIEW_ID` BIGINT NOT NULL AUTO_INCREMENT, + `VIEW_NAME` VARCHAR(128) NOT NULL, + `VIEW_TYPE` VARCHAR(128) NULL, + `VIEW_DATA` LONGTEXT NOT NULL, + `DBS_DB_ID` BIGINT NOT NULL, + PRIMARY KEY (`VIEW_ID`), + INDEX `fk_VIEWS_DBS_idx` (`DBS_DB_ID` ASC), + CONSTRAINT `fk_VIEWS_DBS` + FOREIGN KEY (`DBS_DB_ID`) + REFERENCES `pixels_metadata`.`DBS` (`DB_ID`) + ON DELETE CASCADE + ON UPDATE CASCADE) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8mb4 + COLLATE = utf8mb4_bin; + + +-- ----------------------------------------------------- +-- Table `pixels_metadata`.`RANGE_INDICES` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `pixels_metadata`.`RANGE_INDICES` ( + `RI_ID` BIGINT NOT NULL AUTO_INCREMENT, + `RI_KEY_COLUMNS` TEXT NOT NULL COMMENT 'The ids of the key columns, stored in csv format.', + `TBLS_TBL_ID` BIGINT NOT NULL, + `SCHEMA_VERSIONS_SV_ID` BIGINT NOT NULL, + PRIMARY KEY (`RI_ID`), + INDEX `fk_RANGE_INDICES_TBLS_idx` (`TBLS_TBL_ID` ASC), + INDEX `fk_RANGE_INDICES_SCHEMA_VERSIONS_idx` (`SCHEMA_VERSIONS_SV_ID` ASC), + UNIQUE INDEX `TBL_ID_SV_ID_UNIQUE` (`TBLS_TBL_ID` ASC, `SCHEMA_VERSIONS_SV_ID` ASC) COMMENT 'We ensure every (table, schema_version) has only one range index.', + CONSTRAINT `fk_RANGE_INDICES_TBLS` + FOREIGN KEY (`TBLS_TBL_ID`) + REFERENCES `pixels_metadata`.`TBLS` (`TBL_ID`) + ON DELETE CASCADE + ON UPDATE CASCADE, + CONSTRAINT `fk_RANGE_INDICES_SCHEMA_VERSIONS` + FOREIGN KEY (`SCHEMA_VERSIONS_SV_ID`) + REFERENCES `pixels_metadata`.`SCHEMA_VERSIONS` (`SV_ID`) + ON DELETE CASCADE + ON UPDATE CASCADE) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8mb4 + COLLATE = utf8mb4_bin; + + +-- ----------------------------------------------------- +-- Table `pixels_metadata`.`RANGES` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `pixels_metadata`.`RANGES` ( + `RANGE_ID` BIGINT NOT NULL AUTO_INCREMENT, + `RANGE_MIN` BLOB NOT NULL COMMENT 'The min value of the key column(s).', + `RANGE_MAX` BLOB NOT NULL COMMENT 'The max value of the key column(s).', + `RANGE_PARENT_ID` BIGINT NULL, + `RANGE_INDICES_RI_ID` BIGINT NOT NULL, + PRIMARY KEY (`RANGE_ID`), + INDEX `fk_RANGES_RANGE_INDICES_idx` (`RANGE_INDICES_RI_ID` ASC), + INDEX `fk_RANGES_RANGES_idx` (`RANGE_PARENT_ID` ASC), + CONSTRAINT `fk_RANGES_RANGE_INDICES` + FOREIGN KEY (`RANGE_INDICES_RI_ID`) + REFERENCES `pixels_metadata`.`RANGE_INDICES` (`RI_ID`) + ON DELETE CASCADE + ON UPDATE CASCADE, + CONSTRAINT `fk_RANGES_RANGES` + FOREIGN KEY (`RANGE_PARENT_ID`) + REFERENCES `pixels_metadata`.`RANGES` (`RANGE_ID`) + ON DELETE SET NULL + ON UPDATE CASCADE) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8mb4 + COLLATE = utf8mb4_bin; + + +-- ----------------------------------------------------- +-- Table `pixels_metadata`.`PATHS` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `pixels_metadata`.`PATHS` ( + `PATH_ID` BIGINT NOT NULL AUTO_INCREMENT, + `PATH_URI` VARCHAR(512) NOT NULL COMMENT 'The storage path uri containing the storage scheme prefix.', + `PATH_TYPE` TINYINT NOT NULL COMMENT 'Valid value can be 0 (ordered), 1 (compact), or 2 (projection).', + `LAYOUTS_LAYOUT_ID` BIGINT NOT NULL, + `RANGES_RANGE_ID` BIGINT NULL DEFAULT NULL, + PRIMARY KEY (`PATH_ID`), + INDEX `fk_PATHS_RANGES_idx` (`RANGES_RANGE_ID` ASC), + INDEX `fk_PATHS_LAYOUTS_idx` (`LAYOUTS_LAYOUT_ID` ASC), + UNIQUE INDEX `PATH_URI_UNIQUE` (`PATH_URI` ASC), + CONSTRAINT `fk_PATHS_RANGES` + FOREIGN KEY (`RANGES_RANGE_ID`) + REFERENCES `pixels_metadata`.`RANGES` (`RANGE_ID`) + ON DELETE SET NULL + ON UPDATE CASCADE, + CONSTRAINT `fk_PATHS_LAYOUTS` + FOREIGN KEY (`LAYOUTS_LAYOUT_ID`) + REFERENCES `pixels_metadata`.`LAYOUTS` (`LAYOUT_ID`) + ON DELETE CASCADE + ON UPDATE CASCADE) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8mb4 + COLLATE = utf8mb4_bin; + + +-- ----------------------------------------------------- +-- Table `pixels_metadata`.`USERS` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `pixels_metadata`.`USERS` ( + `USER_ID` BIGINT NOT NULL AUTO_INCREMENT, + `USER_NAME` VARCHAR(128) NOT NULL, + `USER_PASSWORD` VARCHAR(128) NOT NULL, + `USER_EMAIL` VARCHAR(128) NOT NULL, + PRIMARY KEY (`USER_ID`), + UNIQUE INDEX `USER_NAME_UNIQUE` (`USER_NAME` ASC)) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8mb4 + COLLATE = utf8mb4_bin; + + +-- ----------------------------------------------------- +-- Table `pixels_metadata`.`USER_HAS_DB` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `pixels_metadata`.`USER_HAS_DB` ( + `USERS_USER_ID` BIGINT NOT NULL, + `DBS_DB_ID` BIGINT NOT NULL, + `USER_DB_PERMITION` TINYINT NOT NULL, + PRIMARY KEY (`USERS_USER_ID`, `DBS_DB_ID`), + INDEX `fk_USERS_has_DBS_DBS_idx` (`DBS_DB_ID` ASC), + INDEX `fk_USERS_has_DBS_USERS_idx` (`USERS_USER_ID` ASC), + CONSTRAINT `fk_USERS_has_DBS_USERS` + FOREIGN KEY (`USERS_USER_ID`) + REFERENCES `pixels_metadata`.`USERS` (`USER_ID`) + ON DELETE CASCADE + ON UPDATE CASCADE, + CONSTRAINT `fk_USERS_has_DBS_DBS` + FOREIGN KEY (`DBS_DB_ID`) + REFERENCES `pixels_metadata`.`DBS` (`DB_ID`) + ON DELETE CASCADE + ON UPDATE CASCADE) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8mb4 + COLLATE = utf8mb4_bin; + + +-- ----------------------------------------------------- +-- Table `pixels_metadata`.`PEERS` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `pixels_metadata`.`PEERS` ( + `PEER_ID` BIGINT NOT NULL AUTO_INCREMENT, + `PEER_NAME` VARCHAR(128) NOT NULL COMMENT 'The name of the peer, must be unique.', + `PEER_LOCATION` VARCHAR(1024) NOT NULL COMMENT 'The geographic location of the peer.', + `PEER_HOST` VARCHAR(128) NOT NULL COMMENT 'The registered host name or ip address of the peer.', + `PEER_PORT` INT NOT NULL COMMENT 'The registered port of this peer.', + `PEER_STORAGE_SCHEME` VARCHAR(32) NOT NULL, + PRIMARY KEY (`PEER_ID`), + UNIQUE INDEX `PEER_NAME_UNIQUE` (`PEER_NAME` ASC)) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8mb4 + COLLATE = utf8mb4_bin; + + +-- ----------------------------------------------------- +-- Table `pixels_metadata`.`PEER_PATHS` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `pixels_metadata`.`PEER_PATHS` ( + `PEER_PATH_ID` BIGINT NOT NULL AUTO_INCREMENT, + `PEER_PATH_URI` VARCHAR(32) NOT NULL, + `PEER_PATH_COLUMNS` MEDIUMTEXT NOT NULL COMMENT 'The json string that contains the ids of the columns stored in this peer path.', + `PATHS_PATH_ID` BIGINT NOT NULL, + `PEERS_PEER_ID` BIGINT NOT NULL, + PRIMARY KEY (`PEER_PATH_ID`), + INDEX `fk_PEER_PATHS_PATHS_idx` (`PATHS_PATH_ID` ASC), + INDEX `fk_PEER_PATHS_PEERS_idx` (`PEERS_PEER_ID` ASC), + CONSTRAINT `fk_PEER_PATHS_PATHS` + FOREIGN KEY (`PATHS_PATH_ID`) + REFERENCES `pixels_metadata`.`PATHS` (`PATH_ID`) + ON DELETE CASCADE + ON UPDATE CASCADE, + CONSTRAINT `fk_PEER_PATHS_PEERS` + FOREIGN KEY (`PEERS_PEER_ID`) + REFERENCES `pixels_metadata`.`PEERS` (`PEER_ID`) + ON DELETE CASCADE + ON UPDATE CASCADE) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8mb4 + COLLATE = utf8mb4_bin; + + +-- ----------------------------------------------------- +-- Table `pixels_metadata`.`FILES` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `pixels_metadata`.`FILES` ( + `FILE_ID` BIGINT NOT NULL AUTO_INCREMENT, + `FILE_NAME` VARCHAR(128) NOT NULL, + `FILE_TYPE` TINYINT NOT NULL COMMENT "Valid value can be 0 (temporary ingest), 1 (regular), 2 (temporary gc), or 3 (retired).", + `FILE_NUM_RG` INT NOT NULL, + `FILE_MIN_ROW_ID` BIGINT NOT NULL, + `FILE_MAX_ROW_ID` BIGINT NOT NULL, + `FILE_CLEANUP_AT` BIGINT NULL COMMENT "Earliest cleanup deadline in epoch milliseconds; meaningful only when FILE_TYPE = 3 (retired).", + `PATHS_PATH_ID` BIGINT NOT NULL, + PRIMARY KEY (`FILE_ID`), + INDEX `fk_FILES_PATHS_idx` (`PATHS_PATH_ID` ASC), + UNIQUE INDEX `PATH_ID_FILE_NAME_UNIQUE` (`PATHS_PATH_ID` ASC, `FILE_NAME` ASC), + INDEX `FILE_ROW_ID_INDEX` USING BTREE (`FILE_MIN_ROW_ID`, `FILE_MAX_ROW_ID`), + CONSTRAINT `fk_FILES_PATHS` + FOREIGN KEY (`PATHS_PATH_ID`) + REFERENCES `pixels_metadata`.`PATHS` (`PATH_ID`) + ON DELETE CASCADE + ON UPDATE CASCADE) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8mb4 + COLLATE = utf8mb4_bin; + + +-- ----------------------------------------------------- +-- Table `pixels_metadata`.`SINGLE_POINT_INDICES` +-- ----------------------------------------------------- +CREATE TABLE IF NOT EXISTS `pixels_metadata`.`SINGLE_POINT_INDICES` ( + `SPI_ID` BIGINT NOT NULL AUTO_INCREMENT, + `SPI_KEY_COLUMNS` TEXT NOT NULL COMMENT 'The ids of the key columns of this index, stored in json format.', + `SPI_PRIMARY` TINYINT NOT NULL COMMENT 'True (1) if this single point index is the primary index. There can be only one primary index on a table.', + `SPI_UNIQUE` TINYINT NOT NULL COMMENT 'True (1) if this single point index is an unique index.', + `SPI_INDEX_SCHEME` VARCHAR(32) NOT NULL COMMENT 'The index scheme, e.g., rocksdb or rockset, of this single pint index.', + `TBLS_TBL_ID` BIGINT NOT NULL, + `SCHEMA_VERSIONS_SV_ID` BIGINT NOT NULL, + PRIMARY KEY (`SPI_ID`), + INDEX `fk_SINGLE_POINT_INDICES_TBLS_idx` (`TBLS_TBL_ID` ASC), + INDEX `fk_SINGLE_POINT_INDICES_SCHEMA_VERSIONS_idx` (`SCHEMA_VERSIONS_SV_ID` ASC), + CONSTRAINT `fk_SINGLE_POINT_INDICES_TBLS` + FOREIGN KEY (`TBLS_TBL_ID`) + REFERENCES `pixels_metadata`.`TBLS` (`TBL_ID`) + ON DELETE CASCADE + ON UPDATE CASCADE, + CONSTRAINT `fk_SINGLE_POINT_INDICES_SCHEMA_VERSIONS` + FOREIGN KEY (`SCHEMA_VERSIONS_SV_ID`) + REFERENCES `pixels_metadata`.`SCHEMA_VERSIONS` (`SV_ID`) + ON DELETE CASCADE + ON UPDATE CASCADE) + ENGINE = InnoDB + DEFAULT CHARACTER SET = utf8mb4 + COLLATE = utf8mb4_bin; + + +SET SQL_MODE=@OLD_SQL_MODE; +SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS; +SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS; From 53ef0192721b9b26019c470e1fc47dc6594eca73 Mon Sep 17 00:00:00 2001 From: haoyueli Date: Thu, 3 Sep 2026 16:16:43 +0800 Subject: [PATCH 3/4] fix: docs and structure --- docker/Dockerfile | 41 ++-------- docs/INSTALL.md | 56 ++++++++++---- pixels-cli/pom.xml | 38 ++++----- .../pixels/cli/executor/InitMetaExecutor.java | 4 +- .../cli/load}/MetadataSchemaInitializer.java | 3 +- .../common}/metadata/MetadataDbType.java | 2 +- pixels-daemon/pom.xml | 7 -- .../daemon/metadata/dao/DaoFactory.java | 2 +- scripts/docker/entrypoint.sh | 9 +-- skills/pixels-install/claude/agent.md | 19 +++-- skills/pixels-install/codex/SKILL.md | 19 +++-- skills/pixels-install/cursor/SKILL.md | 19 +++-- .../scripts/build_install_pixels.sh | 12 ++- .../scripts/check_prerequisites.sh | 2 +- .../scripts/configure_pixels.sh | 55 +++++++++---- .../pixels-install/scripts/init_metadata.sh | 77 +++++++++++++++++++ .../pixels-install/scripts/install_mysql.sh | 1 + skills/pixels-install/scripts/smoke_test.sh | 50 +++++++++--- skills/pixels-install/skill.yaml | 3 +- 19 files changed, 277 insertions(+), 142 deletions(-) rename {pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata => pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load}/MetadataSchemaInitializer.java (98%) rename {pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon => pixels-common/src/main/java/io/pixelsdb/pixels/common}/metadata/MetadataDbType.java (98%) create mode 100755 skills/pixels-install/scripts/init_metadata.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index 6d4ba08a0d..552f33faea 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,8 +20,8 @@ RUN sed -i 's|http://mirrors.cloud.aliyuncs.com|http://mirrors.aliyun.com|g' /et git maven less g++ cmake make netcat-openbsd\ python-is-python3 \ wget tar unzip openssh-server vim \ - mysql-server golang \ - ca-certificates libmysqlclient-dev \ + golang \ + ca-certificates \ # Additional dependency packages xz-utils \ libx11-6 libxau6 libxcb1 libxdmcp6 \ @@ -50,10 +50,6 @@ RUN sed -i 's|http://mirrors.cloud.aliyuncs.com|http://mirrors.aliyun.com|g' /et && cd $PIXELS_HOME && mvn install \ && bash $PIXELS_HOME/install.sh \ -# Prepare MySQL Connector/J -&& cd $PIXELS_HOME/lib/ \ -&& wget https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/mysql-connector-j-8.0.33.jar \ - # Clone and compile pixels-trino && cd $HOME/opt/ \ && git clone --depth 1 https://github.com/pixelsdb/pixels-trino.git \ @@ -127,34 +123,11 @@ RUN sed -i 's|http://mirrors.cloud.aliyuncs.com|http://mirrors.aliyun.com|g' /et && echo "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED" >> $HOME/opt/trino-server/etc/jvm.config \ && echo "--add-opens=java.base/java.nio=ALL-UNNAMED" >> $HOME/opt/trino-server/etc/jvm.config \ -# Configure MySQL -&& cd $HOME \ -# Initialize root user credentials (username: root, password: password) -&& echo "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'password';" >> $HOME/root.sql \ -# Create pixels user and database -&& echo "CREATE USER 'pixels'@'%' IDENTIFIED BY 'password';" >> $HOME/user.sql \ -&& echo "CREATE DATABASE pixels_metadata;" >> $HOME/user.sql \ -&& echo "GRANT ALL PRIVILEGES ON pixels_metadata.* to 'pixels'@'%';" >> $HOME/user.sql \ -&& echo "FLUSH PRIVILEGES;" >> $HOME/user.sql \ - -# Initialize MySQL service -&& usermod -d /var/lib/mysql/ mysql \ -&& service mysql start \ -&& while ! mysqladmin ping -hlocalhost --silent; do sleep 1; done \ -# Initialize root user -&& mysql < $HOME/root.sql \ -# Create root credential file -&& echo "[client]" >> $HOME/root.cnf \ -&& echo "user=root" >> $HOME/root.cnf \ -&& echo "password=password" >> $HOME/root.cnf \ -# Create pixels user credential file -&& echo "[client]" >> $HOME/pixels.cnf \ -&& echo "user=pixels" >> $HOME/pixels.cnf \ -&& echo "password=password" >> $HOME/pixels.cnf \ -# Initialize metadata schema -&& mysql --defaults-file=root.cnf < $HOME/user.sql \ -&& mysql --defaults-file=pixels.cnf < $HOME/opt/pixels/pixels-daemon/src/main/resources/pixels_metadata_mysql.sql \ -&& service mysql stop \ +# Configure Derby as the metadata database and create tables with INIT-META +&& sed -i 's|^metadata.db.driver=.*|metadata.db.driver=org.apache.derby.jdbc.EmbeddedDriver|' $PIXELS_HOME/etc/pixels.properties \ +&& sed -i "s|^metadata.db.url=.*|metadata.db.url=jdbc:derby:${PIXELS_HOME}/var/pixels_metadata;create=true|" $PIXELS_HOME/etc/pixels.properties \ +&& mkdir -p $PIXELS_HOME/var \ +&& printf 'INIT-META\nexit\n' | java -jar $PIXELS_HOME/sbin/pixels-cli-*-full.jar \ # Install ETCD && cp -v $HOME/opt/pixels/scripts/tars/etcd-v3.3.4-linux-amd64.tar.xz $HOME/opt/ \ diff --git a/docs/INSTALL.md b/docs/INSTALL.md index a0f2366411..6ebfc65512 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -81,6 +81,7 @@ source ~/.bashrc But you still need to modify `PIXELS_HOME/etc/pixels.properties` to ensure the following properties are valid: ```properties pixels.var.dir=/home/pixels/opt/pixels/var/ +metadata.db.driver=org.apache.derby.jdbc.EmbeddedDriver metadata.db.user=pixels metadata.db.password=password metadata.db.url=jdbc:derby:/home/pixels/opt/pixels/var/pixels_metadata;create=true @@ -98,6 +99,10 @@ presto.pixels.jdbc.url=jdbc:trino://localhost:8080/pixels/tpch ``` The hostnames, ports, paths, usernames, and passwords in these properties are to be configured in the following steps of installation. +The default metadata store is the embedded Derby database under `PIXELS_HOME/var`. +No extra database server or JDBC connector is required. After the properties are valid, +create the metadata tables with pixels-cli `INIT-META` (see [Initialize Metadata](#initialize-metadata)). + > **Note:** optionally, you can also set the `PIXEL_CONFIG` system environment variable > to specify a different location of `pixels.properties`. This can be a http or https URL > to a remote location. @@ -134,16 +139,38 @@ mkdir var ``` Put the sh scripts in `scripts/bin` and `scripts/sbin` into `PIXELS_HOME/bin` and `PIXELS_HOME/sbin`, respectively. Put `pixels-daemon-*-full.jar` into `PIXELS_HOME/bin`. -Put `pixels-cli-*-full.jar` into `PIXELS_HOME/sbin` -Put the jdbc connector of MySQL into `PIXELS_HOME/lib`. +Put `pixels-cli-*-full.jar` into `PIXELS_HOME/sbin`. Put `pixels-common/src/main/resources/pixels.properties` into `PIXELS_HOME/etc`. Modify `pixels.properties` to ensure that the URLs, ports, paths, usernames, and passwords are valid. -Leave the other config parameters as default. +Leave the other config parameters as default. Derby is bundled in pixels-cli and pixels-daemon, +so you do not need a MySQL JDBC connector unless you switch the metadata database to MySQL. Set `cache.enabled` to `false` in `PIXELS_HOME/etc/pixels.properties` if you don't use pixels-cache. +Then run `INIT-META` as described in [Initialize Metadata](#initialize-metadata). + +## Initialize Metadata + +Pixels stores table/layout metadata in the database configured by `metadata.db.driver` +and `metadata.db.url`. Create those tables with the `INIT-META` command in pixels-cli. +No need to source the SQL scripts by hand; `INIT-META` selects the matching schema +(`pixels_metadata_derby.sql` or `pixels_metadata_mysql.sql`) from the configured JDBC URL. + +```bash +mkdir -p $PIXELS_HOME/var +java -jar $PIXELS_HOME/sbin/pixels-cli-*-full.jar +# then in the pixels-cli prompt: +INIT-META +``` + +It should print `Initializing metadata tables in DERBY database...` (or `MYSQL` if configured) and +`INIT-META finished`. + +Run `INIT-META` once on the node that hosts the metadata database (the coordinator when +using the default Derby URL). Workers do not need their own metadata schema. + ## Install MySQL* -Mysql is optional. Pixels uses the embedded database Derby to store the metadata by default. +MySQL is optional. Pixels uses the embedded database Derby to store the metadata by default. However, we also support MySQL as the metadata database. MySQL/MariaDB 5.5 or later has been tested. Other forks or variants may also work. @@ -173,18 +200,19 @@ FLUSH PRIVILEGES; Ensure that MySQL server can be accessed remotely. Sometimes the default MySQL configuration binds the server to localhost thus declines remote connections. -Use the `INIT-META` command in pixels-cli to create tables in the configured metadata database -(`pixels-daemon/src/main/resources/pixels_metadata_mysql.sql` for MySQL, or -`pixels-daemon/src/main/resources/pixels_metadata_derby.sql` for Derby): -```bash -java -jar $PIXELS_HOME/sbin/pixels-cli-*-full.jar -# then in the pixels-cli prompt: -INIT-META +Then put the [MySQL JDBC connector](https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/mysql-connector-j-8.0.33.jar) +into `PIXELS_HOME/lib` and switch the metadata properties in `PIXELS_HOME/etc/pixels.properties`: +```properties +metadata.db.driver=com.mysql.cj.jdbc.Driver +metadata.db.user=pixels +metadata.db.password=password +metadata.db.url=jdbc:mysql://localhost:3306/pixels_metadata?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull ``` +Change `localhost` in the URL to the hostname of the MySQL server if it is not running on the same node as the Pixels coordinator. -Then, put the [MySQL JDBC connector](https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/mysql-connector-j-8.0.33.jar) into `PIXELS_HOME/lib` and set `metadata.db.url=jdbc:mysql://localhost:3306/pixels_metadata?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull` -in `PIXELS_HOME/etc/pixels.properties` to enable MySQL as the metadata storage in Pixels. -Change `localhost` in the URL to the hostname of the MySQL server if it is not running on the same node as Pixels coordinator. +After the properties point at MySQL, create the tables with the same `INIT-META` command +described in [Initialize Metadata](#initialize-metadata). It should report +`Initializing metadata tables in MYSQL database...`. ## Install etcd diff --git a/pixels-cli/pom.xml b/pixels-cli/pom.xml index a254751088..a3973897a2 100644 --- a/pixels-cli/pom.xml +++ b/pixels-cli/pom.xml @@ -51,27 +51,6 @@ io.pixelsdb pixels-storage-s3 - - io.pixelsdb - pixels-daemon - ${project.version} - - - * - * - - - - - com.mysql - mysql-connector-j - 8.0.33 - - - org.apache.derby - derby - 10.14.2.0 - com.facebook.presto @@ -118,6 +97,16 @@ ${dep.jline.version} + + org.apache.derby + derby + + + com.mysql + mysql-connector-j + 8.0.33 + + org.apache.logging.log4j @@ -131,6 +120,13 @@ src/main/resources + + ${project.basedir}/../pixels-daemon/src/main/resources + + pixels_metadata_mysql.sql + pixels_metadata_derby.sql + + diff --git a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/InitMetaExecutor.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/InitMetaExecutor.java index f0c93b9b23..931ea42d3b 100644 --- a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/InitMetaExecutor.java +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/InitMetaExecutor.java @@ -19,9 +19,9 @@ */ package io.pixelsdb.pixels.cli.executor; +import io.pixelsdb.pixels.cli.load.MetadataSchemaInitializer; +import io.pixelsdb.pixels.common.metadata.MetadataDbType; import io.pixelsdb.pixels.common.utils.ConfigFactory; -import io.pixelsdb.pixels.daemon.metadata.MetadataDbType; -import io.pixelsdb.pixels.daemon.metadata.MetadataSchemaInitializer; import net.sourceforge.argparse4j.inf.Namespace; /** diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataSchemaInitializer.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/MetadataSchemaInitializer.java similarity index 98% rename from pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataSchemaInitializer.java rename to pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/MetadataSchemaInitializer.java index 46f4c34be2..9717470431 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataSchemaInitializer.java +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/MetadataSchemaInitializer.java @@ -17,8 +17,9 @@ * License along with Pixels. If not, see * . */ -package io.pixelsdb.pixels.daemon.metadata; +package io.pixelsdb.pixels.cli.load; +import io.pixelsdb.pixels.common.metadata.MetadataDbType; import io.pixelsdb.pixels.common.utils.ConfigFactory; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataDbType.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataDbType.java similarity index 98% rename from pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataDbType.java rename to pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataDbType.java index c874581f24..55f1d4dea9 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/MetadataDbType.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/metadata/MetadataDbType.java @@ -17,7 +17,7 @@ * License along with Pixels. If not, see * . */ -package io.pixelsdb.pixels.daemon.metadata; +package io.pixelsdb.pixels.common.metadata; import io.pixelsdb.pixels.common.utils.ConfigFactory; diff --git a/pixels-daemon/pom.xml b/pixels-daemon/pom.xml index 3878b661e9..3d2d0ede2a 100644 --- a/pixels-daemon/pom.xml +++ b/pixels-daemon/pom.xml @@ -158,13 +158,6 @@ software.amazon.awssdk ec2 - - - org.apache.derby - derby - 10.14.2.0 - - org.apache.logging.log4j diff --git a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/DaoFactory.java b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/DaoFactory.java index 560a5315aa..5ff782744e 100644 --- a/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/DaoFactory.java +++ b/pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/metadata/dao/DaoFactory.java @@ -1,6 +1,6 @@ package io.pixelsdb.pixels.daemon.metadata.dao; -import io.pixelsdb.pixels.daemon.metadata.MetadataDbType; +import io.pixelsdb.pixels.common.metadata.MetadataDbType; import io.pixelsdb.pixels.daemon.metadata.dao.impl.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/scripts/docker/entrypoint.sh b/scripts/docker/entrypoint.sh index 35f6100f85..cb3428290c 100644 --- a/scripts/docker/entrypoint.sh +++ b/scripts/docker/entrypoint.sh @@ -35,10 +35,7 @@ run_service() { # [1] SSH service ssh start -# [2] MySQL -service mysql start - -# [3] Etcd +# [2] Etcd echo -n "${SUFFIX}Start Etcd ... " sh $HOME/opt/etcd/start-etcd.sh >/tmp/etcd.log 2>&1 & ETCD_PID=$! @@ -47,10 +44,10 @@ ETCD_PID=$! until nc -z 127.0.0.1 2379; do sleep 1; done print_result 0 -# [4] Pixels +# [3] Pixels run_service "Pixels" "\$PIXELS_HOME/sbin/start-pixels.sh" "/tmp/pixels.log" -# [5] Trino +# [4] Trino run_service "Trino" "\$HOME/opt/trino-server/bin/launcher start" "/tmp/trino.log" diff --git a/skills/pixels-install/claude/agent.md b/skills/pixels-install/claude/agent.md index 1694d68eb2..9d8e4f31c5 100644 --- a/skills/pixels-install/claude/agent.md +++ b/skills/pixels-install/claude/agent.md @@ -17,10 +17,11 @@ Use these helper scripts when they directly fit the current environment and the - `check_prerequisites.sh`: validate OS, architecture, memory, disk, ports, host resolution, privilege, and optional SSH reachability. Runs every check and exits with a structured `=== check_prerequisites result ===` summary (one `ok|warn|fail|skip : ` line per check, plus a final `summary: ok=N warn=N fail=N skip=N status=pass|fail` line) instead of stopping at the first failure, so a single run shows every problem at once. - `install_jdk.sh`: install a Zulu OpenJDK build (default JDK 23) matching the server's CPU architecture (x86_64 or aarch64) under `~/opt`, and persist `JAVA_HOME` only into the current user's shell profile. Looks for a JDK that already satisfies `JDK_VERSION` first — checking `JAVA_HOME` if set, otherwise resolving whatever `java` is on `PATH` (e.g. an `apt`-installed JDK that never exported `JAVA_HOME`) — and if one is found, skips the download entirely and only fixes up the environment variable to point at it. Only downloads/installs a fresh Zulu build when nothing on the system satisfies the version requirement. Falls back to pointing at the manual `.deb` method in `docs/INSTALL.md` if the Azul metadata API lookup fails. - `install_maven.sh`: install or validate Maven 3.8+ (default pinned version `3.9.8`; override with `MAVEN_VERSION` only if the user explicitly asks for a different one) when the current Maven is missing or incompatible with the selected JDK. Same "skip if already satisfied" logic as `install_jdk.sh`: if an existing `mvn` (from `apt`, a prior manual install, etc.) already meets the minimum version, it reuses that installation's home directory instead of downloading anything. Fresh installs go into `~/opt/apache-maven-` with a `~/opt/maven` symlink, never into `/opt`. -- `install_mysql.sh`: install MySQL, then interactively confirm the root password and the `pixels` metadata-DB-user password (default `password` for both, never applied silently), create the `pixels_metadata` database/user, and load `pixels-daemon/src/main/resources/pixels_metadata_mysql.sql`. It validates DB/user identifiers, quotes SQL values, avoids putting the root password in command-line arguments, and writes shell-quoted `STATE_DIR/deployment.secrets.env` (mode 600, untracked) so the confirmed credentials flow into `configure_pixels.sh` automatically. +- `init_metadata.sh`: **default metadata setup**. After Pixels is installed and `pixels.properties` points at the metadata database, it runs pixels-cli `INIT-META` to create the tables. Derby is the default backend (`jdbc:derby:$PIXELS_HOME/var/pixels_metadata;create=true`); no extra server or JDBC connector is required because Derby is packaged with pixels-cli/pixels-daemon. +- `install_mysql.sh`: **optional alternative** to Derby. Use only when the user explicitly wants MySQL as the metadata database. Installs MySQL, then interactively confirm the root password and the `pixels` metadata-DB-user password (default `password` for both, never applied silently), create the `pixels_metadata` database/user, and optionally load `pixels-daemon/src/main/resources/pixels_metadata_mysql.sql`. It validates DB/user identifiers, quotes SQL values, avoids putting the root password in command-line arguments, and writes shell-quoted `STATE_DIR/deployment.secrets.env` (mode 600, untracked) so `configure_pixels.sh` can set `METADATA_DB_TYPE=mysql`. Table creation can still be done with `INIT-META`. - `install_etcd.sh`: install and optionally start the bundled etcd 3.3.4 package under `~/opt`. Always fixes the shipped `conf.yml`'s hardcoded `/home/ubuntu/...` `data-dir` to the real install path. By default (`ETCD_ALLOW_REMOTE=false`) it keeps etcd localhost-only. For a cluster, set `ETCD_ALLOW_REMOTE=true` only after confirming private networking/security-group rules; the script then requires `CONFIRM_ETCD_REMOTE_ACCESS=true`, `ASSUME_YES=true`, or an interactive confirmation before binding the client/peer listeners for remote access. Can also install and enable a `systemd` unit so etcd survives reboots and restarts on failure, but only after asking — it never installs this silently. Leave `INSTALL_ETCD_SYSTEMD_SERVICE` unset to be prompted interactively (`[y/N]`, defaults to no); set it to `true`/`false` to answer non-interactively once the user has actually agreed, or `ASSUME_YES=true` to answer yes to this and other yes/no prompts. Declining (or running non-interactively with no explicit answer) just skips the unit — etcd still starts, as a manually backgrounded process instead. Falls back the same way when `systemctl` isn't available at all. Restarts/re-enables the installed service automatically when the config actually changed. -- `build_install_pixels.sh`: build and install Pixels into the confirmed `PIXELS_HOME` from `deployment.env`, then add the MySQL JDBC connector. It prints the repo, deployment file, and install target before running; in non-interactive mode, set `CONFIRM_PIXELS_INSTALL=true` only after the user confirms the target. If you must bypass the prepared deployment file, pass `USE_DEPLOYMENT_FILE=false PIXELS_HOME=` explicitly. `AUTO_CONFIRM_INSTALL` (default `true`) auto-answers `install.sh`'s prompts only when that is safe; if `PIXELS_HOME/etc/pixels.properties` or `pixels-cpp.properties` already exist, it forces the interactive path instead, because `install.sh` would otherwise prompt to add/remove config options or overwrite `pixels-cpp.properties` outright — exactly what `Claude agent`/`SKILL.md` say not to do without the user's say-so. -- `configure_pixels.sh`: update `PIXELS_HOME/etc/pixels.properties` after database, etcd, host, port, topology, and path values are known. Automatically sources `deployment.env` and `deployment.secrets.env` (if present), so `PIXELS_HOME`, coordinator service hosts, worker names, and MySQL credentials stay in sync with what the earlier scripts confirmed. It writes `$PIXELS_HOME/etc/workers` from `PIXELS_WORKERS` by default. It fails instead of silently writing the documented default DB password unless `METADATA_DB_PASSWORD` is explicit or `ALLOW_DEFAULT_METADATA_PASSWORD=true` is set. +- `build_install_pixels.sh`: build and install Pixels into the confirmed `PIXELS_HOME` from `deployment.env`. It does **not** download the MySQL JDBC connector unless the user chose MySQL (`METADATA_DB_TYPE=mysql` or `INSTALL_MYSQL_CONNECTOR=true`). It prints the repo, deployment file, and install target before running; in non-interactive mode, set `CONFIRM_PIXELS_INSTALL=true` only after the user confirms the target. If you must bypass the prepared deployment file, pass `USE_DEPLOYMENT_FILE=false PIXELS_HOME=` explicitly. `AUTO_CONFIRM_INSTALL` (default `true`) auto-answers `install.sh`'s prompts only when that is safe; if `PIXELS_HOME/etc/pixels.properties` or `pixels-cpp.properties` already exist, it forces the interactive path instead, because `install.sh` would otherwise prompt to add/remove config options or overwrite `pixels-cpp.properties` outright — exactly what `Claude agent`/`SKILL.md` say not to do without the user's say-so. +- `configure_pixels.sh`: update `PIXELS_HOME/etc/pixels.properties` after database, etcd, host, port, topology, and path values are known. Defaults to Derby (`METADATA_DB_TYPE=derby`, URL `jdbc:derby:$PIXELS_HOME/var/pixels_metadata;create=true`). Automatically sources `deployment.env` and `deployment.secrets.env` (if present). It writes `$PIXELS_HOME/etc/workers` from `PIXELS_WORKERS` by default. For the MySQL alternative, set `METADATA_DB_TYPE=mysql` and reuse credentials from `install_mysql.sh`; it fails instead of silently writing the documented default DB password unless `METADATA_DB_PASSWORD` is explicit or `ALLOW_DEFAULT_METADATA_PASSWORD=true` is set. - `install_shell_helpers.sh`: optional convenience step that **asks first** before writing shell functions. By default it installs only `start_pixels`/`stop_pixels`/`restart_pixels` on the Pixels coordinator recorded in `deployment.env`; when invoked from another host, it copies a minimal installer over SSH and writes the coordinator user's `~/.pixels-shell-helpers.sh` plus shell-profile source line there. These wrap `$PIXELS_HOME/sbin/start-pixels.sh` and `$PIXELS_HOME/sbin/stop-pixels.sh`. Set `PIXELS_SHELL_HELPERS_TARGET=local` only when the user explicitly wants the functions on the current host. Leave `INSTALL_PIXELS_SHELL_HELPERS` unset to be prompted `[y/N]`; set `true`/`false` to answer non-interactively once the user has actually agreed, or `ASSUME_YES=true`. - `smoke_test.sh`: verify the installed layout, configuration, Java (`CHECK_JAVA=true`), metadata access, etcd health, Pixels topology (`deployment.env` plus `$PIXELS_HOME/etc/workers`), core service ports (`CHECK_CORE_SERVICES=true`), `$PIXELS_HOME/logs` error patterns (`CHECK_PIXELS_LOGS=true`), and basic CLI behavior when applicable. It also checks Trino layout/catalog files when `CHECK_TRINO=true` and `trino-deployment.env` is present, can separately validate the lightweight Trino-side Pixels client config plus shell-profile `PIXELS_HOME`/`PIXELS_CONFIG` exports (`CHECK_TRINO_PIXELS_CLIENT=true`) without requiring a full Pixels runtime layout (`CHECK_PIXELS_LAYOUT=false`), can check Trino launcher status across the recorded cluster (`CHECK_TRINO_CLUSTER_STATUS=true`), checks that Trino logs did not fall back to classpath `pixels.properties` (`CHECK_TRINO_LOGS=true`), waits for `/v1/info` to report `starting=false` (`CHECK_TRINO_READY=true`), and can run `trino --catalog pixels --execute "SHOW SCHEMAS"` (`CHECK_TRINO_CLI=true`). `SHOW SCHEMAS` only needs to succeed; an empty Pixels catalog is expected before data is loaded. Same structured-summary behavior as `check_prerequisites.sh` above. - `progress.sh`: records which of skill.yaml's `phases` have actually completed, in `STATE_DIR/progress.env` (generated, untracked). `progress.sh mark [note]` after a phase succeeds, `progress.sh show` to see what's already done, `progress.sh is-done ` to check one phase, `progress.sh reset` to start over. Exists so a session interrupted partway through a multi-phase install can resume from the actual state instead of guessing or re-running completed steps. @@ -45,7 +46,7 @@ persisting any `export`. Never assume the deployment user is named `pixels` or `ubuntu`, and never write environment variables into a global file such as `/etc/environment` — `skills/pixels-install/scripts/lib/shell_env.sh` centralizes this logic and is shared by `install_jdk.sh`, `install_maven.sh`, -`install_etcd.sh`, `build_install_pixels.sh`, `configure_pixels.sh`, +`install_etcd.sh`, `init_metadata.sh`, `build_install_pixels.sh`, `configure_pixels.sh`, `check_prerequisites.sh`, `smoke_test.sh`, `install_trino.sh`, `install_trino_cluster.sh`, `prepare_trino_cluster.sh`, `build_pixels_trino_artifacts.sh`, @@ -79,9 +80,10 @@ The phase sequence is fixed in `skill.yaml`'s `phases` list — this section is - **prepare_deployment_config**: run for both single-node and cluster deployments so later scripts read the same confirmed `PIXELS_HOME`, coordinator service hosts, and worker list from `deployment.env`. Prefer `prepare_deployment.sh` with no node arguments for an interactive prompt, or explicit `--coordinator ... --worker ...` arguments when the user has already provided the topology. - **install_or_verify_jdk**: `install_jdk.sh` already skips the download when anything on the system (apt-installed JDK, a previous run, etc.) satisfies `JDK_VERSION`; it only selects and installs a Zulu build under `~/opt` when nothing qualifies. JDK 23+ is required for the documented Pixels + Trino 466 path. Ask before installing a downloaded JDK package. - **install_or_verify_maven**: `install_maven.sh` has the same "skip if already satisfied" behavior for Maven 3.8+ (default pinned `3.9.8`, only overridden if the user asks). Ensure `mvn -v` ends up using the selected JDK. -- **install_or_verify_mysql** / **install_or_verify_etcd**: only after the user confirms credentials/ports (see Guardrails — never apply the default MySQL password silently); `install_etcd.sh` keeps localhost-only by default, asks before remote etcd access, and asks before installing its `systemd` auto-start unit. -- **build_install_pixels**: install Pixels from the repository into the confirmed `PIXELS_HOME` from `deployment.env` and place the MySQL JDBC connector under `PIXELS_HOME/lib`. In a Pixels cluster, worker nodes need this full installed `PIXELS_HOME` runtime layout to run `$PIXELS_HOME/bin/start-daemon.sh worker`, but they do **not** need to rebuild Pixels locally; build/install once, then distribute the installed `PIXELS_HOME` to worker nodes (for example with `$PIXELS_HOME/sbin/rsync-cluster.sh` after `etc/workers` is correct). -- **configure_pixels**: update `pixels.properties` only once paths, hosts, ports, database, etcd, topology, and cache settings are confirmed; it derives `pixels.var.dir` from the resolved `PIXELS_HOME`, writes `$PIXELS_HOME/etc/workers` from `PIXELS_WORKERS`, and reuses the MySQL credentials `install_mysql.sh` already confirmed. If MySQL was configured elsewhere, pass `METADATA_DB_PASSWORD` explicitly. +- **install_or_verify_etcd**: `install_etcd.sh` keeps localhost-only by default, asks before remote etcd access, and asks before installing its `systemd` auto-start unit. +- **build_install_pixels**: install Pixels from the repository into the confirmed `PIXELS_HOME` from `deployment.env`. Do **not** download the MySQL JDBC connector unless the user chose MySQL. In a Pixels cluster, worker nodes need this full installed `PIXELS_HOME` runtime layout to run `$PIXELS_HOME/bin/start-daemon.sh worker`, but they do **not** need to rebuild Pixels locally; build/install once, then distribute the installed `PIXELS_HOME` to worker nodes (for example with `$PIXELS_HOME/sbin/rsync-cluster.sh` after `etc/workers` is correct). +- **configure_pixels**: update `pixels.properties` only once paths, hosts, ports, database, etcd, topology, and cache settings are confirmed; it derives `pixels.var.dir` from the resolved `PIXELS_HOME`, writes `$PIXELS_HOME/etc/workers` from `PIXELS_WORKERS`, and defaults to Derby. If the user explicitly wants MySQL, run `install_mysql.sh` first (see Guardrails — never apply the default MySQL password silently), then set `METADATA_DB_TYPE=mysql` and pass `METADATA_DB_PASSWORD` or reuse `deployment.secrets.env`. +- **init_metadata**: run `init_metadata.sh` so pixels-cli `INIT-META` creates the metadata tables in the configured backend. This is the default table-creation path for both Derby and MySQL. - **start_pixels_optional**: only when the user asks to start services or startup is part of the requested task. - **install_shell_helpers_optional**: only if the user wants convenience functions. Install Pixels helpers on the Pixels coordinator and Trino helpers on the Trino coordinator; do not install them on the agent's current host unless that host is the corresponding coordinator or the user explicitly asks for a local helper install. Both helper scripts ask by default before writing to the target user's shell profile. - **smoke_test**: run after configuration or startup changes; read the full structured summary (see "Failure Handling" below), don't just check the exit code. For a final deployment check, enable the relevant checks from the actual topology, e.g. `CHECK_PIXELS_LOGS=true CHECK_TRINO=true CHECK_TRINO_CLUSTER_STATUS=true CHECK_TRINO_CLI=true`. `SHOW SCHEMAS` success is enough before data is loaded; do not require returned schemas to be non-empty. @@ -160,7 +162,8 @@ Check (and resume from) the recorded install progress: - Do not run scripts that install packages, change services, update credentials, or start daemons unless the user is asking to perform that installation stage. - Do not modify Git state. - Keep changes limited to `skills/pixels-install/` and the shared `shared-scripts/setup_cluster.sh` unless the user explicitly requests more. -- Ask before handling secrets, installing downloaded JDK packages, changing MySQL root authentication, enabling remote database access, installing the etcd `systemd` auto-start unit, or installing the Trino cluster shell helpers. +- Ask before handling secrets, installing downloaded JDK packages, installing MySQL as an alternative metadata backend, changing MySQL root authentication, enabling remote database access, installing the etcd `systemd` auto-start unit, or installing the Trino cluster shell helpers. +- Use Derby plus pixels-cli `INIT-META` as the default metadata path. Do not install MySQL unless the user explicitly asks for it. - Ask before installing Pixels shell helpers or Trino shell helpers into the user's shell profile; do not silently add operational functions. - For management shell helpers, the default target is the relevant coordinator, not the current agent host: Pixels helpers go to `PIXELS_COORDINATOR_SSH_TARGET`, and Trino helpers go to `TRINO_COORDINATOR_SSH_TARGET`. Use `PIXELS_SHELL_HELPERS_TARGET=local` or `TRINO_SHELL_HELPERS_TARGET=local` only after the user explicitly asks for local-only helper functions. - Ask or use the interactive preparation scripts before choosing `PIXELS_HOME`, Trino install/data paths, single-node vs. cluster mode, coordinator role, worker list, or whether a coordinator also runs worker tasks. Never let the agent silently accept topology/path defaults on the user's behalf. diff --git a/skills/pixels-install/codex/SKILL.md b/skills/pixels-install/codex/SKILL.md index 592215bf73..bfc3843b4f 100644 --- a/skills/pixels-install/codex/SKILL.md +++ b/skills/pixels-install/codex/SKILL.md @@ -25,10 +25,11 @@ Use these helpers when they directly fit the current environment and the user-ap - `check_prerequisites.sh`: validate OS, architecture, memory, disk, ports, host resolution, privilege, and optional SSH reachability. Runs every check and exits with a structured `=== check_prerequisites result ===` summary (one `ok|warn|fail|skip : ` line per check, plus a final `summary: ok=N warn=N fail=N skip=N status=pass|fail` line) instead of stopping at the first failure, so a single run shows every problem at once. - `install_jdk.sh`: install a Zulu OpenJDK build (default JDK 23) matching the server's CPU architecture (x86_64 or aarch64) under `~/opt`, and persist `JAVA_HOME` only into the current user's shell profile. Looks for a JDK that already satisfies `JDK_VERSION` first — checking `JAVA_HOME` if set, otherwise resolving whatever `java` is on `PATH` (e.g. an `apt`-installed JDK that never exported `JAVA_HOME`) — and if one is found, skips the download entirely and only fixes up the environment variable to point at it. Only downloads/installs a fresh Zulu build when nothing on the system satisfies the version requirement. Falls back to pointing at the manual `.deb` method in `docs/INSTALL.md` if the Azul metadata API lookup fails. - `install_maven.sh`: install or validate Maven 3.8+ (default pinned version `3.9.8`; override with `MAVEN_VERSION` only if the user explicitly asks for a different one) when the current Maven is missing or incompatible with the selected JDK. Same "skip if already satisfied" logic as `install_jdk.sh`: if an existing `mvn` (from `apt`, a prior manual install, etc.) already meets the minimum version, it reuses that installation's home directory instead of downloading anything. Fresh installs go into `~/opt/apache-maven-` with a `~/opt/maven` symlink, never into `/opt`. -- `install_mysql.sh`: install MySQL, then interactively confirm the root password and the `pixels` metadata-DB-user password (default `password` for both, never applied silently), create the `pixels_metadata` database/user, and load `pixels-daemon/src/main/resources/pixels_metadata_mysql.sql` from the Pixels source tree. It validates DB/user identifiers, quotes SQL values, avoids putting the root password in command-line arguments, and writes shell-quoted `STATE_DIR/deployment.secrets.env` (mode 600) so the confirmed credentials flow into `configure_pixels.sh` automatically. +- `init_metadata.sh`: **default metadata setup**. After Pixels is installed and `pixels.properties` points at the metadata database, it runs pixels-cli `INIT-META` to create the tables. Derby is the default backend (`jdbc:derby:$PIXELS_HOME/var/pixels_metadata;create=true`); no extra server or JDBC connector is required because Derby is packaged with pixels-cli/pixels-daemon. +- `install_mysql.sh`: **optional alternative** to Derby. Use only when the user explicitly wants MySQL as the metadata database. Installs MySQL, then interactively confirm the root password and the `pixels` metadata-DB-user password (default `password` for both, never applied silently), create the `pixels_metadata` database/user, and optionally load `pixels-daemon/src/main/resources/pixels_metadata_mysql.sql`. It validates DB/user identifiers, quotes SQL values, avoids putting the root password in command-line arguments, and writes shell-quoted `STATE_DIR/deployment.secrets.env` (mode 600) so `configure_pixels.sh` can set `METADATA_DB_TYPE=mysql`. Table creation can still be done with `INIT-META`. - `install_etcd.sh`: install and optionally start the bundled etcd 3.3.4 package under `~/opt`. Always fixes the shipped `conf.yml`'s hardcoded `/home/ubuntu/...` `data-dir` to the real install path. By default (`ETCD_ALLOW_REMOTE=false`) it keeps etcd localhost-only. For a cluster, set `ETCD_ALLOW_REMOTE=true` only after confirming private networking/security-group rules; the script then requires `CONFIRM_ETCD_REMOTE_ACCESS=true`, `ASSUME_YES=true`, or an interactive confirmation before binding the client/peer listeners for remote access. Can also install and enable a `systemd` unit so etcd survives reboots and restarts on failure, but only after asking — it never installs this silently. Leave `INSTALL_ETCD_SYSTEMD_SERVICE` unset to be prompted interactively (`[y/N]`, defaults to no); set it to `true`/`false` to answer non-interactively once the user has actually agreed, or `ASSUME_YES=true` to answer yes to this and other yes/no prompts. Declining (or running non-interactively with no explicit answer) just skips the unit — etcd still starts, as a manually backgrounded process instead. Falls back the same way when `systemctl` isn't available at all. Restarts/re-enables the installed service automatically when the config actually changed. -- `build_install_pixels.sh`: build and install Pixels into the confirmed `PIXELS_HOME` from `deployment.env`, then add the MySQL JDBC connector. It prints the repo, deployment file, and install target before running; in non-interactive mode, set `CONFIRM_PIXELS_INSTALL=true` only after the user confirms the target. If you must bypass the prepared deployment file, pass `USE_DEPLOYMENT_FILE=false PIXELS_HOME=` explicitly. `AUTO_CONFIRM_INSTALL` (default `true`) auto-answers `install.sh`'s prompts only when that is safe; if `PIXELS_HOME/etc/pixels.properties` or `pixels-cpp.properties` already exist, it forces the interactive path instead, because `install.sh` would otherwise prompt to add/remove config options or overwrite `pixels-cpp.properties` outright — exactly what `Claude agent`/`SKILL.md` say not to do without the user's say-so. -- `configure_pixels.sh`: update `PIXELS_HOME/etc/pixels.properties` after database, etcd, host, port, topology, and path values are known. Automatically sources `deployment.env` and `deployment.secrets.env` (if present), so `PIXELS_HOME`, coordinator service hosts, worker names, and MySQL credentials stay in sync with what the earlier scripts confirmed. It writes `$PIXELS_HOME/etc/workers` from `PIXELS_WORKERS` by default. It fails instead of silently writing the documented default DB password unless `METADATA_DB_PASSWORD` is explicit or `ALLOW_DEFAULT_METADATA_PASSWORD=true` is set. +- `build_install_pixels.sh`: build and install Pixels into the confirmed `PIXELS_HOME` from `deployment.env`. It does **not** download the MySQL JDBC connector unless the user chose MySQL (`METADATA_DB_TYPE=mysql` or `INSTALL_MYSQL_CONNECTOR=true`). It prints the repo, deployment file, and install target before running; in non-interactive mode, set `CONFIRM_PIXELS_INSTALL=true` only after the user confirms the target. If you must bypass the prepared deployment file, pass `USE_DEPLOYMENT_FILE=false PIXELS_HOME=` explicitly. `AUTO_CONFIRM_INSTALL` (default `true`) auto-answers `install.sh`'s prompts only when that is safe; if `PIXELS_HOME/etc/pixels.properties` or `pixels-cpp.properties` already exist, it forces the interactive path instead, because `install.sh` would otherwise prompt to add/remove config options or overwrite `pixels-cpp.properties` outright — exactly what `Claude agent`/`SKILL.md` say not to do without the user's say-so. +- `configure_pixels.sh`: update `PIXELS_HOME/etc/pixels.properties` after database, etcd, host, port, topology, and path values are known. Defaults to Derby (`METADATA_DB_TYPE=derby`, URL `jdbc:derby:$PIXELS_HOME/var/pixels_metadata;create=true`). Automatically sources `deployment.env` and `deployment.secrets.env` (if present). It writes `$PIXELS_HOME/etc/workers` from `PIXELS_WORKERS` by default. For the MySQL alternative, set `METADATA_DB_TYPE=mysql` and reuse credentials from `install_mysql.sh`; it fails instead of silently writing the documented default DB password unless `METADATA_DB_PASSWORD` is explicit or `ALLOW_DEFAULT_METADATA_PASSWORD=true` is set. - `install_shell_helpers.sh`: optional convenience step that **asks first** before writing shell functions. By default it installs only `start_pixels`/`stop_pixels`/`restart_pixels` on the Pixels coordinator recorded in `deployment.env`; when invoked from another host, it copies a minimal installer over SSH and writes the coordinator user's `~/.pixels-shell-helpers.sh` plus shell-profile source line there. These wrap `$PIXELS_HOME/sbin/start-pixels.sh` and `$PIXELS_HOME/sbin/stop-pixels.sh`. Set `PIXELS_SHELL_HELPERS_TARGET=local` only when the user explicitly wants the functions on the current host. Leave `INSTALL_PIXELS_SHELL_HELPERS` unset to be prompted `[y/N]`; set `true`/`false` to answer non-interactively once the user has actually agreed, or `ASSUME_YES=true`. - `smoke_test.sh`: verify the installed layout, configuration, Java (`CHECK_JAVA=true`), metadata access, etcd health, Pixels topology (`deployment.env` plus `$PIXELS_HOME/etc/workers`), core service ports (`CHECK_CORE_SERVICES=true`), `$PIXELS_HOME/logs` error patterns (`CHECK_PIXELS_LOGS=true`), and basic CLI behavior when applicable. It also checks Trino layout/catalog files when `CHECK_TRINO=true` and `trino-deployment.env` is present, can separately validate the lightweight Trino-side Pixels client config plus shell-profile `PIXELS_HOME`/`PIXELS_CONFIG` exports (`CHECK_TRINO_PIXELS_CLIENT=true`) without requiring a full Pixels runtime layout (`CHECK_PIXELS_LAYOUT=false`), can check Trino launcher status across the recorded cluster (`CHECK_TRINO_CLUSTER_STATUS=true`), checks that Trino logs did not fall back to classpath `pixels.properties` (`CHECK_TRINO_LOGS=true`), waits for `/v1/info` to report `starting=false` (`CHECK_TRINO_READY=true`), and can run `trino --catalog pixels --execute "SHOW SCHEMAS"` (`CHECK_TRINO_CLI=true`). `SHOW SCHEMAS` only needs to succeed; an empty Pixels catalog is expected before data is loaded. Same structured-summary behavior as `check_prerequisites.sh` above. - `progress.sh`: records which of `skill.yaml`'s `phases` have actually completed, in `STATE_DIR/progress.env`. `progress.sh mark [note]` after a phase succeeds, `progress.sh show` to see what's already done, `progress.sh is-done ` to check one phase, `progress.sh reset` to start over. Exists so a session interrupted partway through a multi-phase install can resume from the actual state instead of guessing or re-running completed steps. @@ -53,7 +54,7 @@ persisting any `export`. Never assume the deployment user is named `pixels` or `ubuntu`, and never write environment variables into a global file such as `/etc/environment` — `scripts/lib/shell_env.sh` centralizes this logic and is shared by `install_jdk.sh`, `install_maven.sh`, -`install_etcd.sh`, `build_install_pixels.sh`, `configure_pixels.sh`, +`install_etcd.sh`, `init_metadata.sh`, `build_install_pixels.sh`, `configure_pixels.sh`, `check_prerequisites.sh`, `smoke_test.sh`, `install_trino.sh`, `install_trino_cluster.sh`, `prepare_trino_cluster.sh`, `build_pixels_trino_artifacts.sh`, @@ -87,9 +88,10 @@ The phase sequence is fixed in `skill.yaml`'s `phases` list — this section is - **prepare_deployment_config**: run for both single-node and cluster deployments so later scripts read the same confirmed `PIXELS_HOME`, coordinator service hosts, and worker list from `deployment.env`. Prefer `prepare_deployment.sh` with no node arguments for an interactive prompt, or explicit `--coordinator ... --worker ...` arguments when the user has already provided the topology. - **install_or_verify_jdk**: `install_jdk.sh` already skips the download when anything on the system (apt-installed JDK, a previous run, etc.) satisfies `JDK_VERSION`; it only selects and installs a Zulu build under `~/opt` when nothing qualifies. JDK 23+ is required for the documented Pixels + Trino 466 path. Ask before installing a downloaded JDK package. - **install_or_verify_maven**: `install_maven.sh` has the same "skip if already satisfied" behavior for Maven 3.8+ (default pinned `3.9.8`, only overridden if the user asks). Ensure `mvn -v` ends up using the selected JDK. -- **install_or_verify_mysql** / **install_or_verify_etcd**: only after the user confirms credentials/ports (see Guardrails — never apply the default MySQL password silently); `install_etcd.sh` keeps localhost-only by default, asks before remote etcd access, and asks before installing its `systemd` auto-start unit. -- **build_install_pixels**: install Pixels from the repository into the confirmed `PIXELS_HOME` from `deployment.env` and place the MySQL JDBC connector under `PIXELS_HOME/lib`. In a Pixels cluster, worker nodes need this full installed `PIXELS_HOME` runtime layout to run `$PIXELS_HOME/bin/start-daemon.sh worker`, but they do **not** need to rebuild Pixels locally; build/install once, then distribute the installed `PIXELS_HOME` to worker nodes (for example with `$PIXELS_HOME/sbin/rsync-cluster.sh` after `etc/workers` is correct). -- **configure_pixels**: update `pixels.properties` only once paths, hosts, ports, database, etcd, topology, and cache settings are confirmed; it derives `pixels.var.dir` from the resolved `PIXELS_HOME`, writes `$PIXELS_HOME/etc/workers` from `PIXELS_WORKERS`, and reuses the MySQL credentials `install_mysql.sh` already confirmed. If MySQL was configured elsewhere, pass `METADATA_DB_PASSWORD` explicitly. +- **install_or_verify_etcd**: `install_etcd.sh` keeps localhost-only by default, asks before remote etcd access, and asks before installing its `systemd` auto-start unit. +- **build_install_pixels**: install Pixels from the repository into the confirmed `PIXELS_HOME` from `deployment.env`. Do **not** download the MySQL JDBC connector unless the user chose MySQL. In a Pixels cluster, worker nodes need this full installed `PIXELS_HOME` runtime layout to run `$PIXELS_HOME/bin/start-daemon.sh worker`, but they do **not** need to rebuild Pixels locally; build/install once, then distribute the installed `PIXELS_HOME` to worker nodes (for example with `$PIXELS_HOME/sbin/rsync-cluster.sh` after `etc/workers` is correct). +- **configure_pixels**: update `pixels.properties` only once paths, hosts, ports, database, etcd, topology, and cache settings are confirmed; it derives `pixels.var.dir` from the resolved `PIXELS_HOME`, writes `$PIXELS_HOME/etc/workers` from `PIXELS_WORKERS`, and defaults to Derby. If the user explicitly wants MySQL, run `install_mysql.sh` first (see Guardrails — never apply the default MySQL password silently), then set `METADATA_DB_TYPE=mysql` and pass `METADATA_DB_PASSWORD` or reuse `deployment.secrets.env`. +- **init_metadata**: run `init_metadata.sh` so pixels-cli `INIT-META` creates the metadata tables in the configured backend. This is the default table-creation path for both Derby and MySQL. - **start_pixels_optional**: only when the user asks to start services or startup is part of the requested task. - **install_shell_helpers_optional**: only if the user wants convenience functions. Install Pixels helpers on the Pixels coordinator and Trino helpers on the Trino coordinator; do not install them on the agent's current host unless that host is the corresponding coordinator or the user explicitly asks for a local helper install. Both helper scripts ask by default before writing to the target user's shell profile. - **smoke_test**: run after configuration or startup changes; read the full structured summary (see "Failure Handling" below), don't just check the exit code. For a final deployment check, enable the relevant checks from the actual topology, e.g. `CHECK_PIXELS_LOGS=true CHECK_TRINO=true CHECK_TRINO_CLUSTER_STATUS=true CHECK_TRINO_CLI=true`. `SHOW SCHEMAS` success is enough before data is loaded; do not require returned schemas to be non-empty. @@ -166,7 +168,8 @@ Check (and resume from) the recorded install progress: - Do not run scripts that install packages, change services, update credentials, or start daemons unless the user is asking to perform that installation stage. - Do not modify Git state. - When modifying this skill's source, keep changes limited to `skills/pixels-install/` unless the user explicitly requests more. -- Ask before handling secrets, installing downloaded JDK packages, changing MySQL root authentication, enabling remote database access, installing the etcd `systemd` auto-start unit, or installing the Trino cluster shell helpers. +- Ask before handling secrets, installing downloaded JDK packages, installing MySQL as an alternative metadata backend, changing MySQL root authentication, enabling remote database access, installing the etcd `systemd` auto-start unit, or installing the Trino cluster shell helpers. +- Use Derby plus pixels-cli `INIT-META` as the default metadata path. Do not install MySQL unless the user explicitly asks for it. - Ask before installing Pixels shell helpers or Trino shell helpers into the user's shell profile; do not silently add operational functions. - For management shell helpers, the default target is the relevant coordinator, not the current agent host: Pixels helpers go to `PIXELS_COORDINATOR_SSH_TARGET`, and Trino helpers go to `TRINO_COORDINATOR_SSH_TARGET`. Use `PIXELS_SHELL_HELPERS_TARGET=local` or `TRINO_SHELL_HELPERS_TARGET=local` only after the user explicitly asks for local-only helper functions. - Ask or use the interactive preparation scripts before choosing `PIXELS_HOME`, Trino install/data paths, single-node vs. cluster mode, coordinator role, worker list, or whether a coordinator also runs worker tasks. Never let the skill silently accept topology/path defaults on the user's behalf. diff --git a/skills/pixels-install/cursor/SKILL.md b/skills/pixels-install/cursor/SKILL.md index 4708908877..16448a374b 100644 --- a/skills/pixels-install/cursor/SKILL.md +++ b/skills/pixels-install/cursor/SKILL.md @@ -25,10 +25,11 @@ Use these helpers when they directly fit the current environment and the user-ap - `check_prerequisites.sh`: validate OS, architecture, memory, disk, ports, host resolution, privilege, and optional SSH reachability. Runs every check and exits with a structured `=== check_prerequisites result ===` summary (one `ok|warn|fail|skip : ` line per check, plus a final `summary: ok=N warn=N fail=N skip=N status=pass|fail` line) instead of stopping at the first failure, so a single run shows every problem at once. - `install_jdk.sh`: install a Zulu OpenJDK build (default JDK 23) matching the server's CPU architecture (x86_64 or aarch64) under `~/opt`, and persist `JAVA_HOME` only into the current user's shell profile. Looks for a JDK that already satisfies `JDK_VERSION` first — checking `JAVA_HOME` if set, otherwise resolving whatever `java` is on `PATH` (e.g. an `apt`-installed JDK that never exported `JAVA_HOME`) — and if one is found, skips the download entirely and only fixes up the environment variable to point at it. Only downloads/installs a fresh Zulu build when nothing on the system satisfies the version requirement. Falls back to pointing at the manual `.deb` method in `docs/INSTALL.md` if the Azul metadata API lookup fails. - `install_maven.sh`: install or validate Maven 3.8+ (default pinned version `3.9.8`; override with `MAVEN_VERSION` only if the user explicitly asks for a different one) when the current Maven is missing or incompatible with the selected JDK. Same "skip if already satisfied" logic as `install_jdk.sh`: if an existing `mvn` (from `apt`, a prior manual install, etc.) already meets the minimum version, it reuses that installation's home directory instead of downloading anything. Fresh installs go into `~/opt/apache-maven-` with a `~/opt/maven` symlink, never into `/opt`. -- `install_mysql.sh`: install MySQL, then interactively confirm the root password and the `pixels` metadata-DB-user password (default `password` for both, never applied silently), create the `pixels_metadata` database/user, and load `pixels-daemon/src/main/resources/pixels_metadata_mysql.sql` from the Pixels source tree. It validates DB/user identifiers, quotes SQL values, avoids putting the root password in command-line arguments, and writes shell-quoted `STATE_DIR/deployment.secrets.env` (mode 600) so the confirmed credentials flow into `configure_pixels.sh` automatically. +- `init_metadata.sh`: **default metadata setup**. After Pixels is installed and `pixels.properties` points at the metadata database, it runs pixels-cli `INIT-META` to create the tables. Derby is the default backend (`jdbc:derby:$PIXELS_HOME/var/pixels_metadata;create=true`); no extra server or JDBC connector is required because Derby is packaged with pixels-cli/pixels-daemon. +- `install_mysql.sh`: **optional alternative** to Derby. Use only when the user explicitly wants MySQL as the metadata database. Installs MySQL, then interactively confirm the root password and the `pixels` metadata-DB-user password (default `password` for both, never applied silently), create the `pixels_metadata` database/user, and optionally load `pixels-daemon/src/main/resources/pixels_metadata_mysql.sql`. It validates DB/user identifiers, quotes SQL values, avoids putting the root password in command-line arguments, and writes shell-quoted `STATE_DIR/deployment.secrets.env` (mode 600) so `configure_pixels.sh` can set `METADATA_DB_TYPE=mysql`. Table creation can still be done with `INIT-META`. - `install_etcd.sh`: install and optionally start the bundled etcd 3.3.4 package under `~/opt`. Always fixes the shipped `conf.yml`'s hardcoded `/home/ubuntu/...` `data-dir` to the real install path. By default (`ETCD_ALLOW_REMOTE=false`) it keeps etcd localhost-only. For a cluster, set `ETCD_ALLOW_REMOTE=true` only after confirming private networking/security-group rules; the script then requires `CONFIRM_ETCD_REMOTE_ACCESS=true`, `ASSUME_YES=true`, or an interactive confirmation before binding the client/peer listeners for remote access. Can also install and enable a `systemd` unit so etcd survives reboots and restarts on failure, but only after asking — it never installs this silently. Leave `INSTALL_ETCD_SYSTEMD_SERVICE` unset to be prompted interactively (`[y/N]`, defaults to no); set it to `true`/`false` to answer non-interactively once the user has actually agreed, or `ASSUME_YES=true` to answer yes to this and other yes/no prompts. Declining (or running non-interactively with no explicit answer) just skips the unit — etcd still starts, as a manually backgrounded process instead. Falls back the same way when `systemctl` isn't available at all. Restarts/re-enables the installed service automatically when the config actually changed. -- `build_install_pixels.sh`: build and install Pixels into the confirmed `PIXELS_HOME` from `deployment.env`, then add the MySQL JDBC connector. It prints the repo, deployment file, and install target before running; in non-interactive mode, set `CONFIRM_PIXELS_INSTALL=true` only after the user confirms the target. If you must bypass the prepared deployment file, pass `USE_DEPLOYMENT_FILE=false PIXELS_HOME=` explicitly. `AUTO_CONFIRM_INSTALL` (default `true`) auto-answers `install.sh`'s prompts only when that is safe; if `PIXELS_HOME/etc/pixels.properties` or `pixels-cpp.properties` already exist, it forces the interactive path instead, because `install.sh` would otherwise prompt to add/remove config options or overwrite `pixels-cpp.properties` outright — exactly what `Claude agent`/`SKILL.md` say not to do without the user's say-so. -- `configure_pixels.sh`: update `PIXELS_HOME/etc/pixels.properties` after database, etcd, host, port, topology, and path values are known. Automatically sources `deployment.env` and `deployment.secrets.env` (if present), so `PIXELS_HOME`, coordinator service hosts, worker names, and MySQL credentials stay in sync with what the earlier scripts confirmed. It writes `$PIXELS_HOME/etc/workers` from `PIXELS_WORKERS` by default. It fails instead of silently writing the documented default DB password unless `METADATA_DB_PASSWORD` is explicit or `ALLOW_DEFAULT_METADATA_PASSWORD=true` is set. +- `build_install_pixels.sh`: build and install Pixels into the confirmed `PIXELS_HOME` from `deployment.env`. It does **not** download the MySQL JDBC connector unless the user chose MySQL (`METADATA_DB_TYPE=mysql` or `INSTALL_MYSQL_CONNECTOR=true`). It prints the repo, deployment file, and install target before running; in non-interactive mode, set `CONFIRM_PIXELS_INSTALL=true` only after the user confirms the target. If you must bypass the prepared deployment file, pass `USE_DEPLOYMENT_FILE=false PIXELS_HOME=` explicitly. `AUTO_CONFIRM_INSTALL` (default `true`) auto-answers `install.sh`'s prompts only when that is safe; if `PIXELS_HOME/etc/pixels.properties` or `pixels-cpp.properties` already exist, it forces the interactive path instead, because `install.sh` would otherwise prompt to add/remove config options or overwrite `pixels-cpp.properties` outright — exactly what `Claude agent`/`SKILL.md` say not to do without the user's say-so. +- `configure_pixels.sh`: update `PIXELS_HOME/etc/pixels.properties` after database, etcd, host, port, topology, and path values are known. Defaults to Derby (`METADATA_DB_TYPE=derby`, URL `jdbc:derby:$PIXELS_HOME/var/pixels_metadata;create=true`). Automatically sources `deployment.env` and `deployment.secrets.env` (if present). It writes `$PIXELS_HOME/etc/workers` from `PIXELS_WORKERS` by default. For the MySQL alternative, set `METADATA_DB_TYPE=mysql` and reuse credentials from `install_mysql.sh`; it fails instead of silently writing the documented default DB password unless `METADATA_DB_PASSWORD` is explicit or `ALLOW_DEFAULT_METADATA_PASSWORD=true` is set. - `install_shell_helpers.sh`: optional convenience step that **asks first** before writing shell functions. By default it installs only `start_pixels`/`stop_pixels`/`restart_pixels` on the Pixels coordinator recorded in `deployment.env`; when invoked from another host, it copies a minimal installer over SSH and writes the coordinator user's `~/.pixels-shell-helpers.sh` plus shell-profile source line there. These wrap `$PIXELS_HOME/sbin/start-pixels.sh` and `$PIXELS_HOME/sbin/stop-pixels.sh`. Set `PIXELS_SHELL_HELPERS_TARGET=local` only when the user explicitly wants the functions on the current host. Leave `INSTALL_PIXELS_SHELL_HELPERS` unset to be prompted `[y/N]`; set `true`/`false` to answer non-interactively once the user has actually agreed, or `ASSUME_YES=true`. - `smoke_test.sh`: verify the installed layout, configuration, Java (`CHECK_JAVA=true`), metadata access, etcd health, Pixels topology (`deployment.env` plus `$PIXELS_HOME/etc/workers`), core service ports (`CHECK_CORE_SERVICES=true`), `$PIXELS_HOME/logs` error patterns (`CHECK_PIXELS_LOGS=true`), and basic CLI behavior when applicable. It also checks Trino layout/catalog files when `CHECK_TRINO=true` and `trino-deployment.env` is present, can separately validate the lightweight Trino-side Pixels client config plus shell-profile `PIXELS_HOME`/`PIXELS_CONFIG` exports (`CHECK_TRINO_PIXELS_CLIENT=true`) without requiring a full Pixels runtime layout (`CHECK_PIXELS_LAYOUT=false`), can check Trino launcher status across the recorded cluster (`CHECK_TRINO_CLUSTER_STATUS=true`), checks that Trino logs did not fall back to classpath `pixels.properties` (`CHECK_TRINO_LOGS=true`), waits for `/v1/info` to report `starting=false` (`CHECK_TRINO_READY=true`), and can run `trino --catalog pixels --execute "SHOW SCHEMAS"` (`CHECK_TRINO_CLI=true`). `SHOW SCHEMAS` only needs to succeed; an empty Pixels catalog is expected before data is loaded. Same structured-summary behavior as `check_prerequisites.sh` above. - `progress.sh`: records which of `skill.yaml`'s `phases` have actually completed, in `STATE_DIR/progress.env`. `progress.sh mark [note]` after a phase succeeds, `progress.sh show` to see what's already done, `progress.sh is-done ` to check one phase, `progress.sh reset` to start over. Exists so a session interrupted partway through a multi-phase install can resume from the actual state instead of guessing or re-running completed steps. @@ -53,7 +54,7 @@ persisting any `export`. Never assume the deployment user is named `pixels` or `ubuntu`, and never write environment variables into a global file such as `/etc/environment` — `scripts/lib/shell_env.sh` centralizes this logic and is shared by `install_jdk.sh`, `install_maven.sh`, -`install_etcd.sh`, `build_install_pixels.sh`, `configure_pixels.sh`, +`install_etcd.sh`, `init_metadata.sh`, `build_install_pixels.sh`, `configure_pixels.sh`, `check_prerequisites.sh`, `smoke_test.sh`, `install_trino.sh`, `install_trino_cluster.sh`, `prepare_trino_cluster.sh`, `build_pixels_trino_artifacts.sh`, @@ -87,9 +88,10 @@ The phase sequence is fixed in `skill.yaml`'s `phases` list — this section is - **prepare_deployment_config**: run for both single-node and cluster deployments so later scripts read the same confirmed `PIXELS_HOME`, coordinator service hosts, and worker list from `deployment.env`. Prefer `prepare_deployment.sh` with no node arguments for an interactive prompt, or explicit `--coordinator ... --worker ...` arguments when the user has already provided the topology. - **install_or_verify_jdk**: `install_jdk.sh` already skips the download when anything on the system (apt-installed JDK, a previous run, etc.) satisfies `JDK_VERSION`; it only selects and installs a Zulu build under `~/opt` when nothing qualifies. JDK 23+ is required for the documented Pixels + Trino 466 path. Ask before installing a downloaded JDK package. - **install_or_verify_maven**: `install_maven.sh` has the same "skip if already satisfied" behavior for Maven 3.8+ (default pinned `3.9.8`, only overridden if the user asks). Ensure `mvn -v` ends up using the selected JDK. -- **install_or_verify_mysql** / **install_or_verify_etcd**: only after the user confirms credentials/ports (see Guardrails — never apply the default MySQL password silently); `install_etcd.sh` keeps localhost-only by default, asks before remote etcd access, and asks before installing its `systemd` auto-start unit. -- **build_install_pixels**: install Pixels from the repository into the confirmed `PIXELS_HOME` from `deployment.env` and place the MySQL JDBC connector under `PIXELS_HOME/lib`. In a Pixels cluster, worker nodes need this full installed `PIXELS_HOME` runtime layout to run `$PIXELS_HOME/bin/start-daemon.sh worker`, but they do **not** need to rebuild Pixels locally; build/install once, then distribute the installed `PIXELS_HOME` to worker nodes (for example with `$PIXELS_HOME/sbin/rsync-cluster.sh` after `etc/workers` is correct). -- **configure_pixels**: update `pixels.properties` only once paths, hosts, ports, database, etcd, topology, and cache settings are confirmed; it derives `pixels.var.dir` from the resolved `PIXELS_HOME`, writes `$PIXELS_HOME/etc/workers` from `PIXELS_WORKERS`, and reuses the MySQL credentials `install_mysql.sh` already confirmed. If MySQL was configured elsewhere, pass `METADATA_DB_PASSWORD` explicitly. +- **install_or_verify_etcd**: `install_etcd.sh` keeps localhost-only by default, asks before remote etcd access, and asks before installing its `systemd` auto-start unit. +- **build_install_pixels**: install Pixels from the repository into the confirmed `PIXELS_HOME` from `deployment.env`. Do **not** download the MySQL JDBC connector unless the user chose MySQL. In a Pixels cluster, worker nodes need this full installed `PIXELS_HOME` runtime layout to run `$PIXELS_HOME/bin/start-daemon.sh worker`, but they do **not** need to rebuild Pixels locally; build/install once, then distribute the installed `PIXELS_HOME` to worker nodes (for example with `$PIXELS_HOME/sbin/rsync-cluster.sh` after `etc/workers` is correct). +- **configure_pixels**: update `pixels.properties` only once paths, hosts, ports, database, etcd, topology, and cache settings are confirmed; it derives `pixels.var.dir` from the resolved `PIXELS_HOME`, writes `$PIXELS_HOME/etc/workers` from `PIXELS_WORKERS`, and defaults to Derby. If the user explicitly wants MySQL, run `install_mysql.sh` first (see Guardrails — never apply the default MySQL password silently), then set `METADATA_DB_TYPE=mysql` and pass `METADATA_DB_PASSWORD` or reuse `deployment.secrets.env`. +- **init_metadata**: run `init_metadata.sh` so pixels-cli `INIT-META` creates the metadata tables in the configured backend. This is the default table-creation path for both Derby and MySQL. - **start_pixels_optional**: only when the user asks to start services or startup is part of the requested task. - **install_shell_helpers_optional**: only if the user wants convenience functions. Install Pixels helpers on the Pixels coordinator and Trino helpers on the Trino coordinator; do not install them on the agent's current host unless that host is the corresponding coordinator or the user explicitly asks for a local helper install. Both helper scripts ask by default before writing to the target user's shell profile. - **smoke_test**: run after configuration or startup changes; read the full structured summary (see "Failure Handling" below), don't just check the exit code. For a final deployment check, enable the relevant checks from the actual topology, e.g. `CHECK_PIXELS_LOGS=true CHECK_TRINO=true CHECK_TRINO_CLUSTER_STATUS=true CHECK_TRINO_CLI=true`. `SHOW SCHEMAS` success is enough before data is loaded; do not require returned schemas to be non-empty. @@ -166,7 +168,8 @@ Check (and resume from) the recorded install progress: - Do not run scripts that install packages, change services, update credentials, or start daemons unless the user is asking to perform that installation stage. - Do not modify Git state. - When modifying this skill's source, keep changes limited to `skills/pixels-install/` unless the user explicitly requests more. -- Ask before handling secrets, installing downloaded JDK packages, changing MySQL root authentication, enabling remote database access, installing the etcd `systemd` auto-start unit, or installing the Trino cluster shell helpers. +- Ask before handling secrets, installing downloaded JDK packages, installing MySQL as an alternative metadata backend, changing MySQL root authentication, enabling remote database access, installing the etcd `systemd` auto-start unit, or installing the Trino cluster shell helpers. +- Use Derby plus pixels-cli `INIT-META` as the default metadata path. Do not install MySQL unless the user explicitly asks for it. - Ask before installing Pixels shell helpers or Trino shell helpers into the user's shell profile; do not silently add operational functions. - For management shell helpers, the default target is the relevant coordinator, not the current agent host: Pixels helpers go to `PIXELS_COORDINATOR_SSH_TARGET`, and Trino helpers go to `TRINO_COORDINATOR_SSH_TARGET`. Use `PIXELS_SHELL_HELPERS_TARGET=local` or `TRINO_SHELL_HELPERS_TARGET=local` only after the user explicitly asks for local-only helper functions. - Ask or use the interactive preparation scripts before choosing `PIXELS_HOME`, Trino install/data paths, single-node vs. cluster mode, coordinator role, worker list, or whether a coordinator also runs worker tasks. Never let the skill silently accept topology/path defaults on the user's behalf. diff --git a/skills/pixels-install/scripts/build_install_pixels.sh b/skills/pixels-install/scripts/build_install_pixels.sh index 4c83a6c3e1..621c51cf79 100755 --- a/skills/pixels-install/scripts/build_install_pixels.sh +++ b/skills/pixels-install/scripts/build_install_pixels.sh @@ -29,6 +29,8 @@ fi # load_toolchain_env preserves the confirmed PIXELS_HOME above. load_toolchain_env PIXELS_HOME="${PIXELS_HOME:-$HOME/opt/pixels}" +METADATA_DB_TYPE="${METADATA_DB_TYPE:-derby}" +INSTALL_MYSQL_CONNECTOR="${INSTALL_MYSQL_CONNECTOR:-false}" MYSQL_CONNECTOR_VERSION="${MYSQL_CONNECTOR_VERSION:-8.0.33}" MYSQL_CONNECTOR_JAR="${MYSQL_CONNECTOR_JAR:-$PIXELS_HOME/lib/mysql-connector-j-$MYSQL_CONNECTOR_VERSION.jar}" MYSQL_CONNECTOR_URL="${MYSQL_CONNECTOR_URL:-https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/$MYSQL_CONNECTOR_VERSION/mysql-connector-j-$MYSQL_CONNECTOR_VERSION.jar}" @@ -140,7 +142,9 @@ verify_install() { compgen -G "$PIXELS_HOME/bin/pixels-daemon-*-full.jar" >/dev/null || fail "pixels daemon jar not found in $PIXELS_HOME/bin" compgen -G "$PIXELS_HOME/sbin/pixels-cli-*-full.jar" >/dev/null || fail "pixels cli jar not found in $PIXELS_HOME/sbin" [[ -f "$PIXELS_HOME/etc/pixels.properties" ]] || fail "missing config: $PIXELS_HOME/etc/pixels.properties" - find "$PIXELS_HOME/lib" -maxdepth 1 -name 'mysql-connector-j-*.jar' -print -quit | grep -q . || fail "MySQL JDBC connector not found in $PIXELS_HOME/lib" + if [[ "$METADATA_DB_TYPE" == "mysql" || "$INSTALL_MYSQL_CONNECTOR" == "true" ]]; then + find "$PIXELS_HOME/lib" -maxdepth 1 -name 'mysql-connector-j-*.jar' -print -quit | grep -q . || fail "MySQL JDBC connector not found in $PIXELS_HOME/lib" + fi log "Pixels installation verified at $PIXELS_HOME" } @@ -149,7 +153,11 @@ main() { confirm_pixels_install persist_pixels_home run_pixels_install - install_mysql_connector + if [[ "$METADATA_DB_TYPE" == "mysql" || "$INSTALL_MYSQL_CONNECTOR" == "true" ]]; then + install_mysql_connector + else + log "skipping MySQL JDBC connector (default metadata backend is Derby; set METADATA_DB_TYPE=mysql or INSTALL_MYSQL_CONNECTOR=true to install it)" + fi verify_install } diff --git a/skills/pixels-install/scripts/check_prerequisites.sh b/skills/pixels-install/scripts/check_prerequisites.sh index 8fa1277b4c..c1ac5c07bc 100755 --- a/skills/pixels-install/scripts/check_prerequisites.sh +++ b/skills/pixels-install/scripts/check_prerequisites.sh @@ -22,7 +22,7 @@ source "$SCRIPT_DIR/lib/shell_env.sh" MIN_MEMORY_MB="${MIN_MEMORY_MB:-4096}" MIN_DISK_GB="${MIN_DISK_GB:-20}" -CHECK_PORTS="${CHECK_PORTS:-18888 18889 18893 2379 2380 3306}" +CHECK_PORTS="${CHECK_PORTS:-18888 18889 18893 2379 2380}" CHECK_HOSTS="${CHECK_HOSTS:-}" CHECK_SSH_HOSTS="${CHECK_SSH_HOSTS:-}" SSH_USER="${SSH_USER:-}" diff --git a/skills/pixels-install/scripts/configure_pixels.sh b/skills/pixels-install/scripts/configure_pixels.sh index 0556e4ca3d..aa1da1e337 100755 --- a/skills/pixels-install/scripts/configure_pixels.sh +++ b/skills/pixels-install/scripts/configure_pixels.sh @@ -17,10 +17,9 @@ source "$SCRIPT_DIR/lib/shell_env.sh" # default. load_toolchain_env -# Picks up METADATA_DB_USER/METADATA_DB_PASSWORD/METADATA_DB_NAME written by -# install_mysql.sh, so the MySQL credentials confirmed interactively there -# are the same ones written into pixels.properties below. Explicit env vars -# passed to this script still take precedence over the secrets file. +# Default metadata backend is Derby. When METADATA_DB_TYPE=mysql, this +# script also picks up METADATA_DB_USER/METADATA_DB_PASSWORD/METADATA_DB_NAME +# written by install_mysql.sh. Explicit env vars still take precedence. SKILL_DIR="${SKILL_DIR:-$(skill_dir)}" STATE_DIR="${STATE_DIR:-$(state_dir)}" DEPLOYMENT_FILE="${DEPLOYMENT_FILE:-$STATE_DIR/deployment.env}" @@ -32,6 +31,7 @@ if [[ "$USE_DEPLOYMENT_FILE" == "true" && -f "$DEPLOYMENT_FILE" ]]; then set +a fi PIXELS_HOME="${PIXELS_HOME:-$HOME/opt/pixels}" +PIXELS_HOME="${PIXELS_HOME%/}" CONFIG_FILE="${PIXELS_CONFIG_FILE:-$PIXELS_HOME/etc/pixels.properties}" BACKUP_FILE="${BACKUP_FILE:-$CONFIG_FILE.bak.$(date '+%Y%m%d%H%M%S')}" SECRETS_FILE="${SECRETS_FILE:-$STATE_DIR/deployment.secrets.env}" @@ -44,8 +44,17 @@ if [[ -f "$SECRETS_FILE" ]]; then set +a fi +log() { + printf '[%s] %s\n' "$(date '+%H:%M:%S')" "$*" +} + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + PIXELS_VAR_DIR="${PIXELS_VAR_DIR:-$PIXELS_HOME/var/}" -METADATA_DB_DRIVER="${METADATA_DB_DRIVER:-com.mysql.cj.jdbc.Driver}" +METADATA_DB_TYPE="${METADATA_DB_TYPE:-derby}" METADATA_DB_USER="${METADATA_DB_USER:-pixels}" ALLOW_DEFAULT_METADATA_PASSWORD="${ALLOW_DEFAULT_METADATA_PASSWORD:-false}" if [[ -z "${METADATA_DB_PASSWORD:-}" && "$ALLOW_DEFAULT_METADATA_PASSWORD" == "true" ]]; then @@ -55,7 +64,20 @@ METADATA_DB_PASSWORD="${METADATA_DB_PASSWORD:-}" METADATA_DB_HOST="${METADATA_DB_HOST:-localhost}" METADATA_DB_PORT="${METADATA_DB_PORT:-3306}" METADATA_DB_NAME="${METADATA_DB_NAME:-pixels_metadata}" -METADATA_DB_URL="${METADATA_DB_URL:-jdbc:mysql://$METADATA_DB_HOST:$METADATA_DB_PORT/$METADATA_DB_NAME?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull}" +case "$METADATA_DB_TYPE" in + derby) + METADATA_DB_DRIVER="${METADATA_DB_DRIVER:-org.apache.derby.jdbc.EmbeddedDriver}" + METADATA_DB_PASSWORD="${METADATA_DB_PASSWORD:-pixels}" + METADATA_DB_URL="${METADATA_DB_URL:-jdbc:derby:${PIXELS_HOME}/var/pixels_metadata;create=true}" + ;; + mysql) + METADATA_DB_DRIVER="${METADATA_DB_DRIVER:-com.mysql.cj.jdbc.Driver}" + METADATA_DB_URL="${METADATA_DB_URL:-jdbc:mysql://$METADATA_DB_HOST:$METADATA_DB_PORT/$METADATA_DB_NAME?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull}" + ;; + *) + fail "METADATA_DB_TYPE must be derby or mysql, got: $METADATA_DB_TYPE" + ;; +esac METADATA_SERVER_HOST="${METADATA_SERVER_HOST:-localhost}" METADATA_SERVER_PORT="${METADATA_SERVER_PORT:-18888}" TRANS_SERVER_HOST="${TRANS_SERVER_HOST:-localhost}" @@ -69,15 +91,6 @@ PRESTO_PIXELS_JDBC_URL="${PRESTO_PIXELS_JDBC_URL:-jdbc:trino://localhost:8080/pi CACHE_ENABLED="${CACHE_ENABLED:-false}" CONFIGURE_PIXELS_WORKERS_FILE="${CONFIGURE_PIXELS_WORKERS_FILE:-true}" -log() { - printf '[%s] %s\n' "$(date '+%H:%M:%S')" "$*" -} - -fail() { - printf 'ERROR: %s\n' "$*" >&2 - exit 1 -} - escape_ere() { printf '%s' "$1" | sed 's/[][\\.^$*+?{}()|]/\\&/g' } @@ -109,7 +122,17 @@ validate_boolean() { esac } +validate_metadata_type() { + case "$METADATA_DB_TYPE" in + derby|mysql) ;; + *) fail "METADATA_DB_TYPE must be derby or mysql, got: $METADATA_DB_TYPE" ;; + esac +} + validate_metadata_password() { + if [[ "$METADATA_DB_TYPE" == "derby" ]]; then + return + fi if [[ -n "$METADATA_DB_PASSWORD" ]]; then return fi @@ -201,8 +224,10 @@ verify_config() { main() { validate_boolean + validate_metadata_type validate_metadata_password validate_config_file + mkdir -p "$PIXELS_HOME/var" configure_pixels configure_workers_file verify_config diff --git a/skills/pixels-install/scripts/init_metadata.sh b/skills/pixels-install/scripts/init_metadata.sh new file mode 100755 index 0000000000..9ab8c8ed69 --- /dev/null +++ b/skills/pixels-install/scripts/init_metadata.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +if [ -z "${BASH_VERSION:-}" ]; then + printf 'ERROR: pixels-install scripts must be executed by Bash; do not run this script with zsh.\n' >&2 + exit 1 +fi +if [ "${BASH_SOURCE[0]}" != "$0" ]; then + printf 'ERROR: do not source pixels-install installer scripts; execute this script directly with Bash.\n' >&2 + return 1 2>/dev/null || exit 1 +fi +set -euo pipefail + +# Default metadata setup: run pixels-cli INIT-META against the database +# configured in PIXELS_HOME/etc/pixels.properties. Derby is the default +# backend (configure_pixels.sh writes a jdbc:derby URL). MySQL is an +# optional alternative: run install_mysql.sh and configure_pixels.sh with +# METADATA_DB_TYPE=mysql first. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/shell_env.sh +source "$SCRIPT_DIR/lib/shell_env.sh" +load_toolchain_env + +SKILL_DIR="${SKILL_DIR:-$(skill_dir)}" +STATE_DIR="${STATE_DIR:-$(state_dir)}" +DEPLOYMENT_FILE="${DEPLOYMENT_FILE:-$STATE_DIR/deployment.env}" +if [[ -f "$DEPLOYMENT_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$DEPLOYMENT_FILE" + set +a +fi +PIXELS_HOME="${PIXELS_HOME:-$HOME/opt/pixels}" +PIXELS_HOME="${PIXELS_HOME%/}" +CONFIG_FILE="${PIXELS_CONFIG_FILE:-${PIXELS_CONFIG:-$PIXELS_HOME/etc/pixels.properties}}" + +log() { printf '[%s] %s\n' "$(date '+%H:%M:%S')" "$*"; } +fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } + +prop() { + grep -E "^[[:space:]]*$1=" "$CONFIG_FILE" | tail -n 1 | cut -d= -f2- +} + +[[ -f "$CONFIG_FILE" ]] || fail "missing Pixels config file: $CONFIG_FILE" +CLI_JAR=$(ls -1 "$PIXELS_HOME"/sbin/pixels-cli-*-full.jar 2>/dev/null | head -n 1 || true) +[[ -n "$CLI_JAR" ]] || fail "pixels-cli fat jar not found under $PIXELS_HOME/sbin" + +DRIVER="$(prop metadata.db.driver)" +URL="$(prop metadata.db.url)" +[[ -n "$DRIVER" && -n "$URL" ]] || fail "metadata.db.driver / metadata.db.url not set in $CONFIG_FILE" + +mkdir -p "$PIXELS_HOME/var" + +log "PIXELS_HOME=$PIXELS_HOME" +log "config=$CONFIG_FILE" +log "driver=$DRIVER" +log "url=$URL" +log "running INIT-META" + +CLI_OUT="$(mktemp "${TMPDIR:-/tmp}/pixels-init-meta.XXXXXX.out")" +trap 'rm -f "$CLI_OUT"' EXIT +set +e +printf 'INIT-META\nexit\n' | PIXELS_HOME="$PIXELS_HOME" PIXELS_CONFIG="$CONFIG_FILE" \ + java -jar "$CLI_JAR" >"$CLI_OUT" 2>&1 +CLI_RC=$? +set -e +cat "$CLI_OUT" + +grep -q "INIT-META finished" "$CLI_OUT" || fail "INIT-META did not finish successfully" +[[ "$CLI_RC" -eq 0 ]] || fail "pixels-cli exited with $CLI_RC" + +if grep -qi "DERBY" "$CLI_OUT"; then + log "INIT-META created Derby metadata tables" +elif grep -qi "MYSQL" "$CLI_OUT"; then + log "INIT-META created MySQL metadata tables" +else + log "INIT-META finished" +fi diff --git a/skills/pixels-install/scripts/install_mysql.sh b/skills/pixels-install/scripts/install_mysql.sh index 13cb03a884..fc871b2bb6 100755 --- a/skills/pixels-install/scripts/install_mysql.sh +++ b/skills/pixels-install/scripts/install_mysql.sh @@ -9,6 +9,7 @@ if [ "${BASH_SOURCE[0]}" != "$0" ]; then fi set -euo pipefail +# Optional alternative to the default Derby + INIT-META metadata path. # Installs MySQL, sets the root password, and creates the Pixels metadata # database/user described in docs/INSTALL.md ("Install MySQL"). Both the # MySQL root password and the pixels DB user password default to diff --git a/skills/pixels-install/scripts/smoke_test.sh b/skills/pixels-install/scripts/smoke_test.sh index 2e3d5b2854..3c4eb36123 100755 --- a/skills/pixels-install/scripts/smoke_test.sh +++ b/skills/pixels-install/scripts/smoke_test.sh @@ -72,7 +72,8 @@ CHECK_TRANS_SERVER="${CHECK_TRANS_SERVER:-false}" CHECK_CORE_SERVICES="${CHECK_CORE_SERVICES:-true}" CHECK_JAVA="${CHECK_JAVA:-true}" CHECK_ETCD="${CHECK_ETCD:-true}" -CHECK_MYSQL="${CHECK_MYSQL:-true}" +CHECK_MYSQL="${CHECK_MYSQL:-false}" +CHECK_DERBY="${CHECK_DERBY:-true}" CHECK_PIXELS_CLI="${CHECK_PIXELS_CLI:-true}" CHECK_TRINO="${CHECK_TRINO:-false}" CHECK_PIXELS_LAYOUT="${CHECK_PIXELS_LAYOUT:-true}" @@ -350,20 +351,45 @@ verify_etcd() { check_port_reachable etcd "etcd" "$etcd_host" "$etcd_port" } -verify_mysql() { - if [[ "$CHECK_MYSQL" != "true" ]]; then - result_record mysql skip "set CHECK_MYSQL=true to enable" +verify_metadata_db() { + local metadata_db_url derby_dir + + metadata_db_url="$(property_value metadata.db.url)" + if [[ -z "$metadata_db_url" ]]; then + result_record metadata_db fail "metadata.db.url is missing" return fi - local metadata_db_url mysql_host mysql_port - metadata_db_url="$(property_value metadata.db.url)" - mysql_host="$(printf '%s\n' "$metadata_db_url" | sed -nE 's#^jdbc:mysql://([^:/?]+).*#\1#p')" - mysql_port="$(printf '%s\n' "$metadata_db_url" | sed -nE 's#^jdbc:mysql://[^:/?]+:([0-9]+).*#\1#p')" - mysql_host="${mysql_host:-localhost}" - mysql_port="${mysql_port:-$MYSQL_PORT}" + if [[ "$metadata_db_url" == jdbc:derby:* ]]; then + if [[ "$CHECK_DERBY" != "true" ]]; then + result_record derby skip "set CHECK_DERBY=true to enable" + return + fi + derby_dir="${metadata_db_url#jdbc:derby:}" + derby_dir="${derby_dir%%;*}" + if [[ -d "$derby_dir" ]]; then + result_record derby ok "Derby metadata directory exists: $derby_dir" + else + result_record derby fail "Derby metadata directory not found: $derby_dir (run init_metadata.sh / INIT-META)" + fi + return + fi + + if [[ "$metadata_db_url" == jdbc:mysql:* ]]; then + if [[ "$CHECK_MYSQL" != "true" ]]; then + result_record mysql skip "MySQL URL detected; set CHECK_MYSQL=true to probe the server" + return + fi + local mysql_host mysql_port + mysql_host="$(printf '%s\n' "$metadata_db_url" | sed -nE 's#^jdbc:mysql://([^:/?]+).*#\1#p')" + mysql_port="$(printf '%s\n' "$metadata_db_url" | sed -nE 's#^jdbc:mysql://[^:/?]+:([0-9]+).*#\1#p')" + mysql_host="${mysql_host:-localhost}" + mysql_port="${mysql_port:-$MYSQL_PORT}" + check_port_reachable mysql "MySQL" "$mysql_host" "$mysql_port" + return + fi - check_port_reachable mysql "MySQL" "$mysql_host" "$mysql_port" + result_record metadata_db warn "unrecognized metadata.db.url: $metadata_db_url" } verify_pixels_cli() { @@ -698,7 +724,7 @@ main() { verify_java verify_core_services verify_etcd - verify_mysql + verify_metadata_db verify_pixels_cli verify_pixels_logs verify_trino_pixels_client diff --git a/skills/pixels-install/skill.yaml b/skills/pixels-install/skill.yaml index 209298ba0e..cf2ebbe1f0 100644 --- a/skills/pixels-install/skill.yaml +++ b/skills/pixels-install/skill.yaml @@ -10,6 +10,7 @@ scripts: check: scripts/check_prerequisites.sh jdk: scripts/install_jdk.sh maven: scripts/install_maven.sh + init_metadata: scripts/init_metadata.sh mysql: scripts/install_mysql.sh etcd: scripts/install_etcd.sh pixels: scripts/build_install_pixels.sh @@ -29,10 +30,10 @@ phases: - prepare_deployment_config - install_or_verify_jdk - install_or_verify_maven - - install_or_verify_mysql - install_or_verify_etcd - build_install_pixels - configure_pixels + - init_metadata - start_pixels_optional - install_shell_helpers_optional - smoke_test From 2328d5d9a4e90432440791a70d0c12066d97911f Mon Sep 17 00:00:00 2001 From: haoyueli Date: Fri, 4 Sep 2026 16:06:23 +0800 Subject: [PATCH 4/4] fix: install docs and add initmeta --- docs/INSTALL.md | 2 -- pixels-cli/pom.xml | 1 - .../io/pixelsdb/pixels/cli/executor/InitMetaExecutor.java | 2 +- .../cli/{load => initmeta}/MetadataSchemaInitializer.java | 2 +- pom.xml | 8 ++++++++ 5 files changed, 10 insertions(+), 5 deletions(-) rename pixels-cli/src/main/java/io/pixelsdb/pixels/cli/{load => initmeta}/MetadataSchemaInitializer.java (99%) diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 6ebfc65512..f975c124d6 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -81,7 +81,6 @@ source ~/.bashrc But you still need to modify `PIXELS_HOME/etc/pixels.properties` to ensure the following properties are valid: ```properties pixels.var.dir=/home/pixels/opt/pixels/var/ -metadata.db.driver=org.apache.derby.jdbc.EmbeddedDriver metadata.db.user=pixels metadata.db.password=password metadata.db.url=jdbc:derby:/home/pixels/opt/pixels/var/pixels_metadata;create=true @@ -203,7 +202,6 @@ binds the server to localhost thus declines remote connections. Then put the [MySQL JDBC connector](https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/mysql-connector-j-8.0.33.jar) into `PIXELS_HOME/lib` and switch the metadata properties in `PIXELS_HOME/etc/pixels.properties`: ```properties -metadata.db.driver=com.mysql.cj.jdbc.Driver metadata.db.user=pixels metadata.db.password=password metadata.db.url=jdbc:mysql://localhost:3306/pixels_metadata?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull diff --git a/pixels-cli/pom.xml b/pixels-cli/pom.xml index a3973897a2..b6b18cbabe 100644 --- a/pixels-cli/pom.xml +++ b/pixels-cli/pom.xml @@ -104,7 +104,6 @@ com.mysql mysql-connector-j - 8.0.33 diff --git a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/InitMetaExecutor.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/InitMetaExecutor.java index 931ea42d3b..6781143214 100644 --- a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/InitMetaExecutor.java +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/InitMetaExecutor.java @@ -19,7 +19,7 @@ */ package io.pixelsdb.pixels.cli.executor; -import io.pixelsdb.pixels.cli.load.MetadataSchemaInitializer; +import io.pixelsdb.pixels.cli.initmeta.MetadataSchemaInitializer; import io.pixelsdb.pixels.common.metadata.MetadataDbType; import io.pixelsdb.pixels.common.utils.ConfigFactory; import net.sourceforge.argparse4j.inf.Namespace; diff --git a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/MetadataSchemaInitializer.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/initmeta/MetadataSchemaInitializer.java similarity index 99% rename from pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/MetadataSchemaInitializer.java rename to pixels-cli/src/main/java/io/pixelsdb/pixels/cli/initmeta/MetadataSchemaInitializer.java index 9717470431..a39db7a848 100644 --- a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/MetadataSchemaInitializer.java +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/initmeta/MetadataSchemaInitializer.java @@ -17,7 +17,7 @@ * License along with Pixels. If not, see * . */ -package io.pixelsdb.pixels.cli.load; +package io.pixelsdb.pixels.cli.initmeta; import io.pixelsdb.pixels.common.metadata.MetadataDbType; import io.pixelsdb.pixels.common.utils.ConfigFactory; diff --git a/pom.xml b/pom.xml index 8cc964974b..b82250bc02 100644 --- a/pom.xml +++ b/pom.xml @@ -120,6 +120,7 @@ 0.7.7 10.14.2.0 + 8.0.33 5.13.0 1.3.2 0.16.0 @@ -360,6 +361,13 @@ ${dep.derby.version} + + + com.mysql + mysql-connector-j + ${dep.mysql-connector.version} + + io.prometheus