Skip to content

HIVE-29813: (ACID) Improve the queries of the metrics collection and duplicate deletion by adding indexes #6721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kuczoram wants to merge 1 commit into
base: master
Choose a base branch
from
Open

HIVE-29813: (ACID) Improve the queries of the metrics collection and duplicate deletion by adding indexes #6721

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.3.0.derby.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -535,6 +535,8 @@ CREATE TABLE TXNS (
INSERT INTO TXNS (TXN_ID, TXN_STATE, TXN_STARTED, TXN_LAST_HEARTBEAT, TXN_USER, TXN_HOST)
VALUES(0, 'c', 0, 0, '', '');

CREATE INDEX TXNS_STATE_TYPE_ID_IDX ON TXNS (TXN_STATE, TXN_TYPE, TXN_ID);

CREATE TABLE TXN_COMPONENTS (
TC_TXNID bigint NOT NULL REFERENCES TXNS (TXN_ID),
TC_DATABASE varchar(128) NOT NULL,
Expand All @@ -556,7 +558,7 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS (
CTC_UPDATE_DELETE char(1) NOT NULL
);

CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION);
CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION, CTC_WRITEID, CTC_UPDATE_DELETE);

CREATE TABLE TXN_LOCK_TBL (
TXN_LOCK bigint NOT NULL
Expand Down Expand Up @@ -585,6 +587,7 @@ CREATE TABLE HIVE_LOCKS (
);

CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID);
CREATE INDEX HL_ACQUIRED_AT_IDX ON HIVE_LOCKS (HL_ACQUIRED_AT);

CREATE TABLE NEXT_LOCK_ID (
NL_NEXT bigint NOT NULL
Expand Down Expand Up @@ -620,6 +623,8 @@ CREATE TABLE COMPACTION_QUEUE (
CQ_ORDER_BY varchar(4000)
);

CREATE INDEX CQ_STATE_COMMIT_TIME_IDX ON COMPACTION_QUEUE (CQ_STATE, CQ_COMMIT_TIME);

CREATE TABLE NEXT_COMPACTION_QUEUE_ID (
NCQ_NEXT bigint NOT NULL
);
Expand Down
7 changes: 7 additions & 0 deletions standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.2.0-to-4.3.0.derby.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,12 @@ ALTER TABLE MATERIALIZATION_REBUILD_LOCKS ADD COLUMN MRL_CAT_NAME varchar(128) N

CREATE INDEX MIN_HISTORY_WRITE_ID_IDX ON MIN_HISTORY_WRITE_ID (MH_DATABASE, MH_TABLE, MH_WRITEID);

CREATE INDEX TXNS_STATE_TYPE_ID_IDX ON TXNS (TXN_STATE, TXN_TYPE, TXN_ID);
CREATE INDEX HL_ACQUIRED_AT_IDX ON HIVE_LOCKS (HL_ACQUIRED_AT);
CREATE INDEX CQ_STATE_COMMIT_TIME_IDX ON COMPACTION_QUEUE (CQ_STATE, CQ_COMMIT_TIME);

DROP INDEX COMPLETED_TXN_COMPONENTS_IDX;
CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION, CTC_WRITEID, CTC_UPDATE_DELETE);

-- This needs to be the last thing done. Insert any changes above this line.
UPDATE "APP".VERSION SET SCHEMA_VERSION='4.3.0', VERSION_COMMENT='Hive release version 4.3.0' where VER_ID=1;
9 changes: 8 additions & 1 deletion standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.3.0.mssql.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -997,6 +997,8 @@ PRIMARY KEY CLUSTERED
)
);

CREATE INDEX CQ_STATE_COMMIT_TIME_IDX ON COMPACTION_QUEUE (CQ_STATE, CQ_COMMIT_TIME);

CREATE TABLE COMPLETED_COMPACTIONS (
CC_ID bigint NOT NULL,
CC_DATABASE nvarchar(128) NOT NULL,
Expand Down Expand Up @@ -1052,7 +1054,7 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS(
);


CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION);
CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION, CTC_WRITEID, CTC_UPDATE_DELETE);

CREATE TABLE HIVE_LOCKS(
HL_LOCK_EXT_ID bigint NOT NULL,
Expand All @@ -1079,6 +1081,8 @@ PRIMARY KEY CLUSTERED
)
);

CREATE INDEX HL_ACQUIRED_AT_IDX ON HIVE_LOCKS (HL_ACQUIRED_AT);

CREATE TABLE NEXT_COMPACTION_QUEUE_ID(
NCQ_NEXT bigint NOT NULL
);
Expand Down Expand Up @@ -1113,6 +1117,9 @@ PRIMARY KEY CLUSTERED
TXN_ID ASC
)
);

CREATE INDEX TXNS_STATE_TYPE_ID_IDX ON TXNS (TXN_STATE, TXN_TYPE, TXN_ID);

SET IDENTITY_INSERT TXNS ON;
INSERT INTO TXNS (TXN_ID, TXN_STATE, TXN_STARTED, TXN_LAST_HEARTBEAT, TXN_USER, TXN_HOST)
VALUES(0, 'c', 0, 0, '', '');
Expand Down
7 changes: 7 additions & 0 deletions standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.2.0-to-4.3.0.mssql.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,13 @@ ALTER TABLE MATERIALIZATION_REBUILD_LOCKS ADD MRL_CAT_NAME nvarchar(128) NOT NUL

CREATE INDEX MIN_HISTORY_WRITE_ID_IDX ON MIN_HISTORY_WRITE_ID (MH_DATABASE, MH_TABLE, MH_WRITEID);

CREATE INDEX TXNS_STATE_TYPE_ID_IDX ON TXNS (TXN_STATE, TXN_TYPE, TXN_ID);
CREATE INDEX HL_ACQUIRED_AT_IDX ON HIVE_LOCKS (HL_ACQUIRED_AT);
CREATE INDEX CQ_STATE_COMMIT_TIME_IDX ON COMPACTION_QUEUE (CQ_STATE, CQ_COMMIT_TIME);

DROP INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS;
CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION, CTC_WRITEID, CTC_UPDATE_DELETE);

-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='4.3.0', VERSION_COMMENT='Hive release version 4.3.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 4.2.0 to 4.3.0' AS MESSAGE;
7 changes: 6 additions & 1 deletion standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.3.0.mysql.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -964,6 +964,8 @@ CREATE TABLE TXNS (
INSERT INTO TXNS (TXN_ID, TXN_STATE, TXN_STARTED, TXN_LAST_HEARTBEAT, TXN_USER, TXN_HOST)
VALUES(0, 'c', 0, 0, '', '');

CREATE INDEX TXNS_STATE_TYPE_ID_IDX ON TXNS (TXN_STATE, TXN_TYPE, TXN_ID);

CREATE TABLE TXN_COMPONENTS (
TC_TXNID bigint NOT NULL,
TC_DATABASE varchar(128) NOT NULL,
Expand All @@ -986,7 +988,7 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS (
CTC_UPDATE_DELETE char(1) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=latin1;

CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION) USING BTREE;
CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION, CTC_WRITEID, CTC_UPDATE_DELETE) USING BTREE;

CREATE TABLE TXN_LOCK_TBL (
TXN_LOCK bigint NOT NULL
Expand Down Expand Up @@ -1016,6 +1018,7 @@ CREATE TABLE HIVE_LOCKS (
) ENGINE=InnoDB DEFAULT CHARSET=latin1;

CREATE INDEX HL_TXNID_IDX ON HIVE_LOCKS (HL_TXNID);
CREATE INDEX HL_ACQUIRED_AT_IDX ON HIVE_LOCKS (HL_ACQUIRED_AT);

CREATE TABLE NEXT_LOCK_ID (
NL_NEXT bigint NOT NULL
Expand Down Expand Up @@ -1051,6 +1054,8 @@ CREATE TABLE COMPACTION_QUEUE (
CQ_ORDER_BY varchar(4000)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;

CREATE INDEX CQ_STATE_COMMIT_TIME_IDX ON COMPACTION_QUEUE (CQ_STATE, CQ_COMMIT_TIME);

CREATE TABLE COMPLETED_COMPACTIONS (
CC_ID bigint PRIMARY KEY,
CC_DATABASE varchar(128) NOT NULL,
Expand Down
7 changes: 7 additions & 0 deletions standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.2.0-to-4.3.0.mysql.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,13 @@ ALTER TABLE MATERIALIZATION_REBUILD_LOCKS ADD MRL_CAT_NAME varchar(128) NOT NUL

CREATE INDEX MIN_HISTORY_WRITE_ID_IDX ON MIN_HISTORY_WRITE_ID (MH_DATABASE, MH_TABLE, MH_WRITEID);

CREATE INDEX TXNS_STATE_TYPE_ID_IDX ON TXNS (TXN_STATE, TXN_TYPE, TXN_ID) ALGORITHM=INPLACE LOCK=NONE;
CREATE INDEX HL_ACQUIRED_AT_IDX ON HIVE_LOCKS (HL_ACQUIRED_AT) ALGORITHM=INPLACE LOCK=NONE;
CREATE INDEX CQ_STATE_COMMIT_TIME_IDX ON COMPACTION_QUEUE (CQ_STATE, CQ_COMMIT_TIME) ALGORITHM=INPLACE LOCK=NONE;

ALTER TABLE COMPLETED_TXN_COMPONENTS DROP INDEX COMPLETED_TXN_COMPONENTS_IDX;
CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION, CTC_WRITEID, CTC_UPDATE_DELETE) USING BTREE ALGORITHM=INPLACE LOCK=NONE;

-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='4.3.0', VERSION_COMMENT='Hive release version 4.3.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 4.2.0 to 4.3.0' AS MESSAGE;
7 changes: 6 additions & 1 deletion standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.3.0.oracle.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -955,6 +955,8 @@ CREATE TABLE TXNS (
INSERT INTO TXNS (TXN_ID, TXN_STATE, TXN_STARTED, TXN_LAST_HEARTBEAT, TXN_USER, TXN_HOST)
VALUES(0, 'c', 0, 0, '_', '_');

CREATE INDEX TXNS_STATE_TYPE_ID_IDX ON TXNS (TXN_STATE, TXN_TYPE, TXN_ID);

CREATE TABLE TXN_COMPONENTS (
TC_TXNID NUMBER(19) NOT NULL REFERENCES TXNS (TXN_ID),
TC_DATABASE VARCHAR2(128) NOT NULL,
Expand All @@ -976,7 +978,7 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS (
CTC_UPDATE_DELETE CHAR(1) NOT NULL
) ROWDEPENDENCIES;

CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION);
CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION, CTC_WRITEID, CTC_UPDATE_DELETE);

CREATE TABLE TXN_LOCK_TBL (
TXN_LOCK NUMBER(19) NOT NULL
Expand Down Expand Up @@ -1005,6 +1007,7 @@ CREATE TABLE HIVE_LOCKS (
) ROWDEPENDENCIES;

CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID);
CREATE INDEX HL_ACQUIRED_AT_IDX ON HIVE_LOCKS (HL_ACQUIRED_AT);

CREATE TABLE NEXT_LOCK_ID (
NL_NEXT NUMBER(19) NOT NULL
Expand Down Expand Up @@ -1040,6 +1043,8 @@ CREATE TABLE COMPACTION_QUEUE (
CQ_ORDER_BY varchar(4000)
) ROWDEPENDENCIES;

CREATE INDEX CQ_STATE_COMMIT_TIME_IDX ON COMPACTION_QUEUE (CQ_STATE, CQ_COMMIT_TIME);

CREATE TABLE NEXT_COMPACTION_QUEUE_ID (
NCQ_NEXT NUMBER(19) NOT NULL
);
Expand Down
7 changes: 7 additions & 0 deletions standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.2.0-to-4.3.0.oracle.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,13 @@ ALTER TABLE MATERIALIZATION_REBUILD_LOCKS ADD (MRL_CAT_NAME VARCHAR2(128) DEFAUL

CREATE INDEX MIN_HISTORY_WRITE_ID_IDX ON MIN_HISTORY_WRITE_ID (MH_DATABASE, MH_TABLE, MH_WRITEID);

CREATE INDEX TXNS_STATE_TYPE_ID_IDX ON TXNS (TXN_STATE, TXN_TYPE, TXN_ID);
CREATE INDEX HL_ACQUIRED_AT_IDX ON HIVE_LOCKS (HL_ACQUIRED_AT);
CREATE INDEX CQ_STATE_COMMIT_TIME_IDX ON COMPACTION_QUEUE (CQ_STATE, CQ_COMMIT_TIME);

DROP INDEX COMPLETED_TXN_COMPONENTS_INDEX;
CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION, CTC_WRITEID, CTC_UPDATE_DELETE);

-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='4.3.0', VERSION_COMMENT='Hive release version 4.3.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 4.2.0 to 4.3.0' AS Status from dual;
7 changes: 6 additions & 1 deletion standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.3.0.postgres.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1590,6 +1590,8 @@ CREATE TABLE "TXNS" (
INSERT INTO "TXNS" ("TXN_ID", "TXN_STATE", "TXN_STARTED", "TXN_LAST_HEARTBEAT", "TXN_USER", "TXN_HOST")
VALUES(0, 'c', 0, 0, '', '');

CREATE INDEX "TXNS_STATE_TYPE_ID_IDX" ON "TXNS" USING btree ("TXN_STATE", "TXN_TYPE", "TXN_ID");

CREATE TABLE "TXN_COMPONENTS" (
"TC_TXNID" bigint NOT NULL REFERENCES "TXNS" ("TXN_ID"),
"TC_DATABASE" varchar(128) NOT NULL,
Expand All @@ -1611,7 +1613,7 @@ CREATE TABLE "COMPLETED_TXN_COMPONENTS" (
"CTC_UPDATE_DELETE" char(1) NOT NULL
);

CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON "COMPLETED_TXN_COMPONENTS" USING btree ("CTC_DATABASE", "CTC_TABLE", "CTC_PARTITION");
CREATE INDEX "COMPLETED_TXN_COMPONENTS_INDEX" ON "COMPLETED_TXN_COMPONENTS" USING btree ("CTC_DATABASE", "CTC_TABLE", "CTC_PARTITION", "CTC_WRITEID", "CTC_UPDATE_DELETE");

CREATE TABLE "TXN_LOCK_TBL" (
"TXN_LOCK" bigint NOT NULL
Expand Down Expand Up @@ -1640,6 +1642,7 @@ CREATE TABLE "HIVE_LOCKS" (
);

CREATE INDEX HL_TXNID_INDEX ON "HIVE_LOCKS" USING hash ("HL_TXNID");
CREATE INDEX "HL_ACQUIRED_AT_IDX" ON "HIVE_LOCKS" USING btree ("HL_ACQUIRED_AT");

CREATE TABLE "NEXT_LOCK_ID" (
"NL_NEXT" bigint NOT NULL
Expand Down Expand Up @@ -1675,6 +1678,8 @@ CREATE TABLE "COMPACTION_QUEUE" (
"CQ_ORDER_BY" varchar(4000)
);

CREATE INDEX "CQ_STATE_COMMIT_TIME_IDX" ON "COMPACTION_QUEUE" USING btree ("CQ_STATE", "CQ_COMMIT_TIME");

CREATE TABLE "NEXT_COMPACTION_QUEUE_ID" (
"NCQ_NEXT" bigint NOT NULL
);
Expand Down
7 changes: 7 additions & 0 deletions ...lone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.2.0-to-4.3.0.postgres.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,13 @@ ALTER TABLE "MATERIALIZATION_REBUILD_LOCKS" ADD COLUMN "MRL_CAT_NAME" varchar(12

CREATE INDEX "MIN_HISTORY_WRITE_ID_IDX" ON "MIN_HISTORY_WRITE_ID" ("MH_DATABASE", "MH_TABLE", "MH_WRITEID");

CREATE INDEX "TXNS_STATE_TYPE_ID_IDX" ON "TXNS" USING btree ("TXN_STATE", "TXN_TYPE", "TXN_ID");
CREATE INDEX "HL_ACQUIRED_AT_IDX" ON "HIVE_LOCKS" USING btree ("HL_ACQUIRED_AT");
CREATE INDEX "CQ_STATE_COMMIT_TIME_IDX" ON "COMPACTION_QUEUE" USING btree ("CQ_STATE", "CQ_COMMIT_TIME");

DROP INDEX COMPLETED_TXN_COMPONENTS_INDEX;
CREATE INDEX "COMPLETED_TXN_COMPONENTS_INDEX" ON "COMPLETED_TXN_COMPONENTS" USING btree ("CTC_DATABASE", "CTC_TABLE", "CTC_PARTITION", "CTC_WRITEID", "CTC_UPDATE_DELETE");

-- These lines need to be last. Insert any changes above.
UPDATE "VERSION" SET "SCHEMA_VERSION"='4.3.0', "VERSION_COMMENT"='Hive release version 4.3.0' where "VER_ID"=1;
SELECT 'Finished upgrading MetaStore schema from 4.2.0 to 4.3.0';
Loading