diff --git a/docs/_docs/snapshots/snapshots.adoc b/docs/_docs/snapshots/snapshots.adoc index fe61b3c49234a..8242d4f6e7101 100644 --- a/docs/_docs/snapshots/snapshots.adoc +++ b/docs/_docs/snapshots/snapshots.adoc @@ -287,6 +287,49 @@ control.(sh|bat) --snapshot restore snapshot_09062021 --groups cache-group1,cach control.(sh|bat) --snapshot restore snapshot_09062021 --increment 1 ---- +== Deleting Snapshot + +You can delete a snapshot using the `control.sh|bat` script. + +The deletion is performed on all *online* server nodes of the cluster. +[NOTE] +==== +The snapshot integrity, topology and correctness aren't checked. Snapshot data on offline server nodes aren't deleted. +==== + +[tabs] +-- +tab:Unix[] +[source,shell] +---- +# Delete the snapshot "snapshot_09062021". +control.sh --snapshot delete snapshot_09062021 + +# Delete the snapshot "snapshot_09062021" located in the "/tmp/ignite/snapshots" folder. +control.sh --snapshot delete snapshot_09062021 --src /tmp/ignite/snapshots +---- + +tab:Windows[] +[source,shell] +---- +# Delete the snapshot "snapshot_09062021". +control.bat --snapshot delete snapshot_09062021 + +# Delete the snapshot "snapshot_09062021" located in the "C:\tmp\ignite\snapshots" folder. +control.bat --snapshot delete snapshot_09062021 --src C:\tmp\ignite\snapshots +---- +-- + +=== Delete operation limitations + +The delete operation is subject to the following limitations: + +* The deletion is rejected if any snapshot operation (create, restore, check, delete) is active for the snapshot. +* The operation requires the snapshot administration permissions via `IgniteSecurity` (if configured). +* The operation cannot be undone and the deleted snapshot cannot be restored. The command prompts for a confirmation. +* Before deletion, no validation is performed on the snapshot other than locating and reading its metadata. If the metadata isn't found. +* The deletion operation is independent of the status of the snapshot operation and cancellation requests. + == Getting Snapshot Operation Status The status of the current snapshot operation in the cluster can be obtained using the `control.sh|bat` script or JMX interface: diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/hint/HintUtils.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/hint/HintUtils.java index 40c267ded097a..5597b4c776a1b 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/hint/HintUtils.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/hint/HintUtils.java @@ -99,7 +99,7 @@ public static List nonInheritedRelHints(RelNode rel) { /** * @return Distinct hints within {@code hints} filtered with {@code hintDefs}, {@link HintOptionsChecker} and - * removed inherit pathes. + * removed inherit paths. * @see HintOptionsChecker * @see RelHint#inheritPath */ diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlannerHelper.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlannerHelper.java index d88275fb786e4..39fb078f6b219 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlannerHelper.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlannerHelper.java @@ -134,7 +134,7 @@ public static IgniteRel optimize(SqlNode sqlNode, IgnitePlanner planner, IgniteL rel = planner.trimUnusedFields(root.withRel(rel)).rel; // The following pushed down project can erase top-level hints. We store them to reassign hints for join nodes. - // Clear the inherit pathes to consider the hints as not propogated ones. + // Clear the inherit paths to consider the hints as not propagated ones. List topHints = HintUtils.allRelHints(rel).stream().map(h -> h.inheritPath.isEmpty() ? h : h.copy(Collections.emptyList())).collect(Collectors.toList()); diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java index 0fc350ab7c8fc..99882f4ccc6ff 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java @@ -81,6 +81,7 @@ import org.apache.ignite.internal.management.performancestatistics.PerformanceStatisticsCommand; import org.apache.ignite.internal.management.property.PropertyCommand; import org.apache.ignite.internal.management.snapshot.SnapshotCommand; +import org.apache.ignite.internal.management.snapshot.SnapshotDeleteCommand; import org.apache.ignite.internal.management.snapshot.SnapshotRestoreCommand; import org.apache.ignite.internal.management.tx.TxCommand; import org.apache.ignite.internal.management.tx.TxCommandArg; @@ -529,6 +530,8 @@ else if (cmd.getClass() == EncryptionChangeCacheKeyCommand.class) cmdText = F.concat(cmdText, "cacheGroup1"); else if (cmd.getClass() == SnapshotRestoreCommand.class) cmdText = F.concat(cmdText, "snp1"); + else if (cmd.getClass() == SnapshotDeleteCommand.class) + cmdText = F.concat(cmdText, "snp1"); else if (cmd.getClass() == MetaUpdateCommand.class) return; else if (cmd.getClass() == MetaRemoveCommand.class) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite.java b/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite.java index 6528d919f6105..91a1d67d0dfc0 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite.java @@ -27,6 +27,7 @@ import org.apache.ignite.util.GridCommandHandlerCheckpointTest; import org.apache.ignite.util.GridCommandHandlerClusterByClassTest; import org.apache.ignite.util.GridCommandHandlerClusterByClassWithSSLTest; +import org.apache.ignite.util.GridCommandHandlerDeleteSnapshotTest; import org.apache.ignite.util.GridCommandHandlerIncompatibleSslConfigTest; import org.apache.ignite.util.GridCommandHandlerIndexingCheckSizeTest; import org.apache.ignite.util.GridCommandHandlerIndexingClusterByClassTest; @@ -74,6 +75,7 @@ GridCommandHandlerCheckIndexesInlineSizeTest.class, GridCommandHandlerMetadataTest.class, GridCommandHandlerCheckIncrementalSnapshotTest.class, + GridCommandHandlerDeleteSnapshotTest.class, GridCommandHandlerLegacyClientTest.class, KillCommandsControlShTest.class, diff --git a/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite6.java b/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite6.java new file mode 100644 index 0000000000000..3719e8d887d3e --- /dev/null +++ b/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite6.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testsuites; + +import org.apache.ignite.util.GridCommandHandlerDeleteSnapshotTest; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +/** + * Test suite for control utility. + */ +@RunWith(Suite.class) +@Suite.SuiteClasses({ + GridCommandHandlerDeleteSnapshotTest.class +}) +public class IgniteControlUtilityTestSuite6 { +} diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerDeleteSnapshotTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerDeleteSnapshotTest.java new file mode 100644 index 0000000000000..4c093ac614720 --- /dev/null +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerDeleteSnapshotTest.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.util; + +import java.io.File; +import java.nio.file.DirectoryStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collection; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.management.snapshot.SnapshotDeleteCommand; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.testframework.GridTestUtils; +import org.junit.Test; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import static java.nio.file.Files.newDirectoryStream; +import static org.apache.ignite.cluster.ClusterState.ACTIVE; +import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; +import static org.apache.ignite.internal.processors.cache.persistence.snapshot.AbstractSnapshotSelfTest.snp; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; +import static org.junit.Assume.assumeTrue; + +/** Test for the command '--snapshot delete'. */ +public class GridCommandHandlerDeleteSnapshotTest extends GridCommandHandlerAbstractTest { + /** Flag to add an extra server node after the cluster start and the snapshot creation. */ + @Parameter(1) + public boolean extraServerNode; + + /** */ + @Parameter(2) + public boolean incremental; + + /** */ + @Parameter(3) + public boolean changeBaseline; + + /** */ + @Parameter(4) + public boolean customPath; + + /** */ + @Parameter(5) + public boolean separatedWorkDir; + + /** */ + @Parameters(name = "client={0},useExtraNode={1},inc={2},chBaseln={3},cstSnpPath={4},ownWorkDir={5}") + public static Collection parameters() { + return GridTestUtils.cartesianProduct( + commandHandlers(), + F.asList(false, true), // Use or not extra server node after the cluster start and the snapshot creation; + F.asList(false, true), // Add incremental snapshot; + F.asList(false, true), // Change baseline; + F.asList(false, true), // Use custom snapshot path; + F.asList(false, true) // Separated (own) work directory. + ); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + /** Handy if test running is interrupted and {@link #afterTest()} isn't invoked. */ + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void cleanPersistenceDir() throws Exception { + super.cleanPersistenceDir(); + + // Also cleans separated snapshot working directories and custom snapshot paths. + try (DirectoryStream files = newDirectoryStream(Paths.get(U.defaultWorkDirectory()))) { + for (Path path : files) + U.delete(path); + } + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + if (separatedWorkDir) + cfg.setWorkDirectory(new File(U.defaultWorkDirectory(), igniteInstanceName).getAbsolutePath()); + + return cfg; + } + + /** */ + @Test + public void testSnapshotDelete() throws Exception { + // A custom snapshot path actually puts snapshots in a shared directory. This skews the results when dedicated + // work directories are set. + assumeTrue(!customPath || !separatedWorkDir); + + int entriesCnt = 100; + int initNodes = 3; + + walCompactionEnabled(incremental); + + IgniteEx ig = (IgniteEx)startGridsMultiThreaded(initNodes); + + if (changeBaseline) { + ig.cluster().baselineAutoAdjustEnabled(false); + + ig.cluster().setBaselineTopology(ig.cluster().topologyVersion()); + } + + ig.cluster().state(ACTIVE); + + createCacheAndPreload(ig, entriesCnt); + + File cstSnpsRoot = customPath + ? new File(grid(0).context().pdsFolderResolver().fileTree().snapshotsRoot(), "ex_snapshots") + : null; + File snpDir = new File(customPath ? cstSnpsRoot : ig.context().pdsFolderResolver().fileTree().snapshotsRoot(), "testSnapshot"); + + snp(ig).createSnapshot("testSnapshot", customPath ? cstSnpsRoot.getAbsolutePath() : null, false, false) + .get(getTestTimeout()); + + if (incremental) { + for (int i = 0; i < 3; ++i) { + int dataIdx = entriesCnt + entriesCnt / 4 * i; + + try (IgniteDataStreamer streamer = ig.dataStreamer(DEFAULT_CACHE_NAME)) { + for (int d = dataIdx; d < dataIdx + entriesCnt / 4; ++d) + streamer.addData(d, d); + } + + snp(ig).createSnapshot("testSnapshot", customPath ? cstSnpsRoot.getAbsolutePath() : null, true, false) + .get(getTestTimeout()); + } + } + + String baselineGone = null; + + // Optionally restarts with the same servers number, but changed baseline. The snapshot is kept on the same + // previous nodes independently of the baseline. + if (changeBaseline) { + baselineGone = grid(initNodes - 1).name(); + + ig.destroyCache(DEFAULT_CACHE_NAME); + awaitPartitionMapExchange(); + + stopAllGrids(); + + ig = (IgniteEx)startGridsMultiThreaded(initNodes - 1); + + ig.cluster().setBaselineTopology(ig.cluster().topologyVersion()); + + startGrid(initNodes - 1); + + assertEquals(initNodes - 1, ig.cluster().currentBaselineTopology().size()); + assertEquals(initNodes, ig.cluster().nodes().size()); + } + + // Optionally adds extra server. + if (extraServerNode) + startGrid(initNodes); + + injectTestSystemOut(); + + // Tests missing snapshot deletion. + if (customPath) { + assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "--src", + cstSnpsRoot.getAbsolutePath(), "wrongSnapshot")); + + assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "--src", + cstSnpsRoot.getAbsolutePath() + "_wrongPath", "testSnapshot")); + } + else + assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "wrongSnapshot")); + + String out = testOut.toString(); + + assertFalse(out.contains(SnapshotDeleteCommand.REMOVED_PREF)); + assertFalse(out.contains(SnapshotDeleteCommand.NODE_NOT_FOUND_PREF)); + assertTrue(out.contains(SnapshotDeleteCommand.NOT_FOUND)); + + testOut.reset(); + assertTrue(testOut.toString().isEmpty()); + + if (customPath) { + assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "--src", + cstSnpsRoot.getAbsolutePath(), "testSnapshot")); + } + else + assertEquals(EXIT_CODE_OK, execute(newCommandHandler(), "--snapshot", "delete", "testSnapshot")); + + out = testOut.toString(); + + if (changeBaseline) + assertTrue(out.contains(SnapshotDeleteCommand.MISSING_BASELINES + "[cnt=1]: " + baselineGone)); + + if (separatedWorkDir) { + // When the nodes use own separated work directory, we expect a strict result. + assertTrue(out.contains(SnapshotDeleteCommand.REMOVED_PREF + "[cnt=%d]:".formatted(initNodes))); + + if (extraServerNode) + assertTrue(out.contains(SnapshotDeleteCommand.NODE_NOT_FOUND_PREF + "[cnt=1]:")); + } + else { + // When nodes use a shared work directory, there is a race for the delete operation. One node can get faster + // than others and remove snapshot completely quickly. The others might not find snapshot files. We can be + // only sure that at least one node removes snapshot. + assertTrue(out.contains(SnapshotDeleteCommand.REMOVED_PREF) || out.contains(SnapshotDeleteCommand.UNSURED_DELETION_PREF)); + } + + assertFalse(out.contains("Snapshot not found on current server nodes")); + + assertTrue(waitForCondition(() -> !snpDir.exists(), getTestTimeout())); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java index 923ef50c9375f..4d90506b09889 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java @@ -162,6 +162,8 @@ import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckPartitionHashesResponse; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckProcessRequest; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotCheckResponse; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteRequest; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteResponse; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotFilesFailureMessage; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotFilesRequestMessage; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotHandlerResult; @@ -436,6 +438,8 @@ public CoreMessagesProvider() { register(SnapshotFilesFailureMessage.class); register(IncrementalSnapshotVerifyResult.class); register(IncrementalSnapshotAwareMessage.class); + register(SnapshotDeleteRequest.class); + register(SnapshotDeleteResponse.class); // [6300 - 6400]: Services messages. Most of them originally come from Discovery. msgIdx = 6300; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotCommand.java index deb5416b8e28b..8979958bd8fb6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotCommand.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotCommand.java @@ -28,7 +28,8 @@ public SnapshotCommand() { new SnapshotCancelCommand(), new SnapshotCheckCommand(), new SnapshotRestoreCommand(), - new SnapshotStatusCommand() + new SnapshotStatusCommand(), + new SnapshotDeleteCommand() ); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommand.java new file mode 100644 index 0000000000000..adf1731bbc736 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommand.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.management.snapshot; + +import java.util.Collection; +import java.util.Map; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteProcess; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteProcessResult; +import org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry; +import org.apache.ignite.internal.util.typedef.internal.U; + +/** + * Snapshot deletion command. + * + * @see SupportedFeatureRegistry#SNAPSHOT_DELETE_FEATURE + * @see SnapshotDeleteProcess + */ +public class SnapshotDeleteCommand extends AbstractSnapshotCommand { + /** */ + public static final String DESC = "Deletes the snapshot and all its incremental snapshots from all online server nodes"; + + /** */ + public static final String UNSURED_DELETION_PREF = "WARNING: the following nodes found snapshot data but might not " + + "remove it completely "; + + /** */ + public static final String REMOVED_PREF = "Snapshot removal is completed on "; + + /** */ + public static final String NODE_NOT_FOUND_PREF = "NOTE: the following nodes can't find any snapshot data, " + + "operation skipped "; + + /** */ + public static final String NOT_FOUND = "Snapshot not found on available server nodes."; + + /** */ + public static final String MISSING_BASELINES = "WARNING: the snapshot's baseline nodes with the following consistent " + + "ids are missing in current cluster "; + + /** + * {@inheritDoc} + */ + @Override public String description() { + return DESC; + } + + /** {@inheritDoc} */ + @Override public Class argClass() { + return SnapshotDeleteCommandArg.class; + } + + /** {@inheritDoc} */ + @Override public Class taskClass() { + return SnapshotDeleteTask.class; + } + + /** {@inheritDoc} */ + @Override public void printResult(SnapshotDeleteCommandArg arg, SnapshotDeleteProcessResult res, Consumer printer) { + boolean found = false; + + if (!res.uncompletedNodes().isEmpty()) { + found = true; + + printer.accept(UNSURED_DELETION_PREF + nodeIdPairsStrLst(res.uncompletedNodes())); + + printer.accept(""); + } + + if (!res.completedNodes().isEmpty()) { + found = true; + + printer.accept(REMOVED_PREF + nodeIdPairsStrLst(res.completedNodes())); + printer.accept(""); + } + + if (found) { + if (!res.emptyNodes().isEmpty()) + printer.accept(NODE_NOT_FOUND_PREF + nodeIdPairsStrLst(res.emptyNodes())); + + if (!res.absentBaselines().isEmpty()) + printer.accept(MISSING_BASELINES + nodeIdsStrLst(res.absentBaselines())); + } + else { + assert !res.emptyNodes().isEmpty(); + + printer.accept(NOT_FOUND); + } + } + + /** */ + private static String nodeIdPairsStrLst(Map uuids) { + return "[cnt=" + uuids.size() + "]: " + uuids.entrySet().stream() + .map(e -> e.getValue() + " [uuid=" + e.getKey() + ']') + .collect(Collectors.joining(", ")); + } + + /** */ + private static String nodeIdsStrLst(Collection uuids) { + return "[cnt=" + uuids.size() + "]: " + String.join(", ", uuids); + } + + /** {@inheritDoc} */ + @Override public String confirmationPrompt(SnapshotDeleteCommandArg arg) { + return "This operation will completely remove snapshot: '" + arg.snapshotName() + "' and all its incremental snapshots." + + U.nl() + U.nl() + + "The operation may require the snapshot administration permissions." + + U.nl() + U.nl() + + "Deletion of any data without (valid) snapshot metadata is prohibited." + + U.nl() + U.nl() + + "The operation cannot be reverted." + + U.nl() + U.nl() + + "NOTE: Snapshot data on offline server nodes will remain untouched."; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommandArg.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommandArg.java new file mode 100644 index 0000000000000..ee251f7afd741 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteCommandArg.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.management.snapshot; + +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; +import org.apache.ignite.internal.management.api.Argument; +import org.apache.ignite.internal.management.api.Positional; +import org.jetbrains.annotations.Nullable; + +/** */ +public class SnapshotDeleteCommandArg extends IgniteDataTransferObject { + /** */ + private static final long serialVersionUID = 0; + + /** */ + @Order(0) + @Positional + @Argument(description = "Snapshot name") + @Nullable String snapshotName; + + /** */ + @Order(1) + @Argument(example = "path/to/snapshots", optional = true, description = "Path to snapshot location directory. If not specified " + + "or specified a relative path, the default snapshot configuration directory will be used") + @Nullable String src; + + /** */ + public @Nullable String snapshotName() { + return snapshotName; + } + + /** */ + public void snapshotName(@Nullable String snapshotName) { + this.snapshotName = snapshotName; + } + + /** */ + public @Nullable String src() { + return src; + } + + /** */ + public void src(@Nullable String src) { + this.src = src; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteTask.java new file mode 100644 index 0000000000000..d4c84738ec429 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/snapshot/SnapshotDeleteTask.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.management.snapshot; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.apache.ignite.IgniteException; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.compute.ComputeJob; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteProcess; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDeleteProcessResult; +import org.apache.ignite.internal.processors.task.GridInternal; +import org.apache.ignite.internal.visor.VisorJob; +import org.apache.ignite.internal.visor.VisorOneNodeTask; +import org.apache.ignite.internal.visor.VisorTaskArgument; +import org.apache.ignite.resources.IgniteInstanceResource; + +import static org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry.SNAPSHOT_DELETE_FEATURE; + +/** + * @see IgniteSnapshotManager#deleteSnapshot(String, String) + * @see SnapshotDeleteProcess + */ +@GridInternal +public class SnapshotDeleteTask extends VisorOneNodeTask { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** */ + @IgniteInstanceResource + private IgniteEx ignite; + + /** {@inheritDoc} */ + @Override protected VisorJob job(SnapshotDeleteCommandArg arg) { + return new SnapshotDeleteJob(arg, debug); + } + + /** {@inheritDoc} */ + @Override protected Collection jobNodes(VisorTaskArgument arg) { + return super.jobNodes(arg); + } + + /** {@inheritDoc} */ + @Override protected Map map0( + List subgrid, + VisorTaskArgument arg + ) { + if (!ignite.context().rollingUpgrade().features().isActive(SNAPSHOT_DELETE_FEATURE)) + throw new IgniteException(SnapshotDeleteProcess.OP_REJECT_FEATURE_MSG); + + return super.map0(subgrid, arg); + } + + /** */ + private static class SnapshotDeleteJob extends SnapshotJob { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** + * @param arg Snapshot delete task argument. + * @param debug Flag indicating whether debug information should be printed into node log. + */ + protected SnapshotDeleteJob(SnapshotDeleteCommandArg arg, boolean debug) { + super(arg, debug); + } + + /** {@inheritDoc} */ + @Override protected SnapshotDeleteProcessResult run(SnapshotDeleteCommandArg arg) { + IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr(); + + return snpMgr.deleteSnapshot(arg.snapshotName(), arg.src()).get(); + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SharedFileTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SharedFileTree.java index 33df1ed636040..b4224b9077ef5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SharedFileTree.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SharedFileTree.java @@ -49,7 +49,7 @@ public class SharedFileTree { public static final String MARSHALLER_DIR = "marshaller"; /** Database default folder. */ - protected static final String DB_DIR = "db"; + public static final String DB_DIR = "db"; /** Root(work) directory. */ protected final File root; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotFileTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotFileTree.java index f53abc94ba480..ff9e58d5a9cde 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotFileTree.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotFileTree.java @@ -457,7 +457,6 @@ private NodeFileTree tempFileTree(GridKernalContext ctx) { return res; } - /** {@inheritDoc} */ @Override public String toString() { return S.toString(SnapshotFileTree.class, this); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 4d32bc1d38c09..521f2dd0a8514 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -60,6 +60,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; import java.util.function.BiFunction; @@ -344,6 +345,9 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter /** Snapshot validation distributed process. */ private final SnapshotCheckProcess checkSnpProc; + /** Distributed process to delete cluster snapshot. */ + private final SnapshotDeleteProcess deleteSnpProc; + /** Check previously performed snapshot operation and delete uncompleted files if we need. */ private final DistributedProcess endSnpProc; @@ -446,6 +450,8 @@ public IgniteSnapshotManager(GridKernalContext ctx) { checkSnpProc = new SnapshotCheckProcess(ctx); + deleteSnpProc = new SnapshotDeleteProcess(ctx); + // Manage remote snapshots. snpRmtMgr = new SequentialRemoteSnapshotManager(); } @@ -664,6 +670,7 @@ public IgniteSnapshotManager(GridKernalContext ctx) { restoreCacheGrpProc.interrupt(stopErr); checkSnpProc.interrupt(stopErr); + deleteSnpProc.interrupt(stopErr); // Try stop all snapshot processing if not yet. for (AbstractSnapshotFutureTask sctx : locSnpTasks.values()) @@ -701,46 +708,105 @@ public IgniteSnapshotManager(GridKernalContext ctx) { /** * @param snpDir Snapshot dir. */ - public void deleteSnapshot(File snpDir) { + public void deleteLocalSnapshot(File snpDir) { if (!snpDir.exists()) return; if (!snpDir.isDirectory()) return; - deleteSnapshot(new SnapshotFileTree( + var sft = new SnapshotFileTree( cctx.kernalContext(), snpDir.getName(), snpDir.getParent(), ft.folderName(), - pdsSettings.consistentId().toString())); + pdsSettings.consistentId().toString() + ); + + deleteLocalSnapshot(sft); } - /** */ - public void deleteSnapshot(SnapshotFileTree sft) { - try { - U.delete(sft.binaryMeta()); - sft.allStorages().forEach(U::delete); - U.delete(sft.meta()); + /** + * Tries to delete local snapshot data. + * + * @param sft Snapshot file tree. + * @return A pair of {@code boolean} values. The first indicates whether snapshot was completely deleted. The second + * indicates whether the snapshot was found at all. + */ + public T2 deleteLocalSnapshot(SnapshotFileTree sft) { + T2 res = new T2<>(false, false); + + if (sft.root().exists()) + res.set2(true); + else { + for (File storage : sft.allStorages().toList()) { + if (storage.exists()) + res.set2(true); + } + } + + // Not found at all - nothing to delete. + if (!res.get2()) + return res; + + // Assume we'll successed. + res.set1(true); - deleteDirectory(sft.binaryMetaRoot()); - deleteDirectory(sft.marshaller()); + // The 'exists' checks are for a concurrent deletion when nodes share their working and snapshot directories. + // Nodes may steal removal jobs and the files aren't synchronized. There are gaps between and `exists()` and `delete()`. + // We try to delete first. If snapshot data wasn't deleted because it doesn't already exist is not a removal error here. + try { + if (!sft.meta().delete() && sft.meta().exists()) + res.set1(false); + + for (var s : sft.allStorages().toList()) { + if (!deleteDirectory(s) && s.exists()) + res.set1(false); + + /** + * Deletes "db" directory also beyond the snapshot root in the case when + * {@link DataStorageConfiguration#setExtraSnapshotPaths(String...)} are configured. + */ + s = s.getParentFile(); + + if (!deleteDirectory(s) && s.exists()) + res.set1(false); + + s = s.getParentFile(); + + /** + * Deletes the snapshot root directory beyond the snapshot root in the case when + * {@link DataStorageConfiguration#setExtraSnapshotPaths(String...)} are configured. + */ + if (!sft.root().equals(s) && !s.delete() && s.exists()) + res.set1(false); + } - // Delete parent dir which is {snapshot_root}/db if empty. - sft.marshaller().getParentFile().delete(); - // Delete root dir which is {snapshot_root} if empty. - sft.root().delete(); + for (File p : F.asList(sft.binaryMeta(), sft.binaryMetaRoot(), sft.marshaller(), sft.incrementsRoot(), sft.root())) { + if (!deleteDirectory(p) && p.exists()) + res.set1(false); + } } - catch (IOException e) { - throw new IgniteException(e); + catch (Exception e) { + log.warning("Failed to delete local snapshot [snpName=" + sft.name() + ']', e); + + res.set1(false); } + + return res; } /** Concurrently traverse the directory and delete all files. */ - private void deleteDirectory(File dir) throws IOException { - Files.walkFileTree(dir.toPath(), new SimpleFileVisitor() { + private boolean deleteDirectory(File dir) throws IOException { + var res = new AtomicBoolean(true); + + Files.walkFileTree(dir.toPath(), new SimpleFileVisitor<>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { - U.delete(file); + File f0 = file.toFile(); + + // Check for a concurrent deletion. + if (!U.delete(f0) && f0.exists()) + res.set(false); return FileVisitResult.CONTINUE; } @@ -751,7 +817,11 @@ private void deleteDirectory(File dir) throws IOException { } @Override public FileVisitResult postVisitDirectory(Path dir, IOException e) { - dir.toFile().delete(); + File f0 = dir.toFile(); + + // Check for a concurrent deletion. + if (!f0.delete() && f0.exists()) + res.set(false); if (log.isInfoEnabled() && e != null) log.info("Snapshot directory cleaned with an exception [dir=" + dir + ", e=" + e.getMessage() + ']'); @@ -759,6 +829,8 @@ private void deleteDirectory(File dir) throws IOException { return FileVisitResult.CONTINUE; } }); + + return res.get(); } /** @@ -813,6 +885,11 @@ private IgniteInternalFuture initLocalSnapshotStartSt "re-encryption process is not finished yet.")); } + if (cctx.snapshotMgr().isSnapshotDeleting(req.snapshotName(), req.snapshotPath())) { + return new GridFinishedFuture<>(new IgniteCheckedException("Snapshot operation has been rejected. Snapshot " + + "'%s' is being deleted.".formatted(req.snapshotName()))); + } + List grpIds = new ArrayList<>(F.viewReadOnly(req.groups(), CU::cacheId)); Collection comprGrpIds = F.view(grpIds, i -> { CacheGroupDescriptor desc = cctx.cache().cacheGroupDescriptor(i); @@ -1282,7 +1359,7 @@ private IgniteInternalFuture initLocalSnapshotEndStag if (snpStartReq.incremental()) U.delete(snpOp.snapshotFileTree().incrementalSnapshotFileTree(snpStartReq.incrementIndex()).root()); else - deleteSnapshot(snpOp.snapshotFileTree()); + deleteLocalSnapshot(snpOp.snapshotFileTree()); } else if (!F.isEmpty(endReq.warnings())) { // Pass the warnings further to the next stage for the case when snapshot started from not coordinator. @@ -1446,6 +1523,27 @@ public boolean isSnapshotChecking(String snpName) { return checkSnpProc.isSnapshotChecking(snpName); } + /** + * @return {@code True} if a snapshot {@code snpName} delete operation is in progress. + */ + public boolean isSnapshotDeleting(String snpName, @Nullable String snpPath) { + return deleteSnpProc.isDeleting(snpName, snpPath); + } + + /** + * Deletes the cluster-wide snapshot with the given name. + *

+ * The operation is rejected if a concurrent snapshot operation (create, restore, check, etc...) is in progress + * for the snapshot. + * + * @param name Snapshot name. + * @param snpPath Snapshot directory path. If {@code null}, the default configured snapshot directory will be used. + * @return Future which will be completed when the snapshot is deleted on all the online server nodes. + */ + public IgniteFuture deleteSnapshot(String name, @Nullable String snpPath) { + return deleteSnpProc.start(name, snpPath); + } + /** * Sets the streamer warning flag to current snapshot process if it is active. */ @@ -1478,8 +1576,11 @@ public boolean isRestoring() { * @param snpName Snapshot name. * @return {@code True} if the snapshot restore operation from the specified snapshot is in progress locally. */ - public boolean isRestoring(String snpName) { - return snpName.equals(restoreCacheGrpProc.restoringSnapshotName()); + public boolean isRestoring(@Nullable String snpName) { + if (snpName == null) + return false; + + return snpName.equalsIgnoreCase(restoreCacheGrpProc.restoringSnapshotName()); } /** @@ -1830,6 +1931,21 @@ public T readFromFile(File smf) throws IgniteCheckedException, IOException { * local node will be placed on the first place. */ public List readSnapshotMetadatas(SnapshotFileTree sft) { + return readSnapshotMetadatas(sft, true); + } + + /** + * Note, there can be snapshots from other nodes. + * This method will read all metadata. + * Some instances can return {@link SnapshotMetadata#folderName()} and {@link SnapshotMetadata#consistentId()} that differs from local. + * + * @param sft Snapshot file tree. + * @param failIfCantRead If {@code true}, throws an exception if cannot read a metadata file. + * @return List of snapshot metadata for the given snapshot name on local node. + * If snapshot has been taken from local node the snapshot metadata for given + * local node will be placed on the first place. + */ + public List readSnapshotMetadatas(SnapshotFileTree sft, boolean failIfCantRead) { if (!(sft.root().exists() && sft.root().isDirectory())) return Collections.emptyList(); @@ -1841,8 +1957,8 @@ public List readSnapshotMetadatas(SnapshotFileTree sft) { Map metasMap = new HashMap<>(); SnapshotMetadata prev = null; - try { - for (File smf : smfs) { + for (File smf : smfs) { + try { SnapshotMetadata curr = readSnapshotMetadata(smf); if (prev != null && !prev.sameSnapshot(curr)) { @@ -1854,9 +1970,14 @@ public List readSnapshotMetadatas(SnapshotFileTree sft) { prev = curr; } - } - catch (IgniteCheckedException | IOException e) { - throw new IgniteException(e); + catch (Exception e) { + String err = "Fail to read snapshot metadata [meta=" + smf + ']'; + + if (failIfCantRead) + throw new IgniteException(err, e); + else + log.error(err, e); + } } SnapshotMetadata currNodeSmf = metasMap.remove(cctx.localNode().consistentId().toString()); @@ -2032,7 +2153,7 @@ public IgniteFutureImpl createSnapshot( if (!incremental && snpExists) { throw new IgniteException("Create snapshot request has been rejected. " + - "Snapshot with given name already exists on local node."); + "Snapshot with given name already exists on local node or the path is not empty."); } if (incremental) { @@ -2263,7 +2384,7 @@ public IgniteFutureImpl restoreSnapshot( if (SnapshotFileTree.incrementSnapshotDir(snpDir)) U.delete(snpDir); else - deleteSnapshot(snpDir); + deleteLocalSnapshot(snpDir); } if (log.isInfoEnabled()) { @@ -3957,7 +4078,7 @@ public LocalSnapshotSender(SnapshotFileTree sft) { log.info("The Local snapshot sender closed. All resources released [dbNodeSnpDir=" + sft.nodeStorage() + ']'); } else { - deleteSnapshot(sft); + deleteLocalSnapshot(sft); if (log.isDebugEnabled()) log.debug("Local snapshot sender closed due to an error occurred: " + th.getMessage()); @@ -4317,7 +4438,7 @@ public CancelSnapshotCallable(UUID reqId, String snpName) { } /** {@inheritDoc} */ - @Override public Boolean call() throws Exception { + @Override public Boolean call() { if (reqId != null) return ignite.context().cache().context().snapshotMgr().cancelLocalSnapshotOperations(reqId); else { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotProcessor.java index 9ba4277dd92fa..c1bdb9d69a97b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotProcessor.java @@ -138,7 +138,8 @@ void process( WALRecord rec = walRec.getValue(); - if (rec.type() == CLUSTER_SNAPSHOT && ((ClusterSnapshotRecord)rec).clusterSnapshotName().equals(sft.name())) { + // A filesystem might not support the character case of directory or file name. + if (rec.type() == CLUSTER_SNAPSHOT && ((ClusterSnapshotRecord)rec).clusterSnapshotName().equalsIgnoreCase(sft.name())) { startIdx = walRec.getKey().index(); break; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java index 2f19878617a15..77c7c8eb0b743 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCheckProcess.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -32,6 +33,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteIllegalStateException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.GridKernalContext; @@ -131,7 +133,7 @@ private IgniteInternalFuture reduceValidatePartsAndFinish( return new GridFinishedFuture<>(); try { - contexts.remove(ctx.req.snapshotName()); + contexts.remove(ctx.req.snapshotName().toLowerCase(Locale.ROOT)); GridFutureAdapter clusterOpFut = clusterOpFuts.get(reqId); @@ -473,7 +475,7 @@ private static Exception asException(Throwable th) { private @Nullable SnapshotCheckContext context(@Nullable String snpName, UUID reqId) { return snpName == null ? contexts.values().stream().filter(ctx0 -> ctx0.req.requestId().equals(reqId)).findFirst().orElse(null) - : contexts.get(snpName); + : contexts.get(snpName.toLowerCase(Locale.ROOT)); } /** Phase 1 beginning: prepare, collect and check local metas. */ @@ -487,12 +489,17 @@ private IgniteInternalFuture prepareAndCheckMetas(UUID ig if (nodeStopping) return new GridFinishedFuture<>(new NodeStoppingException("The node is stopping: " + kctx.localNodeId())); - ctx = contexts.computeIfAbsent(req.snapshotName(), snpName -> new SnapshotCheckContext(req)); + ctx = contexts.computeIfAbsent(req.snapshotName().toLowerCase(Locale.ROOT), snpName -> new SnapshotCheckContext(req)); } if (!ctx.req.requestId().equals(req.requestId())) { - return new GridFinishedFuture<>(new IllegalStateException("Validation of snapshot '" + req.snapshotName() - + "' has already started [ctx=" + ctx + ']')); + return new GridFinishedFuture<>(new IgniteIllegalStateException("Validation of snapshot '" + req.snapshotName() + + "' has already started [req=" + req + ']')); + } + + if (kctx.cache().context().snapshotMgr().isSnapshotDeleting(req.snapshotName(), req.snapshotPath())) { + return new GridFinishedFuture<>(new IgniteIllegalStateException("Snapshot '" + req.snapshotName() + + "' is being deleted [req=" + req + ']')); } // Excludes non-baseline initiator. @@ -588,7 +595,7 @@ private void reducePreparationAndMetasCheck( if (ctx != null) { unregisterMetrics(ctx.req.snapshotName()); - contexts.remove(ctx.req.snapshotName()); + contexts.remove(ctx.req.snapshotName().toLowerCase(Locale.ROOT)); } if (clusterOpFut != null) @@ -694,7 +701,7 @@ public IgniteInternalFuture start( /** @return {@code True} if snapshot with specified name is checking. */ boolean isSnapshotChecking(String snpName) { - return contexts.get(snpName) != null; + return contexts.get(snpName.toLowerCase(Locale.ROOT)) != null; } /** @return {@code True} if node with the provided id is in the cluster and is a baseline node. {@code False} otherwise. */ @@ -770,7 +777,7 @@ private static final class SnapshotCheckContext { */ @Nullable private volatile List metas; - /** Map of snapshot pathes per consistent id for {@link #metas}. */ + /** Map of snapshot paths per consistent id for {@link #metas}. */ @GridToStringInclude @Nullable private Map locFileTree; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcess.java new file mode 100644 index 0000000000000..89696114da503 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcess.java @@ -0,0 +1,404 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.ignite.IgniteIllegalStateException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.NodeStoppingException; +import org.apache.ignite.internal.processors.cache.persistence.filename.SnapshotFileTree; +import org.apache.ignite.internal.util.distributed.DistributedProcess; +import org.apache.ignite.internal.util.future.GridCompoundFuture; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.future.IgniteFutureImpl; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.T2; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteFuture; +import org.apache.ignite.lang.IgniteReducer; +import org.jetbrains.annotations.Nullable; + +import static org.apache.ignite.internal.processors.rollingupgrade.feature.SupportedFeatureRegistry.SNAPSHOT_DELETE_FEATURE; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.DELETE_SNAPSHOT; +import static org.apache.ignite.plugin.security.SecurityPermission.ADMIN_SNAPSHOT; + +/** + * Distributed process to delete a cluster snapshot. The operation is rejected if any concurrent snapshot operation is + * active. + */ +public class SnapshotDeleteProcess { + /** Reject operation messages. */ + private static final String OP_REJECT_MSG = "Snapshot deletion was rejected. "; + + /** */ + public static final String OP_REJECT_FEATURE_MSG = OP_REJECT_MSG + "The snapshot deletion feature isn't activated yet."; + + /** */ + private static final String CONCURRENT_OP_PREF = "Snapshot with the same name is being "; + + /** */ + public static final String BEING_CREATED_PREF = CONCURRENT_OP_PREF + "created "; + + /** */ + public static final String BEING_RESTORED_PREF = CONCURRENT_OP_PREF + "restored "; + + /** */ + public static final String BEING_CHECKED_PREF = CONCURRENT_OP_PREF + "checked "; + + /** Kernal context. */ + private final GridKernalContext kctx; + + /** Logger. */ + private final IgniteLogger log; + + /** */ + private volatile boolean interrupted; + + /** Cluster-wide operation futures per request id on certain node. */ + private final Map> clusterOpFuts = new ConcurrentHashMap<>(); + + /** Process requests per snapshot name on each server node. */ + private final Set requests = ConcurrentHashMap.newKeySet(); + + /** The distributed process. */ + private final DistributedProcess distrProc; + + /** + * @param ctx Kernal context. + */ + public SnapshotDeleteProcess(GridKernalContext ctx) { + kctx = ctx; + + log = ctx.log(getClass()); + + distrProc = new DistributedProcess<>(ctx, DELETE_SNAPSHOT, this::deletePhase, this::reducePhase); + } + + /** + * Starts the cluster snapshot delete process. + * + * @param snpName Snapshot name. + * @param snpPath Snapshot directory path (optional). + * @return Future that will be completed when the snapshot is deleted. + */ + public IgniteFuture start(String snpName, @Nullable String snpPath) { + var clusterOpFut = new GridFutureAdapter(); + + if (!kctx.rollingUpgrade().features().isActive(SNAPSHOT_DELETE_FEATURE)) { + clusterOpFut.onDone(new IgniteIllegalStateException(OP_REJECT_FEATURE_MSG)); + + return new IgniteFutureImpl<>(clusterOpFut); + } + + UUID reqId = UUID.randomUUID(); + + clusterOpFut.listen(fut -> clusterOpFuts.remove(reqId)); + + try { + if (interrupted || kctx.isStopping()) + throw new NodeStoppingException("Failed to start snapshot delete process: node is stopping."); + + clusterOpFuts.put(reqId, clusterOpFut); + + SnapshotDeleteRequest req = new SnapshotDeleteRequest(reqId, snpName, snpPath); + + distrProc.start(reqId, req); + } + catch (Throwable t) { + log.error("Failed to start distributed delete snapshot process [snpName=" + snpName + ", snpPath=" + snpPath + ']', t); + + clusterOpFut.onDone(t); + } + + return new IgniteFutureImpl<>(clusterOpFut); + } + + /** */ + private IgniteInternalFuture deletePhase(UUID ignored, SnapshotDeleteRequest req) { + if (interrupted || kctx.isStopping()) { + return new GridFinishedFuture<>(new NodeStoppingException(OP_REJECT_MSG + + " Node is stopping [req=" + req + ']')); + } + + if (kctx.cluster().get().localNode().isClient()) + return new GridFinishedFuture<>(new SnapshotDeleteResponse()); + + kctx.security().authorize(ADMIN_SNAPSHOT); + + IgniteSnapshotManager snpMgr = kctx.cache().context().snapshotMgr(); + + var curCreateRq = snpMgr.currentCreateRequest(); + + if (curCreateRq != null && curCreateRq.snpName.equalsIgnoreCase(req.snpName)) { + return new GridFinishedFuture<>(new IgniteIllegalStateException(OP_REJECT_MSG + BEING_CREATED_PREF + + "[req=" + req + ']')); + } + + if (snpMgr.isRestoring(req.snpName)) { + return new GridFinishedFuture<>(new IgniteIllegalStateException(OP_REJECT_MSG + BEING_RESTORED_PREF + + "[req=" + req + ']')); + } + + if (snpMgr.isSnapshotChecking(req.snpName)) { + return new GridFinishedFuture<>(new IgniteIllegalStateException(OP_REJECT_MSG + BEING_CHECKED_PREF + + "[req=" + req + ']')); + } + + try { + File path = resolvePath(req.snpPath); + + req.resolvedPath = path; + + if (!requests.add(req)) { + return new GridFinishedFuture<>(new IgniteIllegalStateException("Deletion of the snapshot has already " + + "started [req=" + req + ']')); + } + + SnapshotFileTree snpFiles = new SnapshotFileTree(kctx, req.snpName, path.getAbsolutePath()); + + // We need to find and read snapshot metas to ensure the content is a snapshot. Also, the metas contain + // initial cluster topology and actual snapshot folder names. + List locMetas = kctx.cache().context().snapshotMgr().readSnapshotMetadatas(snpFiles, false); + + if (locMetas.isEmpty()) { + requests.remove(req); + + log.warning("Snapshot deletion won't process, no snapshot metadata found [req=" + req + ']'); + + return new GridFinishedFuture<>(new SnapshotDeleteResponse(SnapshotDeleteResponse.DeleteStatus.NOT_FOUND, null)); + } + + // Future to delete snapshot contents according to snapshot metadatas. + GridCompoundFuture resultFut = + new GridCompoundFuture<>(new MetaFuturesReducer()); + + resultFut.listen(fut -> requests.remove(req)); + + File path0 = path; + + for (var meta : locMetas) { + GridFutureAdapter perMetaFut = new GridFutureAdapter<>(); + + kctx.pools().getSnapshotExecutorService().submit(() -> { + try { + // Read file tree of the snapshot. + var byMetaSft = new SnapshotFileTree( + kctx.config(), + kctx.pdsFolderResolver().fileTree(), + req.snpName, + req.snpPath == null ? null : path0.getAbsolutePath(), + meta.folderName(), + meta.consId + ); + + T2 deleted = snpMgr.deleteLocalSnapshot(byMetaSft); + + SnapshotDeleteResponse.DeleteStatus status; + + // If found. + if (deleted.get2()) { + if (deleted.get1() && log.isInfoEnabled()) + log.info("Snapshot successfully deleted [req=" + req + ']'); + else if (!deleted.get1()) + log.warning("Snapshot deleted not completely [req=" + req + ']'); + + status = deleted.get1() + ? SnapshotDeleteResponse.DeleteStatus.DELETED + : SnapshotDeleteResponse.DeleteStatus.PARTLY; + } + else { + if (log.isInfoEnabled()) + log.info("Snapshot not found to delete [req=" + req + ']'); + + status = SnapshotDeleteResponse.DeleteStatus.NOT_FOUND; + } + + perMetaFut.onDone(new SnapshotDeleteResponse(status, meta.bltNodes)); + } + catch (Throwable e) { + perMetaFut.onDone(e); + } + }); + + resultFut.add(perMetaFut); + } + + resultFut.markInitialized(); + + if (log.isInfoEnabled()) + log.info("Deletion of snapshot initialized [req=" + req + ']'); + + return resultFut; + } + catch (Throwable t) { + requests.remove(req); + + log.warning("An error occurred during snapshot deletion [req=" + req + ']', t); + + return new GridFinishedFuture<>(t); + } + } + + /** */ + private File resolvePath(@Nullable String path) throws IOException { + return path == null + ? kctx.pdsFolderResolver().fileTree().snapshotsRoot() + : new File(path).getCanonicalFile(); + } + + /** */ + private void reducePhase(UUID reqId, Map results, Map errors) { + var clusterOpFut = clusterOpFuts.get(reqId); + + if (clusterOpFut == null) + return; + + assert clusterOpFut != null; + + try { + var errP = F.isEmpty(errors) ? null : F.first(errors.entrySet()); + + if (errP != null) { + log.warning("Snapshot deletion finished with an error [reqId=" + reqId + ", nodeId=" + + errP.getKey() + ", err='" + errP.getValue().getMessage() + "']", errP.getValue()); + + clusterOpFut.onDone(errP.getValue()); + + return; + } + + Map completedNodes = U.newHashMap(results.size()); + Map uncompletedNodes = U.newHashMap(results.size()); + Map emptyNodes = U.newHashMap(results.size()); + var snpNodes = new HashSet(); + + results.forEach((nodeId, nodeRes) -> { + if (!F.isEmpty(nodeRes.nodeIds)) + snpNodes.addAll(nodeRes.nodeIds); + + if (nodeRes.status != null) { + switch (nodeRes.status) { + case NOT_FOUND: + emptyNodes.put(nodeId, consistentId(nodeId)); + break; + case DELETED: + completedNodes.put(nodeId, consistentId(nodeId)); + break; + case PARTLY: + uncompletedNodes.put(nodeId, consistentId(nodeId)); + break; + default: + throw new IgniteIllegalStateException("Unknown snapshot deletion node result, [nodeRes=" + + nodeRes + ", nodeId=" + nodeId + ']'); + } + } + }); + + Collection curBaseline = kctx.discovery().discoCache().aliveBaselineNodes(); + + if (!F.isEmpty(curBaseline)) + curBaseline.stream().map(bn -> bn.consistentId().toString()).toList().forEach(snpNodes::remove); + + clusterOpFut.onDone(new SnapshotDeleteProcessResult(completedNodes, uncompletedNodes, emptyNodes, snpNodes)); + } + catch (Throwable t) { + clusterOpFut.onDone(t); + } + } + + /** */ + private String consistentId(UUID nodeId) { + var node = kctx.discovery().node(nodeId); + + if (node == null) + node = kctx.discovery().historicalNode(nodeId); + + return node == null ? "" : node.consistentId().toString(); + } + + /** */ + public boolean isDeleting(String snpName, @Nullable String snpPath) { + var rq = new SnapshotDeleteRequest(null, snpName, snpPath); + + try { + rq.resolvedPath = resolvePath(rq.snpPath); + } + catch (IOException ignored) { + return false; + } + + return requests.contains(rq); + } + + /** + * @param err The interrupt reason. + */ + void interrupt(Throwable err) { + interrupted = true; + + clusterOpFuts.forEach((reqId, clusterOpFut) -> clusterOpFut.onDone(err)); + + clusterOpFuts.clear(); + } + + /** */ + private static class MetaFuturesReducer implements IgniteReducer { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** */ + private @Nullable SnapshotDeleteResponse.DeleteStatus status; + + /** */ + private final Collection nodeIds = new HashSet<>(); + + /** {@inheritDoc} */ + @Override public boolean collect(SnapshotDeleteResponse res) { + assert res != null; + + synchronized (this) { + if (!F.isEmpty(res.nodeIds)) + nodeIds.addAll(res.nodeIds); + + if (status == null || status == res.status) + status = res.status; + else + status = SnapshotDeleteResponse.DeleteStatus.PARTLY; + } + + return true; + } + + /** {@inheritDoc} */ + @Override public SnapshotDeleteResponse reduce() { + return new SnapshotDeleteResponse(status == null ? SnapshotDeleteResponse.DeleteStatus.NOT_FOUND : status, nodeIds); + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcessResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcessResult.java new file mode 100644 index 0000000000000..5bf62b5373cc1 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteProcessResult.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.UUID; +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; +import org.apache.ignite.plugin.extensions.communication.MessageFactory; + +/** Result of {@link SnapshotDeleteProcess}. */ +public final class SnapshotDeleteProcessResult extends IgniteDataTransferObject { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** Nodes which found snapshot data and completely removed it. */ + @Order(0) + Map completedNodes; + + /** Nodes which found snapshot data but didn't remove it completely. */ + @Order(1) + Map uncompletedNodes; + + /** Server nodes which didn't find any snapshot data. */ + @Order(2) + Map emptyNodes; + + /** Snapshot's baseline nodes missing in current cluster. */ + @Order(3) + Collection absentBaselines; + + /** Default constructor for {@link MessageFactory}. */ + public SnapshotDeleteProcessResult() { + // No-op. + } + + /** */ + public SnapshotDeleteProcessResult( + Map completedNodes, + Map uncompletedNodes, + Map emptyNodes, + Collection absentBaselines + ) { + this.completedNodes = completedNodes; + this.uncompletedNodes = uncompletedNodes; + this.emptyNodes = emptyNodes; + this.absentBaselines = absentBaselines; + } + + /** */ + public Map completedNodes() { + return Collections.unmodifiableMap(completedNodes); + } + + /** */ + public Map uncompletedNodes() { + return Collections.unmodifiableMap(uncompletedNodes); + } + + /** */ + public Map emptyNodes() { + return Collections.unmodifiableMap(emptyNodes); + } + + /** */ + public Collection absentBaselines() { + return Collections.unmodifiableCollection(absentBaselines); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteRequest.java new file mode 100644 index 0000000000000..4d227a8b2075c --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteRequest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.io.File; +import java.util.Locale; +import java.util.Objects; +import java.util.UUID; +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.util.tostring.GridToStringExclude; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.plugin.extensions.communication.MessageFactory; +import org.jetbrains.annotations.Nullable; + +/** + * Cluster snapshot delete distributed process request. + * + * @see SnapshotDeleteProcess + */ +public class SnapshotDeleteRequest implements Message { + /** Request ID. */ + @Order(0) + UUID reqId; + + /** Snapshot name. */ + @Order(1) + String snpName; + + /** Snapshot directory path. */ + @Order(2) + @Nullable String snpPath; + + /** Resolved absolute path. Transient */ + @GridToStringExclude + @Nullable File resolvedPath; + + /** Default constructor for {@link MessageFactory}. */ + public SnapshotDeleteRequest() { + // No-op. + } + + /** + * @param reqId Request ID. + * @param snpName Snapshot name. + * @param snpPath Snapshot directory path. + */ + SnapshotDeleteRequest(UUID reqId, String snpName, @Nullable String snpPath) { + this.reqId = reqId; + this.snpName = snpName; + this.snpPath = snpPath; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) + return false; + + SnapshotDeleteRequest other = (SnapshotDeleteRequest)o; + + return snpName.equalsIgnoreCase(other.snpName) && Objects.equals(resolvedPath, other.resolvedPath); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return Objects.hash(snpName.toLowerCase(Locale.ROOT), resolvedPath); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(SnapshotDeleteRequest.class, this); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteResponse.java new file mode 100644 index 0000000000000..24a7878e3dbf4 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDeleteResponse.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.util.Collection; +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.plugin.extensions.communication.MessageFactory; +import org.jetbrains.annotations.Nullable; + +/** + * Single-node result of the snapshot deletion distributed process. + * + * @see SnapshotDeleteProcess + */ +public class SnapshotDeleteResponse implements Message { + /** {@code null} for client node. */ + @Order(0) + @Nullable SnapshotDeleteResponse.DeleteStatus status; + + /** Snapshot's node ids. */ + @Order(1) + @Nullable Collection nodeIds; + + /** Default constructor for {@link MessageFactory}. */ + public SnapshotDeleteResponse() { + // No-op. + } + + /** {@code null} for client node. */ + SnapshotDeleteResponse(DeleteStatus status, Collection nodeIds) { + this.status = status; + this.nodeIds = nodeIds; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(SnapshotDeleteResponse.class, this); + } + + /** */ + enum DeleteStatus { + /** Snapshot found and completely deleted. */ + DELETED, + + /** Snapshot found but some files or directories might not be deleted (locked). */ + PARTLY, + + /** Snapshot not found. */ + NOT_FOUND; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTask.java index 643838ac3c445..4a8fea2b43b73 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataVerificationTask.java @@ -35,7 +35,10 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory; import org.apache.ignite.internal.util.typedef.F; -/** Snapshot task to verify snapshot metadata on the baseline nodes for given snapshot name. */ +/** + * Snapshot task to verify snapshot metadata on the baseline nodes for given snapshot name. + * TODO : Revise in https://issues.apache.org/jira/browse/IGNITE-29062 + */ public class SnapshotMetadataVerificationTask implements Supplier> { /** */ private final IgniteEx ignite; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index 77f56f4d4cc7f..5c4804208fdcb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -659,6 +659,11 @@ private IgniteInternalFuture prepare(UUID igno if (snpMgr.isSnapshotCreating()) throw new IgniteCheckedException(OP_REJECT_MSG + "A cluster snapshot operation is in progress."); + if (snpMgr.isSnapshotDeleting(req.snapshotName(), req.snapshotPath())) { + throw new IgniteException(OP_REJECT_MSG + "A snapshot '" + req.snapshotName() + + "' delete operation is in progress."); + } + if (ctx.encryption().isMasterKeyChangeInProgress()) { return new GridFinishedFuture<>(new IgniteCheckedException(OP_REJECT_MSG + "Master key changing " + "process is not finished yet.")); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/metastorage/persistence/DistributedMetaStorageImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/metastorage/persistence/DistributedMetaStorageImpl.java index 575300772a905..bf7f6f34c2566 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/metastorage/persistence/DistributedMetaStorageImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/metastorage/persistence/DistributedMetaStorageImpl.java @@ -103,7 +103,7 @@ *

*

Whole updates history until some point in the past is stored along with the data, so when an outdated node * connects to the cluster it will receive all the missing data and apply it locally. Listeners will also be invoked - * after such updates. If there's not enough history stored or joining node is clear then it'll receive shapshot of + * after such updates. If there's not enough history stored or joining node is clear then it'll receive snapshot of * distributed metastorage (usually called {@code fullData} in code) so there won't be inconsistencies. *

* diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java index 7b3e55b85d3c4..a14e52331cda0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/SupportedFeatureRegistry.java @@ -93,4 +93,7 @@ public class SupportedFeatureRegistry { /** */ public static final IgniteFeature ROLLING_UPGRADE_FEATURE = new IgniteCoreFeature(0); + + /** */ + public static final IgniteFeature SNAPSHOT_DELETE_FEATURE = new IgniteCoreFeature(1); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java index 3d0ce062f4f75..9c428e0fa748c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/distributed/DistributedProcess.java @@ -519,5 +519,10 @@ public enum DistributedProcessType { * Cluster version finalization abort process. */ RU_ABORT_VERSION_FINALIZATION, + + /** + * Delete snapshot procedure. + */ + DELETE_SNAPSHOT } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java index 8857a4ddd656e..a637cceadffa7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java @@ -36,7 +36,9 @@ import java.util.Queue; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; @@ -90,6 +92,7 @@ import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; @@ -98,6 +101,8 @@ import org.apache.ignite.lang.IgniteFutureCancelledException; import org.apache.ignite.lang.IgniteFutureTimeoutException; import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.plugin.AbstractTestPluginProvider; +import org.apache.ignite.plugin.PluginContext; import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi; @@ -173,6 +178,12 @@ protected Function valueBuilder() { return valBuilder; } + /** */ + protected @Nullable AbstractTestPluginProvider pluginProvider; + + /** */ + protected boolean removeAllWorkingDirectories = true; + /** Enable encryption of all caches in {@code IgniteConfiguration} before start. */ @Parameterized.Parameter public boolean encryption; @@ -182,7 +193,7 @@ protected Function valueBuilder() { public boolean onlyPrimary; /** Parameters. */ - @Parameterized.Parameters(name = "encryption={0}, onlyPrimay={1}") + @Parameterized.Parameters(name = "encryption={0}, onlyPrimary={1}") public static Collection params() { List res = new ArrayList<>(); @@ -217,6 +228,9 @@ protected static Collection encryptionParameters() { if (cfg.isClientMode()) return cfg; + if (pluginProvider != null) + cfg.setPluginProviders(pluginProvider); + return cfg.setConsistentId(igniteInstanceName) .setDataStorageConfiguration(new DataStorageConfiguration() .setDefaultDataRegionConfiguration(new DataRegionConfiguration() @@ -283,6 +297,25 @@ public void afterTestSnapshot() throws Exception { cleanPersistenceDir(); } + /** {@inheritDoc} */ + @Override protected void cleanPersistenceDir() throws Exception { + super.cleanPersistenceDir(); + + if (!removeAllWorkingDirectories()) + return; + + // Clean all: also separated snapshot working directories and custom snapshot paths. + try (DirectoryStream files = newDirectoryStream(Paths.get(U.defaultWorkDirectory()))) { + for (Path path : files) + U.delete(path); + } + } + + /** @return Whether a test removes all content of the working directory in {@link #cleanPersistenceDir()}. */ + protected boolean removeAllWorkingDirectories() { + return removeAllWorkingDirectories; + } + /** * @param evts Events to check. * @throws IgniteInterruptedCheckedException If interrupted. @@ -701,8 +734,8 @@ protected void checkSnapshot(String snpName, String snpPath) { * @param ignite Ignite instance. * @return Snapshot manager related to given ignite instance. */ - public static IgniteSnapshotManager snp(IgniteEx ignite) { - return ignite.context().cache().context().snapshotMgr(); + public static IgniteSnapshotManager snp(Ignite ignite) { + return ((IgniteEx)ignite).context().cache().context().snapshotMgr(); } /** @@ -745,7 +778,7 @@ protected static List setBlockingSnapshotExecutor(List execs = new ArrayList<>(); for (Ignite grid : grids) { - IgniteSnapshotManager mgr = snp((IgniteEx)grid); + IgniteSnapshotManager mgr = snp(grid); Function old = mgr.localSnapshotSenderFactory(); BlockingExecutor block = new BlockingExecutor(mgr.snapshotExecutorService()); @@ -814,6 +847,82 @@ public static void doSnapshotCancellationTest( assertEquals("Snapshot directory must be empty due to snapshot cancelled", 0, snpDir.list().length); } + /** Tests concurrent snapshot deletion. */ + protected void doTestConcurrentSnapshotDeleteOperation( + ExRunnable prepareCluster, + ExRunnable concurrentOp, + @Nullable Function errValidator, + boolean rerunAtTheEnd + ) throws Exception { + CountDownLatch delProcInitLatch = new CountDownLatch(1); + CountDownLatch delProcProceedLatch = new CountDownLatch(1); + + pluginProvider = new AbstractTestPluginProvider() { + @Override public String name() { + return "TestSnpMgrProvider"; + } + + @Override public T createComponent(PluginContext ctx, Class cls) { + if (IgniteSnapshotManager.class.isAssignableFrom(cls)) { + return (T)new IgniteSnapshotManager(((IgniteEx)ctx.grid()).context()) { + @Override public T2 deleteLocalSnapshot(SnapshotFileTree sft) { + delProcInitLatch.countDown(); + + try { + assertTrue(delProcProceedLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + } + catch (InterruptedException e) { + throw new RuntimeException("Interrupted.", e); + } + + return super.deleteLocalSnapshot(sft); + } + }; + } + + return super.createComponent(ctx, cls); + } + }; + + prepareCluster.run(); + + var delFut = snp(grid(0)).deleteSnapshot(SNAPSHOT_NAME, null); + + assertTrue(delProcInitLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + + try { + concurrentOp.run(); + + if (errValidator != null) + throw new IllegalStateException("Exception is not thrown."); + } + catch (Exception e) { + if (errValidator == null || !errValidator.apply(e)) + throw new IllegalStateException("Unexpected exception: " + e.getMessage(), e); + } + + delProcProceedLatch.countDown(); + + delFut.get(getTestTimeout()); + + for (var node : G.allGrids()) + assertFalse(new SnapshotFileTree(((IgniteEx)node).context(), SNAPSHOT_NAME, null).root().exists()); + + if (!rerunAtTheEnd) + return; + + assertThrowsAnyCause( + null, + () -> { + concurrentOp.run(); + + return null; + }, + IllegalArgumentException.class, + "Snapshot does not exists " + ); + } + /** * @param sft Snapshot file tree. * @param parts Collection of pairs group and appropriate cache partition to be snapshot. @@ -994,6 +1103,13 @@ public void waitBlockedSize(int size, long timeout) throws IgniteInterruptedChec } } + /** */ + @FunctionalInterface + protected interface ExRunnable { + /** */ + void run() throws Exception; + } + /** */ protected static class Value { /** */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java index f6ae22c2db4ca..7f426b64498d4 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java @@ -232,6 +232,8 @@ public void testValidatingSnapshotFailsWithNoEncryption() throws Exception { assertTrue(snpDir.renameTo(tmpSnpDir)); + removeAllWorkingDirectories = false; + cleanPersistenceDir(); assertTrue(tmpSnpDir.renameTo(snpDir)); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java index 306d44224a4d6..268ff54775232 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java @@ -46,6 +46,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteIllegalStateException; import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.cluster.BaselineNode; @@ -124,6 +125,7 @@ import static org.apache.ignite.testframework.GridTestUtils.cartesianProduct; import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; /** * Cluster-wide snapshot check procedure tests. @@ -1201,6 +1203,37 @@ public void testConcurrentFullCheckAndFullRestoreDeclined() throws Exception { ); } + /** */ + @Test + public void testConcurrentSnapshotDeleteAndCheckOperations() throws Exception { + doTestConcurrentSnapshotDeleteOperation( + () -> prepareGridsAndSnapshot(4, 3, 1, false), + () -> snp(grid(2)).checkSnapshot(SNAPSHOT_NAME, null).get(), + e -> e.getMessage().contains("Snapshot '%s' is being deleted".formatted(SNAPSHOT_NAME)), + true + ); + } + + /** */ + @Test + public void testConcurrentSnapshotDeleteAndCheckOperationsWithDifferentPath() throws Exception { + // The test uses thread blocking. + assumeTrue(snpThrdPoolSz > 1); + + String snpPath = new File(U.defaultWorkDirectory(), "ex_snapshots").getAbsolutePath(); + + doTestConcurrentSnapshotDeleteOperation( + () -> { + prepareGridsAndSnapshot(4, 3, 1, false); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, snpPath, false, false).get(getTestTimeout()); + }, + () -> snp(grid(2)).checkSnapshot(SNAPSHOT_NAME, snpPath).get(), + null, + false + ); + } + /** Tests that concurrent snapshot full check is declined when the same snapshot is being fully restored (checked). */ @Test public void testConcurrentTheSameSnpFullCheckWhenFullyRestoringDeclined() throws Exception { @@ -1409,8 +1442,8 @@ private void prepareGridsAndSnapshot(int servers, int baseLineCnt, int clients, /** * Tests concurrent snapshot operations related to the snapshot checking. * - * @param originatorOp First snapshot operation on an originator node. - * @param trierOp Second concurrent snapshot operation on a trier node. + * @param firstOp First snapshot operation on an originator node. + * @param secondOp Second concurrent snapshot operation on a trier node. * @param firstDelay First distributed process full message of {@code originatorOp} to delay on the coordinator * to launch {@code trierOp}. * @param secondDelay Second distributed process full message of {@code originatorOp} to delay on the coordinator @@ -1422,8 +1455,8 @@ private void prepareGridsAndSnapshot(int servers, int baseLineCnt, int clients, * @param cleaner If not {@code null}, is executed at the end. */ private void doTestConcurrentSnpCheckOperations( - Supplier> originatorOp, - Supplier> trierOp, + Supplier> firstOp, + Supplier> secondOp, DistributedProcess.DistributedProcessType firstDelay, @Nullable DistributedProcess.DistributedProcessType secondDelay, boolean expectFailure, @@ -1440,17 +1473,17 @@ private void doTestConcurrentSnpCheckOperations( && ((FullMessage)msg).type() == firstDelay.ordinal() && (waitForBothFirstDelays || firstDelayed.compareAndSet(false, true))); - IgniteFuture fut = originatorOp.get(); + IgniteFuture fut = firstOp.get(); discoSpi(grid(0)).waitBlocked(getTestTimeout()); - IgniteFuture fut2 = trierOp.get(); + IgniteFuture fut2 = secondOp.get(); if (expectFailure) { assertThrowsAnyCause( log, fut2::get, - IllegalStateException.class, + IgniteIllegalStateException.class, "Validation of snapshot '" + SNAPSHOT_NAME + "' has already started" ); @@ -1473,7 +1506,7 @@ private void doTestConcurrentSnpCheckOperations( assertThrowsAnyCause( log, fut2::get, - IllegalStateException.class, + IgniteIllegalStateException.class, "Validation of snapshot '" + SNAPSHOT_NAME + "' has already started" ); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteRollingUpgradeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteRollingUpgradeTest.java new file mode 100644 index 0000000000000..5d98357453cb1 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteRollingUpgradeTest.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.io.File; +import java.nio.file.DirectoryStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteIllegalStateException; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.management.snapshot.SnapshotDeleteCommandArg; +import org.apache.ignite.internal.management.snapshot.SnapshotDeleteTask; +import org.apache.ignite.internal.processors.rollingupgrade.AbstractRollingUpgradeTest; +import org.apache.ignite.internal.util.distributed.SingleNodeMessage; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.visor.VisorTaskArgument; +import org.apache.ignite.testframework.GridTestUtils; +import org.junit.Test; + +import static java.nio.file.Files.newDirectoryStream; +import static org.apache.ignite.internal.TestRecordingCommunicationSpi.spi; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RU_PREPARE_VERSION_FINALIZATION; +import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + +/** */ +public class IgniteClusterSnapshotDeleteRollingUpgradeTest extends AbstractRollingUpgradeTest { + /** */ + private static final int ALL_GRIDS = 4; + + /** */ + private static final int CLIENTS = 1; + + /** */ + private static final String SNP_NAME = "testSnapshot"; + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + cleanPersistenceDir(); + + // Clean all: also separated snapshot working directories. + try (DirectoryStream files = newDirectoryStream(Paths.get(U.defaultWorkDirectory()))) { + for (Path path : files) + U.delete(path); + } + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName, String ver) throws Exception { + var cfg = super.getConfiguration(igniteInstanceName, ver); + + cfg.setDataStorageConfiguration( + new DataStorageConfiguration() + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration() + .setPersistenceEnabled(true) + .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE) + ) + ); + + cfg.setWorkDirectory(new File(U.defaultWorkDirectory(), igniteInstanceName).getAbsolutePath()); + + return cfg; + } + + /** */ + @Test + public void testParallelRollingUpgradeInProgress() throws Exception { + for (int i = 0; i < ALL_GRIDS; i++) + startGrid(i, "2.19.0", i >= ALL_GRIDS - CLIENTS); + + grid(0).cluster().active(true); + + int testNodeIx = ALL_GRIDS - CLIENTS - 1; + + createCacheAndSnapshot(testNodeIx); + + ru(grid(testNodeIx)).enableVersionUpgrade(); + + for (int i = 0; i < ALL_GRIDS; i++) { + assertTrue(ru(grid(i)).isVersionUpgradeEnabled()); + + upgradeNodeVersion(i, "2.19.1"); + } + + spi(grid(testNodeIx)).blockMessages((node, msg) -> msg instanceof SingleNodeMessage snm && + snm.type() == RU_PREPARE_VERSION_FINALIZATION.ordinal()); + + var finalizeFut = GridTestUtils.runAsync(() -> ru(testNodeIx).finalizeClusterVersion()); + + assertTrue(spi(grid(testNodeIx)).waitForBlocked(1, getTestTimeout())); + + ensureSnapshotDeletionFailed(false); + + spi(grid(testNodeIx)).stopBlock(); + + assertFalse(spi(grid(testNodeIx)).hasBlockedMessages()); + + finalizeFut.get(getTestTimeout()); + + for (int i = 0; i < ALL_GRIDS; i++) { + int i0 = i; + + assertTrue(waitForCondition(() -> !ru(grid(i0)).isVersionUpgradeEnabled(), getTestTimeout())); + } + + assertFalse(snp(1).deleteSnapshot(SNP_NAME, null).get(getTestTimeout()).completedNodes().isEmpty()); + } + + /** */ + @Test + public void testSnapshotDeleteFeature() throws Exception { + doTestSnapshotDeleteFeature(false); + } + + /** */ + @Test + public void testSnapshotDeleteFeatureWithTask() throws Exception { + doTestSnapshotDeleteFeature(true); + } + + /** */ + private void doTestSnapshotDeleteFeature(boolean useTask) throws Exception { + for (int i = 0; i < ALL_GRIDS; i++) + startGrid(i, "2.19.0", i >= ALL_GRIDS - CLIENTS); + + grid(0).cluster().active(true); + + createCacheAndSnapshot(1); + + ensureSnapshotDeletionFailed(useTask); + + ru(grid(0)).enableVersionUpgrade(); + + for (int i = 0; i < ALL_GRIDS; i++) { + assertTrue(ru(grid(i)).isVersionUpgradeEnabled()); + + upgradeNodeVersion(i, "2.19.1"); + + ensureSnapshotDeletionFailed(useTask); + } + + ru(grid(1)).finalizeClusterVersion(); + + for (int i = 0; i < ALL_GRIDS; i++) { + int i0 = i; + + assertTrue(waitForCondition(() -> !ru(grid(i0)).isVersionUpgradeEnabled(), getTestTimeout())); + } + + SnapshotDeleteProcessResult delRes; + + if (useTask) { + SnapshotDeleteCommandArg args = new SnapshotDeleteCommandArg(); + + args.snapshotName(SNP_NAME); + + IgniteEx ig = grid(1); + + delRes = ig.compute().execute(new SnapshotDeleteTask(), new VisorTaskArgument<>(ig.localNode().id(), args, false)).result(); + + assertFalse(delRes == null); + } + else + delRes = snp(1).deleteSnapshot(SNP_NAME, null).get(getTestTimeout()); + + assertEquals(3, delRes.completedNodes().size()); + } + + /** */ + private void createCacheAndSnapshot(int gridIdx) { + int partsCnt = 5; + int keysCnt = partsCnt * 10; + + grid(gridIdx).createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) + .setCacheMode(CacheMode.REPLICATED) + .setAffinity(new RendezvousAffinityFunction().setPartitions(partsCnt)) + .setAtomicityMode(CacheAtomicityMode.ATOMIC)); + + try (var ds = grid(gridIdx).dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = 0; i < keysCnt; i++) + ds.addData(i, i); + } + + createSnapshot(gridIdx); + } + + /** */ + private void createSnapshot(int gridIdx) { + snp(gridIdx).createSnapshot(SNP_NAME).get(getTestTimeout()); + } + + /** */ + private void ensureSnapshotDeletionFailed(boolean useTask) { + String err = "The snapshot deletion feature isn't activated yet"; + + for (int i = 0; i < ALL_GRIDS; i++) { + int i0 = i; + + if (useTask) { + SnapshotDeleteCommandArg args = new SnapshotDeleteCommandArg(); + + args.snapshotName(SNP_NAME); + + IgniteEx ig = grid(i0); + + assertThrowsAnyCause( + null, + () -> ig.compute().execute(new SnapshotDeleteTask(), new VisorTaskArgument<>(ig.localNode().id(), args, false)), + IgniteException.class, + err + ); + } + else { + assertThrowsAnyCause( + null, + () -> snp(i0).deleteSnapshot(SNP_NAME, null).get(), + IgniteIllegalStateException.class, + err + ); + } + } + } + + /** */ + private IgniteSnapshotManager snp(int gridIdx) { + return grid(gridIdx).context().cache().context().snapshotMgr(); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteTest.java new file mode 100644 index 0000000000000..e8e176cc0f626 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotDeleteTest.java @@ -0,0 +1,1000 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.io.File; +import java.io.RandomAccessFile; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Random; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteIllegalStateException; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; +import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.filename.SnapshotFileTree; +import org.apache.ignite.internal.util.distributed.DistributedProcess; +import org.apache.ignite.internal.util.distributed.SingleNodeMessage; +import org.apache.ignite.internal.util.future.IgniteFutureImpl; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.T2; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteFuture; +import org.apache.ignite.plugin.AbstractTestPluginProvider; +import org.apache.ignite.plugin.PluginContext; +import org.jetbrains.annotations.Nullable; +import org.junit.Test; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; + +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_METAS; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.CHECK_SNAPSHOT_PARTS; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.DELETE_SNAPSHOT; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.END_SNAPSHOT; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_ROLLBACK; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_START; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_INCREMENTAL_SNAPSHOT_START; +import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.START_SNAPSHOT; +import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause; +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; + +/** */ +public class IgniteClusterSnapshotDeleteTest extends AbstractSnapshotSelfTest { + /** */ + private static final int INC_CACHE_KEYS_RANGE = CACHE_KEYS_RANGE + CACHE_KEYS_RANGE / 4; + + /** Extra storage path. */ + private static final String EXT_STORAGE_PATH = "extStorage"; + + /** */ + private static boolean CASE_INSENSETIVE_FS; + + /** */ + private static boolean POSIX_PERMISSIONS; + + /** */ + private boolean separatedWorkDir; + + /** */ + private boolean extraStorages; + + /** */ + @Parameter(2) + public boolean incremental = true; + + /** */ + private @Nullable String cstIdSuffix; + + /** */ + private @Nullable String[] extStoragePaths; + + /** Parameters. */ + @Parameterized.Parameters(name = "encryption={0}, onlyPrimary={1}, incremental={2}") + public static Collection runParams() { + /** Use {@link #incremental} only. */ + return F.asList( + new Object[] {false, false, false}, + new Object[] {false, false, true} + ); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + var cfg = super.getConfiguration(igniteInstanceName); + + String workDir = separatedWorkDir + ? new File(U.defaultWorkDirectory(), igniteInstanceName).getAbsolutePath() + : U.defaultWorkDirectory(); + + cfg.setWorkDirectory(workDir); + + if (cstIdSuffix != null) + cfg.setConsistentId(cfg.getConsistentId().toString() + '_' + cstIdSuffix); + + if (extraStorages) { + cfg.getDataStorageConfiguration().setExtraStoragePaths( + workDir + File.separator, + workDir + File.separator + EXT_STORAGE_PATH + ); + + extStoragePaths = cfg.getDataStorageConfiguration().getExtraStoragePaths(); + + cfg.getDataStorageConfiguration().setExtraSnapshotPaths("", EXT_STORAGE_PATH); + } + + return cfg; + } + + /** {@inheritDoc} */ + @Override public void afterTestSnapshot() throws Exception { + super.afterTestSnapshot(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override public void beforeTestSnapshot() throws Exception { + super.beforeTestSnapshot(); + + /** Handy if test running is interrupted and {@link #afterTestSnapshot()} isn't invoked. */ + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + File workDir = new File(U.defaultWorkDirectory()); + + assertTrue(workDir.exists()); + + CASE_INSENSETIVE_FS = new File(workDir.getAbsolutePath().toLowerCase()).exists() && + new File(workDir.getAbsolutePath().toUpperCase()).exists(); + + Path workPath = workDir.toPath(); + + try { + Files.getPosixFilePermissions(workPath); + + POSIX_PERMISSIONS = true; + } + catch (UnsupportedOperationException ignored) { + // No-op. + } + } + + /** */ + @Test + public void testDeniedPermissions() throws Exception { + assumeTrue(POSIX_PERMISSIONS); + + // Doesn't matter here. + assumeFalse(incremental); + + separatedWorkDir = true; + + AtomicReference> prevPerms = new AtomicReference<>(); + AtomicReference pathRef = new AtomicReference<>(); + + pluginProvider = new AbstractTestPluginProvider() { + @Override public String name() { + return "TestSnpMgrProvider"; + } + + @Override public T createComponent(PluginContext ctx, Class cls) { + if (IgniteSnapshotManager.class.isAssignableFrom(cls)) { + return (T)new IgniteSnapshotManager(((IgniteEx)ctx.grid()).context()) { + @Override public T2 deleteLocalSnapshot(SnapshotFileTree sft) { + if (ctx.localNode().id().equals(grid(1).localNode().id())) { + Path path = sft.root().toPath(); + + pathRef.set(path); + + try { + prevPerms.set(Files.getPosixFilePermissions(path)); + + // Denies writing (deletion). + Files.setPosixFilePermissions(path, PosixFilePermissions.fromString("r-xr-x---")); + } + catch (Exception e) { + throw new IgniteException("Unable to set the posix permissions.", e); + } + } + + return super.deleteLocalSnapshot(sft); + } + }; + } + + return super.createComponent(ctx, cls); + } + }; + + startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME).get(getTestTimeout()); + + try { + SnapshotDeleteProcessResult res = snp(grid(0)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + assertEquals(1, res.uncompletedNodes().size()); + assertEquals(2, res.completedNodes().size()); + } + finally { + if (pathRef.get() != null && pathRef.get() != null) + Files.setPosixFilePermissions(pathRef.get(), prevPerms.get()); + } + } + + /** */ + @Test + public void testExtraStoragesDeleted() throws Exception { + extraStorages = true; + + dfltCacheCfg = null; + + startGridsMultiThreaded(3); + + assertFalse(F.isEmpty(extStoragePaths)); + + assertTrue(grid(0).cache(DEFAULT_CACHE_NAME) == null); + + dfltCacheCfg = defaultCacheConfiguration(); + + // Works only with a shared work directory. + dfltCacheCfg.setStoragePaths(extStoragePaths); + + grid(0).createCache(dfltCacheCfg); + awaitPartitionMapExchange(); + + // Fills the cache. + startGridsWithCache(0, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout()); + + if (incremental) + addIncrementalSnapshot(null); + + for (Ignite ig : G.allGrids()) { + SnapshotFileTree snpTree = new SnapshotFileTree(((IgniteEx)ig).context(), SNAPSHOT_NAME, null); + + assertEquals(2, snpTree.extraStorages().size()); + + for (File extSnpStorage : snpTree.allStorages().toList()) { + assertTrue(extSnpStorage.exists()); + assertTrue(extSnpStorage.isDirectory()); + } + } + + snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + for (Ignite ig : G.allGrids()) { + SnapshotFileTree snpTree = new SnapshotFileTree(((IgniteEx)ig).context(), SNAPSHOT_NAME, null); + + for (File extSnpStorage : snpTree.allStorages().toList()) { + // Snapshot root, even in an external storage. + extSnpStorage = extSnpStorage.getParentFile().getParentFile(); + + assertEquals(SNAPSHOT_NAME, extSnpStorage.getName()); + + assertFalse(extSnpStorage.exists()); + } + } + } + + /** Tests snapshot deletion when one node finds snapshot but fails to delete its data. */ + @Test + public void testUncompletedNodes() throws Exception { + separatedWorkDir = true; + + // Simulates a deletion error on some node. + pluginProvider = new AbstractTestPluginProvider() { + @Override public String name() { + return "TestSnpMgrProvider"; + } + + // Simulates the deletion failure. + @Override public T createComponent(PluginContext ctx, Class cls) { + if (IgniteSnapshotManager.class.isAssignableFrom(cls)) { + return (T)new IgniteSnapshotManager(((IgniteEx)ctx.grid()).context()) { + @Override public T2 deleteLocalSnapshot(SnapshotFileTree sft) { + if (ctx.localNode().id().equals(grid(1).localNode().id())) + return new T2<>(false, true); + + return super.deleteLocalSnapshot(sft); + } + }; + } + + return super.createComponent(ctx, cls); + } + }; + + startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout()); + + if (incremental) + addIncrementalSnapshot(null); + + SnapshotDeleteProcessResult delSnpRes = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + assertTrue(delSnpRes.emptyNodes().isEmpty()); + assertEquals(1, delSnpRes.uncompletedNodes().size()); + assertTrue(delSnpRes.uncompletedNodes().containsKey(grid(1).localNode().id())); + } + + /** Tests the snapshot deletion when cluster restarts with changed nodes consistent ids. */ + @Test + public void testDeleteOtherConsistentId() throws Exception { + startGridsWithSnapshot(3, CACHE_KEYS_RANGE, false); + + stopAllGrids(); + + cstIdSuffix = "_ext"; + + startGridsMultiThreaded(3); + + SnapshotDeleteProcessResult delRes = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + for (var ig : G.allGrids()) { + assertTrue(Files.list(((IgniteEx)ig).context().pdsFolderResolver().fileTree().snapshotsRoot().toPath()) + .findFirst().isEmpty()); + } + + assertEquals(3, delRes.absentBaselines().size()); + } + + /** Tests that snapshot is not deleted if snaphot metadata is not found. Shared working directory. */ + @Test + public void testDeleteSnapshotNoMetaSharedDirectory() throws Exception { + doTestDeleteNotSnapshot(false, false); + } + + /** Tests that snapshot is not deleted if snaphot metadata is not found. Dedicated working directories. */ + @Test + public void testDeleteSnapshotNoMetaDedicatedDirectories() throws Exception { + doTestDeleteNotSnapshot(true, false); + } + + /** Tests that snapshot is not deleted if snaphot metadata cannot be read (corrupted). Shared working directory. */ + @Test + public void testDeleteSnapshotCorruptedMetaSharedDirectory() throws Exception { + doTestDeleteNotSnapshot(false, true); + } + + /** Tests that snapshot is not deleted if snaphot metadata cannot be read (corrupted). Dedicated working directories. */ + @Test + public void testDeleteSnapshotCorruptedMetaDedicatedDirectories() throws Exception { + doTestDeleteNotSnapshot(true, true); + } + + /** + * Tests that snapshot is not deleted if snaphot metadata isn't found or cannot be read (corrupted). + * + * @param separatedWorkDir If {@code true}, the dedicated dirictories is used for the nodes. If {@code false}, + * the nodes use a shared working directory. + * @param corruptSnpMeta If {@code true}, corrupts the snapshot metadata file. If {@code false}, removes the snapshot + * metadata file + */ + protected void doTestDeleteNotSnapshot(boolean separatedWorkDir, boolean corruptSnpMeta) throws Exception { + // Doesn't matter here; + assumeFalse(onlyPrimary); + + this.separatedWorkDir = separatedWorkDir; + + startGridsWithCache(3, CACHE_KEYS_RANGE, valueBuilder(), dfltCacheCfg); + + snp(grid(1)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout()); + + var snpSft = new SnapshotFileTree(grid(1).context(), SNAPSHOT_NAME, null); + + // Ensure that all the snapshot node folders exist. + assertTrue(snpSft.binaryMeta().exists()); + assertTrue(new SnapshotFileTree(grid(0).context(), SNAPSHOT_NAME, null, folderName(0), consistentId(0)) + .binaryMeta().exists()); + assertTrue(new SnapshotFileTree(grid(2).context(), SNAPSHOT_NAME, null, folderName(2), consistentId(2)) + .binaryMeta().exists()); + + assertTrue(snpSft.meta().exists()); + + if (corruptSnpMeta) { + try (var rwf = new RandomAccessFile(snpSft.meta(), "rw")) { + byte[] slop = new byte[128]; + + new Random().nextBytes(slop); + + rwf.write(slop); + } + } + else { + assertTrue(U.delete(snpSft.meta())); + assertFalse(snpSft.meta().exists()); + } + + var delSnpRes = snp(grid(2)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + // Check the result. + if (separatedWorkDir) { + // One node doesn't find meta, decided not a snapshot. + assertTrue(delSnpRes.uncompletedNodes().isEmpty()); + assertEquals(2, delSnpRes.completedNodes().size()); + assertEquals(1, delSnpRes.emptyNodes().size()); + assertTrue(delSnpRes.emptyNodes().containsKey(grid(1).localNode().id())); + assertTrue(snpSft.binaryMeta().exists()); + } + else + assertEquals(3, delSnpRes.uncompletedNodes().size() + delSnpRes.completedNodes().size() + delSnpRes.emptyNodes().size()); + + assertFalse(new SnapshotFileTree(grid(0).context(), SNAPSHOT_NAME, null, folderName(0), consistentId(0)) + .binaryMeta().exists()); + assertFalse(new SnapshotFileTree(grid(2).context(), SNAPSHOT_NAME, null, folderName(2), consistentId(2)) + .binaryMeta().exists()); + } + + /** */ + private String consistentId(int gridIdx) { + return grid(gridIdx).configuration().getConsistentId().toString(); + } + + /** */ + private String folderName(int gridIdx) { + return grid(gridIdx).context().pdsFolderResolver().fileTree().folderName(); + } + + /** Tests snapshot deletion when one node has no snapshot data. */ + @Test + public void testEmptyNodes() throws Exception { + separatedWorkDir = true; + + startGridsWithCache(2, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout()); + + if (incremental) + addIncrementalSnapshot(null); + + IgniteEx additionalNode = startGrid(G.allGrids().size()); + + SnapshotDeleteProcessResult delSnpRes = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + assertFalse(delSnpRes.emptyNodes().isEmpty()); + assertTrue(delSnpRes.emptyNodes().containsKey(additionalNode.localNode().id())); + assertTrue(delSnpRes.uncompletedNodes().isEmpty()); + } + + /** Tests snapshot deletion repeat after an offline node restarts. */ + @Test + public void testDeletionRepeatAfterOfflineNodeStarts() throws Exception { + separatedWorkDir = true; + + startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout()); + + if (incremental) + addIncrementalSnapshot(null); + + int stoppedNodeIdx = G.allGrids().size() - 1; + + UUID stoppedNodeId = grid(stoppedNodeIdx).localNode().id(); + + stopGrid(stoppedNodeIdx); + + SnapshotDeleteProcessResult delSnpRes = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + assertEquals(2, delSnpRes.completedNodes().size()); + assertFalse(delSnpRes.completedNodes().containsKey(stoppedNodeId)); + + assertTrue(delSnpRes.uncompletedNodes().isEmpty()); + assertTrue(delSnpRes.emptyNodes().isEmpty()); + + startGrid(stoppedNodeIdx); + + stoppedNodeId = grid(stoppedNodeIdx).localNode().id(); + + delSnpRes = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + assertEquals(1, delSnpRes.completedNodes().size()); + assertTrue(delSnpRes.completedNodes().containsKey(stoppedNodeId)); + + assertTrue(delSnpRes.uncompletedNodes().isEmpty()); + assertEquals(2, delSnpRes.emptyNodes().size()); + } + + /** Test snapshot deletion process when one node leaves. */ + @Test + public void testNodeStopsInTheMiddle() throws Exception { + separatedWorkDir = true; + + CountDownLatch beginLatch = new CountDownLatch(1); + CountDownLatch proceedLatch = new CountDownLatch(1); + + // Simulates a deletion error on some node. + pluginProvider = new AbstractTestPluginProvider() { + @Override public String name() { + return "TestSnpMgrProvider"; + } + + @Override public T createComponent(PluginContext ctx, Class cls) { + if (IgniteSnapshotManager.class.isAssignableFrom(cls)) { + return (T)new IgniteSnapshotManager(((IgniteEx)ctx.grid()).context()) { + @Override public T2 deleteLocalSnapshot(SnapshotFileTree sft) { + if (ctx.localNode().id().equals(grid(1).localNode().id())) { + beginLatch.countDown(); + + try { + assertTrue(proceedLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + } + catch (InterruptedException e) { + throw new RuntimeException("Interrupted.", e); + } + } + + return super.deleteLocalSnapshot(sft); + } + }; + } + + return super.createComponent(ctx, cls); + } + }; + + startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(getTestTimeout()); + + if (incremental) + addIncrementalSnapshot(null); + + var delFut = snp(grid(2)).deleteSnapshot(SNAPSHOT_NAME, null); + + assertTrue(beginLatch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + + UUID stoppedGridId = grid(1).localNode().id(); + + stopGrid(1); + + proceedLatch.countDown(); + + var delRes = delFut.get(getTestTimeout()); + + assertEquals(2, delRes.completedNodes().size()); + assertFalse(delRes.completedNodes().containsKey(stoppedGridId)); + + startGrid(1); + + delRes = snp(grid(2)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + assertEquals(1, delRes.completedNodes().size()); + assertTrue(delRes.completedNodes().containsKey(grid(1).localNode().id())); + } + + /** Tests that a concurrent deletion of a snapshot with the same name but different path is allowed. */ + @Test + public void testConcurrentDeleteOfTheSameSnapshotDifferentPath() throws Exception { + startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(TIMEOUT); + + if (incremental) + addIncrementalSnapshot(null); + + String snpPath = new File(grid(0).context().pdsFolderResolver().fileTree().snapshotsRoot(), "ex_snapshots").getAbsolutePath(); + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, snpPath, false, onlyPrimary).get(getTestTimeout()); + + if (incremental) + addIncrementalSnapshot(snpPath); + + TestRecordingCommunicationSpi commSpi1 = (TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi(); + + commSpi1.blockMessages((node, msg) -> + msg instanceof SingleNodeMessage msg0 && msg0.type() == DELETE_SNAPSHOT.ordinal()); + + var delFut0 = snp(grid(0)).deleteSnapshot(SNAPSHOT_NAME, null); + var delFut1 = snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, snpPath); + + commSpi1.waitForBlocked(2, getTestTimeout()); + + commSpi1.stopBlock(); + + var delRes0 = delFut0.get(getTestTimeout()); + var delRes1 = delFut1.get(getTestTimeout()); + + assertTrue(!delRes0.completedNodes().isEmpty() || !delRes0.uncompletedNodes().isEmpty()); + assertTrue(!delRes1.completedNodes().isEmpty() || !delRes1.uncompletedNodes().isEmpty()); + } + + /** Tests that a concurrent deletion of the same snapshot is declined. */ + @Test + public void testConcurrentDeleteOfTheSameSnapshot() throws Exception { + // Doesn't matter here. + assumeFalse(onlyPrimary); + + doTestConcurrentSnapshotDeleteOperation( + () -> startGridsWithSnapshot(3, CACHE_KEYS_RANGE, false), + () -> snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()), + e -> e.getMessage().contains("Deletion of the snapshot has already started"), + false + ); + } + + /** Tests that a snapshot deletion is declined when a snapshot check operation is in progress. */ + @Test + public void testSnapshotDeleteWhenCheckInProgress() throws Exception { + doTestConcurrentSnapshotDelete( + () -> new IgniteFutureImpl<>(snp(grid(2)).checkSnapshot(SNAPSHOT_NAME, null, incremental ? 1 : 0)), + F.asList(CHECK_SNAPSHOT_METAS, CHECK_SNAPSHOT_PARTS), + true, + null, + SnapshotDeleteProcess.BEING_CHECKED_PREF, + false, + false + ); + } + + /** + * Tests that a snapshot deletion is declined when a snapshot check operation is in progress when the delete operation + * uses changed character case in the snapshot name. + * + * @see SnapshotCheckProcess#isSnapshotChecking(String) + */ + @Test + public void testSnapshotDeleteWhenCheckInProgressNameCase() throws Exception { + // For case-insensitive file systems only. + assumeTrue(CASE_INSENSETIVE_FS); + + SnapshotPartitionsVerifyResult res = doTestConcurrentSnapshotDelete( + () -> new IgniteFutureImpl<>(snp(grid(2)).checkSnapshot(SNAPSHOT_NAME.toLowerCase(), null, incremental ? 1 : 0)), + F.asList(CHECK_SNAPSHOT_METAS), + true, + null, + SnapshotDeleteProcess.BEING_CREATED_PREF, + false, + true + ); + + assertFalse(res == null); + assertTrue(F.isEmpty(res.exceptions())); + } + + /** Tests that a snapshot deletion is declined when a snapshot create operation is in progress. */ + @Test + public void testSnapshotDeleteWhenCreateInProgress() throws Exception { + doTestConcurrentSnapshotDelete( + () -> snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, incremental, onlyPrimary), + F.asList(START_SNAPSHOT, END_SNAPSHOT), + false, + () -> { + snp(grid(0)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + + if (incremental) + snp(grid(0)).createSnapshot(SNAPSHOT_NAME).get(getTestTimeout()); + }, + "Snapshot with the same name is being created", + false, + false + ); + } + + /** + * Tests that a snapshot deletion is declined when a snapshot create operation is in progress when the delete operation + * uses changed character case in the snapshot name. + */ + @Test + public void testSnapshotDeleteWhenCreateInProgressNameCase() throws Exception { + doTestConcurrentSnapshotDelete( + () -> snp(grid(0)).createSnapshot(SNAPSHOT_NAME.toLowerCase(), null, incremental, onlyPrimary), + F.asList(START_SNAPSHOT), + false, + () -> { + snp(grid(0)).deleteSnapshot(SNAPSHOT_NAME.toLowerCase(), null).get(getTestTimeout()); + + if (incremental) + snp(grid(0)).createSnapshot(SNAPSHOT_NAME.toLowerCase()).get(getTestTimeout()); + }, + "Snapshot with the same name is being created", + false, + true + ); + } + + /** Tests that a snapshot deletion is declined when a snapshot restore begins. */ + @Test + public void testSnapshotDeleteWhenRestoreBegins() throws Exception { + doTestConcurrentSnapshotDelete( + () -> { + if (incremental) + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null, 1); + else + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null); + }, + F.asList(CHECK_SNAPSHOT_METAS, CHECK_SNAPSHOT_PARTS), + true, + () -> { + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + awaitPartitionMapExchange(); + }, + SnapshotDeleteProcess.BEING_CHECKED_PREF, + false, + false + ); + } + + /** + * Tests that a snapshot deletion is declined when a snapshot restore begins when the delete operation uses + * changed character case in the snapshot name. + * + * @see SnapshotCheckProcess#isSnapshotChecking(String) + */ + @Test + public void testSnapshotDeleteWhenRestoreBeginsNameCase() throws Exception { + // For case-insensitive file systems only. + assumeTrue(CASE_INSENSETIVE_FS); + + doTestConcurrentSnapshotDelete( + () -> { + if (incremental) + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME.toLowerCase(), null, 1); + else + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME.toLowerCase(), null); + }, + F.asList(CHECK_SNAPSHOT_METAS, CHECK_SNAPSHOT_PARTS), + true, + () -> { + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + awaitPartitionMapExchange(); + }, + SnapshotDeleteProcess.BEING_CREATED_PREF, + false, + true + ); + + // Ensure that the test exists. + assertEquals(incremental ? INC_CACHE_KEYS_RANGE : CACHE_KEYS_RANGE, grid(0).cache(DEFAULT_CACHE_NAME).size()); + } + + /** Tests that a snapshot deletion is declined when a snapshot restore is in progress. */ + @Test + public void testSnapshotDeleteWhenRestoreInProgress() throws Exception { + var restoreMsgs = F.asList( + RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE, + RESTORE_CACHE_GROUP_SNAPSHOT_START + ); + + if (incremental) { + restoreMsgs = new ArrayList<>(restoreMsgs); + restoreMsgs.add(RESTORE_INCREMENTAL_SNAPSHOT_START); + } + + doTestConcurrentSnapshotDelete( + () -> { + if (incremental) + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null, 1); + else + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null); + }, + restoreMsgs, + true, + () -> { + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + awaitPartitionMapExchange(); + }, + SnapshotDeleteProcess.BEING_RESTORED_PREF, + false, + false + ); + } + + /** + * Tests that a snapshot deletion is declined when a snapshot restore is in progress when the delete operation uses + * changed character case in the snapshot name. + * + * @see IgniteSnapshotManager#isRestoring(String) + */ + @Test + public void testSnapshotDeleteWhenRestoreInProgressNameCase() throws Exception { + // For case-insensitive file systems only. + assumeTrue(CASE_INSENSETIVE_FS); + + var restoreMsgs = F.asList( + RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE, + RESTORE_CACHE_GROUP_SNAPSHOT_START + ); + + if (incremental) { + restoreMsgs = new ArrayList<>(restoreMsgs); + restoreMsgs.add(RESTORE_INCREMENTAL_SNAPSHOT_START); + } + + doTestConcurrentSnapshotDelete( + () -> { + if (incremental) + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME.toLowerCase(), null, 1); + else + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME.toLowerCase(), null); + }, + restoreMsgs, + true, + () -> { + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + awaitPartitionMapExchange(); + }, + SnapshotDeleteProcess.BEING_RESTORED_PREF, + false, + true + ); + + // Ensure that the test exists. + assertEquals(incremental ? INC_CACHE_KEYS_RANGE : CACHE_KEYS_RANGE, grid(0).cache(DEFAULT_CACHE_NAME).size()); + } + + /** Tests that a snapshot deletion is declined when a snapshot restore is in progress but fails. */ + @Test + public void testSnapshotDeleteWhenRestoreProgressFails() throws Exception { + // An in-the-middle failure won't allow to start restoring the incrementals. + assumeFalse(incremental); + + var restoreMsgs = F.asList(RESTORE_CACHE_GROUP_SNAPSHOT_ROLLBACK); + + if (incremental) { + restoreMsgs = new ArrayList<>(restoreMsgs); + restoreMsgs.add(RESTORE_INCREMENTAL_SNAPSHOT_START); + } + + doTestConcurrentSnapshotDelete( + () -> { + if (incremental) + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null, 1); + else + return snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null); + }, + restoreMsgs, + true, + () -> { + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + awaitPartitionMapExchange(); + + SnapshotFileTree sft = snapshotFileTree(grid(1), SNAPSHOT_NAME); + + String failingFilePath = sft.partitionFile(dfltCacheCfg, primaries[0]).getAbsolutePath() + .replace(sft.nodeStorage().getAbsolutePath(), ""); + + grid(1).context().cache().context().snapshotMgr().ioFactory((file, modes) -> { + FileIO delegate = new RandomAccessFileIOFactory().create(file, modes); + + if (file.getPath().endsWith(failingFilePath)) + throw new RuntimeException("Test exception"); + + return delegate; + }); + }, + SnapshotDeleteProcess.BEING_RESTORED_PREF, + true, + false + ); + } + + /** + * @param firstOp First cluster-wide snapshot operation. + * @param msgsToWatch {@link SingleNodeMessage#type()} relating to {@code firstOp} to block on one node. + * @param precreateSnp If {@code true}, creates snapshot after the cluster start. + * @param prepareIteration If not {@code null}, is invoked in the beginning of test iteration at each {@code msgsToWatch}. + * @param concurrentMsgErr Test of failed concurrent to {@code firstOp} delete snapshot operation to watch. + * @param ignoreFirstOpFailure If {@code true}, possible failure of {@code firstOp} is ignored. + * @param ignoreDeleteOpFailure If {@code true}, possible failure of the deletion operation is ignored. Allows to + * focus on the first operation's result. + */ + protected @Nullable T doTestConcurrentSnapshotDelete( + Supplier> firstOp, + Collection msgsToWatch, + boolean precreateSnp, + @Nullable Runnable prepareIteration, + String concurrentMsgErr, + boolean ignoreFirstOpFailure, + boolean ignoreDeleteOpFailure + ) throws Exception { + startGridsWithCache(3, CACHE_KEYS_RANGE, i -> i, dfltCacheCfg); + + if (precreateSnp) { + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary) + .get(TIMEOUT); + + if (incremental) + addIncrementalSnapshot(null); + } + + TestRecordingCommunicationSpi commSpi1 = (TestRecordingCommunicationSpi)grid(1).configuration().getCommunicationSpi(); + + T res = null; + + for (var nodeResMsgType : msgsToWatch) { + if (log.isInfoEnabled()) + log.info("Iteration with message-to-wait-for type: " + nodeResMsgType); + + if (prepareIteration != null) + prepareIteration.run(); + + commSpi1.blockMessages((node, msg) -> + msg instanceof SingleNodeMessage msg0 && msg0.type() == nodeResMsgType.ordinal()); + + IgniteFuture firstFut = firstOp.get(); + + commSpi1.waitForBlocked(1, getTestTimeout()); + + if (ignoreDeleteOpFailure) { + try { + snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()); + } + catch (Exception e) { + if (log.isDebugEnabled()) + log.debug("The deletion operation failed but a failure is expected, err: " + e.getMessage()); + } + } + else { + assertThrowsAnyCause( + null, + () -> snp(grid(1)).deleteSnapshot(SNAPSHOT_NAME, null).get(getTestTimeout()), + IgniteIllegalStateException.class, + concurrentMsgErr + ); + } + + commSpi1.stopBlock(); + + if (ignoreFirstOpFailure) { + try { + res = firstFut.get(getTestTimeout()); + } + catch (Exception e) { + if (log.isDebugEnabled()) + log.debug("The first operation failed but a failure is expected, err: " + e.getMessage()); + + return null; + } + } + else + res = firstFut.get(getTestTimeout()); + } + + return res; + } + + /** */ + private void addIncrementalSnapshot(@Nullable String path) { + try (var ds = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = CACHE_KEYS_RANGE; i < INC_CACHE_KEYS_RANGE; i++) + ds.addData(i, i); + } + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, path, true, onlyPrimary) + .get(getTestTimeout()); + } + + /** {@inheritDoc} */ + @Override protected void awaitPartitionMapExchange() { + try { + super.awaitPartitionMapExchange(); + } + catch (InterruptedException e) { + throw new RuntimeException("Interrupted.", e); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index 78ec729ec931b..84dd1f57c05d8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@ -333,6 +333,45 @@ public void testCreateSnapshotDuringRestore() throws Exception { assertCacheKeys(ignite.cache(DEFAULT_CACHE_NAME), CACHE_KEYS_RANGE); } + /** Tests that snapshot restore is declined when the same snapshot is being deleted. */ + @Test + public void testConcurrentSnapshotDeleteAndRestoreOperations() throws Exception { + doTestConcurrentSnapshotDeleteOperation( + () -> startGridsWithSnapshot(3, CACHE_KEYS_RANGE), + () -> snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, null).get(), + e -> e.getMessage().contains("Snapshot '%s' is being deleted".formatted(SNAPSHOT_NAME)), + true + ); + } + + /** */ + @Test + public void testConcurrentSnapshotDeleteAndRestoreOperationsWithDifferentPath() throws Exception { + String snpPath = new File(U.defaultWorkDirectory(), "ex_snapshots").getAbsolutePath(); + + doTestConcurrentSnapshotDeleteOperation( + () -> { + startGridsWithSnapshot(3, CACHE_KEYS_RANGE); + + grid(0).createCache(DEFAULT_CACHE_NAME); + + try (var ds = grid(0).dataStreamer(DEFAULT_CACHE_NAME)) { + for (int i = 0; i < CACHE_KEYS_RANGE; ++i) + ds.addData(i, i); + } + + snp(grid(0)).createSnapshot(SNAPSHOT_NAME, snpPath, false, false).get(getTestTimeout()); + + grid(0).destroyCache(DEFAULT_CACHE_NAME); + + awaitPartitionMapExchange(); + }, + () -> snp(grid(2)).restoreSnapshot(SNAPSHOT_NAME, snpPath, null).get(), + null, + false + ); + } + /** * Ensures that the cache doesn't start if one of the baseline nodes fails. * diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java index 570761dac227f..31bd938b79f43 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotSelfTest.java @@ -603,7 +603,7 @@ public void testSnapshotExistsException() throws Exception { assertThrowsAnyCause(log, () -> snp(ignite).createSnapshot(SNAPSHOT_NAME, null, false, onlyPrimary).get(), IgniteException.class, - "Snapshot with given name already exists on local node."); + "Snapshot with given name already exists on local node"); stopAllGrids(); @@ -615,6 +615,43 @@ public void testSnapshotExistsException() throws Exception { waitForEvents(EVT_CLUSTER_SNAPSHOT_STARTED, EVT_CLUSTER_SNAPSHOT_FAILED); } + /** + * Tests that snapshot create detects concurrent deletion, or detects still existing snapshot or successfully + * proceeds if snapshot already deleted. + */ + @Test + public void testConcurrentSnapshotDeleteOperation() throws Exception { + doTestConcurrentSnapshotDeleteOperation( + () -> { + startGridsWithCache(3, dfltCacheCfg, CACHE_KEYS_RANGE); + + snp(grid(2)).createSnapshot(SNAPSHOT_NAME).get(); + }, + () -> snp(grid(2)).createSnapshot(SNAPSHOT_NAME).get(), + e -> e.getMessage().contains("Snapshot with given name already exists"), + false + ); + } + + /** + * Tests that a concurrent deletion of a same-named snapshot is allowed if it has a different path. + */ + @Test + public void testConcurrentSnapshotDeleteOperationWithDifferentPath() throws Exception { + String snpPath = new File(U.defaultWorkDirectory(), "ex_snapshots").getAbsolutePath(); + + doTestConcurrentSnapshotDeleteOperation( + () -> { + startGridsWithCache(3, dfltCacheCfg, CACHE_KEYS_RANGE); + + snp(grid(2)).createSnapshot(SNAPSHOT_NAME).get(); + }, + () -> snp(grid(2)).createSnapshot(SNAPSHOT_NAME, snpPath, false, false).get(), + null, + false + ); + } + /** @throws Exception If fails. */ @Test public void testClusterSnapshotCleanedOnLeft() throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotRestoreFromRemoteTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotRestoreFromRemoteTest.java index a31cd9a71376c..b6b4dd80c7b8f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotRestoreFromRemoteTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotRestoreFromRemoteTest.java @@ -127,6 +127,11 @@ public class IgniteSnapshotRestoreFromRemoteTest extends IgniteClusterSnapshotRe return cfg; } + /** {@inheritDoc} */ + @Override protected boolean removeAllWorkingDirectories() { + return false; + } + /** @throws Exception If fails. */ @Before public void prepareDedicatedSnapshot() throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/AbstractIncrementalSnapshotTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/AbstractIncrementalSnapshotTest.java index fcf65fa9b30ed..8055e7795e396 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/AbstractIncrementalSnapshotTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/AbstractIncrementalSnapshotTest.java @@ -132,7 +132,7 @@ protected void awaitSnapshotResourcesCleaned() { try { assertTrue(GridTestUtils.waitForCondition(() -> { for (Ignite g: G.allGrids()) { - if (snp((IgniteEx)g).currentCreateRequest() != null) + if (snp(g).currentCreateRequest() != null) return false; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/IncrementalSnapshotNodeFailureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/IncrementalSnapshotNodeFailureTest.java index 88aad62a4985f..68ff0e443740f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/IncrementalSnapshotNodeFailureTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/incremental/IncrementalSnapshotNodeFailureTest.java @@ -109,7 +109,7 @@ private void runIncrementalSnapshotAndBreak(Supplier breakSnpWithExcp) t awaitSnapshotResourcesCleaned(); for (Ignite g: G.allGrids()) - assertNull(snp((IgniteEx)g).incrementalSnapshotId()); + assertNull(snp(g).incrementalSnapshotId()); stopAllGrids(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_1.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_1.java index 38056daa0b428..c6108f4a28fb8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_1.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/feature/TestIgniteReleaseFeatures_2_19_1.java @@ -21,4 +21,7 @@ public class TestIgniteReleaseFeatures_2_19_1 { /** */ public static final IgniteFeature ROLLING_UPGRADE_FEATURE = TestIgniteReleaseFeatures_2_19_0.ROLLING_UPGRADE_FEATURE; + + /** */ + public static final IgniteFeature SNAPSHOT_DELETE_FEATURE = SupportedFeatureRegistry.SNAPSHOT_DELETE_FEATURE; } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite8.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite8.java index 5673ed2b71af3..857e317599640 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite8.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSnapshotTestSuite8.java @@ -21,6 +21,8 @@ import java.util.Collection; import java.util.List; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotCheckTest; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotDeleteRollingUpgradeTest; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotDeleteTest; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteClusterSnapshotHandlerTest; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.DynamicSuite; @@ -48,6 +50,8 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotCheckTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotHandlerTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotDeleteRollingUpgradeTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, IgniteClusterSnapshotDeleteTest.class, ignoredTests); return suite; } diff --git a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output index dd6ff4e43caf5..e0ddc44cf24d7 100644 --- a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output +++ b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassTest_help.output @@ -269,6 +269,13 @@ This utility can do the following commands: Get the status of the current snapshot operation: control.(sh|bat) --snapshot status + Deletes snapshot and all its incrementals from all the online server nodes: + control.(sh|bat) --snapshot delete snapshot_name [--src path] + + Parameters: + snapshot_name - Snapshot name. + --src path - Path to snapshot location directory. If not specified or specified a relative path, the default snapshot configuration directory will be used. + Change cluster tag to new value: control.(sh|bat) --change-tag newTagValue [--yes] diff --git a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output index 76f0f8c7e0e37..6ddf5d9106536 100644 --- a/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output +++ b/modules/core/src/test/resources/org.apache.ignite.util/GridCommandHandlerClusterByClassWithSSLTest_help.output @@ -269,6 +269,13 @@ This utility can do the following commands: Get the status of the current snapshot operation: control.(sh|bat) --snapshot status + Deletes snapshot and all its incrementals from all the online server nodes: + control.(sh|bat) --snapshot delete snapshot_name [--src path] + + Parameters: + snapshot_name - Snapshot name. + --src path - Path to snapshot location directory. If not specified or specified a relative path, the default snapshot configuration directory will be used. + Change cluster tag to new value: control.(sh|bat) --change-tag newTagValue [--yes] diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java index bff6187c71532..5df1a59847a60 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/ContinuousDataLoadApplication.java @@ -78,7 +78,7 @@ public class ContinuousDataLoadApplication extends IgniteAwareApplication { if (notifyTime + TimeUnit.MILLISECONDS.toNanos(1500) < System.nanoTime()) notifyTime = System.nanoTime(); - // Delayed notify of the initialization to make sure the data load has completelly began and + // Delayed notify of the initialization to make sure the data load has completely begun and // has produced some valuable amount of data. if (!inited() && warmUpCnt == loaded) markInitialized();