-
Notifications
You must be signed in to change notification settings - Fork 1.2k
Avoid duplicate DataNode memory configuration initialization #18467
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -141,7 +141,7 @@ public class IoTDBDescriptor { | |
| } | ||
|
|
||
| protected IoTDBDescriptor() { | ||
| loadProps(); | ||
| boolean hasLoadedProperties = loadProps(); | ||
| ServiceLoader<IPropertiesLoader> propertiesLoaderServiceLoader = | ||
| ServiceLoader.load(IPropertiesLoader.class); | ||
| boolean hasProperties = false; | ||
|
|
@@ -167,8 +167,8 @@ protected IoTDBDescriptor() { | |
| .getConfig() | ||
| .setCustomizedProperties(loader.getCustomizedProperties()); | ||
| } | ||
| // if there are no properties, we need to init memory config | ||
| if (!hasProperties) { | ||
| // If no configuration source initialized the memory config, initialize it with defaults. | ||
| if (!hasLoadedProperties && !hasProperties) { | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Require both configuration sources to be absent before applying defaults. This preserves values loaded from iotdb-system.properties while retaining the existing fallback when neither the system file nor an external loader is available.
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Could we add a regression test that exercises this constructor branch with a real system configuration source? The 11 tests listed in the PR are unchanged from the parent commit: IoTDBDescriptorTest only checks URL resolution, while DataNodeMemoryConfigTest tests the calculation/default paths directly. As a result, removing
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for pointing this out. I added two constructor-level regression tests. One uses a real temporary iotdb-system.properties with datanode_memory_proportion=1:1:1:1:1:5 and verifies the activated RPC buffer budget is maxMemory / 4; the other verifies the no-configuration fallback remains maxMemory / 20. The tests run in separate Surefire forks so the descriptor and memory configuration statics are isolated. |
||
| memoryConfig.init(new TrimProperties()); | ||
| } | ||
| } | ||
|
|
@@ -227,7 +227,7 @@ else if (!urlString.endsWith(".properties")) { | |
|
|
||
| /** load a property file and set TsfileDBConfig variables. */ | ||
| @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning | ||
| private void loadProps() { | ||
| private boolean loadProps() { | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Return a boolean from loadProps so callers can tell whether this method reached the configuration-loading path. The true and false returns mirror the existing URL-present and URL-absent branches without changing their error handling. |
||
| TrimProperties commonProperties = new TrimProperties(); | ||
| // if new properties file exist, skip old properties files | ||
| URL url = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); | ||
|
|
@@ -256,11 +256,13 @@ private void loadProps() { | |
| .getMetricConfig() | ||
| .updateRpcInstance(NodeType.DATANODE, SchemaConstant.SYSTEM_DATABASE); | ||
| } | ||
| return true; | ||
| } else { | ||
| LOGGER.warn( | ||
| DataNodeMiscMessages | ||
| .MISC_LOG_COULDN_T_LOAD_THE_CONFIGURATION_FROM_ANY_OF_THE_KNOWN_SOURCES_EE3ED103, | ||
| CommonConfig.SYSTEM_CONFIG_NAME); | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
| package org.apache.iotdb.db.conf; | ||
|
|
||
| import org.apache.iotdb.commons.conf.IoTDBConstant; | ||
| import org.apache.iotdb.commons.memory.MemoryConfig; | ||
|
|
||
| import org.junit.Test; | ||
|
|
||
| import static org.junit.Assert.assertEquals; | ||
|
|
||
| public class IoTDBDescriptorDefaultMemoryConfigTest { | ||
|
|
||
| @Test | ||
| public void testNoConfigurationSourceInitializesDefaultRpcBufferMemory() { | ||
| String originalConf = System.getProperty(IoTDBConstant.IOTDB_CONF); | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This test keeps the no-configuration fallback covered by forcing the system properties lookup to return no URL. The assertion verifies that the descriptor still installs the default maxMemory / 20 RPC buffer budget when no source initializes memory. |
||
| // An unsupported classpath URL makes getPropsUrl return null without opening a file. | ||
| System.setProperty(IoTDBConstant.IOTDB_CONF, "classpath:/missing-iotdb-system.properties"); | ||
|
|
||
| try { | ||
| // The descriptor and MemoryConfig statics are isolated by surefire's per-class fork. | ||
| IoTDBDescriptor descriptor = new IoTDBDescriptor(); | ||
| descriptor.getMemoryConfig().activateAutoResizingBufferMemoryControl(); | ||
|
|
||
| assertEquals( | ||
| Runtime.getRuntime().maxMemory() / 20, | ||
| MemoryConfig.getInstance().getAutoResizingBufferMemoryTotalSizeInBytes()); | ||
| } finally { | ||
| if (originalConf == null) { | ||
| System.clearProperty(IoTDBConstant.IOTDB_CONF); | ||
| } else { | ||
| System.setProperty(IoTDBConstant.IOTDB_CONF, originalConf); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
| package org.apache.iotdb.db.conf; | ||
|
|
||
| import org.apache.iotdb.commons.conf.CommonConfig; | ||
| import org.apache.iotdb.commons.conf.IoTDBConstant; | ||
| import org.apache.iotdb.commons.memory.MemoryConfig; | ||
|
|
||
| import org.junit.Rule; | ||
| import org.junit.Test; | ||
| import org.junit.rules.TemporaryFolder; | ||
|
|
||
| import java.io.File; | ||
| import java.nio.charset.StandardCharsets; | ||
| import java.nio.file.Files; | ||
|
|
||
| import static org.junit.Assert.assertEquals; | ||
|
|
||
| public class IoTDBDescriptorSystemPropertiesTest { | ||
|
|
||
| @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); | ||
|
|
||
| @Test | ||
| public void testSystemPropertiesInitializeConfiguredRpcBufferMemory() throws Exception { | ||
| String originalConf = System.getProperty(IoTDBConstant.IOTDB_CONF); | ||
| File confDir = temporaryFolder.newFolder(); | ||
| Files.writeString( | ||
| confDir.toPath().resolve(CommonConfig.SYSTEM_CONFIG_NAME), | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This test uses a real temporary system properties file so the constructor executes the configured-property path. It verifies the regression fix through the public RPC buffer memory control after the descriptor initializes memory from datanode_memory_proportion. |
||
| "datanode_memory_proportion=1:1:1:1:1:5\n", | ||
| StandardCharsets.UTF_8); | ||
| System.setProperty(IoTDBConstant.IOTDB_CONF, confDir.getAbsolutePath()); | ||
|
|
||
| try { | ||
| // The descriptor and MemoryConfig statics are isolated by surefire's per-class fork. | ||
| IoTDBDescriptor descriptor = new IoTDBDescriptor(); | ||
| descriptor.getMemoryConfig().activateAutoResizingBufferMemoryControl(); | ||
|
|
||
| assertEquals( | ||
| Runtime.getRuntime().maxMemory() / 4, | ||
| MemoryConfig.getInstance().getAutoResizingBufferMemoryTotalSizeInBytes()); | ||
| } finally { | ||
| if (originalConf == null) { | ||
| System.clearProperty(IoTDBConstant.IOTDB_CONF); | ||
| } else { | ||
| System.setProperty(IoTDBConstant.IOTDB_CONF, originalConf); | ||
| } | ||
| } | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Capture whether the system properties source was loaded because loadProperties initializes memoryConfig as part of that path. The constructor needs this state to distinguish an already configured memory manager from the no-configuration fallback.