diff --git a/.github/scripts/check-docker-images.sh b/.github/scripts/check-docker-images.sh index 23b99147d4..c49c97bcd3 100644 --- a/.github/scripts/check-docker-images.sh +++ b/.github/scripts/check-docker-images.sh @@ -35,8 +35,7 @@ for pair in \ case "$dockerfile" in hugegraph-pd/*) OWNED_DIRS=". logs pd_data" ;; hugegraph-store/*) OWNED_DIRS=". logs storage" ;; - hugegraph-server/Dockerfile) OWNED_DIRS=". logs rocksdb-data" ;; - *) OWNED_DIRS=". logs" ;; + hugegraph-server/*) OWNED_DIRS=". logs rocksdb-data" ;; esac IMAGE_USER=$(docker image inspect --format '{{.Config.User}}' "$IMAGE_ID") RUNTIME_IDS=$(docker run --rm --pull never --entrypoint sh "$IMAGE_ID" -c \ @@ -47,6 +46,20 @@ for pair in \ exit 1 } + if [[ "$dockerfile" == hugegraph-server/* ]]; then + ( + volume=$(docker volume create) + trap 'docker volume rm "$volume" >/dev/null' EXIT + # A new named volume must inherit writable ownership, including + # when the HStore image is explicitly used with local RocksDB. + docker run --rm --pull never --entrypoint sh \ + -v "$volume:/hugegraph-server/rocksdb-data" "$IMAGE_ID" -ec ' + test "$(stat -c %u:%g rocksdb-data)" = 10001:10001 + touch rocksdb-data/ownership-probe + ' + ) + fi + # Helm reuses these images for tool containers instead of pulling curlimages/curl. # Check the same non-root/read-only tool environment as the Server test hook. if [[ "$dockerfile" == hugegraph-store/* || "$dockerfile" == hugegraph-server/* ]]; then diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index 7e1f511111..b20cdaac95 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -89,6 +89,19 @@ def test_ordinary_server_changes_only_run_mainline_server_and_arm_smoke(self): self.assertFalse(plan["macos_full"]) self.assertFalse(plan["pd_store"]) + def test_topling_assets_run_server_native_contracts(self): + scripts = ("test-topling-prepare.sh", "test-topling-selection.sh") + workflow = (Path(__file__).resolve().parents[1] / "workflows/server-tests.yml").read_text() + for script in scripts: + step = next(step for step in workflow.split("\n - ") if script in step) + self.assertIn("if: inputs.backend == 'rocksdb'", step) + paths = [policy.TRAVIS + script for script in scripts] + paths += ["hugegraph-server/hugegraph-dist/src/assembly/static/" + path for path in + ("bin/prepare-topling.sh", "bin/preload-topling.sh", "conf/toplingdb.yaml")] + for path in paths: + with self.subTest(path=path): + self.assertIn("server-rocksdb", self.coordinator_jobs(self.plan_for([path]))) + def test_distributed_server_contracts_keep_hstore_cluster_without_pd_store_units(self): for path in [policy.SERVER_CORE + "backend/store/BackendStore.java", policy.SERVER_CORE + "meta/MetaManager.java", policy.SERVER_CORE + "StandardHugeGraph.java", diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index a556e1cde8..44facb281a 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -188,6 +188,7 @@ jobs: - name: Check explicit Topling launcher selection run: | bash $TRAVIS_DIR/test-topling-selection.sh + bash $TRAVIS_DIR/test-topling-prepare.sh bash hugegraph-store/hg-store-dist/src/assembly/travis/test-store-preloads.sh - name: Run PD docker entrypoint secret override tests diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml index 1f707604ea..13dde6f94e 100644 --- a/.github/workflows/server-tests.yml +++ b/.github/workflows/server-tests.yml @@ -79,6 +79,8 @@ jobs: "$TRAVIS_DIR/test-wait-storage.sh" "$TRAVIS_DIR/test-start-server.sh" "$TRAVIS_DIR/test-riscv64-java-runtime.sh" + bash "$TRAVIS_DIR/test-topling-selection.sh" + bash "$TRAVIS_DIR/test-topling-prepare.sh" bash "$TRAVIS_DIR/test-tinkerpop-reports.sh" - name: Validate Docker integration diff --git a/docker/README.md b/docker/README.md index ee5e39f36f..ee7e5fcbcf 100644 --- a/docker/README.md +++ b/docker/README.md @@ -112,6 +112,11 @@ an explicit `-Xmx` that exceeds the container's limit. This is the recommended quickstart. +For a single Server without Hubble, or an unpacked binary distribution, see +[Standalone Server deployment](../docs/deployment-standalone.md). The standard +image is `hugegraph/hugegraph`; `hugegraph/server` selects HStore defaults and +requires PD/Store unless its backend configuration is explicitly replaced. + Start: ```bash diff --git a/docs/deployment-standalone.md b/docs/deployment-standalone.md new file mode 100644 index 0000000000..738e9be9d2 --- /dev/null +++ b/docs/deployment-standalone.md @@ -0,0 +1,107 @@ +# Standalone Server with local RocksDB + +Standalone Server needs no PD or Store. The standard distribution ships +`backend=rocksdb` and `serializer=binary`; `pd.peers` is commented out. The +HStore image deliberately replaces this graph configuration. Select the image +or distribution before diagnosing a wait for distributed storage. + +| Deployment | Artifact | Default backend | +| --- | --- | --- | +| Single Server container | `hugegraph/hugegraph:` | Local RocksDB | +| Binary distribution | `apache-hugegraph-server-` | Local RocksDB | +| Distributed Server container | `hugegraph/server:` | HStore; requires PD and Store | +| Current Helm chart | `helm/hugegraph` | HStore only | + +The chart's `values-single.yaml` means one PD, one Store and one Server. It does +not mean a standalone RocksDB Server. Vermeer and SeaTunnel each have their own +standalone topology setting, independent of the Server storage backend. + +## One Server container + +Pin a tag built from the revision you want to run; `latest` is mutable. A +published image may not yet contain the corresponding source fixes. + +```bash +docker volume create hugegraph-rocksdb +docker run -d --name hugegraph-standalone \ + -p 127.0.0.1:8080:8080 \ + -e PASSWORD='replace-with-your-password' \ + -v hugegraph-rocksdb:/hugegraph-server/rocksdb-data \ + hugegraph/hugegraph:latest +docker logs -f hugegraph-standalone +``` + +The image initializes authentication from `PASSWORD` on the first startup. +Changing this variable does not rotate an existing account's password. The +named data volume retains its contents when the container is removed. The +default data path is `/hugegraph-server/rocksdb-data`; mounting a volume at +`/hugegraph-server/data` does not persist the default RocksDB database. + +If you need to reuse the HStore Server image for a new, empty standalone +installation, set both `HG_SERVER_BACKEND=rocksdb` and +`HG_SERVER_USE_PD=false`. Its template already uses `serializer=binary`. +The entrypoint then configures local storage before the storage-wait step; +non-HStore backends skip PD/Store polling even if the template retains +`pd.peers`. Backend selection accepts spaces around `=` as well as the ordinary +colon/whitespace property separators, without requiring a backend environment +override. Only recognized local backends bypass the wait; an unread or unknown +backend cannot silently take that path. +Images containing this fix pre-create `rocksdb-data` owned by +10001:10001 so a fresh named volume inherits writable ownership. +No graph configuration bind mount is needed. Older HStore images need a +corrected writable graph configuration and data-directory ownership instead. +Do not switch an existing +HStore installation's backend or reuse its initialization marker this way. + +Custom configuration bind mounts must be writable by UID/GID 10001. Mount a +whole writable configuration directory rather than a single properties file: +the entrypoint edits files with atomic rename. Prefer a named data volume to +changing host-directory permissions recursively. See the +[Docker guide](../docker/README.md) for authentication keys, volume ownership +and a Server plus Hubble Compose quickstart. + +## Binary distribution + +Use Java 17. Server 1.8 bytecode does not run on Java 11. Run these commands +inside a fresh unpacked Server distribution: + +```bash +java -version +# Confirm backend=rocksdb, serializer=binary and no active pd.peers in this file. +cat conf/graphs/hugegraph.properties +bin/init-store.sh +bin/start-hugegraph.sh +``` + +To enable authentication on a fresh distribution instead, set +`auth.admin_pa` in `conf/rest-server.properties`, run `bin/enable-auth.sh`, +and provide the same password on stdin when running `bin/init-store.sh`. +The initializer does not read the container entrypoint's `PASSWORD` variable. +Then start Server normally. Configuration files containing passwords must be +readable only by the service account and its administrators. + +## Verify the graph + +```bash +curl -fsS http://localhost:8080/versions +curl -fsS -u 'admin:replace-with-your-password' \ + http://localhost:8080/graphspaces/DEFAULT/graphs +curl -fsS -u 'admin:replace-with-your-password' \ + -H 'Content-Type: application/json' \ + -d '{"gremlin":"g.V().count()","aliases":{"g":"__g_DEFAULT-hugegraph"}}' \ + http://localhost:8080/gremlin +``` + +Omit `-u` if authentication is disabled. Server 1.8 still uses the `DEFAULT` +GraphSpace in standalone mode; it does not register the legacy +`__g_hugegraph` traversal source. GraphSpace management itself requires PD. +See [REST requests](rest-api-usage.md) for aliases, shard pagination and tasks. + +## Optional Topling runtime + +Keep `backend=rocksdb` and select the native provider separately, following +[Storage provider selection](storage-lifecycle.md#storage-provider-selection-and-toplingdb). +The standard image includes a Java 17 JDK and `unzip` for preparing a compatible +JNI supplied by the operator; it does not bundle or download optional JNI. +The JNI's glibc, libaio and CPU requirements must match the image or host. +Do not assume that a mutable image tag proves which native library is loaded. diff --git a/docs/rest-api-usage.md b/docs/rest-api-usage.md index 06201002ae..f489996bb8 100644 --- a/docs/rest-api-usage.md +++ b/docs/rest-api-usage.md @@ -26,10 +26,46 @@ are query variables; they do not create graph bindings. An alias target must be an existing graph or traversal-source binding, rather than a Gremlin expression such as `graph.traversal()`. -For a standalone legacy graph with no GraphSpace, use `` and -`__g_` instead. The Java client builds the appropriate aliases from its -configured GraphSpace and graph. Verify the configured names before treating a -`Could not rebind` response as a graph-registration failure. +Server 1.8 uses these GraphSpace-scoped bindings in standalone RocksDB mode too: +the default graph is `DEFAULT-hugegraph`, and its traversal source is +`__g_DEFAULT-hugegraph`. Disabling PD does not select legacy aliases. Only older +Servers without GraphSpace bindings use `` and `__g_`. Select the +Java client's GraphSpace to match the Server, and verify the configured names +before treating a `Could not rebind` response as a graph-registration failure. + +## Export a shard with bounded pages + +Both `GET .../traversers/vertices/scan` and `GET .../traversers/edges/scan` +require `start` and `end`. Missing either parameter returns HTTP 400 with its +name on every backend. This also changes HStore and in-memory requests that +previously interpreted missing bounds as zero; callers must now supply both. +See the [upgrade notes](upgrade-notes.md#required-shard-scan-bounds). + +Empty strings represent open bounds in RocksDB and HBase. They do **not** +represent a whole-graph range in HStore or the in-memory backend. The following +example is for **RocksDB**: + +```bash +curl -G 'http://localhost:8080/graphspaces/DEFAULT/graphs/hugegraph/traversers/vertices/scan' \ + --data-urlencode 'start=' --data-urlencode 'end=' \ + --data-urlencode 'page=' --data-urlencode 'page_limit=1000' +``` + +Pass `page=` explicitly for the first page, then URL-encode the returned `page` +token on subsequent requests, keeping the same bounds. Stop when the returned +token is null or empty. Edge scans use the same pagination contract. + +For HStore and in-memory exports, first request the corresponding +`.../vertices/shards?split_size=1048576` or `.../edges/shards?split_size=1048576`. +Iterate **every** returned shard, using its exact `start` and `end` and a fresh +`page=` for each shard. Preserve the backend's boundary encoding; do not reuse +vertex bounds for edges. An empty HStore range can stop after the partition +containing hash zero, and an empty in-memory end can produce no results. +A terminal page token finishes only that shard, not the entire graph. + +Omitting `page` retains the existing unpaged scan behavior: `page_limit` does +not bound that response and the result has no continuation token. Exporters +must pass the empty initial page to avoid reading an entire shard at once. ## Run long queries as tasks diff --git a/docs/storage-lifecycle.md b/docs/storage-lifecycle.md index ddf7731a0a..0aac771e35 100644 --- a/docs/storage-lifecycle.md +++ b/docs/storage-lifecycle.md @@ -503,7 +503,30 @@ Use a stopped Server distribution on Linux x86_64 with a Java 17 JDK. If your distribution lacks these scripts, [build it from this source](#build-and-prepare) first. Prepare the runtime once, select it, then initialize and start the Server. The graph APIs and normal start/stop commands stay the same; Topling's native options are -configured in `conf/toplingdb.yaml`. Use new, empty, service-writable data and WAL +configured in `conf/toplingdb.yaml`. The shipped native profile uses community-compatible SkipList +memtables and BlockBasedTable files; it does not require CSPP, DispatcherTable +or private compression plugins. Its 64 MiB cache and 16 MiB write buffers are +individual native budgets, not a process memory limit. Budget for every database +and column family and tune the profile for the deployment. + +With the tested community JNI, EasyMigrate applies these YAML values when the +database opens, overriding the corresponding Java RocksDBOptions values. +The observed settings are a 64 MiB block cache, 16 MiB write buffers and three +write buffers; Java initializes the latter two to 128 MiB and six beforehand. +Other JNI builds must be checked against their persisted OPTIONS files and +`rocksdb.block-cache-capacity`; Java option getters before open do not establish +the effective native settings. + +Custom profiles can require additional plugins in the chosen JNI. Preparation +imports the selected profile to verify its plugins before installing the +runtime. Existing data written with other table formats still requires those +formats' readers; changing this default does not convert existing SST files. +For existing CSPP installations, preserve the matching JNI and native options; +`conf/toplingdb-legacy-cspp.yaml` retains the prior configuration and readers in +Server, PD and Store distributions. Read the [Topling upgrade notes](upgrade-notes.md#topling-native-profiles) +before applying the new profile or its smaller memory and compaction defaults. + +Use new, empty, service-writable data and WAL directories outside the distribution for this example. **1. Prepare the runtime.** Supply your ToplingDB JNI JAR and its trusted SHA-256: @@ -569,8 +592,18 @@ entry, the ELF architecture and native dependencies. Before installation, it ope closes a disposable database with the selected JNI and checks that a probe configuration changes the persisted write-buffer size. A marker or matching hash alone is insufficient: builds without the EasyMigrate configuration entry point are rejected. The -probe validates the configuration entry point, not existing-data compatibility or every -setting in your production profile. Its checksum receipt binds the probe to the +probe validates the configuration entry point. A second native import validates +`TOPLINGDB_EASY_MIGRATE_CONF` when set, otherwise the component's +`conf/toplingdb.yaml`; unavailable plugins fail preparation before publication. +It does not open databases using production WAL/log paths. +Relative profile overrides are converted to absolute paths against the caller's +working directory by both preparation and runtime selection, before the launcher +changes directories. Resolution ignores `CDPATH` and follows physical directory +semantics for symlinks combined with `..`, matching the file checked for readability. +Use the same file for preparation and service startup. +This is not an existing-data compatibility test or validation of every production +column family. Keep an explicit profile override in the service environment too. +The checksum receipt binds the capability probe to the installed JAR and native library. Launchers reject missing or stale receipts; prepare a fresh distribution when upgrading from an older preparation script. It installs `topling/rocksdbjni.jar`, its matching native library and optional web resources outside diff --git a/docs/upgrade-notes.md b/docs/upgrade-notes.md new file mode 100644 index 0000000000..906964bee2 --- /dev/null +++ b/docs/upgrade-notes.md @@ -0,0 +1,52 @@ +# Storage and scan upgrade notes + +These notes describe development storage changes. A published image can lag the +source revision; check that the deployed build contains the corresponding fixes. + +## Required shard scan bounds + +`traversers/vertices/scan` and `traversers/edges/scan` now reject a missing +`start` or `end` with HTTP 400 on **all backends**. Previously RocksDB could +throw an NPE and return 500, while HStore and in-memory implementations could +interpret omitted values as zero and run an incomplete scan. Both parameters +are now mandatory even when that older behavior appeared to succeed. + +For an export, obtain all ranges from the matching `vertices/shards` or +`edges/shards` endpoint. Keep each range's exact boundary strings and paginate +it independently with an explicit initial `page=`. Only RocksDB and HBase +interpret empty bounds as an open range. Do not use `start=&end=` for a +whole-graph HStore or in-memory export. See [REST usage](rest-api-usage.md#export-a-shard-with-bounded-pages). + +## Topling native profiles + +The new default `conf/toplingdb.yaml` supports the tested community JNI without +CSPP plugins. It changes native formats and tuning from the previous default: + +| Setting | Previous CSPP profile | New community profile | +| --- | --- | --- | +| Memtable/table factories | CSPP with DispatcherTable and CSPPMemTabTable readers | SkipList and BlockBasedTable | +| Block cache | 8 GiB | 64 MiB | +| Write buffer / maximum buffers | 128 MiB / 6 | 16 MiB / 3 | +| Maximum background jobs | 8 | 2 | + +Other prior compaction tuning is also absent from the smaller community profile. +This is a configuration and performance change, not a transparent optimization +or an existing-data format conversion. With the tested community JNI, native +EasyMigrate options override corresponding Java options when opening a database. +Inspect effective OPTIONS and cache capacity when using a different JNI build. + +For an existing Topling installation, stop the component and retain its previous +JNI, configuration and a recoverable data backup. The Server, PD and Store +distributions also ship `conf/toplingdb-legacy-cspp.yaml`, preserving the old +profile unchanged. Select that profile, or the deployment's previous tuned +profile, through an absolute `TOPLINGDB_EASY_MIGRATE_CONF` path during preparation +and startup. Keep the native library that provides the required CSPP readers. +The community JNI used for validation lacks those readers and cannot validate +old CSPP SST data; such an upgrade needs a matching JNI and old-format dataset +before deployment. Do not replace its options with the new community profile. + +The prepare script verifies JNI capabilities with a disposable database and +imports the selected profile before publication. This import checks plugin +availability; it does not read existing business SSTs or certify their +compatibility. Relative profile paths are normalized before a launcher changes +its working directory; absolute service configuration is recommended. diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/descriptor/server-assembly.xml b/hugegraph-pd/hg-pd-dist/src/assembly/descriptor/server-assembly.xml index fcf0d93005..a18bb0929b 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/descriptor/server-assembly.xml +++ b/hugegraph-pd/hg-pd-dist/src/assembly/descriptor/server-assembly.xml @@ -57,6 +57,11 @@ conf 644 + + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb-legacy-cspp.yaml + conf + 644 + diff --git a/hugegraph-server/Dockerfile b/hugegraph-server/Dockerfile index 912ddf8755..12577eee50 100644 --- a/hugegraph-server/Dockerfile +++ b/hugegraph-server/Dockerfile @@ -59,6 +59,7 @@ RUN apt-get -q update \ procps \ curl \ iproute2 \ + unzip \ vim \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/hugegraph-server/Dockerfile-hstore b/hugegraph-server/Dockerfile-hstore index bcf405e42b..896b235945 100644 --- a/hugegraph-server/Dockerfile-hstore +++ b/hugegraph-server/Dockerfile-hstore @@ -59,6 +59,7 @@ RUN apt-get -q update \ procps \ curl \ iproute2 \ + unzip \ vim \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -72,7 +73,8 @@ ARG HG_GID=10001 RUN groupadd --system --gid "${HG_GID}" hugegraph \ && useradd --system --uid "${HG_UID}" --gid hugegraph --no-create-home \ --home-dir /hugegraph-server --shell /usr/sbin/nologin hugegraph \ - && install -d -o "${HG_UID}" -g "${HG_GID}" /hugegraph-server /hugegraph-server/logs + && install -d -o "${HG_UID}" -g "${HG_GID}" \ + /hugegraph-server /hugegraph-server/rocksdb-data /hugegraph-server/logs COPY --from=build --chown=${HG_UID}:${HG_GID} /pkg/hugegraph-server/apache-hugegraph-server-*/ /hugegraph-server/ USER ${HG_UID}:${HG_GID} diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/EdgesAPI.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/EdgesAPI.java index 9ae331a02d..86fc140904 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/EdgesAPI.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/EdgesAPI.java @@ -114,6 +114,9 @@ public String scan(@Context GraphManager manager, LOG.debug("Graph [{}] query edges by shard(start: {}, end: {}, " + "page: {}) ", graph, start, end, page); + E.checkArgument(start != null, "The start parameter is required (use a boundary returned by shards)"); + E.checkArgument(end != null, "The end parameter is required (use a boundary returned by shards)"); + HugeGraph g = graph(manager, graphSpace, graph); ConditionQuery query = new ConditionQuery(HugeType.EDGE_OUT); diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/VerticesAPI.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/VerticesAPI.java index 0dc09256fe..df58968b7c 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/VerticesAPI.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/VerticesAPI.java @@ -116,6 +116,9 @@ public String scan(@Context GraphManager manager, LOG.debug("Graph [{}] query vertices by shard(start: {}, end: {}, " + "page: {}) ", graph, start, end, page); + E.checkArgument(start != null, "The start parameter is required (use a boundary returned by shards)"); + E.checkArgument(end != null, "The end parameter is required (use a boundary returned by shards)"); + HugeGraph g = graph(manager, graphSpace, graph); ConditionQuery query = new ConditionQuery(HugeType.VERTEX); diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh index 3f88216368..84ee514d52 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh @@ -33,7 +33,7 @@ topling_remove_path() { topling_select_runtime() { local top provider native jar conf inherited entry local -a preloads - top="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" || return 1 + top="$(CDPATH= cd -P -- "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" || return 1 provider="${TOPLINGDB_ROCKSDB_PROVIDER:-rocksdb}" case "$provider" in rocksdb | topling) ;; *) echo "Error: invalid TOPLINGDB_ROCKSDB_PROVIDER '$provider'" >&2; return 1 ;; @@ -83,9 +83,12 @@ topling_select_runtime() { conf="${TOPLINGDB_EASY_MIGRATE_CONF:-}" if [ -z "$conf" ]; then conf="$top/conf/toplingdb.yaml" - export TOPLING_AUTO_MIGRATE_CONF="$conf" fi [ -r "$conf" ] || { echo "Error: unreadable Easy Migrate configuration: $conf" >&2; return 1; } + conf="$(CDPATH= cd -P -- "$(dirname "$conf")" && pwd)/$(basename "$conf")" || return 1 + if [ -z "${TOPLINGDB_EASY_MIGRATE_CONF:-}" ]; then + export TOPLING_AUTO_MIGRATE_CONF="$conf" + fi export TOPLINGDB_EASY_MIGRATE_CONF="$conf" export LD_LIBRARY_PATH="$top/topling${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" # The selected JNI must precede inherited preloads during native symbol lookup. diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/prepare-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/prepare-topling.sh index 847924c464..8cdd03c427 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/prepare-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/prepare-topling.sh @@ -17,7 +17,7 @@ # # Run only after stopping this component. Runtime preparation never changes data. set -Eeuo pipefail -TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TOP="$(CDPATH= cd -P -- "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" [ "$(uname -s)/$(uname -m)" = Linux/x86_64 ] || { echo 'Error: Topling requires Linux x86_64' >&2; exit 1; } @@ -30,6 +30,9 @@ TOP="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" [ ! -e "$TOP/topling" ] && [ ! -L "$TOP/topling" ] || { echo 'Error: runtime already prepared; use a fresh stopped distribution to replace it' >&2; exit 1; } +profile="${TOPLINGDB_EASY_MIGRATE_CONF:-$TOP/conf/toplingdb.yaml}" +[ -r "$profile" ] || { echo 'Error: Topling configuration must be readable' >&2; exit 1; } +profile="$(CDPATH= cd -P -- "$(dirname "$profile")" && pwd)/$(basename "$profile")" staging=$(mktemp -d "$TOP/.topling-prepare.XXXXXX") trap 'rm -rf "$staging"' EXIT cp "$TOPLING_JNI_JAR" "$staging/rocksdbjni.jar" @@ -76,6 +79,7 @@ import java.nio.file.Path; import java.util.regex.Pattern; import org.rocksdb.Options; import org.rocksdb.RocksDB; +import org.rocksdb.SidePluginRepo; class ToplingRuntimeProbe { public static void main(String[] args) throws Exception { @@ -83,6 +87,12 @@ class ToplingRuntimeProbe { throw new IllegalStateException("Prepare Topling with Java 17"); } RocksDB.loadLibrary(); + if (args.length == 2 && "--profile".equals(args[0])) { + try (SidePluginRepo repo = new SidePluginRepo()) { + repo.importAutoFile(args[1]); + } + return; + } Path database = Path.of(args[0]); try (Options options = new Options().setCreateIfMissing(true); RocksDB db = RocksDB.open(options, database.toString())) { @@ -112,6 +122,17 @@ if ! LD_LIBRARY_PATH="$staging${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ echo 'Error: Java 17 JDK / Topling EasyMigrate capability probe failed' >&2 exit 1 fi +# Import the selected profile without opening its databases. In particular, do +# not let production WAL/log paths redirect the disposable write probe above. +# EasyMigrate support alone does not prove that every named plugin is present. +if ! LD_LIBRARY_PATH="$staging${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ + LD_PRELOAD="$staging/librocksdbjni-linux64.so" \ + TOPLINGDB_EASY_MIGRATE_CONF="$staging/probe.yaml" \ + java --add-opens=java.base/java.nio=ALL-UNNAMED --source 17 \ + --class-path "$staging/rocksdbjni.jar" "$staging/ToplingRuntimeProbe.java" --profile "$profile"; then + echo 'Error: Topling configuration validation failed; runtime was not installed' >&2 + exit 1 +fi native_hash=$(sha256sum "$staging/librocksdbjni-linux64.so" | awk '{print $1}') printf '%s rocksdbjni.jar\n%s librocksdbjni-linux64.so\n' "$actual" "$native_hash" > "$staging/easy-migrate.sha256" rm -rf "$staging/probe-db" "$staging/probe.yaml" "$staging/ToplingRuntimeProbe.java" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/wait-storage.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/wait-storage.sh index c5bc9c83de..d8b3450589 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/wait-storage.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/wait-storage.sh @@ -101,6 +101,29 @@ while IFS=' ' read -r envvar_key envvar_val; do fi done < <(env | sort -r | awk -F= '{ st = index($0, "="); print $1 " " substr($0, st+1) }') +# A reused HStore configuration can retain pd.peers after switching to a local +# backend. Only HStore depends on PD/Store registration for initialization. +# Accept the ordinary Java-properties separators (including whitespace before +# '=') without relying on the backend environment override to normalize them. +BACKEND=$(awk ' + { + sub(/\r$/, "") + sub(/^[ \t\f]+/, "") + if ($0 !~ /^backend([ \t\f]*[:=]|[ \t\f]+|$)/) next + sub(/^backend[ \t\f]*([:=][ \t\f]*)?/, "") + sub(/[ \t\f]+$/, "") + backend = $0 + } + END { print backend } +' "$GRAPH_CONF") +# Only a recognized local backend can bypass PD. An unrecognized property +# must not silently turn a distributed deployment into a local-backend path. +case "$BACKEND" in + rocksdb | memory | hbase) + log "Backend '$BACKEND' does not use PD/Store, skipping storage wait" + exit 0 ;; +esac + # wait for storage if env | grep '^hugegraph\.' > /dev/null; then if [ -n "${WAIT_STORAGE_TIMEOUT_S:-}" ]; then diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb-legacy-cspp.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb-legacy-cspp.yaml new file mode 100644 index 0000000000..3067b378f1 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb-legacy-cspp.yaml @@ -0,0 +1,157 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# common parameters +http: + # normally parent path of db path + document_root: ./topling/rocksdb_resource + listening_ports: '127.0.0.1:2011' + auto_start_http: false +Cache: + lru_cache: + class: LRUCache + params: + capacity: 8G + num_shard_bits: -1 + strict_capacity_limit: false + high_pri_pool_ratio: 0.5 + use_adaptive_mutex: false + metadata_charge_policy: kFullChargeCacheMetadata +Statistics: + stat: + class: default + params: + discard_tickers: + - rocksdb.block.cache + - rocksdb.block.cachecompressed + - rocksdb.block + - rocksdb.memtable.payload.bytes.at.flush + - rocksdb.memtable.garbage.bytes.at.flush + - rocksdb.txn + - rocksdb.blobdb + - rocksdb.row.cache + - rocksdb.number.block + - rocksdb.bloom.filter + - rocksdb.persistent + - rocksdb.sim.block.cache + discard_histograms: + # comment: .... + - rocksdb.blobdb + - rocksdb.bytes.compressed + - rocksdb.bytes.decompressed + - rocksdb.num.index.and.filter.blocks.read.per.level + - rocksdb.num.data.blocks.read.per.level + - rocksdb.compression.times.nanos + - rocksdb.decompression.times.nanos + - rocksdb.read.block.get.micros + - rocksdb.write.raw.block.micros + # comment end of array + #stats_level: kAll + stats_level: kDisableAll +MemTableRepFactory: + cspp: + class: cspp + params: + mem_cap: 16G + use_vm: false + token_use_idle: true + chunk_size: 16K + convert_to_sst: kFileMmap + sync_sst_file: false + skiplist: + class: SkipList + params: + lookahead: 0 +TableFactory: + cspp_memtab_sst: + class: CSPPMemTabTable + params: # empty params + bb: + class: BlockBasedTable + params: + checksum: kCRC32c + block_size: 4K + block_restart_interval: 16 + index_block_restart_interval: 1 + metadata_block_size: 4K + enable_index_compression: true + block_cache: "${lru_cache}" + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst + block_cache_compressed: + persistent_cache: + filter_policy: + dispatch: + class: DispatcherTable + params: + default: bb + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst + level_writers: [ bb, bb, bb, bb, bb, bb ] +CFOptions: + default: + max_write_buffer_number: 6 + memtable_factory: "${cspp}" + write_buffer_size: 128M + # set target_file_size_base as small as 512K is to make many SST files, + # thus key prefix cache can present efficiency + target_file_size_base: 64M + target_file_size_multiplier: 1 + table_factory: dispatch + max_bytes_for_level_base: 512M + max_bytes_for_level_multiplier: 10 + level_compaction_dynamic_level_bytes: false + level0_slowdown_writes_trigger: 20 + level0_stop_writes_trigger: 36 + level0_file_num_compaction_trigger: 2 + merge_operator: uint64add # support merge + level_compaction_dynamic_file_size: true + optimize_filters_for_hits: true + allow_merge_memtables: true + min_write_buffer_number_to_merge: 2 + compression_per_level: + - kNoCompression + - kNoCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression +DBOptions: + log: + create_if_missing: true + create_missing_column_families: true + default: + create_if_missing: true + create_missing_column_families: false # this is important, must be false to hugegraph + max_background_compactions: -1 + max_subcompactions: 4 + max_level1_subcompactions: 0 + inplace_update_support: false + WAL_size_limit_MB: 0 + statistics: "${stat}" + max_manifest_file_size: 100M + max_background_jobs: 8 + # Java org.rocksdb.WriteBatch has no Topling mmap WAL. With + # memtable_as_log_index enabled, every batch write fails: + # "WriteBatch has no mmap wal". Keep the option false for this client. + # convert_to_sst is still Topling-specific. Do not open data written by + # this profile with the standard RocksDB provider; restore a snapshot + # taken before migration instead. + compaction_readahead_size: 0 + memtable_as_log_index: false diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb.yaml index 3067b378f1..5c0ee4226e 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb.yaml +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb.yaml @@ -14,144 +14,40 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# common parameters + +# Community image defaults. Memory grows with databases and column families; +# these are individual native budgets, not a process memory limit. http: - # normally parent path of db path - document_root: ./topling/rocksdb_resource - listening_ports: '127.0.0.1:2011' auto_start_http: false Cache: lru_cache: class: LRUCache params: - capacity: 8G - num_shard_bits: -1 - strict_capacity_limit: false - high_pri_pool_ratio: 0.5 - use_adaptive_mutex: false - metadata_charge_policy: kFullChargeCacheMetadata -Statistics: - stat: - class: default - params: - discard_tickers: - - rocksdb.block.cache - - rocksdb.block.cachecompressed - - rocksdb.block - - rocksdb.memtable.payload.bytes.at.flush - - rocksdb.memtable.garbage.bytes.at.flush - - rocksdb.txn - - rocksdb.blobdb - - rocksdb.row.cache - - rocksdb.number.block - - rocksdb.bloom.filter - - rocksdb.persistent - - rocksdb.sim.block.cache - discard_histograms: - # comment: .... - - rocksdb.blobdb - - rocksdb.bytes.compressed - - rocksdb.bytes.decompressed - - rocksdb.num.index.and.filter.blocks.read.per.level - - rocksdb.num.data.blocks.read.per.level - - rocksdb.compression.times.nanos - - rocksdb.decompression.times.nanos - - rocksdb.read.block.get.micros - - rocksdb.write.raw.block.micros - # comment end of array - #stats_level: kAll - stats_level: kDisableAll + capacity: 64M MemTableRepFactory: - cspp: - class: cspp - params: - mem_cap: 16G - use_vm: false - token_use_idle: true - chunk_size: 16K - convert_to_sst: kFileMmap - sync_sst_file: false skiplist: class: SkipList params: lookahead: 0 TableFactory: - cspp_memtab_sst: - class: CSPPMemTabTable - params: # empty params - bb: + block_based: class: BlockBasedTable params: - checksum: kCRC32c - block_size: 4K - block_restart_interval: 16 - index_block_restart_interval: 1 - metadata_block_size: 4K - enable_index_compression: true block_cache: "${lru_cache}" - readers: - BlockBasedTable: bb - CSPPMemTabTable: cspp_memtab_sst - block_cache_compressed: - persistent_cache: - filter_policy: - dispatch: - class: DispatcherTable - params: - default: bb - readers: - BlockBasedTable: bb - CSPPMemTabTable: cspp_memtab_sst - level_writers: [ bb, bb, bb, bb, bb, bb ] CFOptions: default: - max_write_buffer_number: 6 - memtable_factory: "${cspp}" - write_buffer_size: 128M - # set target_file_size_base as small as 512K is to make many SST files, - # thus key prefix cache can present efficiency - target_file_size_base: 64M - target_file_size_multiplier: 1 - table_factory: dispatch - max_bytes_for_level_base: 512M - max_bytes_for_level_multiplier: 10 - level_compaction_dynamic_level_bytes: false - level0_slowdown_writes_trigger: 20 - level0_stop_writes_trigger: 36 - level0_file_num_compaction_trigger: 2 - merge_operator: uint64add # support merge - level_compaction_dynamic_file_size: true - optimize_filters_for_hits: true - allow_merge_memtables: true - min_write_buffer_number_to_merge: 2 - compression_per_level: - - kNoCompression - - kNoCompression - - kSnappyCompression - - kSnappyCompression - - kSnappyCompression - - kSnappyCompression - - kSnappyCompression + memtable_factory: "${skiplist}" + table_factory: block_based + write_buffer_size: 16M + max_write_buffer_number: 3 + merge_operator: uint64add DBOptions: log: create_if_missing: true create_missing_column_families: true default: create_if_missing: true - create_missing_column_families: false # this is important, must be false to hugegraph - max_background_compactions: -1 - max_subcompactions: 4 - max_level1_subcompactions: 0 - inplace_update_support: false - WAL_size_limit_MB: 0 - statistics: "${stat}" - max_manifest_file_size: 100M - max_background_jobs: 8 - # Java org.rocksdb.WriteBatch has no Topling mmap WAL. With - # memtable_as_log_index enabled, every batch write fails: - # "WriteBatch has no mmap wal". Keep the option false for this client. - # convert_to_sst is still Topling-specific. Do not open data written by - # this profile with the standard RocksDB provider; restore a snapshot - # taken before migration instead. - compaction_readahead_size: 0 + create_missing_column_families: false + max_background_jobs: 2 + # Java WriteBatch does not carry Topling's mmap WAL references. memtable_as_log_index: false diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-prepare.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-prepare.sh new file mode 100644 index 0000000000..ba10b0e798 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-prepare.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Exercise preparation publication/failure with process fixtures. Real JNI +# compatibility requires a separate native read/write smoke test. +set -Eeuo pipefail +source_bin="$(cd "$(dirname "${BASH_SOURCE[0]}")/../static/bin" && pwd)" +test_root=$(mktemp -d) +trap 'rm -rf "$test_root"' EXIT +mkdir -p "$test_root/tools" +python3 - "$test_root/jni.jar" <<'PY' +import sys +import zipfile + +header = bytearray(20) +header[:6] = b'\x7fELF\x02\x01' +header[18:20] = b'\x3e\x00' +with zipfile.ZipFile(sys.argv[1], 'w') as jar: + jar.writestr('org/rocksdb/SidePluginRepo.class', b'fixture') + jar.writestr('librocksdbjni-linux64.so', header) +PY +cat > "$test_root/tools/ldd" <<'SH' +#!/bin/sh +exit 0 +SH +cat > "$test_root/tools/uname" <<'SH' +#!/bin/sh +case "$1" in -s) echo Linux ;; -m) echo x86_64 ;; *) exit 1 ;; esac +SH +cat > "$test_root/tools/java" <<'SH' +#!/bin/bash +set -eu +profile="$TOPLINGDB_EASY_MIGRATE_CONF" +if [[ "${*: -2:1}" == --profile ]]; then + profile="${@: -1}" +fi +printf '%s\n' "$profile" >> "$PROBE_LOG" +printf '%s\n' "$TOPLINGDB_EASY_MIGRATE_CONF" >> "$PROBE_ENV_LOG" +if grep -q 'missing-plugin' "$profile"; then + exit 1 +fi +SH +chmod +x "$test_root/tools/"* +export PATH="$test_root/tools:$PATH" +export TOPLING_JNI_JAR="$test_root/jni.jar" +TOPLING_JNI_SHA256=$(sha256sum "$TOPLING_JNI_JAR" | awk '{print $1}') +export TOPLING_JNI_SHA256 +unset TOPLINGDB_EASY_MIGRATE_CONF + +for scenario in default invalid override relative cdpath symlink; do + component="$test_root/$scenario" + mkdir -p "$component/bin" "$component/conf" + cp "$source_bin/prepare-topling.sh" "$component/bin/" + printf '%s\n' 'http: {auto_start_http: false}' > "$component/conf/toplingdb.yaml" + export PROBE_LOG="$component/probes" + export PROBE_ENV_LOG="$component/probe-env" + unset TOPLINGDB_EASY_MIGRATE_CONF + if [[ "$scenario" != default ]]; then + printf '%s\n' 'missing-plugin' > "$component/conf/toplingdb.yaml" + fi + if [[ "$scenario" == override || "$scenario" == relative || + "$scenario" == cdpath || "$scenario" == symlink ]]; then + printf '%s\n' 'http: {auto_start_http: false}' > "$component/custom.yaml" + export TOPLINGDB_EASY_MIGRATE_CONF="$component/custom.yaml" + fi + if [[ "$scenario" == relative || "$scenario" == cdpath ]]; then + export TOPLINGDB_EASY_MIGRATE_CONF="$scenario/custom.yaml" + fi + if [[ "$scenario" == symlink ]]; then + mkdir -p "$component/physical/child" + ln -s "$component/physical/child" "$component/alias" + cp "$component/custom.yaml" "$component/physical/custom.yaml" + printf '%s\n' 'missing-plugin' > "$component/custom.yaml" + export TOPLINGDB_EASY_MIGRATE_CONF="$component/alias/../custom.yaml" + fi + script="$component/bin/prepare-topling.sh" + fixture_cdpath="" + if [[ "$scenario" == cdpath ]]; then + fixture_cdpath=. + script="$scenario/bin/prepare-topling.sh" + fi + status=0 + (cd "$test_root" && CDPATH="$fixture_cdpath" bash "$script") > "$component/output" 2>&1 || status=$? + # Both the minimal capability and selected profile must be probed. + [[ $(wc -l < "$PROBE_LOG") == 2 ]] + capability=$(head -1 "$PROBE_LOG") + [[ "$capability" == "$component"/.topling-prepare.*/probe.yaml ]] + # Both JVMs initialize under disposable options. Only the importer's + # explicit --profile argument names the production configuration. + [[ $(head -1 "$PROBE_ENV_LOG") == "$capability" ]] + [[ $(tail -1 "$PROBE_ENV_LOG") == "$capability" ]] + if [[ "$scenario" == invalid ]]; then + [[ "$status" != 0 && ! -e "$component/topling" ]] + grep -q 'configuration validation failed' "$component/output" + else + [[ "$status" == 0 ]] + (cd "$component/topling" && sha256sum -c easy-migrate.sha256 >/dev/null) + [[ ! -e "$component/topling/ToplingRuntimeProbe.java" ]] + expected="${TOPLINGDB_EASY_MIGRATE_CONF:-$component/conf/toplingdb.yaml}" + case "$scenario" in + relative | cdpath) expected="$component/custom.yaml" ;; + symlink) expected="$component/physical/custom.yaml" ;; + esac + [[ $(tail -1 "$PROBE_LOG") == "$expected" ]] + fi + [[ -z $(find "$component" -maxdepth 1 -name '.topling-prepare.*' -print -quit) ]] +done +echo 'Topling capability/profile preparation contract passed' diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-selection.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-selection.sh index de9624cf1f..77469a945f 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-selection.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-selection.sh @@ -106,6 +106,32 @@ for provider in unset rocksdb topling; do grep -q 'competing RocksDB JNI in LD_PRELOAD' "$TEST_ROOT/error" done done +# Relative overrides must survive cd to TOP, CDPATH, and symlink/.. semantics. +mkdir -p "$TEST_ROOT/server/physical/child" +cp "$TEST_ROOT/server/conf/toplingdb.yaml" "$TEST_ROOT/server/physical/custom.yaml" +ln -s "$TEST_ROOT/server/physical/child" "$TEST_ROOT/profile-link" +for scenario in relative cdpath symlink; do +PATH="$TEST_ROOT/fake-bin:$PATH" bash -ec ' + unset TOPLING_ACTIVE_NATIVE TOPLING_RUNTIME_CLASSPATH TOPLING_AUTO_MIGRATE_CONF + export TOPLINGDB_ROCKSDB_PROVIDER=topling + cd "$1" + export TOPLINGDB_EASY_MIGRATE_CONF=server/conf/toplingdb.yaml + expected="$1/server/conf/toplingdb.yaml" + if [[ "$2" == cdpath ]]; then + export CDPATH=. + elif [[ "$2" == symlink ]]; then + export TOPLINGDB_EASY_MIGRATE_CONF=profile-link/../custom.yaml + expected="$1/server/physical/custom.yaml" + fi + source server/bin/preload-topling.sh + if [[ "$2" == cdpath ]]; then + test "$CDPATH" = . + fi + cd "$1/pd" + test "$TOPLINGDB_EASY_MIGRATE_CONF" = "$expected" + test -r "$TOPLINGDB_EASY_MIGRATE_CONF" +' _ "$TEST_ROOT" "$scenario" +done # Older or modified preparations must be verified again before launch. for state in missing stale; do receipt="$TEST_ROOT/server/topling/easy-migrate.sha256" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-wait-storage.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-wait-storage.sh index 046f255d50..6c72687e9c 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-wait-storage.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-wait-storage.sh @@ -75,6 +75,7 @@ assert_contract() { run_case() { local scenario="$1" peers="$2" abort_after="$3" password="${4:-test-password}" local timeout_args=() + local backend_args=('hugegraph.backend=hstore') if [[ "$#" -ge 5 ]]; then timeout_args=("WAIT_STORAGE_TIMEOUT_S=$5") fi @@ -83,9 +84,14 @@ run_case() { : > "${CONFIG_LOG}" : > "${COUNT_FILE}" : > "${TIMEOUT_LOG}" - : > "${DIST_ROOT}/conf/graphs/hugegraph.properties" + if [[ "$#" -ge 6 ]]; then + printf '%s\n' "$6" > "${DIST_ROOT}/conf/graphs/hugegraph.properties" + backend_args=() + else + : > "${DIST_ROOT}/conf/graphs/hugegraph.properties" + fi - CASE_OUTPUT=$(env -u WAIT_STORAGE_TIMEOUT_S \ + CASE_OUTPUT=$(env -u WAIT_STORAGE_TIMEOUT_S -u hugegraph.backend \ ${timeout_args[@]+"${timeout_args[@]}"} \ PATH="${MOCK_BIN}:${PATH}" \ MOCK_SCENARIO="${scenario}" \ @@ -98,7 +104,7 @@ run_case() { HG_SERVER_PD_REST_ENDPOINT="${peers}" \ PD_AUTH_USER="test-user" \ PD_AUTH_PASSWORD="${password}" \ - 'hugegraph.backend=hstore' \ + ${backend_args[@]+"${backend_args[@]}"} \ 'hugegraph.pd.peers=config-only:8686' \ "${DIST_ROOT}/bin/wait-storage.sh" 2>&1) CASE_RC=$? @@ -310,7 +316,7 @@ CASE_OUTPUT=$(env \ "${DIST_ROOT}/bin/wait-storage.sh" 2>&1) CASE_RC=$? assert_equal "no-PD topology rc" "0" "${CASE_RC}" -assert_output "No pd.peers configured, skipping storage wait" +assert_output "does not use PD/Store, skipping storage wait" [[ "${CASE_OUTPUT}" != *"PD_AUTH_PASSWORD is empty"* ]] || \ fail "warned about an unused PD credential with no pd.peers configured" echo " PASS no credential warning without pd.peers" @@ -334,4 +340,47 @@ for storage_timeout in '' 0 -1 1.5 abc 86401 999999999999999999999; do echo " PASS invalid timeout '${storage_timeout}'" done -echo "20 passed, 0 failed" +# With only the PD-peer override, entrypoint leaves backend syntax untouched. +# Exercise ready and unready stores without a backend override that could hide +# the configured-property parsing bug. +for property in 'backend = hstore' $' \tbackend\t=\thstore \t' \ + 'backend:hstore' 'backend hstore' \ + $'# backend=memory\nbackend.other=rocksdb\nbackend=rocksdb\nbackend = hstore'; do + run_case "pd1-up" "pd0:8620,pd1:8620" 6 test-password 5 "$property" + assert_equal "configured HStore rc" 0 "$CASE_RC" + assert_equal "configured HStore polls both peers" "$TWO_CALLS" "$(cat "$CALL_LOG")" + assert_output "Store registration check PASSED via pd1:8620" + assert_contract 5 + + run_case "none" "pd0:8620,pd1:8620" 2 test-password 5 "$property" + [[ "$CASE_RC" -ne 0 ]] || fail "configured HStore proceeded before a Store was Up" + assert_output "ERROR: Timeout waiting for storage backend" + assert_contract 5 +done + +# A local backend override must ignore even the distributed image's active +# pd.peers. A failed fix would reach curl or timeout before initialization. +for backend in rocksdb memory hbase; do + cp "$SOURCE_BIN/../conf/graphs/hstore.properties.template" \ + "$DIST_ROOT/conf/graphs/hugegraph.properties" || fail "cannot copy the HStore graph template" + grep -Eq '^[[:blank:]]*pd[.]peers[[:blank:]]*=[[:blank:]]*[^[:space:]#]' \ + "$DIST_ROOT/conf/graphs/hugegraph.properties" || fail "HStore fixture has no active pd.peers" + : > "$CALL_LOG" + : > "$TIMEOUT_LOG" + CASE_OUTPUT=$(env PATH="$MOCK_BIN:$PATH" \ + MOCK_CALL_LOG="$CALL_LOG" MOCK_ARGS_LOG="$ARGS_LOG" \ + MOCK_CONFIG_LOG="$CONFIG_LOG" MOCK_COUNT_FILE="$COUNT_FILE" \ + MOCK_TIMEOUT_LOG="$TIMEOUT_LOG" MOCK_SCENARIO=auth-401 MOCK_ABORT_AFTER=1 \ + WAIT_STORAGE_TIMEOUT_S=5 "hugegraph.backend=$backend" \ + "$DIST_ROOT/bin/wait-storage.sh" 2>&1) + CASE_RC=$? + assert_equal "$backend ignores stale pd.peers" 0 "$CASE_RC" + [[ ! -s "$CALL_LOG" && ! -s "$TIMEOUT_LOG" ]] || fail "local backend contacted PD" + assert_output "does not use PD/Store, skipping storage wait" + + run_case "auth-401" "pd0:8620,pd1:8620" 1 test-password 5 "backend = $backend" + assert_equal "configured $backend ignores PD" 0 "$CASE_RC" + [[ ! -s "$CALL_LOG" && ! -s "$TIMEOUT_LOG" ]] || fail "configured local backend contacted PD" +done + +echo "36 passed, 0 failed" diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/traversers/ShardScanApiTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/traversers/ShardScanApiTest.java new file mode 100644 index 0000000000..69553d0373 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/traversers/ShardScanApiTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.api.traversers; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hugegraph.api.BaseApiTest; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import com.google.common.collect.ImmutableMap; + +@RunWith(Parameterized.class) +public class ShardScanApiTest extends BaseApiTest { + + @Parameterized.Parameters(name = "{0}") + public static Collection elements() { + return Arrays.asList(new Object[]{"vertices", 6}, new Object[]{"edges", 5}); + } + + private final String elements; + private final String path; + private final int count; + + public ShardScanApiTest(String elements, int count) { + this.elements = elements; + this.path = TRAVERSERS_API + "/" + elements + "/scan"; + this.count = count; + } + + private void prepareRocksDBData() { + Assume.assumeTrue("Shard scan contract requires RocksDB", + "rocksdb".equals(System.getProperty("backend"))); + initPropertyKey(); + initVertexLabel(); + initEdgeLabel(); + initVertex(); + initEdge(); + } + + @Test + public void testMissingBounds() { + String content = assertResponseStatus(400, client().get(this.path, + ImmutableMap.of("page_limit", 2))); + String message = assertJsonContains(content, "message"); + Assert.assertTrue(message, message.contains("start parameter is required")); + } + + @Test + public void testMissingStartWithEmptyEnd() { + String content = assertResponseStatus(400, client().get(this.path, + ImmutableMap.of("end", ""))); + String message = assertJsonContains(content, "message"); + Assert.assertTrue(message, message.contains("start parameter is required")); + } + + @Test + public void testMissingEndWithEmptyStart() { + String content = assertResponseStatus(400, client().get(this.path, + ImmutableMap.of("start", ""))); + String message = assertJsonContains(content, "message"); + Assert.assertTrue(message, message.contains("end parameter is required")); + } + + @Test + public void testExplicitEmptyPageLimitsAndResumesScan() { + this.prepareRocksDBData(); + Map params = new HashMap<>(); + params.put("start", ""); + params.put("end", ""); + params.put("page", ""); + params.put("page_limit", 2); + Set ids = new HashSet<>(); + int requests = 0; + String page; + do { + String content = assertResponseStatus(200, client().get(this.path, params)); + List> result = assertJsonContains(content, this.elements); + Assert.assertTrue(result.size() <= 2); + for (Map element : result) { + Assert.assertTrue("Duplicate scan element", ids.add(element.get("id"))); + } + page = assertJsonContains(content, "page"); + params.put("page", page); + Assert.assertTrue("Scan cursor did not terminate", ++requests < 10); + } while (page != null && !page.isEmpty()); + Assert.assertEquals(this.count, ids.size()); + Assert.assertTrue(requests > 1); + } + + @Test + public void testOmittedPageRetainsUnpagedContract() { + this.prepareRocksDBData(); + String content = assertResponseStatus(200, client().get(this.path, + ImmutableMap.of("start", "", "end", "", "page_limit", 2))); + List result = assertJsonContains(content, this.elements); + Assert.assertEquals(this.count, result.size()); + Assert.assertFalse(content.contains("\"page\"")); + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/traversers/TraversersApiTestSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/traversers/TraversersApiTestSuite.java index e985caf58d..57dbb5ffd0 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/traversers/TraversersApiTestSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/traversers/TraversersApiTestSuite.java @@ -27,6 +27,7 @@ CrosspointsApiTest.class, CustomizedCrosspointsApiTest.class, EdgesApiTest.class, + ShardScanApiTest.class, FusiformSimilarityApiTest.class, JaccardSimilarityApiTest.class, KneighborApiTest.class, diff --git a/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml b/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml index ee2b6af9bf..d0f8494027 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml +++ b/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml @@ -75,6 +75,11 @@ conf 644 + + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb-legacy-cspp.yaml + conf + 644 +