diff --git a/.dockerignore b/.dockerignore index 28d48eb939..e8b92e609a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -22,6 +22,7 @@ # Pre-extracted release dirs / archives apache-hugegraph-*/ +**/apache-hugegraph-*/ **/*.tar **/*.tar.gz **/*.zip diff --git a/.github/configs/settings.xml b/.github/configs/settings.xml index 45c87139a9..6cc7dc435c 100644 --- a/.github/configs/settings.xml +++ b/.github/configs/settings.xml @@ -27,6 +27,13 @@ staged-releases https://repository.apache.org/content/groups/staging/ + + github + https://maven.pkg.github.com/hugegraph/toplingdb + + true + + diff --git a/.github/workflows/check-dependencies.yml b/.github/workflows/check-dependencies.yml index 23efa75ddb..a486dd56c2 100644 --- a/.github/workflows/check-dependencies.yml +++ b/.github/workflows/check-dependencies.yml @@ -25,8 +25,10 @@ jobs: dependency-check: runs-on: ubuntu-latest env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. SCRIPT_DEPENDENCY: install-dist/scripts/dependency + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} steps: - name: Checkout source uses: actions/checkout@v4 diff --git a/.github/workflows/cluster-test-ci.yml b/.github/workflows/cluster-test-ci.yml index c2972cb9f9..8c8b03b4a1 100644 --- a/.github/workflows/cluster-test-ci.yml +++ b/.github/workflows/cluster-test-ci.yml @@ -30,7 +30,9 @@ jobs: cluster-test: runs-on: ubuntu-latest env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} steps: - name: Install JDK 11 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d66dc8cee9..52f3d2a93e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -12,7 +12,9 @@ on: jobs: analyze: env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} name: Analyze runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/commons-ci.yml b/.github/workflows/commons-ci.yml index fd01df1825..55da63d3a8 100644 --- a/.github/workflows/commons-ci.yml +++ b/.github/workflows/commons-ci.yml @@ -9,16 +9,22 @@ on: - 'test-*' pull_request: +permissions: + contents: read + packages: read + jobs: build-commons: runs-on: ubuntu-latest env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} strategy: fail-fast: false matrix: - JAVA_VERSION: ['11'] + JAVA_VERSION: [ '11' ] steps: - name: Install JDK ${{ matrix.JAVA_VERSION }} diff --git a/.github/workflows/docker-build-ci.yml b/.github/workflows/docker-build-ci.yml index 97e2d37a41..47d8cb1411 100644 --- a/.github/workflows/docker-build-ci.yml +++ b/.github/workflows/docker-build-ci.yml @@ -64,7 +64,9 @@ jobs: in_build && seen && /^FROM / { exit } - in_build { + # Compare Docker instructions only. Comments between stages may + # differ when a Dockerfile adds an optional build target. + in_build && $0 !~ /^[[:space:]]*(#.*)?$/ { print seen = 1 } @@ -89,7 +91,11 @@ jobs: run: | set -euo pipefail - docker buildx bake --file docker/bake.hcl --print > /tmp/hugegraph-bake.json + docker buildx bake --file docker/bake.hcl --print \ + > /tmp/hugegraph-bake.json + RUNTIME_VARIANT=topling \ + docker buildx bake --file docker/bake.hcl --print \ + > /tmp/hugegraph-bake-topling.json jq -e ' .group.default.targets == [ "build-cache", @@ -101,23 +107,145 @@ jobs: ([.target[] | .platforms] | all(. == ["linux/amd64", "linux/arm64"])) and .target["build-cache"].target == "build" and + .target.pd.target == "standard" and + .target.store.target == "standard" and + .target["server-standalone"].target == "standard" and + (.target["server-hstore"].target == null) and .target["build-cache"].output[0].type == "cacheonly" and + ([.target[] | .args.SOURCE_REPOSITORY] | + all(. == "https://github.com/apache/hugegraph")) and ([.target | to_entries[] | select(.key != "build-cache") | .value.output[0].type] | all(. == "docker")) ' /tmp/hugegraph-bake.json + jq -e ' + .target.pd.target == "topling" and + .target.store.target == "topling" and + .target["server-standalone"].target == "topling" and + (.target["server-hstore"].target == null) and + ([.target[] | .platforms] | + all(. == ["linux/amd64"])) + ' /tmp/hugegraph-bake-topling.json + + - name: Validate ToplingDB Compose parameters + env: + HUGEGRAPH_ADMIN_PASSWORD: ci-test-password + HG_PD_AUTH_SECRET_KEY: ci-test-pd-secret + HUGEGRAPH_SERVER_PULL_POLICY: always + HUGEGRAPH_PD_PULL_POLICY: always + HUGEGRAPH_STORE_PULL_POLICY: always + run: | + set -euo pipefail + + env \ + HUGEGRAPH_SERVER_IMAGE=local/hugegraph:topling \ + HUGEGRAPH_SERVER_VOLUME=server-topling-data \ + HG_SERVER_ROCKSDB_PROVIDER=topling \ + HG_SERVER_DATA_PATH=/hugegraph-server/topling-data \ + HG_SERVER_ENFORCE_PROVIDER_MARKER=true \ + docker compose -f docker/docker-compose.yml \ + config --format json > /tmp/topling-standalone.json + env \ + HUGEGRAPH_PD_IMAGE=local/hugegraph-pd:topling \ + HUGEGRAPH_PD_VOLUME=pd-topling-data \ + HG_PD_ROCKSDB_PROVIDER=topling \ + HG_PD_DATA_PATH=/hugegraph-pd/topling-pd-data \ + HG_PD_ENFORCE_PROVIDER_MARKER=true \ + HUGEGRAPH_STORE_IMAGE=local/hugegraph-store:topling \ + HUGEGRAPH_STORE_VOLUME=store-topling-data \ + HG_STORE_ROCKSDB_PROVIDER=topling \ + HG_STORE_DATA_PATH=/hugegraph-store/topling-storage \ + HG_STORE_ENFORCE_PROVIDER_MARKER=true \ + docker compose -f docker/docker-compose-hstore.yml \ + -f docker/docker-compose.dev.yml \ + config --format json > /tmp/topling-hstore.json + env \ + HUGEGRAPH_PD_IMAGE=local/hugegraph-pd:topling \ + HUGEGRAPH_PD0_VOLUME=hg-pd0-topling-data \ + HUGEGRAPH_PD1_VOLUME=hg-pd1-topling-data \ + HUGEGRAPH_PD2_VOLUME=hg-pd2-topling-data \ + HG_PD_ROCKSDB_PROVIDER=topling \ + HG_PD_DATA_PATH=/hugegraph-pd/topling-pd-data \ + HG_PD_ENFORCE_PROVIDER_MARKER=true \ + HUGEGRAPH_STORE_IMAGE=local/hugegraph-store:topling \ + HUGEGRAPH_STORE0_VOLUME=hg-store0-topling-data \ + HUGEGRAPH_STORE1_VOLUME=hg-store1-topling-data \ + HUGEGRAPH_STORE2_VOLUME=hg-store2-topling-data \ + HG_STORE_ROCKSDB_PROVIDER=topling \ + HG_STORE_DATA_PATH=/hugegraph-store/topling-storage \ + HG_STORE_ENFORCE_PROVIDER_MARKER=true \ + docker compose -f docker/docker-compose-3pd-3store-3server.yml \ + config --format json > /tmp/topling-3x3.json + + jq -e ' + .name == "hugegraph-standalone" and + .services.server.image == "local/hugegraph:topling" and + .services.server.pull_policy == "always" and + .services.server.environment.HG_SERVER_ROCKSDB_PROVIDER == "topling" and + .services.server.environment.HG_SERVER_DATA_PATH == "/hugegraph-server/topling-data" and + .services.server.environment.HG_SERVER_ENFORCE_PROVIDER_MARKER == "true" and + (.services.server.volumes | + any(.source == "server-topling-data" and + .target == "/hugegraph-server/topling-data")) + ' /tmp/topling-standalone.json + + jq -e ' + .name == "hugegraph-hstore" and + .services.pd.image == "local/hugegraph-pd:topling" and + .services.store.image == "local/hugegraph-store:topling" and + .services.pd.environment.HG_PD_ROCKSDB_PROVIDER == "topling" and + .services.store.environment.HG_STORE_ROCKSDB_PROVIDER == "topling" and + .services.server.environment.HG_SERVER_BACKEND == "hstore" and + (.services.server.environment.HG_SERVER_ROCKSDB_PROVIDER == null) and + .services.server.image == "hugegraph/server:dev" and + .services.server.pull_policy == "always" and + .services.pd.pull_policy == "always" and + .services.store.pull_policy == "always" and + (.services.pd.ports | all(.target != 2012)) and + (.services.store.ports | all(.target != 2013)) and + (.services.pd.volumes | + any(.source == "pd-topling-data")) and + (.services.store.volumes | + any(.source == "store-topling-data")) + ' /tmp/topling-hstore.json + + jq -e ' + .name == "hugegraph-3x3" and + ([.services.pd0, .services.pd1, .services.pd2] | + all(.environment.HG_PD_ROCKSDB_PROVIDER == "topling" and + .image == "local/hugegraph-pd:topling" and + .pull_policy == "always")) and + ([.services.store0, .services.store1, .services.store2] | + all(.environment.HG_STORE_ROCKSDB_PROVIDER == "topling" and + .image == "local/hugegraph-store:topling" and + .pull_policy == "always")) and + ([.services.server0, .services.server1, .services.server2] | + all(.environment.HG_SERVER_BACKEND == "hstore" and + (.environment.HG_SERVER_ROCKSDB_PROVIDER == null) and + .image == "hugegraph/server:latest" and + .pull_policy == "always")) and + ([.services.pd0, .services.pd1, .services.pd2, + .services.store0, .services.store1, .services.store2] | + all(.pull_policy == "always")) + ' /tmp/topling-3x3.json docker-build: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - dockerfile: - - hugegraph-pd/Dockerfile - - hugegraph-store/Dockerfile - - hugegraph-server/Dockerfile - - hugegraph-server/Dockerfile-hstore + include: + - dockerfile: hugegraph-pd/Dockerfile + - dockerfile: hugegraph-pd/Dockerfile + target: topling + - dockerfile: hugegraph-store/Dockerfile + - dockerfile: hugegraph-store/Dockerfile + target: topling + - dockerfile: hugegraph-server/Dockerfile + - dockerfile: hugegraph-server/Dockerfile + target: topling + - dockerfile: hugegraph-server/Dockerfile-hstore steps: - name: Checkout @@ -125,18 +253,125 @@ jobs: - name: Build ${{ matrix.dockerfile }} run: | - IMAGE_ID=$(docker build -q --build-arg SOURCE_REVISION="$GITHUB_SHA" \ - -f ${{ matrix.dockerfile }} .) + TARGET_ARGS=() + if [[ -n "${{ matrix.target }}" ]]; then + TARGET_ARGS+=(--target "${{ matrix.target }}") + fi + IMAGE_ID=$(docker build -q "${TARGET_ARGS[@]}" \ + --build-arg SOURCE_REPOSITORY="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ + --build-arg SOURCE_REVISION="$GITHUB_SHA" \ + -f ${{ matrix.dockerfile }} .) echo "Built: $IMAGE_ID" echo "IMAGE_ID=$IMAGE_ID" >> "$GITHUB_ENV" HC=$(docker inspect --format='{{json .Config.Healthcheck}}' "$IMAGE_ID") echo "Healthcheck: $HC" [[ "$HC" != "null" ]] || { echo "ERROR: HEALTHCHECK missing in ${{ matrix.dockerfile }}"; exit 1; } + SOURCE_LABEL=$(docker inspect --format='{{index .Config.Labels "org.opencontainers.image.source"}}' "$IMAGE_ID") + REVISION_LABEL=$(docker inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' "$IMAGE_ID") + RUNTIME_LABEL=$(docker inspect --format='{{index .Config.Labels "org.apache.hugegraph.rocksdb-runtime"}}' "$IMAGE_ID") + [[ "$SOURCE_LABEL" == "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" ]] + [[ "$REVISION_LABEL" == "$GITHUB_SHA" ]] + if [[ "${{ matrix.dockerfile }}" == "hugegraph-server/Dockerfile-hstore" ]]; then + [[ "$RUNTIME_LABEL" == "hstore" ]] + elif [[ "${{ matrix.target }}" == "topling" ]]; then + [[ "$RUNTIME_LABEL" == "topling" ]] + else + [[ "$RUNTIME_LABEL" == "standard" ]] + fi - name: Test server entrypoint property mapping - if: matrix.dockerfile == 'hugegraph-server/Dockerfile' + if: matrix.dockerfile == 'hugegraph-server/Dockerfile' && matrix.target != 'topling' run: bash hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint.sh + - name: Verify ToplingDB image runtime + if: matrix.target == 'topling' + run: | + case "${{ matrix.dockerfile }}" in + hugegraph-pd/Dockerfile) + EXPECTED_ENV='HG_PD_ROCKSDB_PROVIDER=topling' + DATA_ENV_PREFIX='HG_PD_DATA_PATH=' + STANDARD_VOLUME='/hugegraph-pd/pd_data' + TOPLING_VOLUME='/hugegraph-pd/topling-pd-data' + ;; + hugegraph-store/Dockerfile) + EXPECTED_ENV='HG_STORE_ROCKSDB_PROVIDER=topling' + DATA_ENV_PREFIX='HG_STORE_DATA_PATH=' + STANDARD_VOLUME='/hugegraph-store/storage' + TOPLING_VOLUME='/hugegraph-store/topling-storage' + ;; + hugegraph-server/Dockerfile) + EXPECTED_ENV='HG_SERVER_ROCKSDB_PROVIDER=topling' + DATA_ENV_PREFIX='HG_SERVER_DATA_PATH=' + STANDARD_VOLUME='/hugegraph-server/rocksdb-data' + TOPLING_VOLUME='/hugegraph-server/topling-data' + ;; + *) + echo "ERROR: unsupported Topling Dockerfile" >&2 + exit 1 + ;; + esac + docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \ + "$IMAGE_ID" | grep -qx "$EXPECTED_ENV" + if docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \ + "$IMAGE_ID" | grep -q "^${DATA_ENV_PREFIX}"; then + echo "ERROR: data path must follow the selected provider" >&2 + exit 1 + fi + VOLUMES=$(docker inspect --format='{{json .Config.Volumes}}' \ + "$IMAGE_ID") + jq -e --arg standard "$STANDARD_VOLUME" --arg topling "$TOPLING_VOLUME" \ + 'has($standard) and has($topling)' <<<"$VOLUMES" + docker run --rm --entrypoint bash "$IMAGE_ID" -c ' + set -euo pipefail + test -n "$(find lib/topling -maxdepth 1 -name "rocksdbjni*.jar" -print -quit)" + test -r library/librocksdbjni-linux64.so + LDD_OUTPUT=$(ldd library/librocksdbjni-linux64.so) + echo "$LDD_OUTPUT" + if grep -q "not found" <<<"$LDD_OUTPUT"; then + exit 1 + fi + TOPLINGDB_ROCKSDB_PROVIDER=topling source bin/preload-topling.sh + test "$TOPLING_ACTIVE_NATIVE" = \ + "$PWD/library/librocksdbjni-linux64.so" + /bin/true + ' + + - name: Preserve standard image volume compatibility + if: matrix.target != 'topling' && matrix.dockerfile != 'hugegraph-server/Dockerfile-hstore' + run: | + case "${{ matrix.dockerfile }}" in + hugegraph-pd/Dockerfile) + DATA_ENV_PREFIX='HG_PD_DATA_PATH=' + LEGACY_VOLUME='/hugegraph-pd' + ;; + hugegraph-store/Dockerfile) + DATA_ENV_PREFIX='HG_STORE_DATA_PATH=' + LEGACY_VOLUME='/hugegraph-store' + ;; + hugegraph-server/Dockerfile) + DATA_ENV_PREFIX='HG_SERVER_DATA_PATH=' + LEGACY_VOLUME='/hugegraph-server' + ;; + esac + if docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \ + "$IMAGE_ID" | grep -q "^${DATA_ENV_PREFIX}"; then + echo "ERROR: data path must follow the selected provider" >&2 + exit 1 + fi + VOLUMES=$(docker inspect --format='{{json .Config.Volumes}}' \ + "$IMAGE_ID") + jq -e --arg legacy "$LEGACY_VOLUME" 'has($legacy)' <<<"$VOLUMES" + + - name: Verify HStore image has no local Topling runtime + if: matrix.dockerfile == 'hugegraph-server/Dockerfile-hstore' + run: | + docker run --rm --entrypoint bash "$IMAGE_ID" -c ' + set -euo pipefail + test -z "$(find lib -path "*/topling/rocksdbjni*.jar" \ + -print -quit)" + test ! -e library/librocksdbjni-linux64.so + ' + # The startup preflight needs a socket-table tool, and the base image # ships none of its own. Without one every start reports "unknown" and # a duplicate start is no longer refused, so assert the image can diff --git a/.github/workflows/licence-checker.yml b/.github/workflows/licence-checker.yml index a6e6990a64..e1b8ee9f93 100644 --- a/.github/workflows/licence-checker.yml +++ b/.github/workflows/licence-checker.yml @@ -11,7 +11,9 @@ jobs: check-license: runs-on: ubuntu-latest env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index 79217cfc58..1008829cc3 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -26,8 +26,160 @@ on: - '!**/AGENTS.md' - '!**/CLAUDE.md' +permissions: + contents: read + packages: read + # TODO: consider merge to one ci.yml file jobs: + distributed-rocksdb-runtime: + name: distributed-rocksdb-runtime (${{ matrix.check_name }}) + runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-24.04 + component: pd + provider: rocksdb + check_name: pd, rocksdb + experimental: false + - os: ubuntu-24.04 + component: pd + provider: topling + check_name: pd, topling + experimental: false + - os: ubuntu-24.04 + component: store + provider: rocksdb + check_name: store, rocksdb + experimental: false + - os: ubuntu-24.04 + component: store + provider: topling + check_name: store, topling + experimental: false + - os: ubuntu-26.04 + component: pd + provider: topling + check_name: pd, topling, ubuntu-26.04 + experimental: true + - os: ubuntu-26.04 + component: store + provider: topling + check_name: store, topling, ubuntu-26.04 + experimental: true + env: + TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Java 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'zulu' + + - name: Use staged maven repo settings + run: | + mkdir -p "$HOME/.m2" + cp -vf .github/configs/settings.xml "$HOME/.m2/settings.xml" + + - name: Package distributions + run: | + mvn clean package -Dmaven.test.skip=true -ntp + + - name: Select ${{ matrix.component }} ${{ matrix.provider }} distribution + run: | + VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + SERVER_DIR="hugegraph-server/apache-hugegraph-server-$VERSION" + if [ "${{ matrix.component }}" = "pd" ]; then + COMPONENT_DIR="hugegraph-pd/apache-hugegraph-pd-$VERSION" + if [ "${{ matrix.provider }}" = "rocksdb" ]; then + CONFIG_FILE="$COMPONENT_DIR/conf/application.yml" + sed -i 's/^# rocksdb:/rocksdb:/' "$CONFIG_FILE" + sed -i 's/^# provider: rocksdb/ provider: rocksdb/' \ + "$CONFIG_FILE" + fi + else + COMPONENT_DIR="hugegraph-store/apache-hugegraph-store-$VERSION" + if [ "${{ matrix.provider }}" = "rocksdb" ]; then + CONFIG_FILE="$COMPONENT_DIR/conf/application-pd.yml" + sed -i 's/^ # provider: topling/ provider: rocksdb/' \ + "$CONFIG_FILE" + fi + fi + if [ "${{ matrix.provider }}" = "topling" ]; then + touch "$COMPONENT_DIR/bin/pid" + if [ "${{ matrix.component }}" = "pd" ]; then + ln -s /missing/topling-ci-pd-data "$COMPONENT_DIR/pd_data" + else + touch "$COMPONENT_DIR/storage" + fi + install-dist/scripts/build-topling-distribution.sh \ + "${{ matrix.component }}" "$VERSION" + COMPONENT_DIR="$COMPONENT_DIR-topling" + fi + if [ "${{ matrix.component }}" = "pd" ]; then + STANDARD_COMPONENT_DIR="hugegraph-pd/apache-hugegraph-pd-$VERSION" + else + STANDARD_COMPONENT_DIR="hugegraph-store/apache-hugegraph-store-$VERSION" + fi + { + echo "VERSION=$VERSION" + echo "SERVER_DIR=$SERVER_DIR" + echo "STANDARD_COMPONENT_DIR=$STANDARD_COMPONENT_DIR" + echo "COMPONENT_DIR=$COMPONENT_DIR" + } >> "$GITHUB_ENV" + + - name: Verify clean ToplingDB distribution + if: matrix.provider == 'topling' + env: + TOPLING_EXPECT_DIRTY_STANDARD: 'true' + run: | + "$TRAVIS_DIR/test-topling-distribution.sh" \ + "${{ matrix.component }}" "$STANDARD_COMPONENT_DIR" "$COMPONENT_DIR" + + - name: Test standard RocksDB runtime + if: matrix.provider == 'rocksdb' + run: | + "$TRAVIS_DIR/test-rocksdb-runtime.sh" \ + rocksdb "$COMPONENT_DIR" + + - name: Run Topling native lifecycle diagnostic + id: topling-native-diagnostic + if: matrix.provider == 'topling' + continue-on-error: true + run: | + "$TRAVIS_DIR/test-rocksdb-runtime.sh" \ + topling "$COMPONENT_DIR" + + - name: Report Topling native lifecycle diagnostic + if: >- + matrix.provider == 'topling' && + steps.topling-native-diagnostic.outcome == 'failure' + run: | + { + echo "### Topling native lifecycle diagnostic" + echo + echo "The synthetic CF lifecycle reproducer hit the known native" + echo "assertion tracked by hugegraph/hugegraph#212." + echo "Image-backed service lifecycle tests remain required." + } >> "$GITHUB_STEP_SUMMARY" + + - name: Verify standard runtime packaging + if: matrix.component == 'pd' && matrix.provider == 'rocksdb' + run: | + "$TRAVIS_DIR/test-topling-runtime-packaging.sh" \ + "$SERVER_DIR" \ + "hugegraph-pd/apache-hugegraph-pd-$VERSION" \ + "hugegraph-store/apache-hugegraph-store-$VERSION" + struct: runs-on: ubuntu-latest env: @@ -86,11 +238,13 @@ jobs: runs-on: ubuntu-latest env: # TODO: avoid duplicated env setup in pd & store - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. # TODO: remove outdated env TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis REPORT_FILE: hugegraph-pd/hg-pd-test/target/site/jacoco/jacoco.xml TEST_REPORT_DIR: hugegraph-pd/hg-pd-test/target/surefire-reports + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} steps: - name: Install JDK 11 @@ -117,10 +271,15 @@ jobs: cp $HOME/.m2/settings.xml /tmp/settings.xml mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml + - name: Install deps + run: | + "${TRAVIS_DIR}/install-deps.sh" || exit 1 + - name: Package # todo remove --fail-at-end after test run: | mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp --fail-at-end + source "${TRAVIS_DIR}/install-rocksdb.sh" pd - name: Check source formatting run: | @@ -224,11 +383,13 @@ jobs: needs: struct runs-on: ubuntu-latest env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. # TODO: remove outdated env TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis REPORT_FILE: hugegraph-store/hg-store-test/target/site/jacoco/jacoco.xml TEST_REPORT_DIR: hugegraph-store/hg-store-test/target/surefire-reports + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} steps: - name: Install JDK 11 @@ -255,10 +416,15 @@ jobs: cp $HOME/.m2/settings.xml /tmp/settings.xml mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml + - name: Install deps + run: | + "${TRAVIS_DIR}/install-deps.sh" || exit 1 + - name: Package # todo remove --fail-at-end after test run: | mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp --fail-at-end + source "${TRAVIS_DIR}/install-rocksdb.sh" store - name: Check source formatting run: | @@ -368,11 +534,13 @@ jobs: needs: struct runs-on: ubuntu-latest env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis REPORT_DIR: target/site/jacoco BACKEND: hstore RELEASE_BRANCH: ${{ startsWith(github.ref_name, 'release-') || startsWith(github.ref_name, 'test-') || startsWith(github.base_ref, 'release-') }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} steps: - name: Install JDK 11 @@ -393,6 +561,10 @@ jobs: with: fetch-depth: 2 + - name: Install deps + run: | + "${TRAVIS_DIR}/install-deps.sh" || exit 1 + - name: use staged maven repo settings if: ${{ env.USE_STAGE == 'true' }} run: | @@ -403,6 +575,9 @@ jobs: # todo remove --fail-at-end after test run: | mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp --fail-at-end + source "${TRAVIS_DIR}/install-rocksdb.sh" pd + source "${TRAVIS_DIR}/install-rocksdb.sh" store + source "${TRAVIS_DIR}/install-rocksdb.sh" hstore - name: Prepare env and service run: | diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index b5f5efc270..82c8d6f4f7 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -9,6 +9,121 @@ on: pull_request: jobs: + server-rocksdb-runtime: + name: server-rocksdb-runtime (${{ matrix.check_name }}) + runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.experimental }} + permissions: + contents: read + packages: read + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-24.04 + provider: rocksdb + check_name: rocksdb + experimental: false + - os: ubuntu-24.04 + provider: topling + check_name: topling + experimental: false + - os: ubuntu-26.04 + provider: topling + check_name: topling, ubuntu-26.04 + experimental: true + env: + TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Java 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'zulu' + + - name: Use staged maven repo settings + run: | + mkdir -p "$HOME/.m2" + cp -vf .github/configs/settings.xml "$HOME/.m2/settings.xml" + + - name: Package Server + run: | + mvn clean package -pl hugegraph-server/hugegraph-dist -am \ + -Dmaven.test.skip=true -ntp + + - name: Select ${{ matrix.provider }} distribution + run: | + VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + SERVER_DIR="hugegraph-server/apache-hugegraph-server-$VERSION" + if [ "${{ matrix.provider }}" = "topling" ]; then + touch "$SERVER_DIR/bin/pid" + mkdir -p "$SERVER_DIR/logs" + touch "$SERVER_DIR/logs/topling-ci-marker" + install-dist/scripts/build-topling-distribution.sh server "$VERSION" + SERVER_DIR="$SERVER_DIR-topling" + else + GRAPH_CONF="$SERVER_DIR/conf/graphs/hugegraph.properties" + sed -i '/^[#]*rocksdb\.provider=/d' "$GRAPH_CONF" + echo "rocksdb.provider=rocksdb" >> "$GRAPH_CONF" + fi + echo "STANDARD_SERVER_DIR=hugegraph-server/apache-hugegraph-server-$VERSION" \ + >> "$GITHUB_ENV" + echo "SERVER_DIR=$SERVER_DIR" >> "$GITHUB_ENV" + + - name: Verify clean ToplingDB distribution + if: matrix.provider == 'topling' + env: + TOPLING_EXPECT_DIRTY_STANDARD: 'true' + run: | + "$TRAVIS_DIR/test-topling-distribution.sh" \ + server "$STANDARD_SERVER_DIR" "$SERVER_DIR" + + - name: Test standard RocksDB runtime + if: matrix.provider == 'rocksdb' + run: | + source "$SERVER_DIR/bin/preload-topling.sh" + "$TRAVIS_DIR/test-rocksdb-runtime.sh" \ + rocksdb "$SERVER_DIR" + + - name: Run Topling native lifecycle diagnostic + id: topling-native-diagnostic + if: matrix.provider == 'topling' + continue-on-error: true + run: | + source "$SERVER_DIR/bin/preload-topling.sh" + "$TRAVIS_DIR/test-rocksdb-runtime.sh" \ + topling "$SERVER_DIR" + + - name: Report Topling native lifecycle diagnostic + if: >- + matrix.provider == 'topling' && + steps.topling-native-diagnostic.outcome == 'failure' + run: | + { + echo "### Topling native lifecycle diagnostic" + echo + echo "The synthetic CF lifecycle reproducer hit the known native" + echo "assertion tracked by hugegraph/hugegraph#212." + echo "Image-backed service lifecycle tests remain required." + } >> "$GITHUB_STEP_SUMMARY" + + topling-runtime-shell-test: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Test strict component-local runtime selection + run: | + hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-runtime-selection.sh + hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-docker-entrypoints.sh + wait-storage-shell-test: permissions: contents: read @@ -24,8 +139,11 @@ jobs: build-server: runs-on: ubuntu-24.04 + permissions: + contents: read + packages: read env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis REPORT_DIR: target/site/jacoco BACKEND: ${{ matrix.BACKEND }} @@ -35,6 +153,8 @@ jobs: TARGET_BRANCH_NAME: ${{ github.base_ref != '' && github.base_ref || github.ref_name }} RELEASE_BRANCH: ${{ startsWith(github.ref_name, 'release-') || startsWith(github.ref_name, 'test-') }} RAFT_MODE: ${{ startsWith(github.head_ref, 'test') || startsWith(github.head_ref, 'raft') }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} strategy: fail-fast: false @@ -55,16 +175,16 @@ jobs: java-version: '8' distribution: 'zulu' - - name: Prepare backend environment - run: | - $TRAVIS_DIR/install-backend.sh $BACKEND && jps -l - - name: Install Java ${{ matrix.JAVA_VERSION }} uses: actions/setup-java@v4 with: java-version: ${{ matrix.JAVA_VERSION }} distribution: 'zulu' + - name: Install deps + run: | + ${TRAVIS_DIR}/install-deps.sh + - name: Cache Maven packages uses: actions/cache@v4 with: @@ -78,9 +198,14 @@ jobs: cp $HOME/.m2/settings.xml /tmp/settings.xml cp -vf .github/configs/settings.xml $HOME/.m2/settings.xml && cat $HOME/.m2/settings.xml - - name: Compile + - name: Package + run: | + mvn clean package -Dmaven.test.skip=true -ntp + source "$TRAVIS_DIR/install-rocksdb.sh" server + + - name: Prepare backend environment run: | - mvn clean compile -U -Dmaven.javadoc.skip=true -ntp + $TRAVIS_DIR/install-backend.sh $BACKEND && jps -l - name: Validate Docker integration if: ${{ env.BACKEND == 'rocksdb' }} @@ -132,6 +257,8 @@ jobs: - name: Run start-hugegraph.sh foreground mode tests if: ${{ env.BACKEND == 'rocksdb' && steps.server-preflight.outputs.can_run == 'true' }} run: | + mvn package -Dmaven.test.skip=true -pl hugegraph-server/hugegraph-dist -am -ntp + source "$TRAVIS_DIR/install-rocksdb.sh" server VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) SERVER_DIR=hugegraph-server/apache-hugegraph-server-$VERSION/ $TRAVIS_DIR/test-start-hugegraph.sh $SERVER_DIR diff --git a/.goal-task/toplingdb-linux-closure/evidence-index.md b/.goal-task/toplingdb-linux-closure/evidence-index.md new file mode 100644 index 0000000000..da061706cc --- /dev/null +++ b/.goal-task/toplingdb-linux-closure/evidence-index.md @@ -0,0 +1,28 @@ +# 本地证据索引 + +本文件只记录本地证据的逻辑位置和用途，不提交原始日志、大 JSON、镜像或数据集。原始证据位于忽略的 `.goal-task/toplingdb-linux-closure/evidence/`，仅本机可读；PR 中只保留本索引。 + +| 逻辑证据 | 用途 | 当前结论 | +| --- | --- | --- | +| `repaired-full-core-summary.json` | standalone 标准 RocksDB CoreTestSuite | 818 tests，0 failures/errors，42 skips | +| `repaired-full-api-summary.json` | standalone 标准 RocksDB API suite | 161 tests，0 failures/errors，14 skips | +| `helm-standard-111-full-api-summary.json` | Helm 1+1+1 标准 RocksDB API | 155 tests，0 failures/errors，50 skips | +| `helm-standard-333-runtime.json` | Helm 3+3+3 pod 与 JNI 采样 | 9 pods Ready；HA 未验证 | +| `pd-common-core-standard-summary.json` | PD common/core | 83/104，0 failures/errors | +| `pd-client-rest-standard-summary.json` | PD client/rest 与服务退出 | 83/22，0 failures/errors | +| `commons-struct-current-summary.json` | Commons/RPC/Struct | 351/24/8，0 failures/errors | +| `helm-standard-333-store-majority.json` | 历史标准 3+3+3 两个 Store Pod 同时删除 | 已提交数据保持 200；期间写入超时且最终 404；恢复后写入成功 | +| `helm-standard-333-store-leader.json` | 历史标准 3+3+3 Store leader Pod 恢复 | 旧写入保持 200；删除瞬间新写入失败；恢复后写入成功 | +| `helm-standard-333-server-replica.json` | 历史标准 3+3+3 Server 副本切换 | 删除一个 Server 后其余副本继续读写；新 Pod 可读 | +| `helm-standard-333-store-pod-restart.json` | 历史标准 3+3+3 Store-2 Pod 恢复 | 142.869 秒 Ready；PVC 不变；恢复前后读取 200 | +| `helm-standard-333-auth-function.json` | 历史标准 3+3+3 认证和边一致性 | 错误口令 401；三 Server 读边 200；图空间 auth=false | +| `helm-standard-333-write-consistency.json` | 历史标准 3+3+3 三 Server 写入一致性 | Server-0 写 201，三个 Server 读 200；未认证 401 | +| `helm-standard-111-lifecycle.json` | 历史标准 1+1+1 删图、truncate、新卷恢复 | 删图和 truncate 通过；新卷恢复后已确认顶点 404 | +| `helm-standard-111-stop-restart-confirmed.json` | 历史标准 1+1+1 停止/重启和已确认写入 | 写 201，重启前后读 200；PVC 保持；JNI 为标准 RocksDB | +| `store-standard-audit-summary.json` | Store RocksDB/client/core audit | 3/50/22，0 failures/errors | + +## 证据使用规则 + +- 历史证据可能没有在文件内部嵌入 commit SHA；使用前先核对对应 build input、日志和源 hash。 +- 不得把上述标准 RocksDB 结果写成 Topling、HStore HA 或 benchmark 结果。 +- 后续每轮新增证据时，优先写一个小的 summary JSON 或 Markdown，并在文档中记录 commit、镜像 digest、JNI hash、命令和边界。 diff --git a/.goal-task/toplingdb-linux-closure/goal.md b/.goal-task/toplingdb-linux-closure/goal.md new file mode 100644 index 0000000000..ebc65de6cc --- /dev/null +++ b/.goal-task/toplingdb-linux-closure/goal.md @@ -0,0 +1,5 @@ +# 已退出活动真相 + +本文件不再是执行合同。唯一执行入口是 [state.md](state.md)，分项状态只维护在 [todo.md](todo.md)。 + +历史目标、协作边界和验证绑定规则已并入 `state.md`。不要根据本文件启动测试或判断完成。 diff --git a/.goal-task/toplingdb-linux-closure/local-status.md b/.goal-task/toplingdb-linux-closure/local-status.md new file mode 100644 index 0000000000..8f61da0b81 --- /dev/null +++ b/.goal-task/toplingdb-linux-closure/local-status.md @@ -0,0 +1,43 @@ +活动执行合同见 [state.md](state.md)。本文件只保留 2026-09-24 的本机环境快照。 + +# 本机实测状态 + +核对时间：2026-09-24，Asia/Singapore。 + +## kind + +- context：`kind-kind` +- 节点：`kind-control-plane`，`Ready` +- Kubernetes：`v1.37.0` +- container runtime：`containerd://2.3.4` +- 不执行全局清理，不影响任务外 namespace。 + +## 当前保留的 namespace + +| namespace | 当前状态 | 用途 | +| --- | --- | --- | +| `hg-closure-standard-111` | PD/Server/Store 各 1 个，全部 Running | 标准 RocksDB 1+1+1 生命周期和 API 验证 | +| `hg-closure-standard-333` | PD/Server/Store 各 3 个，全部 Running | 标准 RocksDB 3+3+3 部署与 JNI 采样 | + +当前镜像 tag 为 `closure-std-9abae9dbaaa1`。该 tag 只标识本地标准 RocksDB 验证镜像，不能当作最终提交 SHA 或 Topling 候选。 + +## 已完成 + +- 1+1+1 的 API 套件曾完成 155 tests、0 failures/errors、50 skips。 +- 1+1+1 的 Store 同 PVC recreate 和数据复查曾完成。 +- 3+3+3 的 9 个 pod 全部 Ready，PD/Store 标准 JNI hash 已采样。 +- PD、Store、Commons、Struct 的模块级标准 RocksDB 回归有本地证据。 + +## 未完成 + +- 3+3+3 的 leader/follower 退出、网络分区、多数派丢失恢复和写入一致性。 +- Topling 单机、1+1+1、3+3+3 的真实服务与 JNI mapping。 +- PD/Store 混合 provider 组合。 +- Loader 固定子集导入与邻接/重启验证。 +- 任何 benchmark 结论。 + +## 交接注意 + +- 另一台机器应先 fetch `toplingdb`，核对当前 HEAD 与远端关系，再决定是否合并。 +- 本机实测只写文档、测试状态和证据索引；产品代码改动仍回到开发机完成。 +- 每轮测试前先固定源码 SHA 和镜像 digest，测试后把未覆盖边界写清楚。 diff --git a/.goal-task/toplingdb-linux-closure/state.md b/.goal-task/toplingdb-linux-closure/state.md new file mode 100644 index 0000000000..c9636bf680 --- /dev/null +++ b/.goal-task/toplingdb-linux-closure/state.md @@ -0,0 +1,479 @@ +# ToplingDB Linux 实测执行合同 + +## 基线与环境 + +- 仓库 `hugegraph/hugegraph`；fetch/push 远端 `org`；唯一分支 `toplingdb`；PR #179。不新建分支或 PR，不 force-push，不直接合入 master。 +- 执行工作树：`/home/soc-baidu/.codex/worktrees/f29e/hugegraph`。本地分支 `codex/toplingdb-linux-validation`，推送目标 `org/toplingdb`。主 checkout `/home/soc-baidu/github/hugegraph` 停在较旧的 `f9829899c`，不要在那里继续。 +- 2026-09-25 刷新：`git fetch org toplingdb` 后，推送前 fetch 发现 `org/toplingdb` 前进到 `a35ebeb17`，包含 Store 空地址重入、follower partitions GET 和 PD task/balanceLeaders 异常体修复。本地文档提交已 rebase 到其上。当时的镜像和功能证据还绑定 `e109012a07e2e9918f4a98d3faa23e21b93435d1`。这不是当前状态。后面的 a35 镜像和实测绑定干净的 `a35ebeb17`。 +- 工作区另有未提交的 channel refresh：`AbstractGrpcClient.java`、`KvPageScanner.java`、`KvBatchScanner.java`、`KvBatchScanner5.java`、`GrpcStoreStreamClient.java`、`query/CommonKvStreamObserver.java`、`query/QueryExecutor.java`、`NotifyingExecutor.java`，以及未跟踪的 `hg-store-client/src/test/`（含 `AbstractGrpcClientChannelRefreshTest.java`）。3 名独立审查通过且没有未解决的高严重度问题之前，禁止提交或推送这些文件。不要把它们和文档提交混在一起。 +- 工作区另有未提交的 WAL 失败路径修复：`RocksDBStdSessions.java` 和 `RocksDBSessionsTest.java`。`RocksDBSessionsTest` 20 个通过、0 失败、0 跳过。最新差异没有新的三份审查，禁止提交或推送，也不要和 channel refresh 或文档混在同一个提交里。 +- 不要提交 `evidence/`、`.codex-handoff/`、RocksDB 数据、`tmp/` 或 `cacerts.jks`。 +- `e109012a0` 已包含 gRPC 沙箱白名单、嵌套 WAL 恢复和三份 Topling profile 的 `memtable_as_log_index: false`。`closure-top-mmapfix` 仍是旧的脏工作区镜像，不能代表这个 SHA。 +- 2026-09-25 的 a35 镜像来自干净目录 `/home/soc-baidu/.codex/validation-runtime/toplingdb-linux-closure/build-context-a35ebeb17`，detached HEAD `a35ebeb17`，状态为空。不要用脏的 f29e 工作树构建镜像。 +- Kubernetes 只用 `KUBECONFIG=/home/soc-baidu/.kube/config` 和 context `kind-kind`。不用 k3s，不清理无关资源。只有 snapshot/restore 或网络分区门禁需要时，才安装对应 CRD，并且不能改动两个历史 namespace。 +- 历史 namespace `hg-closure-standard-111`、`hg-closure-standard-333` 在 2026-09-24 仍为 Running，镜像是 `closure-std-9abae9dbaaa1`。它们不是当前 SHA 的通过证据，禁止原位升级。 + +## 活动真相与优先级 + +最新用户确认 > [AGENTS.md](../../AGENTS.md) 与产品设计 > [todo.md](todo.md) > 本文件。 + +- 本文件是唯一执行合同和恢复入口。 +- [todo.md](todo.md) 独占分项状态、等待和后置标记。 +- 同目录的 `local-status.md`、`evidence-index.md` 和 `goal.md` 都不是活动真相。 +- 不创建 `design.md`。只有出现可复用证据时才更新 `lessons.md`。 +- 原始证据留在本地 `evidence/`。不得提交 `evidence/`、`.codex-handoff/`、RocksDB 数据、`tmp/`、`cacerts.jks`、镜像或 benchmark 原始大文件。 + +## 用户确认的本机范围 + +本机持续完成 todo.md 的全部 Linux 实测。每项最终只能是绑定证据的通过、失败，或写明解除条件的后置。允许修复可复现的阻塞缺陷并推送；行为修复必须有回归测试。另一台机器可以在同一分支提交代码。每次提交或推送前先 `git fetch org toplingdb`；远端前进时整合双方改动，禁止 force-push 或丢弃另一侧提交。`state.md` 与 `todo.md` 冲突时保留双方证据，再按本地证据改写当前状态。 + +2026-09-24 用户再次确认：继续用本机 goal 做完 todo.md 的全部 Linux 实测，复用本目录和 `toplingdb`，不新开分支或 PR。状态文档在阶段边界和会话结束前推送，不为每一条笔记单独推送。可复现阻塞缺陷可以先审查、提交并推送。本机可以修改代码。 + +2026-09-25 用户确认：继续本机 goal，复用本目录、当前工作树、`org/toplingdb` 和 PR #179。 同日再次确认不新开 goal-task 目录；本机 goal 从当前 `state.md` 与 `todo.md` 继续。本机记录并持续完成 todo.md 里全部 Linux 实测；阶段边界和会话结束前更新并推送 `state.md` 与 `todo.md`。明显且可复现的阻塞缺陷可以直接修复、补回归测试，审查通过后提交并推送。这不是只读限制，但不是授权实现新的跨分区图快照协议。 + +2026-09-25 用户确认生成 goal 并立即复用本目录：本机持续完成 todo.md 的全部 Linux 实测，阶段边界和会话结束前更新并推送 `state.md` 与 `todo.md`。明显且可复现的阻塞缺陷可以直接修复、补回归测试，审查通过后与文档分开提交并推送。不新开分支、PR 或 goal-task 目录，不 force-push。 +同日确认可执行 goal 时，上述范围不变。初始化只补记已经发生、但当时还没写入合同的标准 a35 子集导入，不启动测试，也不提交。 + +2026-09-25 用户回来后再次确认：继续本机 goal，只复用本目录、工作树 `/home/soc-baidu/.codex/worktrees/f29e/hugegraph`、`org/toplingdb` 和 PR #179。主 checkout `/home/soc-baidu/github/hugegraph` 仍停在较旧的 `f9829899c`，不要在那里继续。本机把 todo.md 里还能在单节点 kind 上执行的 Linux 实测做完；每项最终只能是绑定证据的通过、失败，或写明解除条件的后置。阶段边界和会话结束前更新并推送 `state.md` 与 `todo.md`。明显且可复现的阻塞缺陷可以直接修复并补回归测试，审查通过后与文档分开提交、再非强制推送。不新开分支、PR 或 goal-task 目录，不 force-push，不实现新的跨分区图快照协议。 + +## 阶段 + +| 阶段 | 状态 | 依赖 | +| --- | --- | --- | +| P0 合同与修复 | 已推送文档 `04e642ff6`。标准 `a35ebeb17` Store 已证明 follower `GET /v1/partitions`。channel refresh 与未提交 WAL 仍不提交 | 不构建脏工作区镜像，不自动开第四轮审查 | +| P1 历史集群 | `9aba` 有 Pod 级证据；snapshot 失败一次；网络分区后置 | 不阻塞当前 SHA | +| P2 当前 SHA 镜像与 JNI | 两个 provider 的 e109 单机镜像和 JNI 已证明。HStore server 两个 tag 是同一镜像。e109 的 1+1+1 与 3+3+3 namespace 已存在 | 不升级 `9aba`、cc143 或 mmapfix namespace | +| P3 当前 SHA 功能 | 四项已勾选。未修改 Server 现已覆盖批量、索引、Gremlin/Cypher、多图、口令认证、边更新删除和角色 403 | 不重复已通过项 | +| P4 生命周期与 provider | 混合 provider 和错误 provider 已勾选。生命周期总项未勾选：e109 崩溃、删图和 truncate 已有证据，HStore `snapshot_create` 仍是 500 | 不新做跨分区图快照，不重复已有 kill -9 | +| P5 当前 SHA HA | 三项都未勾选。单节点进程或 Pod 恢复不能代替网络分区。Compose 与 Helm 差异已记录且故意不改 | 不把 kind 写成物理多机，不顺手改 HA 配置 | +| P6 Loader | Topling 3+3+3 更换 Store IP 后曾连接旧地址 `10.244.0.93:8500`。后来 `nwjsq` 和未重启的 `qkpz5` 都能返回 `1000000:2098771`，但不能写成已修复。失败重试没有触发。全量 LAW 后置 | 不重跑删除；channel refresh 不自动开第四轮 | +| P7 Benchmark | 未开始 | 核心功能未收口前禁止性能结论 | + +分项计数和完成标记以 todo.md 为准。当前 SHA 测试使用新 namespace。同一时间只运行一个重任务；Maven 全量、镜像构建、Helm 变更和故障注入不叠加。 + +## 已核实的镜像事实 + +标准构建单元 `hg-closure-std-image-cc14333f0-r2` 已结束：`ActiveState=inactive`、`Result=success`、`ExecMainStatus=0`。同名第一单元是 failed，不能当作成功结果。日志是 `evidence/build/current-sha-standard-image-build-r2.log`。 + +| 镜像 | ID | 结论 | +| --- | --- | --- | +| `hugegraph/hugegraph:closure-std-cc14333f0` | `35267423defa` | 2026-09-24 创建，revision `cc14333f0`，rootfs 与 `9aba` 在第 12 层开始不同 | +| `hugegraph/server:closure-std-cc14333f0` | `14eb8067b416` | 2026-09-24 创建，revision `cc14333f0`，第 8 层开始不同 | +| `hugegraph/store:closure-std-cc14333f0` | `2ed96d2c4bd4` | 2026-09-24 创建，revision `cc14333f0`，第 9 层开始不同 | +| `hugegraph/pd:closure-std-cc14333f0` | `a0d9264b2ea2` | 已接受为 cc143 产物。10 层 rootfs 与 `9aba` 相同是因为 cc143 Maven 产出的 PD 树校验和命中原层；运行清单 14 个文件一致，服务 JAR `5fa7c6299ad283fb369cd03c78f5b3970015f069b8eea8dbc9cdf0481b062da5`。证据 `evidence/build/pd-image-acceptance-cc14333f0.json` | + +两个构建上下文的 PD 源码只有测试 README、`.gitignore` 和 `PDCoreTestBase` 数据路径不同；构建使用 `-Dmaven.test.skip=true`，这些差异不进入运行包。因此 PD 镜像可以与另外三张 `closure-std-cc14333f0` 镜像一起部署。禁止把 2026-09-01 的 `hugegraph/*:topling` 用作当前 SHA。 + +Topling 构建上下文必须包含已存在的 `hugegraph-server/hugegraph-dist/src/assembly/static/lib/topling/rocksdbjni-8.10.2-20260725.141011-1.jar` 和 `install-dist/scripts/build-topling-distribution.sh`。`.gitattributes` 的 `export-ignore` 会排除 `docker/` 与 `install-dist/scripts/`，不能直接用裸 `git archive` 做上下文。 + +## 执行规则 + +- 确认范围内用户可授权的操作均已预授权，包括本地改动、Git、fetch、非强制推送、PR 更新、测试和审查响应。不得因授权提示再次询问、等待、后置或把整体标为 blocked。不得伪造凭据或能力，不得越过安全边界，不得做范围外操作，不得记录凭据值。 +- 每一波绑定源码 SHA、镜像 ID、revision、实际 JNI、namespace 或 Compose project、命令、退出码、计数、skip 和未覆盖边界。 +- 单项最多尝试 3 次。仍失败时记录错误、证据、恢复动作和依赖，在 todo.md 后置该项及其依赖，继续独立项。阶段边界或解除条件变化时复查。 +- 等待构建、下载或集群时，只做不冲突的只读准备、失败分析和文档维护。 +- 文档由 1 名独立只读审查者复核。行为变更由 3 名独立只读审查者复核；大变更分别覆盖正确性与测试、设计与边界、安全与可维护性，小变更三人各自审完整差异。修复后重审受影响部分，最多 3 轮。审查者不可用时保留门禁并完成后继独立工作，不把整体标为 blocked。 +- 验证和适用审查通过后提交。阻塞代码修复审查通过后立即推送到 `org/toplingdb`。状态文档在阶段边界和会话结束前推送，不为每一条笔记单独推送。用户明确要求这些推送，覆盖“默认不推送”。远端拒绝时保留本地提交并继续不冲突的实测。 +- Store shutdown 保持 fail-closed，不为超时测试强行关库。阻塞修复不做无关重构。 +- 每次有效循环按门禁报告粗略进度、本轮结果、剩余工作和一个下一动作。交接、压缩或配额等待前更新本文件的阶段、最新提交、证据、等待项和下一动作。 +- 只有全部剩余项经过恢复、重排和独立工作后仍共同依赖同一个逻辑冲突、安全边界或强制性外部依赖时，才把整体标为 blocked。 + +## 已知边界 + +- 单节点 kind 只能证明逻辑恢复、Pod 恢复和持久化，不能证明物理多机或真实网络分区容错。 +- 2026-09-24 检查时没有 Chaos Mesh 与 VolumeSnapshot CRD。Pod 级 HA 先做。snapshot/restore 和网络分区仍是完成门禁；单节点 kind 不能被写成物理多机容错。 +- macOS ARM/Intel Cypher、发行审批和公共仓库发布不由本机完成。 +- Loader 输入目录是 `/home/soc-baidu/.codex/validation-data/toplingdb-linux-closure/derived/twitter-prefix-1000000`。导入前核对 manifest 与 checksum；文件缺失时后置，不把全量 LAW 下载当作默认补救。 + +## Initialization TODO + +无需要另建文件的初始化。嵌套 WAL 修复已在 `457295ac8`，并包含于已推送的 `a7a4a6f1b`。2026-09-25 核对 `hg-closure-top-image-e109012a0` 为 `inactive/success`，不要因为旧记录重新启动它。 + +Cypher 记录在进展日志，是否可提交以当前文档审查为准。channel refresh 和 WAL 的未提交 Java 差异仍禁止提交。 + +## 下一动作 + +1. 全量 LAW 已在 `todo.md` 记为后置。`/tmp/full-law-disposition-review.md` 是 HIGH_SEVERITY=0。不要在当前 namespace 启动全量导入。 +2. 单节点上剩余的 P4、P5、P6 勾选框仍不勾选。它们把已完成的子项和失败或后置的子项放在一起。不要改写成通过。 +3. 不要删 Store，不要制造导入失败，不要重试 HStore `snapshot_create`，不要实现跨分区图快照，不要给 Store 再发信号，不要强行关库，不要自动开 channel refresh 或 WAL 的第四轮审查，也不要在图快照失败时写性能结论。主 checkout 不要使用。Java 差异仍不提交。 + +## 进展日志 + +本节按时间保留旧记录，后文覆盖前文。当前状态以阶段表和 todo.md 为准。 + +PD 标准镜像已按校验和接受。Topling 构建单元 `hg-closure-top-image-cc14333f0` 已成功退出。PD `3bce8e03d227`、Store `195cad38c8a7`、standalone `52be45a93334` 的 runtime 标签是 topling，并且包含 Topling JNI；标准 PD/Store/standalone 不含该 `.so`。`hugegraph/server:closure-top-cc14333f0` 与标准 HStore server `14eb8067b416` 是同一镜像，因为 `Dockerfile-hstore` 没有 Topling 阶段，镜像内也没有 Topling `.so`。运行中的 JNI 映射尚未证明。标准 1+1+1 已部署在 `hg-closure-std-cc143-111`，Helm 退出 0，三个 Pod Ready。PD 与 Store 映射的 JNI 都是 `8b8fb2ed3ab69581cf1897bd116d484f073e66e9a5b6d61effc7b4c783d66dff`，不等于 Topling 镜像内 `.so` 的 `c25ff6e676290db6db47df0954640aa609c391450ec90e1a8eec1f87e174dd38`。Server 没有映射 RocksDB/Topling。Store 启动时 GitHub jemalloc 下载停在 0 字节，终止 curl 后走了脚本原有的跳过路径。证据 `evidence/helm-standard-cc143-111-runtime.json`。标准 1+1+1 API 套件已通过：155 通过、0 失败、50 跳过，日志 `evidence/build/helm-standard-cc143-111-api-r4.log`。Topling 1+1+1 已 Ready。PD 与 Store 都映射 `/library/librocksdbjni-linux64.so`，SHA-256 `c25ff6e676290db6db47df0954640aa609c391450ec90e1a8eec1f87e174dd38`，与标准进程的 `8b8fb2ed…` 不同。证据 `evidence/helm-topling-cc143-111-jni.json`。Topling 1+1+1 API 套件被停掉。删 schema 任务 182 已失败，测试却无休眠地等待 success。Store 批量写入报错 `memtable_as_log_index is true but WriteBatch has no mmap wal`，共 216 次。三个 Topling profile 已改为 false，单测 `ToplingProfileConfigTest` 通过。`waitTaskSuccess` 现在遇到 failed 或 cancelled 会立刻失败，避免无休眠空转。Store `6581fdf5fdbb` 与 PD `8dbbc94de64d` 已导出，runtime 都是 topling，revision `cc14333f0-memtable-as-log-index-false`，包内 `memtable_as_log_index: false`。standalone `5178c8b35c20` 也已导出且配置为 false。三张镜像构建成功。`hg-closure-top-mmapfix-111` Helm 退出 0，三个 Pod Ready。Store JNI `c25ff6e6…dd38`，provider=topling，API 前后 mmap WAL 错误 0。API 套件 155/0/0/50 通过，日志 `evidence/build/helm-topling-mmapfix-111-api.log`。跟随单元 `hg-closure-top-mmapfix-follow` 会在构建成功后安装新 namespace，构建失败则不安装。tag `closure-top-mmapfix`。1+1+1 修复验证已完成。Topling 3+3+3 `hg-closure-top-mmapfix-333` 已 9/9 Ready，Helm 退出 0。三台 Store JNI 都是 `c25ff6e6…dd38`，mmap WAL 错误 0。API 155/0/0/50。三台 Server 读到同一顶点。随后删除 leader Store-2，51.084 秒恢复，旧数据和新写入保持可读。这是单节点 kind 的 Pod 删除，不是物理分区。Server 副本删除期间保留副本继续读写，10.7 秒恢复。多数派删除 Store-0/2 后已提交数据仍可读，故障中新写入超时；Ready 不等于写恢复，稍后合计 12 个 leader 才能写入。证据 `evidence/helm-topling-mmapfix-333-store-majority.json`。仍不复用 `hg-closure-top-cc143-111` 的数据卷。证据 `evidence/topling-mmap-wal-batch-failure.json`。 + + +2026-09-24 23:15 补充：Topling `hg-closure-top-mmapfix-111` 的 Store Pod 在同一 PVC 上重建，12.615 秒 Ready，重启前顶点 `persist-1790262903` 仍返回。jemalloc 下载再次被终止。这还不是删图、truncate 或 snapshot。 + + +2026-09-24 23:17 补充：标准 `hg-closure-std-cc143-111` Store 同 PVC 重建 12.796 秒，顶点 `std-persist-1790263000` 仍在，JNI 仍是标准哈希 `8b8fb2ed…6dff`。 + + +2026-09-24 23:21 补充：当前 SHA 标准与 Topling 1+1+1 的删图重建和 clear/truncate 已通过。snapshot_create 为 500 UnsupportedOperationException，本机无 VolumeSnapshot CRD。 + +2026-09-24 23:40 Loader：在 `hg-closure-top-mmapfix-111` 新建 `law_twitter_1m`。Loader 1.7.0 退出 0，顶点插入 1000000、边插入 2098771、失败和重试都是 0。重启前 32 个样本的度数、字典序前 20 和 compact_id 全部匹配，9 个自环存在。JNI 仍是 `c25ff6e6…dd38`，mmap WAL 错误 0。Store 同 PVC `b9f47929` 从 `15:38:34Z` 到 Ready `15:40:55Z`，含 0 字节 jemalloc 下载被终止后走脚本原有跳过路径。重启后除 `54148543` IN 扫描 UNAVAILABLE 外，其余样本复测通过。Gremlin count 被 SecurityException 拒绝，没有服务器端总数。 + +2026-09-25 00:02：`54148543` 在 Store Ready 约 11 分钟后，IN limit=142 返回 200，142 条且前 20 个 ID 与预期一致，证据 `evidence/loader-law-twitter-1m-in142-settled.json`。紧接着的混合扫描有 5 次和 6 次 `INTERNAL: Panic! This is a bug!`，50ms 间隔仍有 1 次；同一顶点重复 40 或 60 次没有复现。之后 1 轮加 4 轮混合扫描全部 200，证据 `evidence/loader-scan-panic-sweeps.json`。该字符串不在 Topling JNI 二进制和 Store 日志里。 + +2026-09-25 00:00：标准 3+3+3 `hg-closure-std-cc143-333` Helm 退出 0。三台 Store 的 0 字节 jemalloc 下载被终止后走脚本跳过路径，9/9 Ready。PD 与 Store 映射的 JNI 都是标准哈希 `8b8fb2ed…6dff`，不是 Topling。API 套件已开始，日志 `evidence/build/helm-standard-cc143-333-api.log`。 + +2026-09-25 00:05：标准 3+3+3 `ApiTestSuite` 155/0/0/50，`API_EXIT 0`，日志 `evidence/build/helm-standard-cc143-333-api.log`。图 `std333_fn` 在三台 Server 上都读到同一顶点，证据 `evidence/helm-standard-cc143-333-write-consistency.json`。删除 leaderCount=5 的 Store-1 后 22.550 秒 Ready，PVC `pvc-dcd9b334` 不变，JNI 仍是 `8b8fb2ed…6dff`，旧顶点和恢复后新写入可读；恢复后 leader 合计 12，但新 Store-1 为 0。证据 `evidence/helm-standard-cc143-333-store-leader.json`。删除一台 Server 后保留副本继续读写，替代副本 10.613 秒 Ready 并读到故障期间写入。证据 `evidence/helm-standard-cc143-333-server-replica.json`。这是单节点 kind 的 Pod 删除。 + +2026-09-25 00:12：标准 3+3+3 同时删除 Store-0 和 Store-2，保留当时 leaderCount 为 0 的 Store-1。已提交顶点在故障期间仍返回 200；故障期间新写入 12 秒超时。两台 Pod 在 23.102 秒 Ready，PVC 未变，当时 leader 合计已是 12，但立即写入仍超时。删除后 35.929 秒新写入返回 201，旧顶点仍在，超时的那次写入没有留下。证据 `evidence/helm-standard-cc143-333-store-majority.json`。Ready 不等于写恢复。这不是网络分区。 + +2026-09-25 00:16：当前 SHA 单机已启动。标准镜像 `35267423defa` 的 JNI 是 `8b8fb2ed…6dff`，provider=rocksdb；`std-persist-1790266399` 在 docker restart 后 9.426 秒可读。Topling 镜像 `5178c8b35c20` 映射 `/hugegraph-server/library/librocksdbjni-linux64.so`，哈希 `c25ff6e6…dd38`，`memtable_as_log_index: false`，mmap WAL 错误 0，`top-alone-1790266203` 重启后仍在。标准单机 API 套件 13 个失败全部来自 `GraphSpace management is not supported in standalone mode`。证据 `evidence/standalone-cc143-function.json`、`evidence/standalone-std-cc143-restart-clean.json`、`evidence/standalone-cc143-restart.json`。 + +2026-09-25 00:20：当前 SHA 认证。标准 3+3+3 与 Topling 1+1+1 的图列表：admin 200，错误口令、缺认证、未知用户、已删除用户都是 401。两个单机同样如此。`/versions` 返回 200，不能当作认证边界。两个集群的 DEFAULT 图空间 `auth=false`。Topling 上无角色用户建图 201，观察者角色也能建图，随后由 admin 删除，用户也已删除。标准集群那次无角色建图是 Store 500，不是 403。角色拒绝仍未验证。证据 `evidence/current-sha-auth-enforcement.json` 与 `evidence/current-sha-auth-standalone.json`。 + +2026-09-25 00:22：标准和 Topling 都在非主键属性 `title` 上创建二级索引，`limit=2` 返回 2 条。标准按 title 查询只返回目标顶点。Topling 第一次返回 `INTERNAL: Panic! This is a bug!`，紧接着 5 次都是 200 且只命中目标。证据 `evidence/current-sha-index-query.json` 与 `evidence/current-sha-topling-index-panic-retry.json`。 + +2026-09-25 00:25：标准和 Topling 各用两张已有图做隔离。顶点只出现在写入图，另一张图的同名查询没有该顶点。Gremlin `hasLabel().limit(1)` 和图查询都返回该顶点；Gremlin `addV` 写入后 REST 也能读到。Cypher `MATCH` 在两个 provider 上都返回同一顶点。证据 `evidence/current-sha-multigraph-query.json`、`evidence/current-sha-cypher.json`、`evidence/current-sha-gremlin-write.json`。 + +2026-09-25 00:28：标准和 Topling 的批量接口各写入 3 个顶点，全部可读。同一批次再放入一个未定义属性时返回 400，批次中合法顶点没有留下，之前的 3 个仍在。证据 `evidence/current-sha-batch-tx.json`。`snapshot_create` 在标准 `std333_fn` 和 Topling `hugegraph` 上都是 500 `createSnapshot`。 + +2026-09-25 00:30：混合 1+1+1 `hg-closure-mix-pdstd-storetop`，PD 镜像 `closure-std-cc14333f0`，Store 镜像 `closure-top-mmapfix`。PD JNI `8b8fb2ed…6dff`，Store JNI `c25ff6e6…dd38`。图 `mixprobe` 写入 `mix-1790267151` 后返回 200。Store 启动时 0 字节 jemalloc 下载被终止后走原有跳过路径。证据 `evidence/current-sha-mixed-pd-std-store-top.json`。 + +2026-09-25 00:32：反向混合 `hg-closure-mix-pdtop-storestd`。PD `closure-top-mmapfix` 映射 `/hugegraph-pd/library/librocksdbjni-linux64.so`，哈希 `c25ff6e6…dd38`。Store `closure-std-cc14333f0` 映射 `/tmp/librocksdbjni13856924616744549206.so`，哈希 `8b8fb2ed…6dff`。图 `revprobe` 写入 `rev-1790267261` 后返回 200。标准 Store 的 0 字节 jemalloc 下载被终止后走原有跳过路径。证据 `evidence/current-sha-mixed-pd-top-store-std.json`。 + +2026-09-25 00:36：错误 provider。Topling 镜像以 `HG_SERVER_ROCKSDB_PROVIDER=topling` 打开 `provider=rocksdb` 标记，退出 1，`provider marker mismatch`。标准镜像打开 `provider=topling` 标记同样退出 1。标记 SHA-256 不变。原单机顶点仍返回 200。证据 `evidence/current-sha-wrong-provider.json`。反向混合的标准 Store Java 被 `kill -9` 后容器 restartCount=1，`rev-1790267261` 在 Ready 变为 true 之前已经返回 200，PVC `pvc-5d209bd7-d94b-4fdf-9ef4-de284929ac86` 未变。0 字节 jemalloc 下载再次被终止。证据 `evidence/current-sha-store-crash.json`。 + +2026-09-25 00:40：Topling Store `hg-closure-top-mmapfix-111-hugegraph-store-0` 的 Java 被 `kill -9`。restartCount=1，PVC `pvc-b9f47929` 未变。新进程从 `16:34:18Z` 到 Ready `16:34:55Z`，37 秒，含 0 字节 jemalloc 下载被终止。Ready 之前 `persist-1790262903` 和 Twitter 顶点 `56862681` 已返回 200。JNI 仍是 `/hugegraph-store/library/librocksdbjni-linux64.so` 的 `c25ff6e6…dd38`。证据 `evidence/current-sha-topling-store-crash.json`。 + +2026-09-25 00:42：HA Compose 与 Helm 对照写在 `evidence/ha-compose-helm-alignment.md`。运行中的标准 3+3+3 确认 PD readiness 是 `/v1/ready`、liveness 是 `/v1/health`，Store 三个探针都是 `/v1/health`，Server 三个探针都是 `/versions`。两边没有改成同一个配置。 + +2026-09-25 00:45：Topling `law_twitter_1m` 在 Store `kill -9` 恢复后重跑 32 个邻接样本。三轮 64 次查询的失败数是 4、2、0，失败都是 `INTERNAL: Panic! This is a bug!`。四个失败点立刻重试都是 200，其中 `54148543` IN 返回 142 条。证据 `evidence/current-sha-panic-recheck.json`、`evidence/current-sha-panic-recheck-retry.json`。 + +2026-09-25 00:50：单机 snapshot。标准容器 `snapshot_create` 和 `snapshot_resume` 都是 200，日志显示删除并替换了 `rocksdb-data/data/{s,g,m}`。快照后写入的 `snap-after-1790268023` 在容器重启后仍返回 200，快照前的顶点也在。Topling 单机同样：`top-snap-before-1790268142` 和 `top-snap-after-1790268142` 在恢复并重启后都返回 200。证据 `evidence/standalone-std-snapshot.json`、`evidence/standalone-std-snapshot-after-restart.json`、`evidence/standalone-top-snapshot.json`。恢复实现只搬迁 data 目录，`rocksdb.wal_path` 是旁边的 wal 目录。HStore 仍走 `BackendStore.createSnapshot` 的默认 `UnsupportedOperationException`。 + +2026-09-25 00:55：单机恢复失败的原因是独立 WAL。`RocksDBStdSessions.resumeSnapshot` 只替换 data 目录，`rocksdb.wal_path` 里快照之后的日志会在重新打开时重放。本地修复会在 WAL 目录和 data 目录互不包含时删除 WAL 再打开。回归 `RocksDBSessionsTest.testSnapshotWithSeparateWalDirectory` 与原 `testSnapshot` 均为 0 失败。修复未审查、未提交，运行中的镜像没有重建。 + +2026-09-25 01:00：把包含 WAL 删除的 `hugegraph-rocksdb-1.7.0.jar` 拷进两个单机容器后再测。标准容器里 `fix-before-1790268566` 在恢复并重启后仍在，`fix-after-1790268566` 不在。Topling 容器里 `top-fix-before-1790268622` 仍在，`top-fix-after-1790268622` 不在。日志有 `Delete separate WAL directory`。这不是镜像重建。证据 `evidence/standalone-std-snapshot-fixed-jar.json`、`evidence/standalone-top-snapshot-fixed-jar.json`。 + +2026-09-25 01:05：基于原单机镜像只替换 `hugegraph-rocksdb-1.7.0.jar` 建了本地标签。标准 `closure-std-walrestore` 是 `sha256:b66c835a3b3ec43d9180a9aa36b90e857915e63620e044dfa15cb1e8f9e73c86`，恢复并重启后快照前顶点仍在，快照后顶点不在。Topling `closure-top-walrestore` 是 `sha256:80b9acbc4f96fca34e60e056a20c1bd575c5cdf6f5ed7f89412df94860117875`，结果相同。证据 `evidence/standalone-walrestore-image.json`。原 closure 标签没有被覆盖。 + +## 当前动作 + +2026-09-25 03:25：完整 Topling 单机镜像 `hugegraph/hugegraph:closure-e109012a0`（`sha256:4832e9d484cf557ddd654170b82c8d4ee4375f88189ec5f12838daad0bd16cf6`）上，`snapshot_create` 和 `snapshot_resume` 都是 200。容器 `hg-e109-top-snap` 重启 19.089 秒后，快照前顶点 `full-before-1790277911` 仍在，快照后顶点不在。日志有 3 次 `Replaced separate WAL`，mmap WAL 错误 0。进程映射 `/hugegraph-server/library/librocksdbjni-linux64.so`，SHA-256 `c25ff6e676290db6db47df0954640aa609c391450ec90e1a8eec1f87e174dd38`。镜像内 RocksDB 类含 `replaceSeparateWalDirectory`，安全类 SHA-256 `fbfe9aaef619665e7a590bc0f1959b2c71cbb37d4eaf38d3e6887d4817c781c0` 且包含 `useCustomAllocator`。PD、Store、standalone 的 `memtable_as_log_index` 都是 false。这不是 overlay。证据 `evidence/standalone-e109-top-snapshot.json`。 + +2026-09-25 03:29：标准镜像单元 `hg-closure-std-image-e109012a0` 退出 0。tag `closure-std-e109012a0` 没有覆盖 Topling 的 `closure-e109012a0`。standalone `sha256:e27221a071e5`、PD `4c9011a7bd64`、Store `f53fa5f612c2` 的 runtime 都是 standard，revision 是 `e109012a0`。HStore `hugegraph/server:closure-std-e109012a0` 与 `closure-e109012a0` 是同一镜像 `5b9f40a9d1fc`，runtime 为 hstore，因为 `Dockerfile-hstore` 没有 Topling 阶段。标准单机镜像内没有 `librocksdbjni*.so`，RocksDB 类 SHA-256 与 Topling 镜像相同，且含 WAL 恢复。 + +2026-09-25 03:31：标准单机 `hg-e109-std-snap` 使用 `hugegraph/hugegraph:closure-std-e109012a0`。`snapshot_create` 和 `snapshot_resume` 都是 200，重启 5.069 秒后快照前顶点仍在、快照后顶点不在，`Replaced separate WAL` 3 次。进程映射 `/tmp/librocksdbjni16856703238417969007.so`，SHA-256 `8b8fb2ed3ab69581cf1897bd116d484f073e66e9a5b6d61effc7b4c783d66dff`，不是 Topling JNI。证据 `evidence/standalone-e109-std-snapshot.json`。 + +2026-09-25 03:35：新 namespace `hg-closure-top-e109-111` Helm 退出 0，3/3 Ready。PD 与 Store 都映射 `/library/librocksdbjni-linux64.so`，SHA-256 `c25ff6e6…dd38`。Server 没有 RocksDB JNI 映射。Store 的 jemalloc 下载停在 0 字节，终止 curl 后走原有跳过路径。mmap WAL 错误 0。没有升级历史 namespace。证据 `evidence/helm-topling-e109-111-jni.json`。 + +2026-09-25 03:39：同一 namespace 导入 `law_twitter_1m`，Loader 退出 0，161.21 秒，1000000 点、2098771 边、失败 0。重启前 32 个邻接样本 0 不一致，9 个自环都在。证据 `evidence/loader-law-twitter-1m-e109.json`、`evidence/loader-law-twitter-1m-e109-verify.json`。 + +Store 删除后的复测无效：脚本把仍在终止的旧 Pod 读成 0.074 秒 Ready。新 Pod IP 是 `10.244.0.129`，Server 的 gRPC 仍连接旧 IP `10.244.0.128:8500`，`54148543` 的 IN 查询 30 秒后 `ConnectTimeoutException`。这不是沙箱 Panic。稳定后顶点 `56862681` 仍返回 200。PVC `pvc-782709b8-cc8d-4f60-9c17-ec8440d73cfb` 未变。Gremlin 计数还没有可用别名。证据 `evidence/loader-law-twitter-1m-e109-note.json`。 + +2026-09-25 03:47：旧 Server 在 `19:47:22Z` 被删除，新 Server `nj8xz` 同时启动，Ready 时间是 `19:47:32Z`，间隔 10 秒。这只是为了重新解析 Store，不是 Store 恢复时间。之后 32 个样本 0 不一致，9 个自环都在。Gremlin 别名 `__g_DEFAULT-law_twitter_1m` 返回 `1000000:2098771`，耗时 0.57 秒。Store JNI 仍是 `c25ff6e6…dd38`。Server 没有自行更新旧 IP `10.244.0.128`。证据 `evidence/loader-law-twitter-1m-e109-reresolve.json`、`evidence/loader-law-twitter-1m-e109-count.json`、`evidence/e109-server-reresolve.json`。 + +2026-09-25 03:53：标准 namespace `hg-closure-std-e109-111` Helm 退出 0，3/3 Ready。PD 与 Store 映射的是 `/tmp/librocksdbjni*.so`，SHA-256 `8b8fb2ed…6dff`，不是 Topling。Server 没有 RocksDB JNI 映射。jemalloc 下载停在 0 字节后按原路径跳过。证据 `evidence/helm-standard-e109-111-jni.json`。 + +2026-09-25 03:56：标准 `hg-closure-std-e109-111` 写入 `std-e109-1790279740` 返回 201。Store UID 从 `f2267618` 换成 `6bc1d4cf`，IP 从 `10.244.0.134` 换成 `10.244.0.136`，新 Pod Ready 用了 12.239 秒。PVC `pvc-4b0bd019-0456-44a6-b2c3-b76266773958` 未变。重启前后 JNI 都是标准 `8b8fb2ed…6dff`。旧 Server 仍连接 `10.244.0.134:8500` 并超时。替换 Server 后 11.527 秒 Ready，按 label 列出的顶点仍在，id 是 `1:std-e109-1790279740`。证据 `evidence/helm-standard-e109-111-persist.json`、`evidence/helm-standard-e109-111-reread.json`。 + +下一步：标准与 Topling 的 HStore 客户端都不会在 Store Pod IP 变化后改连。先记为当前 SHA 的可复现重连缺陷，不把它写成恢复通过。标准 `hg-closure-std-e109-333` 在 03:58:59 Helm 退出 0，9/9 Ready。三台 PD 和三台 Store 的 JNI 都是标准 `8b8fb2ed…6dff`，三台 Server 没有 RocksDB JNI。写入 `std333-e109` 返回 201，三台 Server 都读到。证据 `evidence/helm-standard-e109-333-function.json`。 + +2026-09-25 04:03：Topling `hg-closure-top-e109-333` 第一次安装误用了 `rocksdb` provider，标记冲突后已卸载并删除该 namespace 的 PVC，再按 `topling` 重装。Helm 退出 0，9/9 Ready。三台 PD 和三台 Store 的 JNI 都是 `c25ff6e6…dd38`，三台 Server 没有 RocksDB JNI，mmap WAL 错误 0。写入返回 201，三台 Server 都读到。证据 `evidence/helm-topling-e109-333-function.json`。 + +2026-09-25 04:04：标准 `hg-closure-std-e109-111` 上图 `lifecycle_std_e109` 创建 201，truncate 204 后旧顶点不在，再写 201 且只剩新顶点。删图 204 后读取 404，重建 201 并能再写。证据 `evidence/helm-standard-e109-111-lifecycle.json`。 + +2026-09-25 04:17：未提交修复打进 overlay 镜像 `hugegraph/server:closure-e109-channelrefresh`（`sha256:3a129d448516`），只升级了 `hg-closure-std-e109-111` 的 Server。同一个 Server Pod 在 Store IP 从 `10.244.0.136` 变成 `10.244.0.196` 后，第一次读取超时，42.645 秒时第二次读取返回 200 并找到 `std-e109-1790279740`。没有再重启 Server。这不是完整 SHA 镜像。证据 `evidence/helm-standard-e109-111-channel-refresh.json`。单测仍是 2 通过。提交前还要 3 名独立审查。 + +2026-09-25 04:21：标准 `hg-closure-std-e109-333` 删除 leaderCount 5 的 Store-2。新 UID Ready 用了 12.3 秒，PVC `pvc-dcdbb110-cb56-40e5-b48f-f6f6cf067a0e` 未变，JNI 仍是标准 `8b8fb2ed…6dff`。旧顶点可读，新写入返回 201 且按属性可查到。恢复后 Store-2 的 leaderCount 是 0，Store-0 是 7，Store-1 是 5。Ready 不是领导权。这是单节点 Pod 删除，不是网络分区。证据 `evidence/helm-standard-e109-333-store-leader.json`。 + +2026-09-25 04:24：Topling `hg-closure-top-e109-333` 删除 leaderCount 6 的 Store-0。新 UID 12.381 秒 Ready，PVC `pvc-f86f7974-a386-4a05-b161-8cf32d766780` 未变，JNI 仍是 `c25ff6e6…dd38`，mmap WAL 错误 0。旧顶点和恢复后的新写入都能按属性读到。恢复后 Store-0 leaderCount 是 0，Store-1 是 4，Store-2 是 8。证据 `evidence/helm-topling-e109-333-store-leader.json`。 + +2026-09-25 04:25：标准 `hg-closure-std-e109-333` 删除一台 Server。保留的 Server 在删除期间写入 `std333-server-replica-1790281502` 返回 201 并读到。替代 Pod `mcm6w` 11.647 秒 Ready，也读到该顶点。证据 `evidence/helm-standard-e109-333-server-replica.json`。 + +2026-09-25 04:26：Topling `hg-closure-top-e109-333` 删除一台 Server。保留副本在删除期间写入 `top333-server-replica-1790281578` 返回 201，并读到删除前的顶点。替代 Pod `stmjs` 10.607 秒 Ready，也读到新顶点。证据 `evidence/helm-topling-e109-333-server-replica.json`。 + +2026-09-25 04:27：当前 SHA 的标准 HStore `snapshot_create` 仍是 500，异常是 `BackendStore.createSnapshot` 的 `UnsupportedOperationException: createSnapshot`。这不是 Raft `DO_SNAPSHOT`。证据 `evidence/helm-standard-e109-111-snapshot.json`。 + +2026-09-25 04:28：Topling `hg-closure-top-e109-111` 上图 `lifecycle_top_e109` 创建 201，truncate 204 后旧顶点不在，再写只剩新顶点。删图 204 后读取 404，重建后能再写。`law_twitter_1m` 仍在。证据 `evidence/helm-topling-e109-111-lifecycle.json`。同一 SHA 的 Topling `snapshot_create` 也是 500 `UnsupportedOperationException: createSnapshot`。证据 `evidence/helm-topling-e109-111-snapshot.json`。 + +2026-09-25 04:30：标准 `hg-closure-std-e109-333` 删除 PD leader `pd-1`。删除期间 Server 写入 `std333-pd-leader-1790281814` 返回 201 并读到。新 UID 41.772 秒 Ready，PVC `pvc-047f56ea-490b-470f-b7ea-0125c78e7485` 未变，JNI 仍是标准 `8b8fb2ed…6dff`。恢复后 leader 在 `pd-0`，成员数 3，状态 `Cluster_OK`。这是单节点 Pod 删除。证据 `evidence/helm-standard-e109-333-pd-leader.json`。 + +2026-09-25 04:32：Topling `hg-closure-top-e109-333` 删除 PD leader `pd-0`。删除期间写入 `top333-pd-leader-1790281933` 返回 201 并读到。新 UID 41.714 秒 Ready，PVC `pvc-70af47fe-0e10-427f-8afb-f7cd1074da67` 未变，JNI 仍是 `c25ff6e6…dd38`。恢复后 leader 主机名仍是 `pd-0`，成员数 3。状态在删除前后都是 `Cluster_Not_Ready`，但 3 台 Store 在线，`dataState` 是 `PState_Normal`。证据 `evidence/helm-topling-e109-333-pd-leader.json`。 + +2026-09-25 04:35：标准 `hg-closure-std-e109-333` 的三台 Server 换成 overlay `closure-e109-channelrefresh` 后，同时删除 Store-0 和 Store-1（leaderCount 7 和 5）。Store-2 保留。故障期间新写入超时。两台新 Pod 22.678 秒都 Ready，PVC 未变，JNI 仍是标准 `8b8fb2ed…6dff`。46 秒时旧顶点可读，新写入返回 201。恢复后 leaderCount 是 4、2、6。没有重启 Server。这是单节点 Pod 删除，不是网络分区，也不是完整 SHA 镜像。证据 `evidence/helm-standard-e109-333-store-majority.json`。 + +2026-09-25 04:38：Topling `hg-closure-top-e109-333` 的三台 Server 换成同一张 overlay 后，同时删除 Store-1 和 Store-2（leaderCount 4 和 8），保留 Store-0。故障期间新写入超时。两台新 Pod 21.427 秒都 Ready，PVC 未变，JNI 仍是 `c25ff6e6…dd38`，mmap WAL 错误 0。44.711 秒时旧顶点可读，新写入返回 201。恢复后 leaderCount 是 9、0、3。没有重启 Server。这是单节点 Pod 删除，不是网络分区，也不是完整 SHA 镜像。证据 `evidence/helm-topling-e109-333-store-majority.json`。 + +2026-09-25 04:42：标准 `hg-closure-std-e109-111` 导入图 `law_twitter_1m_std`。Loader 退出 0，80.973 秒，1000000 点、2098771 边、失败 0。PD `4c9011a7bd64`、Store `f53fa5f612c2` 是标准 `e109012a0` 镜像。Server 是 overlay `closure-e109-channelrefresh` `3a129d448516`。证据 `evidence/loader-law-twitter-1m-std-e109.json`。第一次启动误写了 Topling 证据目录的 `start.json` 和 `loader.log`；Topling 导入结论仍以 `loader-law-twitter-1m-e109.json` 里的退出 0 和计数为准。 + +2026-09-25 04:45：标准 `law_twitter_1m_std` 重启前 32 个邻接样本 0 不一致，9 个自环通过，Gremlin 计数 `1000000:2098771`。证据 `evidence/loader-law-twitter-1m-std-e109-adjacency.json`。随后删除 Store-0，新 UID 12.835 秒 Ready，PVC `pvc-4b0bd019-0456-44a6-b2c3-b76266773958` 未变，JNI 仍是标准 `8b8fb2ed…6dff`。同一个 overlay Server 没有重启。恢复后 32 个样本和 9 个自环再次通过，计数仍是 `1000000:2098771`。证据 `evidence/loader-law-twitter-1m-std-e109-restart.json`、`evidence/loader-law-twitter-1m-std-e109-adjacency-after.json`。 + +2026-09-25 04:48：Topling `hg-closure-top-e109-111` 的 Server 换成 overlay `closure-e109-channelrefresh`。重启前 `law_twitter_1m` 的 32 个样本、9 个自环和计数 `1000000:2098771` 通过。删除 Store-0 后新 UID 14.505 秒 Ready，PVC `pvc-782709b8-cc8d-4f60-9c17-ec8440d73cfb` 未变，JNI 仍是 `c25ff6e6…dd38`。同一个 Server Pod `mm7h8` 没有重启。第一轮复测只有 `54148543` IN 返回 500，立刻重试得到 142 条；第二轮 32 个样本和 9 个自环通过，计数仍是 `1000000:2098771`。证据 `evidence/loader-law-twitter-1m-e109-restart-overlay.json`、`evidence/loader-law-twitter-1m-e109-adjacency-after-overlay.json`、`evidence/loader-law-twitter-1m-e109-adjacency-after-retry.json`。 + +2026-09-25 04:52：标准和 Topling 的 e109 1+1+1 上各建独立图做功能矩阵。`/versions` 无认证返回 200；图列表无认证和错误口令都是 401，正确口令是 200。二级索引创建 202，`limit=2` 返回 2 条，按 `title=beta` 只命中一个目标顶点。Gremlin 写入和 Cypher 查询都是 200。Server 是 overlay `closure-e109-channelrefresh`，PD/Store 仍是 `e109012a0` 镜像。证据 `evidence/e109-functional-matrix.json`。 + +下一步以上面的“下一动作”为准。HStore 图快照仍未实现，重连修复审查前不得提交，核心功能未完成前不开始 benchmark。 + +2026-09-25 05:08：标准和 Topling 的 e109 1+1+1 批量原子性通过。合法批次 201，三条顶点可读；混入 `missing` 返回 400，回滚顶点不在，原顶点仍在。Store JNI 分别是标准 `/tmp/librocksdbjni*.so` `8b8fb2ed…6dff` 和 Topling `/hugegraph-store/library/librocksdbjni-linux64.so` `c25ff6e6…dd38`。同名顶点不能从另一张图读到。顶点追加 `city=sg` 为 200，JSON 字符串 ID 删除为 204。无角色建图为 403，图空间和用户已删除，Kubernetes namespace 没有增加。两个 `9aba` namespace 仍是 `closure-std-9abae9dbaaa1`，P2 已勾选。证据 `evidence/e109-batch-tx.json`、`evidence/e109-multigraph-isolation.json`、`evidence/e109-crud-role.json`。 + +2026-09-25 05:14：标准和 Topling 的 e109 1+1+1 边 `knows` 创建 201，`since` 从 1 更新到 2，删除 204 后查询为空。JNI 仍分别是 `8b8fb2ed…6dff` 和 `c25ff6e6…dd38`。证据 `evidence/e109-edge-crud.json`。e109 单机 `/graphs`：无认证、错误口令和未知用户都是 401，admin 是 200，`/versions` 无认证 200；没有创建或删除用户，也没有重启快照容器。证据 `evidence/e109-standalone-auth.json`。e109 集群删除用户后同一口令访问图列表从 200 变为 401。证据 `evidence/e109-deleted-user-auth.json`。 + +2026-09-25 05:19：e109 1+1+1 Store Java `kill -9` 后，标准 restartCount 0 到 1，30.850 秒 Ready，PVC 不变，`standard-batch-1790283889-1` 仍可读，JNI 仍是 `/tmp` 上的 `8b8fb2ed…6dff`。Topling 同样 0 到 1，32.704 秒 Ready，`topling-batch-1790283890-1` 仍可读，JNI 仍是 `library/librocksdbjni-linux64.so` 的 `c25ff6e6…dd38`。两边启动时 0 字节 jemalloc curl 被终止后走原有跳过路径。Topling 重启后的日志里 mmap WAL 错误是 0。第一次检测误匹配了脚本自身命令行，没有杀掉 Java，结果作废。证据 `evidence/e109-store-kill9.json`。 + +2026-09-25 05:22：标准 e109 3+3+3 只杀掉 Store-2 的 Java。杀掉前 leaderCount 是 4、2、6，目标是 Store-2。同一 Pod UID `0673df3e-232d-4857-85c8-16927a7fbad7` 和 PVC `pvc-dcdbb110-cb56-40e5-b48f-f6f6cf067a0e` 未变，restartCount 0 到 1，33.144 秒 Ready。0 字节 jemalloc 被终止。JNI 仍是 `/tmp` 上的 `8b8fb2ed…6dff`。图 `hugegraph` 顶点 `std333-e109-1790279980` 在三台 Server 上都可读。Ready 后 leader 合计仍是 12，Store-2 为 0。这是领导权转移，不是数据丢失。证据 `evidence/e109-std-333-store-kill9.json`。 + +2026-09-25 05:24：Topling e109 3+3+3 只杀掉 Store-0 的 Java。杀掉前 leaderCount 是 9、0、3。同一 Pod UID `f9b50cb1-82bc-42ac-b5a0-d9dd7ba32f87` 和 PVC `pvc-f86f7974-a386-4a05-b161-8cf32d766780` 未变，restartCount 0 到 1，37.334 秒 Ready。0 字节 jemalloc 被终止。JNI 仍是 `library/librocksdbjni-linux64.so` 的 `c25ff6e6…dd38`，mmap WAL 错误 0。顶点 `top333-e109-1790280210` 在三台 Server 上都可读。Ready 后 leader 合计仍是 12，Store-0 为 0。随后标准和 Topling 都写入新顶点并被三台 Server 读到。证据 `evidence/e109-top-333-store-kill9.json`、`evidence/e109-333-write-after-kill9.json`。 + +2026-09-25 05:28：标准 e109 3+3+3 只杀掉 PD-0 的 Java。杀掉前它是 leader，`Cluster_OK`，成员数 3。同一 Pod UID `ee7cfd5c-d267-462b-84e1-07a264a3b0ab` 和 PVC `pvc-bb325c48-0449-4117-8630-6b4436869cf5` 未变，restartCount 0 到 1，14.514 秒 Ready。JNI 仍是 `/tmp` 上的 `8b8fb2ed…6dff`。Ready 当下状态一度是 `Cluster_Not_Ready`，leader 转到 PD-1，成员数仍是 3。随后复查为 `Cluster_OK`、`PState_Normal`、3 个 Store 在线。旧顶点 `std333-e109-1790279980` 仍可读，恢复后新顶点写入 201 并可读。故障期间写入也返回 201。这是进程崩溃，不是 Pod 删除。证据 `evidence/e109-std-333-pd-kill9.json`。 + +2026-09-25 05:29：Topling e109 3+3+3 只杀掉 PD-0 的 Java。杀掉前它是 leader，`Cluster_OK`，成员数 3。同一 Pod UID `781cc001-b38b-4214-8f3f-dff283e61caf` 和 PVC `pvc-70af47fe-0e10-427f-8afb-f7cd1074da67` 未变，restartCount 0 到 1，12.534 秒 Ready。JNI 是 `/hugegraph-pd/library/librocksdbjni-linux64.so` 的 `c25ff6e6…dd38`，mmap WAL 错误 0。Ready 当下是 `Cluster_Not_Ready`，leader 转到 PD-1。随后复查为 `Cluster_OK`、`PState_Normal`、3 个 Store 在线。旧顶点仍可读，恢复后新写入 201 并可读。证据 `evidence/e109-top-333-pd-kill9.json`。 + +2026-09-25 05:31：标准 e109 3+3+3 杀掉 Server `smq6m` 的 Java。同一 Pod UID `e28ea87b-1af7-418e-be4c-860804a1eecf`，restartCount 0 到 1，12.514 秒 Ready。镜像仍是 overlay `closure-e109-channelrefresh`。保留的 Server 在故障期间读到旧顶点，故障期间和恢复后的写入都是 201。恢复后的 Server 和第三台 Server 也能读到旧顶点和新顶点。Topling 杀掉 Server `plp4x`，UID `b2ac6461-96c2-4463-8fad-d07a890125a4` 不变，9.441 秒 Ready，同样的读写结果。这是进程崩溃，不是 Pod 删除或网络分区。证据 `evidence/e109-std-333-server-kill9.json`、`evidence/e109-top-333-server-kill9.json`、`evidence/e109-333-server-kill9-third-read.json`。 +2026-09-25 05:32：复查 kind 仍只有 `kind-control-plane` 一个节点，Kubernetes v1.37.0。没有 Chaos Mesh 或 VolumeSnapshot CRD。网络分区和卷快照继续后置。 + +2026-09-25 05:40：核对 P6。Topling 完整 Server `sha256:5b9f40a9d1fc` 导入退出 0，1000000 点、2098771 边，重启前 32 个样本和 9 个自环通过。Store Pod IP 从 `10.244.0.128` 变为 `10.244.0.129` 后，`54148543` IN 返回 500，原因是连接旧地址超时。overlay `sha256:3a129d448516` 上，标准重启 12.835 秒、Topling 14.505 秒，重试后都是 32/0/9，Gremlin `1000000:2098771`。标准导入本身也用了这张 overlay。channel refresh 仍是未提交差异。证据 `evidence/e109-loader-restart-gap.json`。 + +另外，一次性单机容器补了已删除用户：创建 201，删除前图列表 200，删除 204，之后 401。标准 JNI `8b8fb2ed…6dff`，Topling JNI `c25ff6e6…dd38`。没有改动快照容器，容器已删除。图空间管理在单机模式仍是 400。证据 `evidence/e109-standalone-deleted-user.json`、`evidence/e109-standalone-graphspace-support.json`。 + +2026-09-25 05:45：`AbstractGrpcClientChannelRefreshTest` 直接用 JUnit 4.13.2 跑了 2 个测试，0.404 秒，结果 OK。普通 `mvn -pl hg-store-client` 因已安装 POM 里的 `${revision}` 无法解析依赖；reactor classpath 编译后由 `JUnitCore` 执行。证据 `evidence/build/channel-refresh-junit.txt`。测试通过仍不能代替 3 名独立审查。 + +全量 LAW 容量：原始图 41652230 节点、1468365182 条弧，压缩图约 2.6GB；固定子集 1000000 点、2098771 边。主机剩余约 1.1T。Store PVC 请求是 50Gi，但容器里的 df 看到的是主机磁盘。全量大约是子集边数的 700 倍，不放进当前 e109 namespace。证据 `evidence/full-law-capacity.json`。 + +源码复核：`HstoreProvider` 没有覆盖 `createSnapshot`，`BackendStore` 的默认实现抛出 `UnsupportedOperationException("createSnapshot")`。这不是可在本机补的一行修复。 + +2026-09-25 05:50：复核 HA Compose 与 Helm。Compose 的 PD 健康检查仍是 `/v1/health`，Store 等待 3 个 PD，Server 等待 3 个 Store。Helm 在 3 副本时 readiness 是 `/v1/ready`，startup/liveness 是 `/v1/health`；只有 `pd.replicas == 1` 时 liveness helper 才变成 `/v1/ready`。`values-cluster.yaml` 仍要求反亲和和 5Gi/8Gi Store 内存，并且不启用 Hubble。没有改配置，也没有改运行中的集群。P5 对齐项继续不勾选。 + +2026-09-25 05:57：用未修改的 `hugegraph/server:closure-e109012a0`（`sha256:5b9f40a9d1fc`）分别在标准和 Topling 的 e109 1+1+1 里短时启动一个带 Server 标签的 Pod。网络策略要求 `app.kubernetes.io/component=server`，并且要显式设置 `HG_SERVER_USE_PD=true`。两边批量写入 3 个顶点都是 201，混入未定义属性返回 400，回滚顶点不在，原来的顶点仍在。图随后删除，Pod 也删除。这不是 overlay，也不证明 Store IP 变化后的重连。证据 `evidence/e109-fullserver-std-batch.json`、`evidence/e109-fullserver-top-batch.json`。 + +2026-09-25 06:05：未修改的 `hugegraph/server:closure-e109012a0`（`sha256:5b9f40a9d1fc`）临时 Pod 上，标准和 Topling 都创建了二级索引。`limit=2` 返回 2 条。按 `title=two` 只返回 `beta` 和 `gamma`。Gremlin 写入任务成功，顶点可读。Cypher 查询 `alpha` 返回 200 且结果包含 `alpha`。测试图已删除，Pod 已删除。Store JNI 仍分别是 `/tmp` 上的 `8b8fb2ed…6dff` 和 `library/librocksdbjni-linux64.so` 的 `c25ff6e6…dd38`。证据 `evidence/e109-fullserver-index-query.json`。 + +2026-09-25 06:12：未修改 Server 镜像的临时 Pod 上，标准和 Topling 的 `/versions` 无认证都是 200，图列表无认证和错误口令是 401，正确口令是 200。新建用户删除前访问为 200，删除后为 401。两个图互相隔离，A 图顶点在 B 图不可见，两个图都已删除，Pod 也已删除。Store JNI 仍分别是 `8b8fb2ed…6dff` 和 `c25ff6e6…dd38`。P3 因此勾选。证据 `evidence/e109-fullserver-isolation-auth.json`。 + +2026-09-25 06:13：当前源码上跑了 `GrpcShutdownBarrierTest` 和 `ScanShutdownTest`，JUnit 4.13.1，11 个测试 OK，用时 0.938 秒。其中扫描迭代器失败会打出预期的 error 日志，测试仍通过，表示资源关闭没有被当成成功完成。证据 `evidence/build/store-shutdown-junit.txt`。没有为了让测试通过而强行关库。 + +2026-09-25 06:16：标准和 Topling 的 e109 3+3+3 都在现有 overlay Server 上完成删图重建和 clear/truncate。删图 204 后读取 404，重建后旧顶点不在、新顶点可读；clear 204 后图仍在且数据清空，之后可以再写。`snapshot_create` 仍是 500 `createSnapshot`。另一轮写入被三台 Server 读到，从第二台删图后三台都返回 404。Store JNI 仍分别是 `8b8fb2ed…6dff` 和 `c25ff6e6…dd38`。证据 `evidence/e109-333-drop-truncate.json`。 + +2026-09-25 06:20：未修改的 `hugegraph/server:closure-e109012a0`（`sha256:5b9f40a9d1fc`）临时 Pod 连到标准和 Topling 的 e109 3+3+3。两边删图 204 后读取 404，重建后旧顶点不在、新顶点可读；clear 204 后图仍在且数据清空，之后可以再写。`snapshot_create` 仍是 500。图和 Pod 已删除。Store JNI 仍分别是 `8b8fb2ed…6dff` 和 `c25ff6e6…dd38`。证据 `evidence/e109-333-fullserver-drop-truncate.json`。 + +2026-09-25 06:24：历史 namespace `hg-closure-standard-333` 仍是 `closure-std-9abae9dbaaa1`，没有升级。复用该 namespace 创建 auth 图空间后，无角色用户建图返回 403 `User not authorized`。图空间和用户已删除，没有新增 Kubernetes namespace。这不是当前 SHA 证据。证据 `evidence/helm-standard-333-role-denial.json`。 + +2026-09-25 06:28：历史 `hg-closure-standard-111` 仍是 `closure-std-9abae9dbaaa1`。对已有图 `hugegraph` 调用 `snapshot_create` 返回 500 `UnsupportedOperationException: createSnapshot`。`evidence/helm-standard-111-lifecycle.json` 里的新卷恢复是文件系统拷贝，1120 个文件一致，但恢复后已确认顶点全部 404。禁止再做这种目录拷贝。这不是当前 SHA，也没有 VolumeSnapshot CRD。证据 `evidence/helm-standard-111-snapshot-api.json`。 + +2026-09-25 06:32：复查 4 个 e109 namespace 里全部 16 个 PD 和 Store Java 进程。8 个标准进程都映射 `/tmp/librocksdbjni*.so`，SHA-256 是 `8b8fb2ed…6dff`，没有映射 Topling 的 `library/librocksdbjni-linux64.so`。8 个 Topling 进程都映射该库，SHA-256 是 `c25ff6e6…dd38`。没有发现静默 fallback。证据 `evidence/e109-jni-after-restarts.json`。 +2026-09-25 05:39：`KvBatchScanner.KvBatchReceiver`、`KvBatchScanner5.OrderAgent` 和 `CommonKvStreamObserver` 在 `UNAVAILABLE` 时会丢掉按地址缓存的 channel。`QueryExecutor` 把查询地址交给 observer。`GrpcStoreStreamClient.doBatchScan3` 把 Store 地址交给批量扫描。JUnit 4.13.2 跑 `AbstractGrpcClientChannelRefreshTest`，6 个测试 OK，0.473 秒。日志里的 ERROR 是测试故意送入的 `INVALID_ARGUMENT` 和 `UNAVAILABLE`。这不是独立审查，不能提交，也不能用来勾选 P6。证据 `evidence/build/channel-refresh-junit.txt`。 + +2026-09-25 06:55：标准 `hg-closure-std-e109-111` 新增临时 Pod `hg-e109-std-fullserver-load`，镜像 `hugegraph/server:closure-e109012a0`（`sha256:5b9f40a9d1fc`），不是 overlay。新图 `law_twitter_1m_std_fullsha` Loader 退出 0，57.254 秒，1000000 点、2098771 边、失败 0。32 个邻接样本 0 不一致，9 个自环都在。Store Pod 没有删除或重启，JNI 是 `/tmp/librocksdbjni4527535612841869204.so`，SHA-256 `8b8fb2ed…6dff`。临时 Pod 已删除。Store IP 变化后的复测仍未做，P6 不勾选。证据 `evidence/loader-law-twitter-1m-std-e109-fullserver.json`。 +2026-09-25 07:05：`KvBatchScanner5.refreshUnavailable` 被 `OrderAgent.onError` 调用，并用会话代理补了 3 个回归。`AbstractGrpcClientChannelRefreshTest` 现在是 9 个测试 OK，0.467 秒。证据 `evidence/build/channel-refresh-junit.txt`。三份只读审查还没结束，不能提交。 +2026-09-25 07:12：审查 1 结论 HIGH_SEVERITY=1，证据 `evidence/build/channel-refresh-review-1.md`。修复后 `closeChannelIfUnavailable` 只移除仍拥有失败 channel 的池；旧 channel 的再次 UNAVAILABLE 不会删除新池。`AbstractGrpcClientChannelRefreshTest` 12 个测试 OK，0.482 秒。第二轮审查进行中，不能提交。 +2026-09-25 07:20：`ContextClosedListenerTest` 2 个 OK，0.487 秒。关闭线程在 worker 清理完成前不会结束，gRPC 回调未结束时 disposable bean 没有关库。没有为了测试强行关库。证据 `evidence/build/context-closed-listener-junit.txt`。channel refresh 的第二轮审查仍在进行，不能提交。 +2026-09-25 07:19：`git fetch org toplingdb` 后本地 HEAD 与 `org/toplingdb` 仍是 `a7a4a6f1b`，ahead 0、behind 0。channel refresh 第二轮三份只读审查的进程仍在运行，最终结论文件还没写出，所以仍然不能提交。 +2026-09-25 07:25：第二轮审查 1 和 2 是 HIGH_SEVERITY=none，审查 3 是 HIGH_SEVERITY=1。问题是关掉旧池后的 `Channel shutdown` UNAVAILABLE 会经 `evictNode` 再删掉新池。现已不把这种状态当成节点摘除；没有 ManagedChannel 的失败也不再删除当前池。`KvPageScannerTest` 3 个通过，`AbstractGrpcClientChannelRefreshTest` 14 个通过。证据 `evidence/build/channel-refresh-rereview-1.md`、`channel-refresh-rereview-2.md`、`channel-refresh-rereview-3.md`、`channel-refresh-junit.txt`、`kv-page-scanner-junit.txt`。第三轮审查已启动，仍不能提交。 +2026-09-25 07:34：第三轮三份只读审查的进程仍在运行，约 7 分钟，最终结论文件还没写出。没有改 Java，也没有提交。 +2026-09-25 07:40：第三轮审查 3 结论 HIGH_SEVERITY=1，证据 `evidence/build/channel-refresh-round3-3.md`。`evictsOnUnavailable` 现在拒绝 `Subchannel shutdown invoked`。`shutdownChannels` 改为 `shutdownNow`。`AbstractGrpcClientChannelRefreshTest` 14 个 OK，0.46 秒，证据 `evidence/build/channel-refresh-junit.txt`。这是第 3 轮审查后的修复，按合同不再自动开始第 4 轮，Java 提交后置。 +2026-09-25 07:45：标准和 Topling 的 e109 3+3+3 各对一台未被本次 kill -9 过的 Store 发送 SIGTERM，没有再发 kill -9。标准 store-1 同一 Pod UID 和 PVC，restartCount 0 到 1，71.680 秒后 Ready，顶点仍返回 200，JNI 仍是 `/tmp` 上的 `8b8fb2ed…6dff`。上一容器日志有 `closing all rocksdb`，没有 `db not closed`。证据 `evidence/e109-std-333-store-sigterm.json`。Topling store-1 同样同一 UID 和 PVC，restartCount 0 到 1，83.606 秒后 Ready，顶点仍返回 200，JNI 是 `library/librocksdbjni-linux64.so` 的 `c25ff6e6…dd38`。上一容器在关闭末尾仍打印 `SidePluginRepo` `db not closed`。证据 `evidence/e109-top-333-store-sigterm.json`、`evidence/e109-top-333-store-sigterm-shutdown.txt`。两边的 Ready 时间都包含 0 字节 jemalloc curl 被终止后走原有跳过路径。这不是 snapshot，P4 仍不勾选。 +2026-09-25 07:50：标准和 Topling 的 e109 3+3+3 各对一台 restartCount 为 0 的 Server 发送 SIGTERM，没有 kill -9。标准 `cq2ss` 9.371 秒后 Ready，Topling `5zmlm` 14.616 秒后 Ready。Pod UID 不变，restartCount 0 到 1。另一台副本在重启前后都能读到原顶点，状态都是 200。这两台 Server 镜像是 overlay `closure-e109-channelrefresh`，不是未修改的 `sha256:5b9f40a9d1fc`，不能勾选当前 SHA。证据 `evidence/e109-std-333-server-sigterm.json`、`evidence/e109-top-333-server-sigterm.json`。 +2026-09-25 07:55：标准和 Topling 的 e109 3+3+3 各对 PD-1 发送 SIGTERM，没有 kill -9。镜像都是 `hugegraph/pd:closure-e109012a0`。标准 PD-1 同一 UID 和 PVC，restartCount 0 到 1，8.459 秒 Ready，JNI 仍是 `/tmp` 上的 `8b8fb2ed…6dff`。顶点在另一台 Server 上重启前后都是 200。信号前集群已经是 `Cluster_Not_Ready`，成员数 3、在线 Store 3、`PState_Normal`，信号后仍是 `Cluster_Not_Ready`，所以不能把 Not_Ready 归因于这次 SIGTERM，也不能写成恢复到 OK。Topling PD-1 同样同一 UID 和 PVC，60.214 秒 Ready，JNI 是 `library/librocksdbjni-linux64.so` 的 `c25ff6e6…dd38`。信号前也是 `Cluster_Not_Ready`，信号后复查为 `Cluster_OK`，成员数 3，在线 Store 3。顶点仍是 200。证据 `evidence/e109-std-333-pd-sigterm.json`、`evidence/e109-top-333-pd-sigterm.json`。这不是网络分区，P5 仍不勾选。 +2026-09-25 08:00：第三轮审查 2 结论 HIGH_SEVERITY=1，证据 `evidence/build/channel-refresh-round3-2.md`。`getChannels` 在某个创建线程失败时仍会发布含 null 的数组，然后因为不可用而无限重建。现改为创建失败时关闭本次已创建的 channel、不发布数组，并向调用方抛出异常。创建移出 `channels` 锁。`AbstractGrpcClientChannelRefreshTest` 15 个 OK，0.505 秒。证据 `evidence/build/channel-refresh-junit.txt`。这仍没有第 4 轮审查，不提交。 +2026-09-25 08:00：标准 e109 3+3+3 的 `/v1/cluster` 仍是 `Cluster_Not_Ready`，但 API message 是 OK，3 个 Store 都是 Up，在线 Store 3，`PState_Normal`。PD-0/2 最近 2000 行和 PD-1 现有 81 行都没有 `cluster is not ready`。这个状态来自缓存的 `getClusterStats()`，GET 本身不重算。不是网络分区。证据 `evidence/e109-std-333-cluster-not-ready.json`。 +2026-09-25 08:05：标准和 Topling 的 e109 1+1+1 唯一 PD 都做了 SIGTERM，没有 kill -9。镜像分别是 `hugegraph/pd:closure-std-e109012a0` 和 `hugegraph/pd:closure-e109012a0`。标准 8.433 秒、Topling 60.063 秒后 Ready，Pod UID 和 PVC 不变，restartCount 0 到 1。JNI 分别仍是 `/tmp` 上的 `8b8fb2ed…6dff` 和 `library/librocksdbjni-linux64.so` 的 `c25ff6e6…dd38`。信号前两边都是 `Cluster_OK`、成员数 1、在线 Store 1。信号后 `/v1/cluster` 都变成 `Cluster_Not_Ready`，但在线 Store 仍是 1，`PState_Normal`。脚本先查了 `hugegraph` 图，顶点 404，这是查错图，不是数据丢失。恢复后查 `batch_std_e109` 和 `batch_top_e109` 的 `1:` 前缀 ID 都是 200。证据 `evidence/e109-std-111-pd-sigterm.json`、`evidence/e109-top-111-pd-sigterm.json`。这不是当前 SHA 的 snapshot，也不证明物理分区。 +2026-09-25 08:10：按 PD 成员重查 `/v1/cluster`。标准 3+3+3 的 leader `pd-0` 是 `Cluster_OK`，follower `pd-1` 和 `pd-2` 仍报 `Cluster_Not_Ready`。Topling 3+3+3 的 leader `pd-2` 是 `Cluster_OK`，follower `pd-0` 和 `pd-1` 仍报 `Cluster_Not_Ready`。两边在线 Store 都是 3，`initial-store-count` 是 3，`PState_Normal`。1+1+1 唯一 PD 现在都是 `Cluster_OK`，`initial-store-count` 是 1，在线 Store 1。SIGTERM 刚结束时的 `Cluster_Not_Ready` 没有保持。因此 follower 的 Not_Ready 不能当成集群故障或网络分区。证据 `evidence/e109-pd-cluster-state-by-member.json`。 + +2026-09-25：用户确认继续本机 goal，复用本目录和 PR #179。刷新下一动作。channel refresh 的重审与第 3 轮结论文件都已落盘，但 Java 在 07:54 之后又改过，JUnit 现为 15 个 OK，不能把旧审查当成最终差异通过。1+1+1 Store SIGTERM 证据仍不存在，作为下一独立实测。本次只更新合同，没有提交、推送或启动测试。 + +2026-09-25：标准和 Topling 的 e109 1+1+1 Store 各发一次 SIGTERM，没有 kill -9。标准 store-0 同一 UID `add022ef` 和 PVC `pvc-4b0bd019`，restartCount 1 到 2，12.692 秒 Ready。JNI 是 `/tmp/librocksdbjni*.so` 的 `8b8fb2ed…6dff`。上一容器有 `closing all rocksdb`，没有 `db not closed`。Topling store-0 同一 UID `0ffd5049` 和 PVC `pvc-782709b8`，restartCount 1 到 2，60.533 秒 Ready。JNI 是 `library/librocksdbjni-linux64.so` 的 `c25ff6e6…dd38`。上一容器在 `closing all rocksdb` 后仍有 `SidePluginRepo` `db not closed`。两边信号前顶点都是 200；信号后第一次读取也是 200。PD `/v1/cluster` 前后都是 `Cluster_OK`、成员数 1、在线 Store 1、`PState_Normal`。Ready 时间包含终止 0 字节 jemalloc curl，没有再杀 Java。读取用的 Server 都是 overlay `closure-e109-channelrefresh`，不是未修改 SHA。HStore snapshot 仍是缺口，P4 不勾选。证据 `evidence/e109-std-111-store-sigterm.json`、`evidence/e109-top-111-store-sigterm.json`。 + +2026-09-25：核对 `hugegraph/server:closure-e109012a0`。主机 Docker ID 是 `sha256:5b9f40a9d1fcfe84a3012a63b9b04f2825a5162de5f0dd9c8d317a310062ab3b`。kind 使用同 tag 启动的探针 Pod 算出 `docker-entrypoint.sh` 为 `4e416ba9…9b1d`、`bin/start-hugegraph.sh` 为 `ed274913…4fca`，与主机镜像一致。kubelet 报告 import digest `sha256:d0c5346b760fa244dc1cce3228fadcd7511a1c625c2fdda5bdf9fa2de16a4b81`，不是 Docker engine ID。证据 `evidence/e109-server-image-identity.json`。随后只把 e109 3+3+3 两个 Server Deployment 从 overlay 切回该镜像，没有改 `9aba`。标准 `dsdqd` SIGTERM 后 10.872 秒 Ready，UID `fe7d50b2` 不变，restartCount 0 到 1。保留副本 `fzf99` UID 不变，顶点 `1:std333-e109-1790279980` 在重启前、期间和之后都是 200。Topling `7zjgx` 11.971 秒，UID `69657dda` 不变，restartCount 0 到 1。保留副本 `92njw` 读 `1:top333-e109-1790280210`，三次都是 200。两边都不是 kill -9，上一容器没有 `db not closed`。P4 和 P5 仍不勾选。证据 `evidence/e109-std-333-server-sigterm-fullsha.json`、`evidence/e109-top-333-server-sigterm-fullsha.json`。 + +2026-09-25：e109 1+1+1 两台 Server 从 overlay 切回 `hugegraph/server:closure-e109012a0`，kubelet image ID 仍是 import digest `sha256:d0c5346b760f`。切换前 overlay 和切换后的新 Pod 都读到原顶点 200。标准 `zmpkw` SIGTERM 后 15.218 秒 Ready，UID 不变，restartCount 0 到 1，图 `batch_std_e109` 顶点 `1:standard-batch-1790283889-1` 恢复后为 200。Topling `vct4k` 14.416 秒，UID 不变，restartCount 0 到 1，图 `batch_top_e109` 顶点 `1:topling-batch-1790283890-1` 恢复后为 200。不是 kill -9。证据 `evidence/e109-std-111-server-sigterm-fullsha.json`、`evidence/e109-top-111-server-sigterm-fullsha.json`。P4、P5 仍不勾选。 + +2026-09-25：`git fetch org toplingdb` 后 HEAD 与远端仍是 `a7a4a6f1b`，ahead/behind 0/0。单节点 kind 仍没有 Chaos Mesh CRD。通过当前未修改 Helm Server 重读固定子集，没有重启 Store。标准 `law_twitter_1m_std_fullsha` 与 Topling `law_twitter_1m` 都是 32 个样本 0 不一致、9 个自环通过，Gremlin `1000000:2098771`。标准 JNI 是 `/tmp/librocksdbjni*.so` 的 `8b8fb2ed…6dff`，Topling JNI 是 `library/librocksdbjni-linux64.so` 的 `c25ff6e6…dd38`。Server 镜像是 `hugegraph/server:closure-e109012a0`，import digest `sha256:d0c5346b760f`。这不覆盖 Store IP 变化，P6 不勾选。证据 `evidence/loader-law-twitter-1m-std-e109-fullsha-helmserver.json`、`evidence/loader-law-twitter-1m-top-e109-fullsha-helmserver.json`。 + +2026-09-25：为已提交的嵌套 WAL 修复 `457295ac8` 启动 3 个只读 `codex exec review`，沙箱 read-only，不审未提交的 channel refresh。进程是 `1223803`、`1223805`、`1223807`。最终文件尚未写出，日志里的草稿不能当结论。 + +2026-09-25：WAL 初审结束。`wal-review-1.md` 未列高严重度，成功路径被接受。`wal-review-2.md` 和 `wal-review-3.md` 各有一个 P1，都是失败路径会在快照数据已经换入后留下或丢失 WAL tail。本地修复先退役全部活动 `*.log`，再发布 checkpoint tail；分开目录改名失败时把 tail 复制进 WAL 目录；WAL 路径是符号链接时不替换链接本身。`RocksDBSessionsTest` 17 个通过、0 失败、0 跳过，耗时 1.303 秒，新增 `testSymlinkedWalDirectoryKeepsLink`、`testNestedWalMoveFailureDoesNotReplayLaterLog`、`testSeparateWalPublishFailureStillInstallsTail`。证据 `evidence/build/rocksdb-sessions-junit.txt`。重审已启动且未完成，所以不提交。 + +2026-09-25：第一轮 WAL 重审里，`wal-rereview-1.md` 和 `wal-rereview-2.md` 都是 HIGH_SEVERITY=2。问题是同名旧日志只按文件名被当成 checkpoint tail，以及复制中断后的短文件会在源 tail 删除前被接受。本地改为先在同一目录把旧 `*.log` 改名为 `.aside-`，再把 checkpoint 写到 `.partial-` 临时文件，核对长度后改成正式日志。失败时不删除 data 目录里的源文件。`RocksDBSessionsTest` 19 个通过、0 失败、0 跳过，耗时 1.274 秒。证据 `evidence/build/rocksdb-sessions-junit.txt`。第二轮重审已启动，尚未落盘，所以不提交。 + +2026-09-25：第二轮重审 2 完成，HIGH_SEVERITY=1。它确认短复制不会再以正式日志名发布，但同长度、不同内容的旧 WAL 仍可能被 `matchesCheckpointLogs` 接受，随后删除 checkpoint 源文件。证据 `evidence/build/wal-round2-2.md`。重审 1 和 3 的进程 `1322802`、`1322806` 还活着，日志分别停在 09:23:12 和 09:28:14，最终文件未写出。 + +2026-09-25：迟到的第一轮重审 3 是 HIGH_SEVERITY=3，见 `evidence/build/wal-rereview-3.md`。它和 `wal-round2-2.md` 的 HIGH_SEVERITY=1 一起说明：只比长度不够，rename 失败不能把旧 WAL 留在活动目录，WAL 目录挪走失败时也不能直接放弃安装。本地已改为比对文件字节；同目录 rename 失败时删除活动 `*.log` 再写入核对过的 tail；目录级 rename 失败时改走原地安装。`RocksDBSessionsTest` 20 个通过、0 失败、0 跳过，耗时 1.355 秒。证据 `evidence/build/rocksdb-sessions-junit.txt`。按三轮上限不再自动开审查，因此不提交。 + +2026-09-25：第二轮重审 3 在字节比对写入后才落盘，`evidence/build/wal-round2-3.md` 仍写 HIGH_SEVERITY=1，理由是只按同名和长度接受旧 WAL。当前 `matchesCheckpointLogs` 已调用 `sameBytes`，所以这份结论对不上最新源码。重审 1 进程 `1322802` 仍可能在跑，它同样开始于这次修改之前。不把这两份当成最新差异的通过或失败结论，也不再开下一轮。 + +2026-09-25：基线补上未提交的 WAL 文件 `RocksDBStdSessions.java` 和 `RocksDBSessionsTest.java`，避免文档把工作区写成只有 channel refresh。已启动一份只读文档审查，进程 `1358552`，最终文件尚未写出。WAL 审查 `1322802` 仍卡在重连，没有新结论。 + +2026-09-25：文档审查 HIGH_SEVERITY=2，见 `evidence/build/doc-review-1.md`。P3 原先把边更新/删除和角色 403 算进未修改 Server，但那些证据在 overlay 或 mmapfix。已在当前未修改 Helm Server 上补做：标准和 Topling 边创建 201、更新 200、`since=2`、删除 204 后为空；无角色建图 403，没有新增 Kubernetes namespace。证据 `evidence/e109-edge-crud-fullsha.json`、`evidence/e109-role-denial-fullsha.json`。channel refresh 的 todo 时间线改为 07:40 的 14 个测试和 08:00 的 15 个测试，不再把它们写成同一份最终差异。文档重审未做，所以还不提交。 + +2026-09-25：文档重审完成，HIGH_SEVERITY=0，证据 `evidence/build/doc-rereview-1.md`。P3 的边更新和角色 403 已改由未修改 Server 证据支持。channel refresh 的 14 个测试和 15 个测试不再写成同一份差异。图空间 `closure_role_std` 和 `closure_role_top` 复查都是 400 Cannot find graph space。随后只提交这两份合同文件。 + +2026-09-25：文档重审 HIGH_SEVERITY=0 后提交 `04e642ff6`，只含两份合同文件，并 rebase 到远端新提交 `a35ebeb17`。推送 `org/toplingdb` 时执行策略直接拒绝，远端仍停留在 `a35ebeb17`。没有改用 force-push。 + +2026-09-25：再次 `git fetch org toplingdb` 后仍是 ahead 1、behind 0。推送仍被执行策略拒绝。在不重建镜像的前提下跑了 `a35ebeb17` 新增单测：`BalanceLeadersAPITest` 4 个、`PartitionAPITest` 2 个、`StoreIdChangeTest` 8 个，全部 0 失败，Maven BUILD SUCCESS。运行中的 e109 镜像不包含这些提交。证据 `evidence/build/upstream-a35ebeb-unit.txt`。 + +2026-09-25：推送再次被执行策略拒绝，ahead 1、behind 0。在未重建镜像的 e109 3+3+3 上查 `GET /v1/partitions`。标准 store-0 和 Topling store-2 返回 200，各 12 个 `STATE_LEADER`。标准 store-1/2 与 Topling store-0/1 返回 500，标准 store-1 日志是 `IllegalStateException: Not leader`，来自 `NodeImpl.listPeers`。证据 `evidence/e109-follower-partitions-baseline.json`。 + +2026-09-25：`git fetch org toplingdb` 后仍是 ahead 1、behind 0，随后 `git push org HEAD:toplingdb` 成功，远端从 `a35ebeb17` 到 `04e642ff6`。没有 force-push，WAL 和 channel refresh 未纳入。已从干净 worktree 启动标准 Store bake，单元 `hg-closure-std-store-a35ebeb17`。这次笔记还没提交。 + +2026-09-25：标准 follower 分区查询已在干净 `a35ebeb17` Store 镜像上证明。`hg-closure-std-a35-s3` 是 1 个 e109 PD、3 个新 Store、1 个 e109 Server。三台 Store 的 `GET /v1/partitions` 都是 HTTP 200，各 12 个 raft group，合计 12 个 `STATE_LEADER` 和 24 个 `STATE_FOLLOWER`。最近两分钟 store-0 没有 `Not leader`。JNI 都是 `/tmp/librocksdbjni*.so` 的 `8b8fb2ed…6dff`。首次启动时本地引擎为空，因为 PD 在 Server 创建图期间被 Helm 升级重启；删除并重建三台 Store Pod 后才出现分区。jemalloc curl 没有写出文件，只终止了 curl。证据 `evidence/a35-std-follower-partitions.json`。这次笔记尚未提交。 + +2026-09-25：Topling Store 镜像 `hugegraph/store:closure-a35ebeb17` 也从同一个干净 worktree 建成。Docker ID `sha256:2117096ad897`，标签 revision `a35ebeb17`，runtime `topling`。包内 JNI `c25ff6e6…dd38`，`PartitionAPI.class` 含 follower guard。`hg-closure-top-a35-s3` Helm 退出 0：1 个 e109 Topling PD、3 个新 Store、1 个 e109 Server。三台 Store 的 `GET /v1/partitions` 都是 HTTP 200，合计 12 个 `STATE_LEADER` 和 24 个 `STATE_FOLLOWER`。进程映射 `/hugegraph-store/library/librocksdbjni-linux64.so`，没有 `Not leader`，mmap WAL 错误 0。kind import digest `sha256:b72eedece1a8`。证据 `evidence/a35-top-follower-partitions.json`。随后标准和 Topling 都通过 e109 Server 写入并读回顶点，证据 `evidence/a35-vertex-write-read.json`。笔记尚未审查或提交。 + +2026-09-25：a35 标准 Store-0 收到 SIGTERM，不是 kill -9。Pod UID `98d6eb18` 和 PVC `store-data-hg-closure-std-a35-s3-hugegraph-store-0` 不变，restartCount 0 到 1，12.233 秒 Ready。JNI 仍是 `/tmp/librocksdbjni*.so` 的 `8b8fb2ed…6dff`。顶点 `1:a35probe` 前后都是 200。本机分区查询仍是 HTTP 200，重启后 12 个 `STATE_FOLLOWER`；三台合计仍是 12 个 `STATE_LEADER` 和 24 个 `STATE_FOLLOWER`。上一容器有 `closing all rocksdb`，没有 `db not closed`。jemalloc 文件不存在时只杀了 curl。证据 `evidence/a35-std-store0-sigterm.json`。 + +2026-09-25：a35 Topling Store-0 同样只发 SIGTERM。UID `c2f1a298` 和 PVC `store-data-hg-closure-top-a35-s3-hugegraph-store-0` 不变，restartCount 0 到 1，67.414 秒 Ready。JNI 仍是 `library/librocksdbjni-linux64.so` 的 `c25ff6e6…dd38`。顶点 `1:a35probe2` 前后都是 200。本机重启后是 12 个 `STATE_FOLLOWER`，三台合计仍是 12/24。上一容器有 `closing all rocksdb`，随后 `sideplugin/rockside/src/topling/side_plugin_repo.cc:199` 的 `db not closed`。没有 mmap WAL 错误，也没有 `Not leader`。证据 `evidence/a35-top-store0-sigterm.json`。 + +2026-09-25：两个 a35 最小集群对默认图 `hugegraph` 执行 `DELETE /clear`，确认消息匹配后都是 204。标准旧顶点 `a35probe` 和 Topling 旧顶点 `a35probe2` 查询都变成空 200。随后各自写入 `a35afterclear`，都是 201，再读 200。Server 镜像仍是 e109，Store 是 a35。这不是 HStore snapshot。证据 `evidence/a35-graph-clear.json`。 + +2026-09-25：干净 `a35ebeb17` 标准 PD 镜像 `hugegraph/pd:closure-std-a35ebeb17`，Docker `sha256:937b14bc3b5f`，runtime `standard`。已替换 `hg-closure-std-a35-s3` 的 PD，Helm 退出 0。运行 JNI 是 `/tmp/librocksdbjni*.so` 的 `8b8fb2ed…6dff`。`/v1/cluster` 200，`Cluster_OK`，3 个 Store Up。`/v1/task/balanceLeaders` 200，返回 7 个分区到 Store 的 JSON，不是异常体。顶点 `a35afterclear` 仍是 200。Store 分区合计 12 个 `STATE_LEADER`、24 个 `STATE_FOLLOWER`。证据 `evidence/a35-std-pd-balance-leaders.json`。 + +2026-09-25：干净 `a35ebeb17` Topling PD 镜像 `hugegraph/pd:closure-a35ebeb17`，Docker `sha256:9ca5b5fb8495`，runtime `topling`，包内 JNI `c25ff6e6…dd38`。替换 `hg-closure-top-a35-s3` 的 PD 时 Helm 也滚动了 Server，新 Server 就绪，旧 Server 为 Completed。PD JNI 映射 `library/librocksdbjni-linux64.so` 的同一哈希，mmap WAL 错误 0。`balanceLeaders` 200，JSON 长度 9。顶点仍在，Store 分区合计仍是 12/24。证据 `evidence/a35-top-pd-balance-leaders.json`。这次调用同样没有进入 `PDException`。笔记尚未审查或提交。 + +2026-09-25：两个 a35 最小集群删除并重建默认图 `hugegraph`。drop 都是 204，之后图列表为空；create 都是 201。标准顶点 `2:a35rebuilt`、Topling 顶点 `10001:a35rebuilt` 都是写入 201、读取 200。删图后三台 Store 的 `GET /v1/partitions` 仍是 HTTP 200，合计 12/24。Server 当时还是 e109 镜像。证据 `evidence/a35-graph-drop-recreate.json`。 + +2026-09-25：干净 `a35ebeb17` 的 HStore Server 镜像 `hugegraph/server:closure-a35ebeb17`，Docker `sha256:61db7557d753`，标签 runtime `hstore`，revision `a35ebeb17`。镜像内没有 `librocksdbjni`。`e109012a0` 到 `a35ebeb17` 的 `hugegraph-server` 源码无差异。两个 a35 namespace 已换上该镜像。标准 Server SIGTERM 后 15.183 秒 Ready，UID `164c0d85` 不变，restartCount 0 到 1，顶点仍是 200，进程没有 RocksDB JNI，上一容器没有 `db not closed`。Topling Server 12.165 秒，UID `58368eb6` 不变，顶点 `10001:a35rebuilt` 仍在，同样没有 JNI 和 `db not closed`。kind import digest 都是 `sha256:b6ee25851186`。证据 `evidence/a35-std-server-sigterm.json`、`evidence/a35-top-server-sigterm.json`。笔记尚未审查或提交。 + +2026-09-25：标准 a35 最小集群同时删除 Store-1 和 Store-2。Store-0 的 UID `98d6eb18` 和三台 PVC 都不变，Store-1、Store-2 是新 UID。两台大约 32.8 秒 Ready。故障期间旧顶点 `a35rebuilt` 返回 200，新写入 12 秒超时且之后查询为空。恢复后分区合计 12 个 `STATE_LEADER`、24 个 `STATE_FOLLOWER`，JNI 仍是 `8b8fb2ed…6dff`。刚 Ready 的写入也超时，但 `a35aftermajority` 随后可读；再写入 `a35aftermajority2` 为 201，耗时约 0.1 秒。证据 `evidence/a35-std-store-majority.json`。 + +2026-09-25：Topling 同样删除 Store-1 和 Store-2。记录脚本在删除后因格式化错误退出，没有第二次删除；后续只观察这次恢复。故障期间旧顶点可读，12 秒写入超时且没有留下顶点。新 Pod 启动时间是 `03:14:56Z`，观察循环里两台在 18.6 秒变为 Ready，距离删除大约 70 秒。PVC 不变，JNI 仍是 `c25ff6e6…dd38`。分区合计回到 12/24，其中 Store-1 当时 12 个都是 `STATE_FOLLOWER`。Ready 后第一次写入 30.261 秒返回 500 `HgStoreClientException`，顶点不在；下一次 `a35aftermajority2` 为 201，耗时 0.098 秒。证据 `evidence/a35-top-store-majority.json`。 + +2026-09-25：标准 a35 PD `hg-closure-std-a35-s3-hugegraph-pd-0` 收到 SIGTERM，不是 kill -9。UID `03ef756e` 和 PVC `pd-data-hg-closure-std-a35-s3-hugegraph-pd-0` 不变，restartCount 0 到 1，18.8 秒 Ready。镜像 `hugegraph/pd:closure-std-a35ebeb17`，JNI 仍是 `/tmp/librocksdbjni*.so` 的 `8b8fb2ed…6dff`。前后 `/v1/cluster` 都是 `Cluster_OK`、`PState_Normal`、3 个 Store Up。顶点 `a35aftermajority2` 复查为 200。完整上一日志有 `shutdown completed`，没有 `db not closed`。证据 `evidence/a35-std-pd-sigterm.json`。 + +2026-09-25：Topling a35 PD 同样只发 SIGTERM。UID 和 PVC `pd-data-hg-closure-top-a35-s3-hugegraph-pd-0` 不变，restartCount 0 到 1，58.618 秒 Ready。JNI 仍是 `library/librocksdbjni-linux64.so` 的 `c25ff6e6…dd38`。上一日志先有 `shutdown completed`，随后 `sideplugin/rockside/src/topling/side_plugin_repo.cc:199` 的 `db not closed`。Ready 当下 `/v1/cluster` 是 `Cluster_Not_Ready`，3 个 Store 已 Up；随后复查为 `Cluster_OK`、`PState_Normal`。顶点 `a35aftermajority2` 为 200。证据 `evidence/a35-top-pd-sigterm.json`。笔记尚未审查或提交。 + +2026-09-25：干净 `a35ebeb17` 标准单机镜像 `hugegraph/hugegraph:closure-std-a35ebeb17`，Docker `sha256:4f1db2033ac3`，runtime `standard`，镜像内没有 `librocksdbjni`。容器 `hg-a35-std-snap` 使用 2Gi 内存上限和空的 `rocksdb-data` 挂载，因为镜像里没有该目录，入口会拒绝不存在的路径。启动 8.126 秒后 schema 202/201，快照前顶点 201，`snapshot_create` 200，快照后顶点 201，`snapshot_resume` 200。重启 10.403 秒后快照前顶点仍在、快照后顶点不在。日志 `Replaced separate WAL` 3 次，mmap WAL 错误 0，`db not closed` 0。进程 JNI 是 `/tmp/librocksdbjni*.so` 的 `8b8fb2ed…6dff`。容器已删除。证据 `evidence/a35-standalone-std-snapshot.json`。 + +2026-09-25：Topling 单机镜像 `hugegraph/hugegraph:closure-a35ebeb17`，Docker `sha256:7100b2807112`，runtime `topling`，包内 JNI `c25ff6e6…dd38`。同样的 snapshot/resume 流程都是 200。启动 20.142 秒，重启 24.571 秒。快照前顶点仍在，快照后顶点不在。`Replaced separate WAL` 3 次，mmap WAL 错误 0，日志有 1 次 `db not closed`。进程映射 `/hugegraph-server/library/librocksdbjni-linux64.so`。容器已删除。证据 `evidence/a35-standalone-top-snapshot.json`。这不是 HStore 的 `snapshot_create`。笔记尚未审查或提交。 + +2026-09-25：主机可用内存大约只剩 8.5Gi，swap 接近满。停止了 8 个已经完成测试的单机 Docker 容器，没有删除，可用内存回到约 19Gi。随后把 `hg-closure-std-a35-s3` 和 `hg-closure-top-a35-s3` 的 Server 从 1 扩到 3，Helm 都退出 0。两边都写入 `a35three` 返回 201，三台 Server 读取都是 200；旧顶点 `a35rebuilt` 也是三台 200。Server 镜像仍是 `hugegraph/server:closure-a35ebeb17`，进程没有 RocksDB JNI。PD 仍各是 1 个。证据 `evidence/a35-std-server3-read.json`、`evidence/a35-top-server3-read.json`。笔记尚未审查或提交。 + +2026-09-25：Helm 拒绝把 `hg-closure-std-a35-s3` 的 PD 从 1 升到 3。错误写明初始 peer list 只在 Raft 首次建立时生效，已有投票配置不会跟着 Helm 变。values 已改回 1，与运行中的 StatefulSet 一致。 + +2026-09-25：新装标准 `hg-closure-std-a35-333`，3 PD、3 Store、3 Server，镜像分别是 `hugegraph/pd:closure-std-a35ebeb17`、`hugegraph/store:closure-std-a35ebeb17`、`hugegraph/server:closure-a35ebeb17`。Helm 退出 0。三台 PD 和三台 Store 的 JNI 都是 `/tmp/librocksdbjni*.so` 的 `8b8fb2ed…6dff`，Server 没有 RocksDB JNI。PD-0 `/v1/cluster` 200，`Cluster_OK`，成员数 3，3 个 Store Up。Store `GET /v1/partitions` 合计 12 个 `STATE_LEADER` 和 24 个 `STATE_FOLLOWER`。顶点 `a35-333` 写入 201，三台 Server 读取都是 200。证据 `evidence/a35-std-333-install.json`。 + +2026-09-25：同一 namespace 删除 PD-1 和 PD-2。新 Pod 17.4 秒 Ready，PVC 都不变，PD-0 UID `444626ea` 不变。故障中 `a35-333` 仍是 200，`a35pdloss` 超时且之后查询为空。Ready 后 `a35pdback` 的请求超时 20 秒，但顶点随后可读。复查集群为 `Cluster_OK`、成员数 3、3 个 Store Up。`a35pdback2` 为 201，耗时 0.096 秒。证据 `evidence/a35-std-333-pd-majority.json`。笔记尚未审查或提交。 + +2026-09-25：标准 `hg-closure-std-a35-333` 同时删除 Store-1 和 Store-2。Store-2 23.795 秒 Ready，Store-1 32.948 秒 Ready。Store-0 UID 不变，三台 PVC 不变。故障中 `a35-333` 返回 200，`a35storeloss` 超时且之后查询为空。Ready 后 `a35storeback` 的请求超时 30 秒，但顶点随后可读。`a35storeback2` 为 201，耗时 0.093 秒。分区合计回到 12/24。三台 JNI 仍是 `/tmp/librocksdbjni*.so` 的 `8b8fb2ed…6dff`。证据 `evidence/a35-std-333-store-majority.json`。 + +2026-09-25：同一集群删除一台 Server。删除期间另一台 Server 读取 `a35-333` 仍是 200。替补 Pod 11.365 秒 Ready，三台 Server 随后都读到该顶点。证据 `evidence/a35-std-333-server-loss.json`。同一镜像上的 HStore `PUT /graphspaces/DEFAULT/graphs/hugegraph/snapshot_create` 返回 500，消息是 `UnsupportedOperationException: createSnapshot`。没有做目录拷贝。 + +2026-09-25：Topling 3+3+3 仍没有新装。`kubectl top` 后主机可用内存约 9.4Gi，再加 9 个 Pod 会压满。没有清理 `9aba`、e109 或 a35 namespace。笔记尚未审查或提交。 + +2026-09-25：标准 `hg-closure-std-a35-333` 删除并重建默认图 `hugegraph`。drop 204，图列表变为空；create 201，backend 为 hstore。属性 202，顶点标签 201，`a35rebuilt333` 写入 201。clear 204 后，创建图的那台 Server 查询旧顶点为空。`a35cleared333` 写入 201。另外两台 Server 立即读取返回超时或 500，消息是 `Graph 'standardhugegraph[DEFAULT-hugegraph]' has been closed`。对这两台发 SIGTERM 后，同一 UID，restartCount 0 到 1，三台随后都读到 `a35cleared333`，HTTP 200。Store 分区合计仍是 12 个 `STATE_LEADER` 和 24 个 `STATE_FOLLOWER`。证据 `evidence/a35-std-333-drop-clear.json`。笔记尚未审查或提交。 + +2026-09-25：Topling `hg-closure-top-a35-s3` 删除并重建默认图。drop 204，图列表为空；create 201。属性 202，顶点标签 201，`a35rebuilt-top` 写入 201。三台 Server 立即读取都是 200。clear 204 后，执行 clear 的 Server 查询旧顶点为空，另外两台仍返回 `20001:a35rebuilt-top`。三台都能读到新顶点 `a35cleared-top`。对仍返回旧顶点的两台发 SIGTERM，Pod UID 不变。重启后三台查询旧顶点都为空，新顶点都是 200。Store 分区合计仍是 12/24。这不是 3 个 PD。证据 `evidence/a35-top-s3-drop-clear.json`。笔记尚未审查或提交。 + +2026-09-25：Topling `hg-closure-top-a35-s3` 删除一台 Server。删除期间另一台读取 `a35cleared-top` 返回 200。替补 Pod 11.507 秒 Ready，三台随后都读到该顶点。Server 镜像仍是 `hugegraph/server:closure-a35ebeb17`。PD 仍是 1 个。证据 `evidence/a35-top-s3-server-loss.json`。当时可用内存约 8.4Gi，没有新装 Topling 3+3+3。笔记尚未审查或提交。 + +2026-09-25：新装 Topling `hg-closure-top-a35-111`，1 PD、1 Store、1 Server，镜像都是 `closure-a35ebeb17`。Helm 退出 0。PD 与 Store 进程映射 `/hugegraph-pd` 和 `/hugegraph-store` 下的 `library/librocksdbjni-linux64.so`，SHA-256 `c25ff6e6…dd38`。Server 没有 RocksDB JNI。mmap WAL 错误 0。PD `/v1/cluster` 200，`Cluster_OK`，成员数 1，1 个 Store Up。Store `GET /v1/partitions` 200，12 个 `STATE_LEADER`，没有 follower，因为只有一个副本。顶点 `a35-111` 写入 201、读取 200。证据 `evidence/a35-top-111-function.json`。 + +2026-09-25：对这个唯一 Store 发 SIGTERM。UID `c0856d04` 和 PVC `store-data-hg-closure-top-a35-111-hugegraph-store-0` 不变，restartCount 0 到 1，日志显示约 77 秒后 Ready。顶点 `a35-111` 仍是 200。JNI 哈希没变。上一容器有 `closing all rocksdb` 和 `db not closed`，没有 mmap WAL 错误。证据 `evidence/a35-top-111-store-sigterm.json`。笔记尚未审查或提交。 + +2026-09-25：Topling `hg-closure-top-a35-111` 删除并重建默认图。drop 204，图列表为空；create 201。`a35rebuilt111` 写入 201，clear 前读取 200。clear 204 后该顶点查询为空。`a35cleared111` 写入 201、读取 200。`snapshot_create` 返回 500，消息是 `UnsupportedOperationException: createSnapshot`。Store 分区仍是 HTTP 200 的 12 个 `STATE_LEADER`。证据 `evidence/a35-top-111-drop-clear.json`。 + +2026-09-25：同一集群唯一 PD 收到 SIGTERM。UID `541bbbc4` 和 PVC `pd-data-hg-closure-top-a35-111-hugegraph-pd-0` 不变，restartCount 0 到 1，54.757 秒 Ready。顶点 `a35cleared111` 仍是 200。Ready 当下 `/v1/cluster` 是 `Cluster_Not_Ready`、1 个 Store Up；随后复查为 `Cluster_OK`、`PState_Normal`。上一日志有 `shutdown completed`，随后有 `db not closed`。证据 `evidence/a35-top-111-pd-sigterm.json`。笔记尚未审查或提交。 + +2026-09-25：Topling `hg-closure-top-a35-111` 的唯一 Server 收到 SIGTERM。UID 不变，restartCount 0 到 1，16.427 秒 Ready。进程没有 RocksDB JNI，上一容器没有 `db not closed`。重启后图 `hugegraph` 仍在列表里，但 `a35cleared111`、`a35rebuilt111` 和 `a35-111` 查询都是空 200。Store 数据目录 `/hugegraph-store/storage` 约 1.9G，`topling-storage` 只有 4K。证据 `evidence/a35-top-111-server-sigterm.json`。 + +2026-09-25：同一 Server 再写入 `a35persist111`，返回 201，读取 200。第二次 SIGTERM 后 24.811 秒 Ready，restartCount 1 到 2，该顶点仍在。因此第一次重启后的空结果不能写成所有 1+1+1 写入都会丢失。`a35-111` 已在更早的删图中去掉，`a35rebuilt111` 已在 clear 中去掉。只有 `a35cleared111` 是 clear 之后写入、却在第一次 Server 重启后消失。证据 `evidence/a35-top-111-server-persist.json`。笔记尚未审查或提交。 + +2026-09-25：标准 a35 `hg-closure-std-a35-333` 导入固定子集到新图 `law_a35std_1m`。源码 `a35ebeb17`，Server `hugegraph/server:closure-a35ebeb17`，证据里的 image id 是 `sha256:b6ee25851186d078d5e141ff21f0bb273af6bb3deebfdfef0359e3b866e192ac`。建图 201，backend 为 hstore，checksums_ok 为 true。Loader 退出 0，141.323 秒；顶点插入 1000000、边插入 2098771，解析和插入失败都是 0。证据 `evidence/a35-std-333-loader.json`。没有做计数、邻接或重启复测，P6 的复核项不勾选。笔记尚未审查或提交。 + +2026-09-25：标准 `hg-closure-std-a35-333` 图 `law_a35std_1m` 复测。Server `hg-closure-std-a35-333-hugegraph-server-54958cdd4d-ggkzf`，镜像 `hugegraph/server:closure-a35ebeb17`，import digest `sha256:b6ee25851186d078d5e141ff21f0bb273af6bb3deebfdfef0359e3b866e192ac`。重启前 Gremlin `1000000:2098771`，32 个邻接样本 0 不一致，9 个自环都在。随后只对 Store-0 的 Java 发 SIGTERM，不是 kill -9。Pod UID `79850a31-9ab6-4486-891e-b1679eb00b3e`、IP `10.244.0.60` 和 PVC 不变，restartCount 0 到 1，12.252 秒 Ready。0 字节 jemalloc curl 被终止，没有杀 Java。上一容器有 1 次 `closing all rocksdb`，`db not closed` 为 0。重启后 Gremlin 仍是 `1000000:2098771`，邻接和自环结果相同。三台 Store 重启前后都映射 `/tmp/librocksdbjni*.so`，SHA-256 `8b8fb2ed…6dff`，没有 Topling JNI。Store 镜像 import digest `sha256:4edeb22fb4de9376a0349c7d31596aa416802da562c1d5c2f301b3217cf9e8ff`。这不是 Store IP 变化，P6 不勾选。证据 `evidence/a35-std-333-loader-verify.json`。当时 available 3.9Gi，节点内存 91%，swap 7.4/8Gi，没有启动 Topling 导入。笔记尚未审查或提交。 + +2026-09-25：主机 available 只有 3.8Gi，节点内存 91%，swap 7.4/8Gi。为了给当前 SHA 的 Topling 实测腾内存，把 `hg-closure-std-cc143-333`、`hg-closure-std-cc143-111`、`hg-closure-top-cc143-111`、`hg-closure-top-mmapfix-333`、`hg-closure-top-mmapfix-111` 的 StatefulSet 和 Deployment 缩到 0。没有删除 namespace 或 PVC，也没有动 `9aba`、e109、mix 和 a35。缩容后 available 约 40Gi，swap 仍约 7.6Gi。证据 `evidence/a35-scale-down-old-clusters.json`。 + +2026-09-25：Topling `hg-closure-top-a35-s3` 导入新图 `law_a35top_1m`。源码 `a35ebeb17`，1 个 PD、3 个 Store、3 个 Server。Server `djgdl` 镜像 `hugegraph/server:closure-a35ebeb17`，import digest `sha256:b6ee25851186`。建图 201，backend 为 hstore，四个输入校验和匹配。Loader 退出 0，耗时 201.39 秒；脚本自身固定 `-Xmx10g`，实际 RSS 约 1.5Gi。顶点插入 1000000，边插入 2098771，失败 0。Gremlin 重启前后都是 `1000000:2098771`。32 个邻接样本 0 不一致，9 个自环通过。Store-0 Java SIGTERM 后 UID `c2f1a298-5245-496b-8d1d-5e29c6829384` 和 IP `10.244.0.16` 不变，restartCount 1 到 2，76.611 秒 Ready。0 字节 jemalloc curl 被终止。上一容器有 1 次 `closing all rocksdb` 和 1 次 `SidePluginRepo` `db not closed`，mmap WAL 错误 0。三台 Store 前后都映射 `library/librocksdbjni-linux64.so`，SHA-256 `c25ff6e6…dd38`。Store import digest `sha256:b72eedece1a8`。这不是 3 个 PD，也不是 Store IP 变化，P6 不勾选。证据 `evidence/a35-top-s3-loader.json`。随后开始新装 `hg-closure-top-a35-333`，Helm 还没结束。笔记尚未审查或提交。 + +2026-09-25：新装 Topling `hg-closure-top-a35-333`，Helm 退出 0。三台 Store 启动时 0 字节 jemalloc curl 被终止，没有杀 Java。PD-0 `/v1/cluster` 200，`Cluster_OK`，成员数 3，3 个 Store 在线，`PState_Normal`。三台 Store 映射 `/hugegraph-store/library/librocksdbjni-linux64.so`，三台 PD 映射 `/hugegraph-pd/library/librocksdbjni-linux64.so`，SHA-256 都是 `c25ff6e6…dd38`。属性 202，顶点标签 201，顶点 `1:a35top333` 写入 201，三台 Server 读取都是 200。这只是安装后的写入一致性，不是多数派、重启或 Loader。证据 `evidence/a35-top-333-install.json`、`evidence/build/helm-topling-a35-333-install.log`。笔记尚未审查或提交。 + +2026-09-25：Topling `hg-closure-top-a35-333` 同时删除 Store-1 和 Store-2，delete 退出 0。这是单节点 kind 的 Pod 删除，不是网络分区。故障期间 `1:a35top333` 仍是 200，`a35top333loss` 超时 12 秒且之后查询 404。Store-1 23.788 秒 Ready，Store-2 23.876 秒 Ready。PVC 不变，UID 和 IP 都变了：Store-1 从 `10.244.0.88` 到 `10.244.0.93`，Store-2 从 `10.244.0.91` 到 `10.244.0.92`。Store-0 UID `434437bd-f2fa-4ead-8eed-fc7b590968ce`、IP `10.244.0.89` 不变。0 字节 jemalloc curl 被终止。Ready 后 `a35top333back` 超时 30 秒，随后查询 404，没有留下。`a35top333back2` 为 201，耗时 0.097 秒，查询 200。分区合计 12 个 `STATE_LEADER`、24 个 `STATE_FOLLOWER`；Store-0 当时 leader 为 0。三台 JNI 仍是 `library/librocksdbjni-linux64.so` 的 `c25ff6e6…dd38`。新 Pod 没有上一容器日志，不能据此声称没有 `db not closed`。P5 不勾选。证据 `evidence/a35-top-333-store-majority.json`。笔记尚未审查或提交。 + +2026-09-25：Topling `hg-closure-top-a35-333` 同时删除 PD-1 和 PD-2，delete 退出 0。这是单节点 kind 的 Pod 删除，不是网络分区。删除前 PD-0 `/v1/cluster` 200，`Cluster_OK`，成员数 3，3 个 Store Up，`PState_Normal`。`1:a35top333back2` 在删除前和删除后都是 200。`a35pd333loss` 在删除命令返回后 0.08 秒就得到 201，之后查询也是 200，所以这次没有证明 PD 多数派丢失会拒绝写入。PD-1 12.653 秒 Ready，PD-2 12.684 秒 Ready。PVC 不变，UID 和 IP 改变：PD-1 从 `10.244.0.84` 到 `10.244.0.95`，PD-2 从 `10.244.0.90` 到 `10.244.0.94`。PD-0 UID `10e8cf09-8a06-43e0-bbd8-199a133d40d3`、IP `10.244.0.86` 不变。Ready 当下对 PD-0 的集群查询没有返回 HTTP 码。随后 `a35pd333back` 超时 30 秒，但稍后查询为 200。`a35pd333back2` 为 201，耗时 0.069 秒。复查集群仍是 `Cluster_OK`、成员数 3、3 个 Store Up、`PState_Normal`。三台 PD JNI 都是 `/hugegraph-pd/library/librocksdbjni-linux64.so` 的 `c25ff6e6…dd38`。新 Pod 没有上一容器，不能判断 `db not closed`。P5 不勾选。证据 `evidence/a35-top-333-pd-majority.json`。笔记尚未审查或提交。 + +2026-09-25：Topling `hg-closure-top-a35-333` 删除一台 Server `vqcvn`，delete 退出 0。保留副本 `nwjsq` 在删除期间读取 `1:a35pd333back2` 为 200，写入 `a35srv333` 为 201，耗时 0.074 秒。替补 `vv5sv` 10.706 秒 Ready，IP `10.244.0.96`。`nwjsq` UID `0c84ecdc-ddfa-4930-8266-cfe73c4f6924`、IP `10.244.0.77` 不变。替补和 `nwjsq` 随后都能读到旧顶点和 `a35srv333`。另一台未删除的 `qkpz5` 当时两次读取都超时 12 秒；Pod 仍是 Ready。稍后重试两个顶点都是 200。这是单节点 kind 的副本替换，不是网络分区。P5 不勾选。证据 `evidence/a35-top-333-server-loss.json`。笔记尚未审查或提交。 + +2026-09-25：Topling `hg-closure-top-a35-333` 对默认图 `hugegraph` 调用一次 `PUT /graphspaces/DEFAULT/graphs/hugegraph/snapshot_create`。Server `nwjsq`，镜像 `hugegraph/server:closure-a35ebeb17`，import digest `sha256:b6ee25851186d078d5e141ff21f0bb273af6bb3deebfdfef0359e3b866e192ac`。返回 500，异常是 `UnsupportedOperationException`，消息是 `createSnapshot`，栈顶是 `BackendStore.createSnapshot`。没有做目录拷贝。调用后 `1:a35srv333` 仍是 200。这不是单机 RocksDB snapshot。P4 不勾选。证据 `evidence/a35-top-333-snapshot.json`。笔记尚未审查或提交。 + +2026-09-25：Topling `hg-closure-top-a35-333` 新建图 `life_a35top333`，没有删除默认图 `hugegraph`。建图 201，backend 为 hstore。属性 202，顶点标签 201，`life-before` 写入 201。三台 Server 立即读取都是 200。`DELETE .../clear?confirm_message=I'm sure to delete all data` 返回 204。执行 clear 的 `nwjsq` 随后查询 `life-before` 为 404。`qkpz5` 和 `vv5sv` 仍返回 200。之后 `life-after` 写入 201，三台读取都是 200。默认图顶点 `1:a35srv333` 仍是 200。没有重启，也没有做 snapshot。P4 不勾选。证据 `evidence/a35-top-333-clear.json`。笔记尚未审查或提交。 + +2026-09-25：Topling `hg-closure-top-a35-333` 只对 `qkpz5` 和 `vv5sv` 的 Java 发 SIGTERM，不是 kill -9，也没有动 Store 或 PD。信号前 `life-before` 在 `nwjsq` 是 404，在这两台是 200；`life-after` 三台都是 200。`qkpz5` 15.158 秒 Ready，`vv5sv` 12.052 秒 Ready。两台 UID 和 IP 不变，restartCount 0 到 1，IP 分别是 `10.244.0.79` 和 `10.244.0.96`。重启后三台查询 `life-before` 都是 404，`life-after` 都是 200。`nwjsq` 没有收到信号。P4 仍不勾选。证据 `evidence/a35-top-333-clear-sigterm.json`。笔记尚未审查或提交。 + +2026-09-25：Topling `hg-closure-top-a35-333` 删除并重建图 `life_a35top333`，没有删除默认图 `hugegraph`。`nwjsq` 上 drop 204。三台立即查询该图都是 404，旧顶点 `1:life-after` 也都是 404。重建 201，backend 为 hstore。属性 202，顶点标签 id 是 2，`life-rebuilt` 写入 201，返回 id `2:life-rebuilt`。按错误 id `1:life-rebuilt` 查询三台都是 404，这不能当成写入丢失。按 `2:life-rebuilt` 查询三台都是 200，图也是 200。重建完成并写入新顶点之后，再次查询 `1:life-after`：`qkpz5` 和 `vv5sv` 返回 200，`nwjsq` 是 404。这是重建之后重新出现的旧 id，不是 drop 当下的 404。默认图 `1:a35srv333` 仍是 200。没有重启。P4 不勾选。证据 `evidence/a35-top-333-drop.json`。笔记尚未审查或提交。 + +2026-09-25：Topling `hg-closure-top-a35-333` 只对 `qkpz5` 和 `vv5sv` 的 Java 再发一次 SIGTERM，不是 kill -9。信号前 `1:life-after` 在 `nwjsq` 是 404，在这两台是 200；`2:life-rebuilt` 三台都是 200。`qkpz5` 23.621 秒 Ready，`vv5sv` 24.698 秒 Ready。UID 和 IP 不变，restartCount 1 到 2，IP 分别是 `10.244.0.79` 和 `10.244.0.96`。重启后三台查询 `1:life-after` 都是 404，`2:life-rebuilt` 都是 200。没有动 Store、PD 或默认图。P4 仍不勾选。证据 `evidence/a35-top-333-drop-sigterm.json`。笔记尚未审查或提交。 + +2026-09-25：标准 `hg-closure-std-a35-333` 只删除 Store-1。删除前 IP `10.244.0.68`，新 Pod IP `10.244.0.97`，UID 改变，PVC `store-data-hg-closure-std-a35-333-hugegraph-store-1` 不变，12.768 秒 Ready。0 字节 jemalloc curl 被终止。替换过程中有一次容器尚未就绪的 exec 错误，不作为读取结果。删除前 Gremlin 是 `1000000:2098771`，样本 `56862681` 的 compact_id、出边 `57606609` 和入边 `15767971` 匹配。新 IP Ready 后 Gremlin 仍是 `1000000:2098771`，同一样本仍匹配。JNI 是 `/tmp/librocksdbjni*.so` 的 `8b8fb2ed…6dff`，没有 Topling JNI。这只是一个邻接样本，不是 32 个样本，也不是 Topling。P6 不勾选。证据 `evidence/a35-std-333-store-ip.json`。笔记尚未审查或提交。 + +2026-09-25：Topling `hg-closure-top-a35-s3` 只删除 Store-1。旧 IP `10.244.0.33`，新 IP `10.244.0.98`，UID 改变，PVC `store-data-hg-closure-top-a35-s3-hugegraph-store-1` 不变，12.797 秒 Ready。这是 1 个 PD，不是 3+3+3。0 字节 jemalloc curl 被终止。新 Pod 没有上一容器，不能判断 `db not closed`。删除前和 Ready 后 Gremlin 都是 `1000000:2098771`。样本 `56862681` 的 compact_id、出边和入边在新 IP 后仍匹配。JNI 是 `/hugegraph-store/library/librocksdbjni-linux64.so` 的 `c25ff6e6…dd38`。证据 `evidence/a35-top-s3-store-ip.json`。 + +随后不再删除 Pod，只重读两个已经更换过 Store IP 的百万图。标准 `law_a35std_1m` 和 Topling `law_a35top_1m` 都是 32 个样本 0 不一致，9 个自环通过。证据 `evidence/a35-std-333-store-ip-adjacency.json`、`evidence/a35-top-s3-store-ip-adjacency.json`。导入失败数仍是 0，失败重试路径没有被触发。P6 不勾选。笔记尚未审查或提交。 + +2026-09-25：文档审查 `evidence/build/doc-review-a35-notes.md` 是 HIGH_SEVERITY=3。已改掉 e109 当前绑定、数据线索和 drop 时间顺序这三处矛盾。重审 `evidence/build/doc-rereview-a35-notes.md` 是 HIGH_SEVERITY=0。随后只提交 `state.md` 和 `todo.md`，提交 `1a495ad43`，并推送到 `org/toplingdb`。没有提交 Java、WAL、channel refresh 或 evidence。 + +2026-09-25：Topling `hg-closure-top-a35-333` 导入新图 `law_a35top333_1m`。3 个 PD、3 个 Store、3 个 Server。Server `nwjsq`，镜像 `hugegraph/server:closure-a35ebeb17`。建图 201，backend 为 hstore，四个输入校验和匹配。Loader 使用本地 `-Xmx3g` 副本，退出 0，141.109 秒。Gremlin `1000000:2098771`。32 个邻接样本 0 不一致，9 个自环通过。三台 Store 都映射 `/hugegraph-store/library/librocksdbjni-linux64.so`，SHA-256 `c25ff6e6…dd38`。没有删除 Store，所以这不覆盖 IP 变化。失败数是 0，重试路径没有触发。P6 不勾选。证据 `evidence/a35-top-333-loader.json`。笔记尚未审查或提交。 + +2026-09-25：Topling `hg-closure-top-a35-333` 只删除 Store-1。旧 IP `10.244.0.93`，新 IP `10.244.0.99`，UID 改变，PVC `store-data-hg-closure-top-a35-333-hugegraph-store-1` 不变，12.883 秒 Ready。0 字节 jemalloc curl 被终止。Server `nwjsq` 的 DNS 把 Store-1 解析到新 IP `10.244.0.99`，不是旧地址。删除前 Gremlin 是 `1000000:2098771`。Ready 后全图计数 HTTP 500，消息是 `SocketTimeoutException: Read timed out`。32 个邻接样本里 31 个匹配，9 个自环通过；`54148543` 的 IN 扫描 curl 超时。当时的异常文本只截到主机名，所以这里曾误写成不是旧 IP 缓存。下面的复查推翻了这句话。JNI 仍是 `library/librocksdbjni-linux64.so` 的 `c25ff6e6…dd38`。P6 不勾选。证据 `evidence/a35-top-333-store-ip.json`。笔记尚未审查或提交。 + +2026-09-25：不再删除 Store，复查 `law_a35top333_1m`。Server `nwjsq` 把 Store-1 的 DNS 解析到 `10.244.0.99`，并且到 `10.244.0.99:8500` 的新 TCP 连接成功。`54148543` IN 仍是 HTTP 500 `UNAVAILABLE: io exception`。全图计数仍是 HTTP 500，完整消息是 `ConnectTimeoutException: connection timed out: hg-closure-top-a35-333-hugegraph-store-1...svc/10.244.0.93:8500`。客户端仍在连接删除前的 IP。运行中的镜像是 `a35ebeb17`，不包含未提交的 channel refresh。P6 不勾选。证据 `evidence/a35-top-333-store-ip-settled.json`。笔记尚未审查或提交。 + +2026-09-25：只对 `nwjsq` 的 Java 发 SIGTERM，没有删 Store。UID `0c84ecdc-ddfa-4930-8266-cfe73c4f6924` 和 IP `10.244.0.77` 不变，restartCount 0 到 1，容器结束并再次启动都在 `2026-09-25T05:39:22Z`。重启后的全图计数是 HTTP 200 `1000000:2098771`，`54148543` IN 也是 200，响应里没有 `10.244.0.93`。对照的 `qkpz5` 这次没有收到信号，restartCount 仍是 2，启动时间仍是 `2026-09-25T04:48:51Z`，同样返回 `1000000:2098771` 和 IN 200。因此不能证明必须重启 Server 才能恢复，也不能把运行中的 `a35ebeb17` 镜像写成已修复。P6 不勾选。证据 `evidence/a35-top-333-server-refresh.json`。笔记尚未审查或提交。 + +2026-09-25：`hg-closure-top-a35-333` 的 Store-0 第二次尝试记录 41 次 HTTP 200，没有非 200。第一次 shell 循环因为 `sh` 拒绝 `SECONDS` 没有发出请求。UID `434437bd-f2fa-4ead-8eed-fc7b590968ce` 和 IP `10.244.0.89` 不变，restartCount 1 到 2，74.674 秒 Ready，所以 Pod 没有被替换。上一容器日志有 `closed gRPC callbacks`、`closing all rocksdb`，然后是 Topling `SidePluginRepo` `db not closed`。`still_waiting` 是 0。`vertex_after` 只有 `http=200` 和 `found=true`。`signal.stdout` 只有 `PID:37` 和 `SENT`，没有信号名。`forced_db_close` 是 false。证据 JSON 没有请求 URL、顶点 ID 或发信号瞬间的计数。这还不是卡住 worker 的超时演练，关闭项不勾选。证据 `evidence/a35-top-333-shutdown-inflight.json`。1 名只读审查结论是 HIGH_SEVERITY=0。 + +2026-09-25：只读复查 `hg-closure-top-a35-333` 的三台 Server。证据 purpose 是 Store-0 SIGTERM 之后的只读计数，并写明 no pod delete。`nwjsq` IP `10.244.0.77`、restartCount 1，`qkpz5` IP `10.244.0.79`、restartCount 2，`vv5sv` IP `10.244.0.96`、restartCount 2，三台都 Ready。图 `law_a35top333_1m` 的计数响应都是 HTTP 200，data 是 `1000000:2098771`。三段响应的 OLDIP 都是 NO。这不能写成地址缓存已修复，P6 不勾选。证据 `evidence/a35-top-333-post-shutdown-count.json`。1 名只读审查结论是 HIGH_SEVERITY=0。 + +2026-09-25：只读复查 `hg-closure-top-a35-333` 图 `law_a35top333_1m` 的邻接。没有删除 Pod，也没有新发信号；证据 purpose 写了这一点。主查 Server `nwjsq`，镜像 `hugegraph/server:closure-a35ebeb17`，import digest `sha256:b6ee25851186d078d5e141ff21f0bb273af6bb3deebfdfef0359e3b866e192ac`。32 个样本里 63 个方向在 4000 字节截断前解析通过，真实不一致数是 0。`54148543` IN 因截断没有解析；完整重读后 `nwjsq`、`qkpz5`、`vv5sv` 的 IN 都是 HTTP 200、count 142、expected 142、first_ok true，OUT 都是 count 4、expected 4。9 个自环都是 HTTP 200 且存在。`old_ip_needle` 是 `10.244.0.93`，`old_ip_seen` 是 false。Store-0 镜像 `hugegraph/store:closure-a35ebeb17`，import digest `sha256:b72eedece1a8165799afe4a950a3630470f7c277c128eba511ea242ec0c8e999`，JNI 路径 `/hugegraph-store/library/librocksdbjni-linux64.so`，SHA-256 `c25ff6e676290db6db47df0954640aa609c391450ec90e1a8eec1f87e174dd38`。运行镜像 revision 是 `a35ebeb17`，不是当时的文档 HEAD `c63ea7d8d`。P6 不勾选，`address_cache_fixed` 是 false。证据 `evidence/a35-top-333-post-shutdown-adjacency-summary.json`。1 名只读审查结论是 HIGH_SEVERITY=0。 + +2026-09-25：只读核对五个 Server 的固定子集邻接，没有删除 Pod，也没有新发信号；证据 purpose 写了这一点。标准 `law_a35std_1m` 的 `ggkzf`、`mcf26`、`zggsl`，以及 Topling `law_a35top333_1m` 的 `qkpz5`、`vv5sv`，都是 rc 0、64 个方向、mismatch_count 0、old_ip_seen false。mismatch_total 是 0。`old_ip_needle` 是 `10.244.0.93`。9 个自环在这五台上共 45 行，self_loops_all_http_200 和 self_loops_all_present 都是 true，self_loop_old_ip_seen 是 false。这次没有重查 `nwjsq`。`server_images.std` 的镜像是 `docker.io/hugegraph/server:closure-a35ebeb17`，imageID `docker.io/library/import-2026-09-25@sha256:b6ee25851186d078d5e141ff21f0bb273af6bb3deebfdfef0359e3b866e192ac`，restarts 0。`server_images.top` 是同一 imageID，restarts 2。这两项没有写明对应的 Server Pod。标准 Store-0 镜像 `docker.io/hugegraph/store:closure-std-a35ebeb17`，imageID `docker.io/library/import-2026-09-25@sha256:4edeb22fb4de9376a0349c7d31596aa416802da562c1d5c2f301b3217cf9e8ff`，restarts 1。Topling Store-0 镜像 `docker.io/hugegraph/store:closure-a35ebeb17`，imageID `docker.io/library/import-2026-09-25@sha256:b72eedece1a8165799afe4a950a3630470f7c277c128eba511ea242ec0c8e999`，restarts 2。标准 JNI 路径是 `/tmp/librocksdbjni2791709567627684767.so`，SHA-256 `8b8fb2ed3ab69581cf1897bd116d484f073e66e9a5b6d61effc7b4c783d66dff`。Topling JNI 路径是 `/hugegraph-store/library/librocksdbjni-linux64.so`，SHA-256 `c25ff6e676290db6db47df0954640aa609c391450ec90e1a8eec1f87e174dd38`。运行镜像 revision 是 `a35ebeb17`。`failure_retry_triggered` 是 false，`address_cache_fixed` 是 false，P6 不勾选。证据 `evidence/a35-cross-server-adjacency-summary.json`。1 名只读审查结论是 HIGH_SEVERITY=0。 + +2026-09-25：检查现有 kind 集群能否不重建就增加节点。`kind get clusters` 的 stdout 是 `kind`，rc 0。`kind get nodes --name kind` 的 stdout 是 `kind-control-plane`，rc 0。kubectl 节点表只有 `kind-control-plane`，状态 Ready，角色 control-plane，AGE 25d，版本 v1.37.0。kind 的命令是 `build`、`completion`、`create`、`delete`、`export`、`get`、`help`、`load`、`version`。`kind create` 的子命令只有 `cluster`。`has_add_node_command` 是 false。CRD 查询 rc 0，stdout 为空，stderr 是 `No resources found`。`chaos_or_snapshot_crd` 是 false。`cluster_recreated` 和 `second_cluster_created` 都是 false。没有新增节点，网络分区的解除条件仍不满足，P5 不勾选。证据 `evidence/kind-node-add-feasibility.json`。1 名只读审查结论是 HIGH_SEVERITY=0。 + +2026-09-25：只读扫描 `hg-closure-std-a35-333` 和 `hg-closure-top-a35-333` 的 PD、Store、Server 进程映射。没有删 Pod，也没有发信号；证据 purpose 写了这一点。`summary.row_count` 是 18，`all_no_silent_fallback` 是 true，`bad_count` 是 0。`standard_pd_store_count` 是 6，`standard_pd_store_hash_match` 是 true，哈希等于 `standard_jni` `8b8fb2ed3ab69581cf1897bd116d484f073e66e9a5b6d61effc7b4c783d66dff`，`standard_pd_store_paths_under_prefix` 是 true，前缀是 `/tmp/librocksdbjni`。`topling_pd_count` 是 3，`topling_pd_path_uniform` 是 true，路径是 `/hugegraph-pd/library/librocksdbjni-linux64.so`。`topling_store_count` 是 3，`topling_store_path_uniform` 是 true，路径是 `/hugegraph-store/library/librocksdbjni-linux64.so`。`topling_pd_store_hash_match` 是 true，哈希等于 `topling_jni` `c25ff6e676290db6db47df0954640aa609c391450ec90e1a8eec1f87e174dd38`。`server_count` 是 6，`server_jni_none` 是 true。`standard_engine_image_tag_match`、`topling_engine_image_tag_match` 和 `server_image_tag_match` 都是 true。标签分别是 `closure-std-a35ebeb17`、`closure-a35ebeb17` 和 `server:closure-a35ebeb17`。`running_image_revision` 是 `a35ebeb17`。这不勾选 P4、P5、P6 或 P7。证据 `evidence/a35-333-jni-sweep.json`。1 名只读审查结论是 HIGH_SEVERITY=0。 + +2026-09-25：对 `hg-closure-std-a35-333` 和 `hg-closure-top-a35-333` 的 PD leader 各做一次带认证的 GET `/v1/task/balanceLeaders`。没有删 Pod，也没有发信号；证据 purpose 写了这一点。两边 `cluster_state` 都是 `Cluster_OK`，`member_size` 是 3，`leader_role` 是 `Leader`，leader Pod 都是各自 namespace 的 `hugegraph-pd-1`。`balance_http` 都是 200，`exception_text` 是 false。标准 `key_count` 是 10，Topling `key_count` 是 8。之后 `leader_sum` 是 12，`follower_sum` 是 24，三台 Store 的原始行都是 `HTTP:200 LEADER:4 FOLLOWER:8`。图顶点 `56862681` 是 HTTP 200。`running_image_revision` 是 `a35ebeb17`。这不是网络分区，P5 不勾选。证据 `evidence/a35-333-balance-leaders.json`。1 名只读审查结论是 HIGH_SEVERITY=0。 + +2026-09-25：`balanceLeaders` 之后，在两个 a35 3+3+3 的 `hugegraph` 图各新建一个 `person` 顶点，再从三台 Server 读取。没有删 Pod，也没有发信号；证据 purpose 写了这一点。标准顶点名 `bal333-std-a4fd`，id `10001:bal333-std-a4fd`，`create_http` 是 201。Topling 顶点名 `bal333-top-a4fd`，id `1:bal333-top-a4fd`，`create_http` 是 201。标准 `ggkzf`、`mcf26`、`zggsl` 和 Topling `nwjsq`、`qkpz5`、`vv5sv` 的读取都是 HTTP 200、FOUND 1。purpose 还写了第一次读取因为 id 末尾多了引号而返回 HTTP 500，那次不是这六次有效读取。`running_image_revision` 是 `a35ebeb17`。这不是网络分区，P5 不勾选。证据 `evidence/a35-333-write-after-balance.json`。1 名只读审查结论是 HIGH_SEVERITY=0。 + +2026-09-25：`balanceLeaders` 之后，从两个 a35 3+3+3 的三台 PD 读取 `/v1/cluster`。没有删 Pod，也没有发信号；证据 purpose 写了这一点。标准和 Topling 的 pd-0、pd-1 都是 HTTP 200、`Cluster_OK`、`PState_Normal`、`memberSize` 3、`onlineStoreSize` 3、`partitionSize` 12、leader `pd-1`、角色 Leader 1 和 Follower 2。两边的 pd-2 都是 HTTP 200、`Cluster_Not_Ready`，其余字段相同，leader 仍是 `pd-1`。`running_image_revision` 是 `a35ebeb17`。这不是网络分区，P5 不勾选。证据 `evidence/a35-333-pd-cluster-by-member.json`。1 名只读审查结论是 HIGH_SEVERITY=0。 + +2026-09-25：重读 `hg-closure-top-a35-333` Store-0 的上一容器日志。没有新发信号，也没有删 Pod；证据 purpose 写了这一点。`line_count` 是 430。`event_order` 先是 `grpc`，接着 12 个 `pe`，然后 `closeall`、`shutdown-db`、`native`。`partition_engine_shutdown_count` 是 12，`group_ids` 是 0 到 11。`segment_log_storage_true_count` 是 12。`still_waiting_count` 是 0。`db_not_closed_count` 是 1。唯一的 shutdown db 行是 `hgstore-metadata`，路径 `/hugegraph-store/storage/hgstore-metadata`。`native_names_db` 是 false，`close_all_db_called` 是 false，`p4_checked` 是 false。证据 `evidence/a35-top-333-shutdown-log-order.json`。1 名只读审查结论是 HIGH_SEVERITY=0。 + +2026-09-25：重读标准 `hg-closure-std-a35-333` Store-0 的上一容器日志。没有新发信号，也没有删 Pod；证据 purpose 写了这一点。`kubectl_rc` 是 0，`line_count` 是 1360。`partition_engine_shutdown_count` 是 12，`segment_log_storage_true_count` 是 12，`still_waiting_count` 是 0，`db_not_closed_count` 是 0，`shutdown_db_count` 是 1，名称是 `hgstore-metadata`。`event_order` 以 `grpc` 开始，然后是 12 个 `pe`，最后是 `closeall` 和 `shutdown-db`，没有 `native`。`close_all_db_called` 是 false，`p4_checked` 是 false。证据 `evidence/a35-std-333-shutdown-log-order.json`。1 名只读审查结论是 HIGH_SEVERITY=0。 + +2026-09-25：在两个 a35 3+3+3 的运行中 Server 上，对 `hugegraph` 图发送原始 Cypher。没有删 Pod，也没有发信号；证据 purpose 写了这一点。标准查询名是 `bal333-std-a4fd`，Topling 查询名是 `bal333-top-a4fd`。标准 `ggkzf`、`mcf26`、`zggsl` 和 Topling `nwjsq`、`qkpz5`、`vv5sv` 都是 HTTP 200、`inner_code` 200、`hit` true。`quoted_body_control` 是 HTTP 200、`inner_code` 400、`hit` false。`running_image_revision` 是 `a35ebeb17`。这不勾选 P4、P5、P6 或 P7。证据 `evidence/a35-333-cypher.json`。笔记尚未审查或提交。 + +2026-09-25：只读复查两个 a35 3+3+3 的三台 PD。没有删 Pod，也没有发信号。标准和 Topling 的 pd-1 `/v1/ready` 都是 `STATE_LEADER`，`/v1/cluster` 是 `Cluster_OK`，`onlineStoreSize` 是 3。两边 pd-2 的 `/v1/ready` 都是 HTTP 200、`STATE_FOLLOWER`，`/v1/cluster` 仍是 `Cluster_Not_Ready`；日志里 `Raft becomes leader`、`Store register`、`update cluster state` 和 `The cluster is not ready` 都是 0。两边 pd-0 都有 1 次 `Raft becomes leader` 和 1 次 `Raft lost leader`，现在是 follower，但 `/v1/cluster` 仍是 `Cluster_OK`。镜像分别是 `hugegraph/pd:closure-std-a35ebeb17` 和 `hugegraph/pd:closure-a35ebeb17`，revision `a35ebeb17`。这不是 Topling 特有，也不是网络分区，P5 不勾选。证据 `evidence/a35-333-pd-local-cluster-state.json`。笔记尚未审查或提交。 + +本地未提交修复：`IndexAPI.clusterState()` 每次 REST 读取前都调用 `checkStoreStatus()`，避免 follower 留下构造时的 `Cluster_Not_Ready` 或旧任期的 `Cluster_OK`。读取抛 `PDException` 时不再把初始 `Cluster_OK` 写回缓存。`/v1/ready` 不走这个方法。测试次数和审查结论分别见 `hugegraph-pd/hg-pd-test/target/surefire-reports/org.apache.hugegraph.pd.rest.IndexAPIClusterStateTest.txt` 与 `evidence/build/pd-cluster-state-review-1.md`、`pd-cluster-state-review-2.md`、`pd-cluster-state-review-3.md`。没有和 channel refresh 或 WAL 改动混在同一个提交里。代码已单独提交并推送为 `327737f16`，没有混入 channel refresh、WAL 或文档。写这句话时，运行中的 a35 镜像仍是 `a35ebeb17`，还没有这个修复，所以当时不把 Service 抽查改写成已通过。后面的 PD 替换记录覆盖这一句。 + +2026-09-25：从 Server Pod 再查。标准 `ggkzf` 直连 pd-0/1 是 `Cluster_OK`，pd-2 是 `Cluster_Not_Ready`；Service 18 次里 `Cluster_OK` 8 次、`Cluster_Not_Ready` 10 次。Topling `nwjsq` 同样 pd-2 为 `Cluster_Not_Ready`，Service 18 次里 `Cluster_OK` 10 次、`Cluster_Not_Ready` 8 次。没有删 Pod，也没有发信号。P5 不勾选。证据 `evidence/a35-333-pd-service-cluster-mix.json`。笔记尚未审查或提交。 + +2026-09-25：PD Service 抽查之后，在 Pod 内只读执行 Gremlin 计数。Topling `nwjsq` 图 `law_a35top333_1m` 和标准 `ggkzf` 图 `law_a35std_1m` 都是 HTTP 200，data 是 `1000000:2098771`。耗时含 kubectl exec，分别 0.066 秒和 0.078 秒，不是全图扫描基准，也不勾选 P6。没有删 Pod，也没有发信号。证据 `evidence/a35-333-count-after-pd-service.json`。笔记尚未审查或提交。 + +2026-09-25：开始用干净 detached worktree `build-context-327737f16` 构建标准 PD 镜像 `hugegraph/pd:closure-std-327737f16`。上下文 revision 是 `327737f16`，工作区为空，源码含 `clusterState()`。systemd 单元 `hg-pd-bake-327737f16-std.service` 正在执行 Maven。日志 `evidence/build/bake-327737f16-standard-pd.log`。没有覆盖 `closure-std-a35ebeb17`，没有从脏的 f29e 工作树构建，也还没有加载到 kind。Topling PD 镜像还没开始。 + +2026-09-25：标准 PD 构建单元 `hg-pd-bake-327737f16-std.service` 仍在运行，不要另起一份。Maven 已到 `hugegraph-dist` `[20/27]`，正在打包；日志停在 assembly 的 parent POM 警告，进程还在。还没有镜像，没有加载 kind，没有滚动 PD。 + +2026-09-25：标准 PD 镜像 `hugegraph/pd:closure-std-327737f16` 构建成功，Docker id `sha256:31c5659ca3353b125c14da190ef6a273bad1a8c9e7ce6d79de4408e84b30f198`。已加载到 kind，并逐台替换 `hg-closure-std-a35-333` 的三台 PD。最终三台镜像都是 `closure-std-327737f16` 且 Ready。leader 是 pd-2，pd-0 和 pd-1 是 follower，三台 `/v1/cluster` 都是 `Cluster_OK`，Service 18 次都是 `Cluster_OK`。PVC UID 与替换前相同，字段在 `pvc_unchanged_vs_pre_roll`。pd-2 JNI 是 `/tmp/librocksdbjni*.so` 的 `8b8fb2ed…6dff`，没有 Topling `.so`。`law_a35std_1m` 计数仍是 `1000000:2098771`。这不勾选 P5。证据 `evidence/a35-std-333-pd-327737f16.json`。Topling PD 构建单元 `hg-pd-bake-327737f16-top.service` 已开始，标签 `closure-327737f16`，还没有加载或替换。 + +2026-09-25：Topling PD 镜像 `hugegraph/pd:closure-327737f16` 构建成功，Docker id `sha256:acec1af25e9d4e011167048252d7eebc7bde31205718462b75380e43ab2382f6`。已加载到 kind，并逐台替换 `hg-closure-top-a35-333` 的三台 PD。最终三台镜像都是 `closure-327737f16` 且 Ready。pd-1 是 leader，pd-0 和 pd-2 是 follower，三台 `/v1/cluster` 都是 `Cluster_OK`，`onlineStoreSize` 是 3，Service 18 次都是 `Cluster_OK`。PVC UID 与替换前相同，字段在 `pvc_unchanged_vs_pre_roll`。pd-2 JNI 是 `/hugegraph-pd/library/librocksdbjni-linux64.so` 的 `c25ff6e6…dd38`，`/tmp/librocksdbjni*` 不存在。`law_a35top333_1m` 计数仍是 `1000000:2098771`。这不勾选 P5。证据 `evidence/a35-top-333-pd-327737f16.json`。笔记尚未审查或提交。 + +2026-09-25：PD 换成 `327737f16` 之后，在两个 a35 3+3+3 的 `hugegraph` 图各新建一个 person 顶点。标准 `pd327-std` 创建 HTTP 201，id `10001:pd327-std`，`ggkzf`、`mcf26`、`zggsl` 用 JSON 引号编码的 id 读取都是 HTTP 200。Topling `pd327-top` 创建 HTTP 201，id `1:pd327-top`，`nwjsq`、`qkpz5`、`vv5sv` 同样都是 HTTP 200。未加引号的 id 是 HTTP 400，不是顶点丢失。没有删 Pod，也没有发信号。Server 镜像仍是 `a35ebeb17`。这不勾选 P4、P5、P6 或 P7。证据 `evidence/a35-333-write-after-pd-327737f16.json`。笔记尚未审查或提交。 + +2026-09-25：对 `hg-closure-std-a35-333` 和 `hg-closure-top-a35-333` 的 PD-0 follower 向 pid 1 发 SIGTERM。没有删 Pod，也没有给 Store 发信号。标准镜像 `hugegraph/pd:closure-std-327737f16`，UID `23329160-9d13-42c0-9fc5-b1d85d06fc13`，PVC `a8e6d13a-df66-4755-b648-fc82881b21e3`，restartCount 从 0 到 1，16.526 秒后 Ready，UID 和 PVC 不变，信号前后都是 follower。记录的 `vertex_http` 前后都是 `200`。`cluster_after` 的 pd-0、pd-1、pd-2 都是 `Cluster_OK`。Topling 镜像 `hugegraph/pd:closure-327737f16`，UID `213e1ab5-c772-4f3b-a840-9bfadb16109e`，PVC `982da1a1-aff1-4790-a608-83075e879530`，restartCount 从 0 到 1，14.458 秒后 Ready，UID 和 PVC 不变，信号前后都是 follower。记录的 `vertex_http` 前后都是 `200`。`cluster_after` 的 pd-0、pd-1、pd-2 都是 `Cluster_OK`。这不勾选 P5。证据 `evidence/a35-std-333-pd0-sigterm-327737f16.json`、`evidence/a35-top-333-pd0-sigterm-327737f16.json`。笔记尚未审查或提交。 + +2026-09-25：对 `hg-closure-std-a35-333` 的 `mcf26` 和 `hg-closure-top-a35-333` 的 `qkpz5` 向 pid 1 发 SIGTERM。没有删 Pod，也没有给 Store 发信号。两边镜像都是 `hugegraph/server:closure-a35ebeb17`。标准 UID `ceab236b-688a-4617-adfb-861119da7f1f`，restartCount 从 1 到 2，16.426 秒后 Ready，UID 不变。`vertex_http` 前后都是 `200`，`ggkzf` 的 `during_reader_http` 是 `200`。Topling UID `b5c2d043-ce9e-4bb3-98b9-1245465f79be`，restartCount 从 2 到 3，16.386 秒后 Ready，UID 不变。`vertex_http` 前后都是 `200`，`nwjsq` 的 `during_reader_http` 是 `200`。这不勾选 P5。证据 `evidence/a35-std-333-server-sigterm-inplace.json`、`evidence/a35-top-333-server-sigterm-inplace.json`。笔记尚未审查或提交。 + +Server 原地 SIGTERM 之后做只读 Gremlin 计数。没有删 Pod，也没有新发信号。`hg-closure-std-a35-333` 的 `ggkzf` 图 `law_a35std_1m` 是 HTTP 200，data 是 `1000000:2098771`，rc 0。`hg-closure-top-a35-333` 的 `nwjsq` 图 `law_a35top333_1m` 是 HTTP 200，data 是 `1000000:2098771`，rc 0。这不勾选 P6。证据 `evidence/a35-333-count-after-server-sigterm.json`。笔记尚未审查或提交。 + +对两台原地重启后的 Server 做只读邻接。没有删 Pod，也没有新发信号。`hg-closure-std-a35-333` 的 `mcf26` 图 `law_a35std_1m`：samples 32，directions 64，mismatch_count 0，http_fail 0。`hg-closure-top-a35-333` 的 `qkpz5` 图 `law_a35top333_1m`：samples 32，directions 64，mismatch_count 0，http_fail 0。这不勾选 P6。证据 `evidence/a35-333-adjacency-after-server-sigterm.json`。 + +对 `hg-closure-std-a35-333` 的 `hg-closure-std-a35-333-hugegraph-pd-2` 向 pid 1 发 SIGTERM。没有删 Pod，也没有给 Store 发信号。镜像 `docker.io/hugegraph/pd:closure-std-327737f16`，source_revision `327737f16`。UID `a6621897-8d89-4e2f-a2ab-acf9e1162a04`，PVC `6fbc3792-e7f1-4855-88ad-f687788c8895`，restartCount 从 0 到 1，same_uid 和 same_pvc 都是 true。信号前 ready_body 是 `{"ready":true,"state":"STATE_LEADER","isLeader":true}`，信号后是 `{"ready":true,"state":"STATE_FOLLOWER","isLeader":false}`。signal_rc 是 0，signal_stdout 是 SENT。recovery_s 是 13.963，`recovery_s_includes_during_vertex_http` 是 true。`during_vertex_http` 是 200，`during_vertex_http_is_after_ready` 是 true。顶点 `10001:pd327-std` 由 `hg-closure-std-a35-333-hugegraph-server-54958cdd4d-ggkzf` 读取，vertex_http 前后都是 200。cluster_before 和 cluster_after 的 0、1、2 都是 HTTP 200、`Cluster_OK`。leaders_after 是 `1`。这不勾选 P5。证据 `evidence/a35-std-333-pd-leader-sigterm-327737f16.json`。 + +对 `hg-closure-top-a35-333` 的 `hg-closure-top-a35-333-hugegraph-pd-1` 向 pid 1 发 SIGTERM。没有删 Pod，也没有给 Store 发信号。镜像 `docker.io/hugegraph/pd:closure-327737f16`，source_revision `327737f16`。UID `ddce86f8-2583-4e99-a716-e64504ff9249`，PVC `ca20600c-9bb1-484c-86a8-5119a95b503c`，restartCount 从 0 到 1，same_uid 和 same_pvc 都是 true。信号前 ready_body 是 `{"ready":true,"state":"STATE_LEADER","isLeader":true}`，信号后是 `{"ready":true,"state":"STATE_FOLLOWER","isLeader":false}`。signal_rc 是 0，signal_stdout 是 SENT。recovery_s 是 14.978，`recovery_s_includes_during_vertex_http` 是 true。`during_vertex_http` 是 200，`during_vertex_http_is_after_ready` 是 true。顶点 `1:pd327-top` 由 `hg-closure-top-a35-333-hugegraph-server-6549c45474-nwjsq` 读取，vertex_http 前后都是 200。cluster_before 和 cluster_after 的 0、1、2 都是 HTTP 200、`Cluster_OK`。leaders_after 是 `0`。这不勾选 P5。证据 `evidence/a35-top-333-pd-leader-sigterm-327737f16.json`。 + +对 `hg-closure-std-a35-333` 的 `ggkzf` 和 `hg-closure-top-a35-333` 的 `nwjsq` 做只读 Gremlin 计数。没有删 Pod，也没有新发信号。purpose 写了这是 `327737f16` PD leader pid 1 SIGTERM 之后，别名使用 `__g_DEFAULT-`。标准别名 `__g_DEFAULT-law_a35std_1m`，图 `law_a35std_1m`，HTTP 200，rc 0，inner_code 200，data 是 `1000000:2098771`，elapsed_s 是 0.826。Topling 别名 `__g_DEFAULT-law_a35top333_1m`，图 `law_a35top333_1m`，HTTP 200，rc 0，inner_code 200，data 是 `1000000:2098771`，elapsed_s 是 0.71。两边镜像都是 `docker.io/hugegraph/server:closure-a35ebeb17`，image_id 都是 `docker.io/library/import-2026-09-25@sha256:b6ee25851186d078d5e141ff21f0bb273af6bb3deebfdfef0359e3b866e192ac`。purpose 写了更早的 `__g_` 别名返回 HTTP 400。这不勾选 P6 或 P7。证据 `evidence/a35-333-count-after-pd-leader-alias.json`。 + +对 `hg-closure-std-a35-333` 的 `ggkzf` 图 `law_a35std_1m` 和 `hg-closure-top-a35-333` 的 `nwjsq` 图 `law_a35top333_1m` 做只读 32 样本邻接。没有删 Pod，也没有新发信号。purpose 写了这是 `327737f16` PD leader pid 1 SIGTERM 之后。expectation_samples 是 32。两边都是 samples 32、directions 64、mismatch_count 0、http_fail 0、self_loop_ids 9、self_loop_present 9、self_loop_http_fail 0。mismatches 的 `ggkzf` 和 `nwjsq` 都是 `[]`。两边镜像都是 `docker.io/hugegraph/server:closure-a35ebeb17`，image_id 都是 `docker.io/library/import-2026-09-25@sha256:b6ee25851186d078d5e141ff21f0bb273af6bb3deebfdfef0359e3b866e192ac`。这不勾选 P6。证据 `evidence/a35-333-adjacency-after-pd-leader.json`。 + +只读核对 `hg-closure-std-a35-333` 和 `hg-closure-top-a35-333` 的六台 PD。没有删 Pod，也没有发信号。purpose 写了这是 `327737f16` PD leader pid 1 SIGTERM 之后。row_count 是 6，bad_count 是 0，all_no_silent_fallback 是 true。标准 `pd-0`、`pd-1`、`pd-2` 的镜像都是 `docker.io/hugegraph/pd:closure-std-327737f16`，jni_sha256 都是 `8b8fb2ed3ab69581cf1897bd116d484f073e66e9a5b6d61effc7b4c783d66dff`，map_rc 都是 0，ready 都是 true，no_silent_fallback 都是 true。`pd-0` restartCount 是 1，路径 `/tmp/librocksdbjni10467293666204976291.so`。`pd-1` restartCount 是 0，路径 `/tmp/librocksdbjni3374695603669534692.so`。`pd-2` restartCount 是 1，路径 `/tmp/librocksdbjni16116505736585664052.so`。Topling `pd-0`、`pd-1`、`pd-2` 的镜像都是 `docker.io/hugegraph/pd:closure-327737f16`，jni_sha256 都是 `c25ff6e676290db6db47df0954640aa609c391450ec90e1a8eec1f87e174dd38`，路径都是 `/hugegraph-pd/library/librocksdbjni-linux64.so`，map_rc 都是 0，ready 都是 true，no_silent_fallback 都是 true。`pd-0` 和 `pd-1` 的 restartCount 是 1，`pd-2` 是 0。这不勾选 P4、P5 或 P6。证据 `evidence/a35-333-pd-jni-after-leader.json`。 + +只读核对 `hg-closure-std-a35-333` 和 `hg-closure-top-a35-333` 的六台 Store。没有删 Pod，也没有发信号。purpose 写了这是 `327737f16` PD leader pid 1 SIGTERM 之后。row_count 是 6，bad_count 是 0，all_no_silent_fallback 是 true。标准 `store-0`、`store-1`、`store-2` 的镜像都是 `docker.io/hugegraph/store:closure-std-a35ebeb17`，image_id 都是 `docker.io/library/import-2026-09-25@sha256:4edeb22fb4de9376a0349c7d31596aa416802da562c1d5c2f301b3217cf9e8ff`，jni_sha256 都是 `8b8fb2ed3ab69581cf1897bd116d484f073e66e9a5b6d61effc7b4c783d66dff`，map_rc 都是 0，ready 都是 true，no_silent_fallback 都是 true。`store-0` restartCount 是 1，路径 `/tmp/librocksdbjni2791709567627684767.so`。`store-1` restartCount 是 0，路径 `/tmp/librocksdbjni9771030718649491973.so`。`store-2` restartCount 是 0，路径 `/tmp/librocksdbjni2026719956457113633.so`。Topling `store-0`、`store-1`、`store-2` 的镜像都是 `docker.io/hugegraph/store:closure-a35ebeb17`，image_id 都是 `docker.io/library/import-2026-09-25@sha256:b72eedece1a8165799afe4a950a3630470f7c277c128eba511ea242ec0c8e999`，jni_sha256 都是 `c25ff6e676290db6db47df0954640aa609c391450ec90e1a8eec1f87e174dd38`，路径都是 `/hugegraph-store/library/librocksdbjni-linux64.so`，map_rc 都是 0，ready 都是 true，no_silent_fallback 都是 true。`store-0` restartCount 是 2，`store-1` 和 `store-2` 是 0。这不勾选 P4、P5 或 P6。证据 `evidence/a35-333-store-jni-after-leader.json`。 + +只读核对 `hg-closure-std-a35-333` 和 `hg-closure-top-a35-333` 的六台 Server。没有删 Pod，也没有发信号。purpose 写了这是 `327737f16` PD leader pid 1 SIGTERM 之后。row_count 是 6，jni_present_count 是 0，all_jni_none 是 true。六台镜像都是 `docker.io/hugegraph/server:closure-a35ebeb17`，image_id 都是 `docker.io/library/import-2026-09-25@sha256:b6ee25851186d078d5e141ff21f0bb273af6bb3deebfdfef0359e3b866e192ac`。六台 ready 都是 true，map_rc 都是 0，jni_none 都是 true，jni_paths 都是 `[]`。`hg-closure-std-a35-333` 的 `ggkzf` restartCount 是 0，`mcf26` 是 2，`zggsl` 是 1。`hg-closure-top-a35-333` 的 `nwjsq` restartCount 是 1，`qkpz5` 是 3，`vv5sv` 是 2。这不勾选 P4、P5 或 P6。证据 `evidence/a35-333-server-jni-after-leader.json`。 diff --git a/.goal-task/toplingdb-linux-closure/todo.md b/.goal-task/toplingdb-linux-closure/todo.md new file mode 100644 index 0000000000..e24045b3cf --- /dev/null +++ b/.goal-task/toplingdb-linux-closure/todo.md @@ -0,0 +1,113 @@ +# 本机实测分项 + +状态标记只放本文件。未勾选表示当前 SHA 尚无完成证据。`9aba` 历史结果仅是线索。 + +## 执行顺序 + +2026-09-25 标准 `a35ebeb17` Store 镜像已证明 follower `GET /v1/partitions`：三台都是 HTTP 200，合计 12 个 `STATE_LEADER`、24 个 `STATE_FOLLOWER`，JNI `8b8fb2ed…6dff`。这不勾选 P4、P5 或 P6，也不覆盖 Topling。证据 `evidence/a35-std-follower-partitions.json`。 +Topling 同一检查也已通过：`hugegraph/store:closure-a35ebeb17`，HTTP 200，12 个 `STATE_LEADER`、24 个 `STATE_FOLLOWER`，JNI `c25ff6e6…dd38`，mmap WAL 错误 0。证据 `evidence/a35-top-follower-partitions.json`。两个最小集群还各有一次顶点写入 201 和读取 200，证据 `evidence/a35-vertex-write-read.json`。这些都不勾选 P3、P4、P5 或 P6。 +2026-09-25 a35 最小集群各做了一次 Store-0 SIGTERM 和默认图 clear。标准 12.233 秒、Topling 67.414 秒，UID 与 PVC 不变，顶点恢复后仍可读；Topling 上一容器仍有 `SidePluginRepo` `db not closed`。clear 都是 204，新顶点 201/200。证据 `evidence/a35-std-store0-sigterm.json`、`evidence/a35-top-store0-sigterm.json`、`evidence/a35-graph-clear.json`。这仍不勾选完整生命周期或 P5。 +同日两个 a35 PD 镜像已替换进最小集群。标准 JNI `8b8fb2ed…6dff`，Topling JNI `c25ff6e6…dd38`。两边 `balanceLeaders` 都是 HTTP 200 的成功 JSON，不是异常体。 +2026-09-25 两个 a35 最小集群删除并重建了默认图，drop 204、create 201、新顶点 201/200，分区仍是 12/24。随后换上 `hugegraph/server:closure-a35ebeb17`。标准 Server SIGTERM 15.183 秒，Topling 12.165 秒，UID 不变，顶点仍在，Server 没有 RocksDB JNI。证据 `evidence/a35-graph-drop-recreate.json`、`evidence/a35-std-server-sigterm.json`、`evidence/a35-top-server-sigterm.json`。`hugegraph-server` 源码相对 e109 没有差异。这仍不是完整 1+1+1 或 3+3+3。 +2026-09-25 两个 a35 最小集群同时删除了 Store-1 和 Store-2。旧顶点在故障中和恢复后都可读，故障中的写入没有留下。标准约 32.8 秒 Ready，Topling 大约 70 秒。Ready 后的第一次写入分别是超时和 500，随后的写入都是 201。分区回到 12/24，JNI 没有换 provider。证据 `evidence/a35-std-store-majority.json`、`evidence/a35-top-store-majority.json`。这不是 3+3+3，也不是网络分区。 +2026-09-25 两个 a35 单 PD 做了 SIGTERM。标准 18.8 秒，没有 `db not closed`。Topling 58.618 秒，`shutdown completed` 后仍有 `SidePluginRepo` `db not closed`，Ready 当下 `Cluster_Not_Ready`，随后 `Cluster_OK`。顶点都还在，JNI 没有换 provider。证据 `evidence/a35-std-pd-sigterm.json`、`evidence/a35-top-pd-sigterm.json`。这不是 PD 多数派。 +2026-09-25 a35 单机 snapshot：标准和 Topling 的 `snapshot_create`、`snapshot_resume` 都是 200，重启后快照前顶点仍在、快照后顶点不在。标准 JNI `8b8fb2ed…6dff`，Topling JNI `c25ff6e6…dd38`，没有 mmap WAL 错误。Topling 日志有 1 次 `db not closed`。证据 `evidence/a35-standalone-std-snapshot.json`、`evidence/a35-standalone-top-snapshot.json`。这不是 HStore snapshot。 +2026-09-25 两个 a35 namespace 的 Server 扩到 3。标准与 Topling 都写入 `a35three` 201，三台 Server 读取新顶点和旧顶点 `a35rebuilt` 都是 200，没有 RocksDB JNI。PD 仍是 1 个。 +2026-09-25 新装标准 `hg-closure-std-a35-333`，Helm 退出 0。PD-0 为 `Cluster_OK`、成员数 3、3 个 Store Up。JNI 都是标准 `8b8fb2ed…6dff`。顶点 `a35-333` 三台 Server 都读到。随后删除 PD-1 和 PD-2，17.4 秒恢复，旧顶点仍在，故障中的写入没有留下，恢复后的写入最终可读。证据 `evidence/a35-std-333-install.json`、`evidence/a35-std-333-pd-majority.json`。Topling 3+3+3 还没有。这不是网络分区。 +2026-09-25 标准 `hg-closure-std-a35-333` 删除 Store-1 和 Store-2 后约 33 秒恢复，旧顶点仍在，故障写入没有留下，分区回到 12/24。删除一台 Server 后其余副本仍可读，替补 11.365 秒 Ready。HStore `snapshot_create` 是 500 `createSnapshot`。 +2026-09-25 同一集群 drop 204、create 201、clear 204。重建后另外两台 Server 读取 500，图已关闭；SIGTERM 重启后三台都读到新顶点。分区仍是 12/24。证据 `evidence/a35-std-333-drop-clear.json`。 +2026-09-25 Topling `hg-closure-top-a35-s3` 也做了 drop 204、create 201 和 clear 204。三台 Server 在重建后都能读到新图。clear 后另外两台仍返回旧顶点，SIGTERM 重启后才和执行 clear 的 Server 一致。证据 `evidence/a35-top-s3-drop-clear.json`。这是 1 个 PD，不是 3+3+3。 +2026-09-25 同一 Topling 集群删除一台 Server，其余副本在删除期间仍读到 `a35cleared-top`。替补 11.507 秒 Ready，三台随后都是 200。证据 `evidence/a35-top-s3-server-loss.json`。 +2026-09-25 新装 Topling `hg-closure-top-a35-111`。Helm 退出 0，JNI `c25ff6e6…dd38`，没有静默 fallback。单副本分区 12 个 `STATE_LEADER`。顶点 `a35-111` 201/200。唯一 Store SIGTERM 后约 77 秒恢复，UID 和 PVC 不变，顶点仍在，上一容器仍有 `db not closed`。证据 `evidence/a35-top-111-function.json`、`evidence/a35-top-111-store-sigterm.json`。这不是 3+3+3。 +2026-09-25 Topling `hg-closure-top-a35-111` drop 204、create 201、clear 204，旧顶点清空后新顶点 201/200。`snapshot_create` 500 `createSnapshot`。唯一 PD SIGTERM 54.757 秒恢复，UID 与 PVC 不变，Ready 当下 `Cluster_Not_Ready`，随后 `Cluster_OK`，仍有 `db not closed`。证据 `evidence/a35-top-111-drop-clear.json`、`evidence/a35-top-111-pd-sigterm.json`。 +2026-09-25 同一 1+1+1 的 Server 第一次 SIGTERM 后 16.427 秒恢复，`a35cleared111` 查询为空。之后写入的 `a35persist111` 在第二次 SIGTERM 后仍在。证据 `evidence/a35-top-111-server-sigterm.json`、`evidence/a35-top-111-server-persist.json`。 + + + + + +证据 `evidence/a35-std-333-store-majority.json`、`evidence/a35-std-333-server-loss.json`。 + +证据 `evidence/a35-std-server3-read.json`、`evidence/a35-top-server3-read.json`。 + + + + +证据 `evidence/a35-std-pd-balance-leaders.json`、`evidence/a35-top-pd-balance-leaders.json`。 + + + +完成标记仍只看下面的复选框。2026-09-25 再次确认后，执行顺序只以 `state.md` 的“下一动作”为准。部分通过、`9aba`、cc143、mmapfix 或 overlay 都不能单独勾选当前 SHA。已勾选的 P2、P3 和不依赖 channel refresh 的已有证据不重跑。 + +## P1 历史标准集群 + +- [x] 1+1+1 停止、重启，并核对重启前已确认写入。仅绑定历史镜像 `closure-std-9abae9dbaaa1`，不算当前 SHA。证据 `evidence/helm-standard-111-stop-restart-confirmed.json`。 +- [ ] 1+1+1 删图重建、truncate、snapshot/restore 到新卷。删图重建和 truncate 已在 `9aba` 通过。2026-09-25 同一历史镜像的 `snapshot_create` 是 500 `createSnapshot`。先前的新卷恢复是文件系统拷贝，文件数一致但顶点全部 404，禁止再做目录拷贝。这不是当前 SHA。证据 `evidence/helm-standard-111-lifecycle.json`、`evidence/helm-standard-111-snapshot-api.json`。 +- [x] 3+3+3 三个 Server 的功能、认证和写入一致性。顶点和边已在三个 Server 上读到；错误口令返回 401。2026-09-25 在未升级的 `closure-std-9abae9dbaaa1` 上复用 namespace 做角色拒绝：无角色用户建图 403，图空间和用户已删除，没有新增 Kubernetes namespace。这只绑定历史镜像，不是当前 SHA。证据 `evidence/helm-standard-333-write-consistency.json`、`evidence/helm-standard-333-auth-function.json`、`evidence/helm-standard-333-role-denial.json`。 +- [ ] 3+3+3 leader/follower 退出、Server 副本切换、多数派丢失与恢复、恢复时间。Store 多数派 Pod 删除已有证据。网络分区未做：单节点 kind 没有独立网络路径，本机也没有 Chaos Mesh。解除条件是多节点集群和 Chaos Mesh CRD。证据 `evidence/helm-standard-333-store-majority.json`。 +- [ ] 3+3+3 网络分区。本机没有 Chaos Mesh CRD 或 namespace，后置；不阻止其他测试。 + +历史线索：1+1+1 API 155/0/0/50，Store 同 PVC 重建一次；3+3+3 为 9/9 Ready 和 JNI 采样。镜像均为 `closure-std-9abae9dbaaa1`。 + +## P2 当前 SHA 构建 + +- [x] 从 cc143 上下文构建标准与 Topling 镜像。标准 PD/Store/standalone 与 Topling PD `3bce8e03d227`、Store `195cad38c8a7`、standalone `52be45a93334` 已分开。HStore server 的 `closure-top` tag 只是标准镜像 `14eb8067b416` 的别名，不含 Topling `.so`。证据 `evidence/build/topling-image-acceptance-cc14333f0.json`。 +- [x] 证明 `hugegraph/pd:closure-std-cc14333f0`（`a0d9264b2ea2`）等于 cc143 产物。Maven 产出校验和命中原层，14 个运行文件与 `9aba` 清单一致；源码差异仅在跳过的测试。证据 `evidence/build/pd-image-acceptance-cc14333f0.json`。 +- [x] 证明 Topling 进程实际映射 Topling JNI，标准镜像不包含或不会静默加载 Topling。标准 PD/Store 映射 `8b8fb2ed3ab69581cf1897bd116d484f073e66e9a5b6d61effc7b4c783d66dff`；Topling PD/Store 映射 `library/librocksdbjni-linux64.so` 的 `c25ff6e676290db6db47df0954640aa609c391450ec90e1a8eec1f87e174dd38`。证据 `evidence/helm-standard-cc143-111-runtime.json`、`evidence/helm-topling-cc143-111-jni.json`。 +- [x] 用新 namespace 加载镜像，不升级两个 `9aba` namespace。e109 的 1+1+1 与 3+3+3 已存在；两个 `9aba` namespace 仍是 `closure-std-9abae9dbaaa1`。e109 Server 后来换成 overlay，没有回写历史 namespace。证据 `evidence/e109-namespaces-not-upgraded.json`、`evidence/helm-standard-e109-111-jni.json`、`evidence/helm-topling-e109-111-jni.json`。 + +## P3 当前 SHA 功能 + +- [x] 标准 RocksDB 单机功能。镜像 `hugegraph/hugegraph:closure-std-cc14333f0`（`35267423defa`），JNI `8b8fb2ed…6dff`，provider=rocksdb。写入可读，docker restart 后 9.426 秒仍读到 `std-persist-1790266399`。API 套件 13 个失败都是单机不支持 GraphSpace。证据 `evidence/standalone-cc143-function.json`、`evidence/standalone-std-cc143-restart-clean.json`。 +- [x] 标准 RocksDB 1+1+1 与 3+3+3 功能。1+1+1 与 3+3+3 API 都是 155/0/0/50，镜像 `closure-std-cc14333f0`，JNI `8b8fb2ed…6dff`。3+3+3 三台 Server 读到 `std333-1790265733` 所在图 `std333_fn` 的同一顶点。证据 `evidence/build/helm-standard-cc143-333-api.log`、`evidence/helm-standard-cc143-333-write-consistency.json`。 +- [x] Topling 单机、1+1+1、3+3+3 功能。1+1+1 与 3+3+3 API 都是 155/0/0/50，三个 Store 无 mmap WAL 错误。3+3+3 三台 Server 读到同一顶点 `mmapfix-333-1790262197`。单机镜像 `5178c8b35c20` 的 JNI 是 `c25ff6e6…dd38`，`top-alone-1790266203` 重启后仍在。单机 GraphSpace 套件未重跑，标准单机已证明该模式会拒绝。证据 `evidence/standalone-cc143-function.json`。 +- [x] 两个 provider 分别覆盖 schema、CRUD、事务、索引、分页或批量、Gremlin/Cypher、多图和认证正反例。认证：标准和 Topling 的集群与单机上，正确口令访问图列表为 200，错误口令、缺认证、未知用户和已删用户都是 401。`/versions` 不鉴权。DEFAULT 图空间 `auth=false`，无角色用户在 Topling 上建图返回 201，角色拒绝没有生效；开启 auth 会创建 Kubernetes namespace，本次没有改。索引和分页：两个 provider 都创建了非主键二级索引，limit=2 返回 2 条。标准按 title 查询只命中目标顶点。Topling 第一次同样查询返回 Panic，随后 5 次都是 200 且只命中目标。多图隔离、Gremlin 读写和 Cypher 查询已通过。批量写入 3 条都可读；同一批次里夹一条非法属性时返回 400，合法的那条没有留下，之前的 3 条仍在。角色拒绝：在 `hg-closure-top-mmapfix-111` 复用现有 namespace 创建 `closure_roleprobe`（auth=true）后，无角色用户建图返回 403 `User not authorized`，图空间和用户已删除，没有新增 Kubernetes namespace。证据 `evidence/current-sha-role-denial.json`。DEFAULT 图空间 auth=false 的历史行为仍在。证据 `evidence/current-sha-auth-enforcement.json`、`evidence/current-sha-auth-standalone.json`、`evidence/current-sha-index-page.json`、`evidence/current-sha-index-query.json`、`evidence/current-sha-topling-index-panic-retry.json`。 2026-09-25 e109 1+1+1：批量 3 顶点返回 201 且可读；混入未定义属性返回 400，回滚顶点不在，原顶点仍在。本图顶点在另一图不可见。顶点追加 `city=sg` 返回 200，按 JSON 字符串 ID 删除返回 204。复用现有 namespace 创建 auth 图空间后，无角色用户建图 403，清理 204，没有新增 Kubernetes namespace。Server 是 overlay `closure-e109-channelrefresh`，PD/Store 是 e109 镜像。边 `knows` 创建 201、`since` 从 1 更新到 2、删除 204 后为空。e109 单机图列表无认证、错误口令和未知用户都是 401，admin 是 200；e109 单机一次性容器上，已删除用户再次访问图列表为 401；图空间管理仍是 400。证据 `evidence/e109-standalone-deleted-user.json`。长期运行的集群 Server 仍是 overlay。2026-09-25 另用未修改镜像 `sha256:5b9f40a9d1fc` 的临时 Pod 复测了两边的批量原子性，图和 Pod 已删除。证据 `evidence/e109-fullserver-std-batch.json`、`evidence/e109-fullserver-top-batch.json`。2026-09-25 这张未修改镜像上，两边的二级索引、`limit=2`、Gremlin 和 Cypher 已通过，证据 `evidence/e109-fullserver-index-query.json`。多图隔离和集群认证也已在这张未修改镜像上通过：无认证和错误口令 401，正确口令 200，删除用户后 401；A 图顶点在 B 图不可见。证据 `evidence/e109-fullserver-isolation-auth.json`。长期运行的 Server Pod 仍是 overlay，不作为这项证明。证据 `evidence/e109-batch-tx.json`、`evidence/e109-multigraph-isolation.json`、`evidence/e109-crud-role.json`。 2026-09-25 文档审查指出边更新/删除和角色 403 原先只在 overlay 或 mmapfix，不能单独支撑当前 SHA。随后在未修改 Helm Server `closure-e109012a0`（import digest `sha256:d0c5346b760f`）上重做：标准和 Topling 的 `knows` 边创建 201，`since` 从 1 改为 2，删除 204 后为空。复用现有 namespace 创建 `auth=true` 图空间后，无角色用户建图 403，没有新增 Kubernetes namespace，图空间和用户已删除。证据 `evidence/e109-edge-crud-fullsha.json`、`evidence/e109-role-denial-fullsha.json`。P3 因此仍勾选，并且这项不再依赖 overlay。 + +## P4 生命周期与 provider + +- [ ] 两个 provider 的停止、重启、异常退出、同数据恢复、删图重建、truncate、snapshot/restore。反向混合的标准 Store 被 `kill -9` 后 restartCount=1，顶点 `rev-1790267261` 在 Ready 翻成 true 前已返回 200，PVC `pvc-5d209bd7` 未变。证据 `evidence/current-sha-store-crash.json`。Topling Store `kill -9` 后 37 秒 Ready，`persist-1790262903` 和 `56862681` 在 Ready 前已可读，JNI 仍是 `c25ff6e6…dd38`。证据 `evidence/current-sha-topling-store-crash.json`。Topling 1+1+1 Store 同 PVC `b9f47929` 重建 12.615 秒后仍读到 `persist-1790262903`，mmap WAL 错误 0。标准 `closure-std-cc14333f0` 的 1+1+1 Store 同 PVC `4f1945d3` 重建 12.796 秒后仍读到 `std-persist-1790263000`，重启前后 JNI 都是 `8b8fb2ed…6dff`，不是 Topling。标准和 Topling 1+1+1 都完成删图重建与 clear/truncate：drop 204 后图 404，重建后旧顶点不在、新顶点可读；clear 204 后图仍在、数据清空、可再写。HStore `snapshot_create` 仍是 500 `createSnapshot`。单机标准和 Topling 的 create/resume 都返回 200，但快照后的写入在重启后仍可读，恢复没有回滚。 2026-09-25 03:25 完整 Topling 镜像 `closure-e109012a0` 上，快照前顶点仍在、快照后顶点不在，证据 `evidence/standalone-e109-top-snapshot.json`。03:31 标准完整镜像 `closure-std-e109012a0` 同样通过，证据 `evidence/standalone-e109-std-snapshot.json`。HStore 仍未通过。证据 `evidence/current-sha-snapshot.json`、`evidence/standalone-std-snapshot-after-restart.json`、`evidence/standalone-top-snapshot.json`。证据 `evidence/helm-current-sha-drop-truncate.json` 与 `evidence/helm-topling-mmapfix-111-drop.json`。证据 `evidence/helm-topling-mmapfix-111-persist.json` 与 `evidence/helm-standard-cc143-111-persist.json`。 2026-09-25 e109 1+1+1 两个 provider 的 Store Java `kill -9`：restartCount 0 到 1，标准 30.850 秒、Topling 32.704 秒 Ready，PVC 不变，原顶点仍可读，JNI 分别保持 `8b8fb2ed…6dff` 和 `c25ff6e6…dd38`。证据 `evidence/e109-store-kill9.json`。HStore snapshot 仍未通过，整项不勾选。 2026-09-25 e109 3+3+3 标准和 Topling 的删图重建、clear/truncate 已通过，三台 Server 在删图后都是 404。Server 是 overlay。`snapshot_create` 仍是 500。证据 `evidence/e109-333-drop-truncate.json`。 同日再用未修改镜像 `sha256:5b9f40a9d1fc` 的临时 Pod 复测，结果相同，图和 Pod 已删除。证据 `evidence/e109-333-fullserver-drop-truncate.json`。 标准 e109 3+3+3 Store-2 `kill -9` 后 33.144 秒 Ready，PVC 和 Pod UID 不变，三台 Server 仍读到 `std333-e109-1790279980`，JNI 仍是 `8b8fb2ed…6dff`。证据 `evidence/e109-std-333-store-kill9.json`。 Topling e109 3+3+3 Store-0 `kill -9` 后 37.334 秒 Ready，PVC 和 Pod UID 不变，三台 Server 仍读到 `top333-e109-1790280210`，JNI 仍是 `c25ff6e6…dd38`，mmap WAL 错误 0。恢复后的新顶点也被三台 Server 读到。证据 `evidence/e109-top-333-store-kill9.json`、`evidence/e109-333-write-after-kill9.json`。 2026-09-25 SIGTERM：标准 store-1 71.680 秒恢复，Topling store-1 83.606 秒恢复，PVC 和 Pod UID 不变，原顶点仍可读。标准关闭没有 `db not closed`；Topling 关闭末尾仍有 `SidePluginRepo` `db not closed`。证据 `evidence/e109-std-333-store-sigterm.json`、`evidence/e109-top-333-store-sigterm.json`。snapshot 仍未通过，整项不勾选。 2026-09-25 Server SIGTERM 只绑定 overlay `closure-e109-channelrefresh`：标准 9.371 秒、Topling 14.616 秒，另一副本仍可读。不能当作未修改 SHA。证据 `evidence/e109-std-333-server-sigterm.json`、`evidence/e109-top-333-server-sigterm.json`。 2026-09-25 未修改 Server `hugegraph/server:closure-e109012a0` 的 SIGTERM已补上。主机镜像 `sha256:5b9f40a9d1fc` 与 kind 中同 tag 的 `docker-entrypoint.sh`、`bin/start-hugegraph.sh` 校验和一致；kubelet 记录的是 import digest `sha256:d0c5346b760f`。标准 `dsdqd` 10.872 秒，Topling `7zjgx` 11.971 秒，restartCount 0 到 1，UID 不变，不是 kill -9。另一副本 UID 不变，重启前、期间和之后顶点都是 200。上一容器没有 `db not closed`。历史 namespace 未改。证据 `evidence/e109-server-image-identity.json`、`evidence/e109-std-333-server-sigterm-fullsha.json`、`evidence/e109-top-333-server-sigterm-fullsha.json`。这不是网络分区，P4 和 P5 都不勾选。 2026-09-25 1+1+1 唯一 PD SIGTERM：标准 8.433 秒、Topling 60.063 秒，UID 和 PVC 不变，JNI 没有静默 fallback。信号后 `/v1/cluster` 从 OK 变为 `Cluster_Not_Ready`，在线 Store 仍是 1。正确图的旧顶点恢复后为 200。证据 `evidence/e109-std-111-pd-sigterm.json`、`evidence/e109-top-111-pd-sigterm.json`。snapshot 仍未通过，整项不勾选。 2026-09-25 1+1+1 Store SIGTERM，不是 kill -9。标准 store-0 12.692 秒，Topling store-0 60.533 秒，restartCount 1 到 2，Pod UID 和 PVC 不变。标准 PVC `pvc-4b0bd019`，Topling PVC `pvc-782709b8`。标准 JNI `8b8fb2ed…6dff`，没有 `db not closed`；Topling JNI `c25ff6e6…dd38`，上一容器有 `SidePluginRepo` `db not closed`。两边顶点 200，PD 都是 `Cluster_OK`、在线 Store 1。Ready 含 0 字节 jemalloc curl 终止。Server 是 overlay，不能当成未修改 SHA。证据 `evidence/e109-std-111-store-sigterm.json`、`evidence/e109-top-111-store-sigterm.json`。 2026-09-25 1+1+1 未修改 Server SIGTERM：两台 Server 从 overlay 切到 `closure-e109012a0` 后，原顶点仍是 200。标准 `zmpkw` 15.218 秒，Topling `vct4k` 14.416 秒，UID 不变，restartCount 0 到 1，不是 kill -9。恢复后顶点仍是 200。证据 `evidence/e109-std-111-server-sigterm-fullsha.json`、`evidence/e109-top-111-server-sigterm-fullsha.json`。snapshot 仍未通过，整项不勾选。 2026-09-25 Topling a35 `hg-closure-top-a35-333` 的 HStore `snapshot_create` 仍是 500 `createSnapshot`，顶点还在，没有目录拷贝。证据 `evidence/a35-top-333-snapshot.json`。整项仍不勾选。 2026-09-25 同一集群新图 `life_a35top333`：建图 201，三台读到 `life-before`。clear 204 后只有执行 Server 变为 404，另外两台仍返回旧顶点。新顶点 `life-after` 三台都是 200。默认图未删。证据 `evidence/a35-top-333-clear.json`。整项仍不勾选。 2026-09-25 对仍返回旧顶点的 `qkpz5` 和 `vv5sv` 发 SIGTERM。Ready 15.158 秒和 12.052 秒，UID 与 IP 不变。重启后三台对 `life-before` 都是 404，`life-after` 都是 200。证据 `evidence/a35-top-333-clear-sigterm.json`。整项仍不勾选。 2026-09-25 删除并重建 `life_a35top333`：drop 204，重建 201，新顶点 id 是 `2:life-rebuilt`，三台读取 200。重建之后另外两台再次读到旧 id `1:life-after`，这不是 drop 当下的结果。默认图未删。证据 `evidence/a35-top-333-drop.json`。整项仍不勾选。 2026-09-25 对仍返回 `1:life-after` 的两台 Server 发 SIGTERM。Ready 23.621 秒和 24.698 秒，UID 与 IP 不变。重启后三台对 `1:life-after` 都是 404，对 `2:life-rebuilt` 都是 200。证据 `evidence/a35-top-333-drop-sigterm.json`。整项仍不勾选。 +- [x] PD/Store 两种混合组合可按预期运行。标准 PD + Topling Store 读到 `mix-1790267151`，JNI 分别是 `8b8fb2ed…6dff` 和 `c25ff6e6…dd38`。Topling PD + 标准 Store 读到 `rev-1790267261`，JNI 分别是 `c25ff6e6…dd38` 和 `8b8fb2ed…6dff`。证据 `evidence/current-sha-mixed-pd-std-store-top.json`、`evidence/current-sha-mixed-pd-top-store-std.json`。 +- [x] 错误 provider 复用原数据被拒绝，原数据不变。Topling 镜像打开 `provider=rocksdb` 标记退出 1，标准镜像打开 `provider=topling` 标记退出 1，标记哈希不变。原单机顶点 `std-persist-1790266399` 和 `top-alone-1790266203` 仍返回 200。证据 `evidence/current-sha-wrong-provider.json`。 + +## P5 当前 SHA HA + +- [ ] 标准 RocksDB 3+3+3 的副本退出、切换、分区、多数派恢复、写入一致性和恢复时间。Store-1 leaderCount 5 删除后 22.550 秒 Ready，PVC 不变，旧数据和新写入可读。Server 副本删除期间保留副本继续读写，10.613 秒恢复。多数派：同时删除 Store-0/2 后已提交数据仍可读，故障中新写入超时；Ready 23.102 秒时 leader 合计已是 12，写入仍超时，35.929 秒才写入成功。分区未做：同样只记录单节点限制，解除条件是多节点集群和 Chaos Mesh。证据 `evidence/helm-standard-cc143-333-store-leader.json`、`evidence/helm-standard-cc143-333-server-replica.json`、`evidence/helm-standard-cc143-333-store-majority.json`。 2026-09-25 标准 e109 3+3+3 PD-0 Java `kill -9`：14.514 秒 Ready，成员数保持 3，leader 从 PD-0 转到 PD-1，复查 `Cluster_OK`。旧顶点和恢复后新写入都可读。证据 `evidence/e109-std-333-pd-kill9.json`。分区仍未做。2026-09-25 Server `smq6m` Java `kill -9` 后 12.514 秒 Ready，保留副本完成读写，第三台也能读到新顶点。证据 `evidence/e109-std-333-server-kill9.json`。整项不勾选。 2026-09-25 PD-1 SIGTERM 8.459 秒，镜像 `closure-std-e109012a0`，JNI 标准哈希。信号前已是 `Cluster_Not_Ready`，信号后仍是，不能写成这次恢复到 OK。证据 `evidence/e109-std-333-pd-sigterm.json`。分区仍未做。 2026-09-25 `/v1/cluster` 仍是 `Cluster_Not_Ready`，但 3 个 Store 为 Up、在线 3、`PState_Normal`，GET 不重算缓存状态。证据 `evidence/e109-std-333-cluster-not-ready.json`。 2026-09-25 leader `pd-0` 是 `Cluster_OK`，follower 仍报 `Cluster_Not_Ready`。证据 `evidence/e109-pd-cluster-state-by-member.json`。 2026-09-25 a35 两个 3+3+3 复查：pd-1 是 leader 且 `Cluster_OK`；pd-2 的 `/v1/ready` 是 200 follower，`/v1/cluster` 仍是 `Cluster_Not_Ready`，`Raft becomes leader` 和 `Store register` 都是 0。pd-0 曾经是 leader，现在是 follower 但仍报 `Cluster_OK`。不是 Topling 特有，也不是网络分区。以上是替换前的快照，不是当前结果。证据 `evidence/a35-333-pd-local-cluster-state.json` 和 `evidence/a35-333-pd-service-cluster-mix.json`。标准集群随后换成 `closure-std-327737f16`：leader 是 pd-2，两台 follower 也是 `Cluster_OK`，Service 18 次都是 `Cluster_OK`。证据 `evidence/a35-std-333-pd-327737f16.json`。Topling 集群随后换成 `closure-327737f16`：leader 是 pd-1，pd-0 和 pd-2 是 follower，三台都是 `Cluster_OK`，Service 18 次都是 `Cluster_OK`。证据 `evidence/a35-top-333-pd-327737f16.json`。这仍不是网络分区，不勾选。 +- [ ] Topling 3+3+3 的同样证据。Store-2（leaderCount 5）删除后 51.084 秒 Ready，PVC `52fd8dad` 未变，领导权转到 Store-0/1，旧顶点和恢复后新写入都可读，mmap WAL 错误 0。恢复时间含 GitHub jemalloc 下载卡死后被终止。网络分区未做。多数派：同时删除 Store-0 和 Store-2 后，已提交顶点在故障期间仍返回 200，故障期间新写入超时；Pod Ready 21.596 秒时领导权尚未恢复，稍后 leaderCount 合计 12，新写入 201。Server 副本 `d8dp9` 删除期间，保留的 `vw8q2` 仍读到旧顶点并写入新顶点，10.7 秒后替代副本 Ready。证据 `evidence/helm-topling-mmapfix-333-store-leader.json` 与 `evidence/helm-topling-mmapfix-333-server-replica-direct.json`。 2026-09-25 Topling e109 3+3+3 PD-0 Java `kill -9`：12.534 秒 Ready，成员数保持 3，leader 转到 PD-1，复查 `Cluster_OK`，JNI 仍是 `c25ff6e6…dd38`。证据 `evidence/e109-top-333-pd-kill9.json`。分区仍未做。2026-09-25 Server `plp4x` Java `kill -9` 后 9.441 秒 Ready，保留副本完成读写。证据 `evidence/e109-top-333-server-kill9.json`。整项不勾选。 2026-09-25 PD-1 SIGTERM 60.214 秒，镜像 `closure-e109012a0`，JNI `c25ff6e6…dd38`。信号前 `Cluster_Not_Ready`，信号后 `Cluster_OK`。证据 `evidence/e109-top-333-pd-sigterm.json`。分区仍未做。 2026-09-25 leader `pd-2` 是 `Cluster_OK`，follower 仍报 `Cluster_Not_Ready`。证据 `evidence/e109-pd-cluster-state-by-member.json`。 2026-09-25 Topling a35 `hg-closure-top-a35-333` 同时删除 Store-1 和 Store-2。旧顶点故障中和恢复后都是 200，故障写入超时且之后 404。Ready 23.788 秒和 23.876 秒，PVC 不变，IP 改变。Ready 后第一次写入超时且 404，随后写入 201，耗时 0.097 秒。分区 12/24，JNI `c25ff6e6…dd38`。不是网络分区，不勾选。证据 `evidence/a35-top-333-store-majority.json`。 2026-09-25 同一集群同时删除 PD-1 和 PD-2。旧顶点仍是 200。删除后 0.08 秒的写入返回 201 并留下，不能写成被拒绝。Ready 12.653 秒和 12.684 秒，PVC 不变，IP 改变。Ready 后第一次写入超时 30 秒但顶点后来可读，下一次 201 耗时 0.069 秒。集群恢复为 `Cluster_OK`、成员数 3。JNI `c25ff6e6…dd38`。不是网络分区，不勾选。证据 `evidence/a35-top-333-pd-majority.json`。 2026-09-25 同一集群删除一台 Server。保留副本在删除期间读取 200、写入 201。替补 10.706 秒 Ready。未删除的另一台当时两次读取超时，稍后重试 200。不是网络分区，不勾选。证据 `evidence/a35-top-333-server-loss.json`。 2026-09-25 a35 两个 3+3+3 复查：pd-1 是 leader 且 `Cluster_OK`；pd-2 的 `/v1/ready` 是 200 follower，`/v1/cluster` 仍是 `Cluster_Not_Ready`，`Raft becomes leader` 和 `Store register` 都是 0。pd-0 曾经是 leader，现在是 follower 但仍报 `Cluster_OK`。不是 Topling 特有，也不是网络分区。以上是替换前的快照，不是当前结果。证据 `evidence/a35-333-pd-local-cluster-state.json` 和 `evidence/a35-333-pd-service-cluster-mix.json`。标准集群随后换成 `closure-std-327737f16`：leader 是 pd-2，两台 follower 也是 `Cluster_OK`，Service 18 次都是 `Cluster_OK`。证据 `evidence/a35-std-333-pd-327737f16.json`。Topling 集群随后换成 `closure-327737f16`：leader 是 pd-1，pd-0 和 pd-2 是 follower，三台都是 `Cluster_OK`，Service 18 次都是 `Cluster_OK`。证据 `evidence/a35-top-333-pd-327737f16.json`。这仍不是网络分区，不勾选。 +- [ ] 现有 HA Compose 与 Helm 配置约定对齐并分别留证。对照已写入 `evidence/ha-compose-helm-alignment.md`。端口、3+3+3、Store `/v1/health`、Server `/versions` 和默认 RocksDB 一致。PD readiness、启动依赖、反亲和、资源、provider marker、Hubble 和凭据来源不一致，所以不勾选。2026-09-25 复核：3 副本 Helm 的 startup/liveness 仍是 `/v1/health`，单副本 helper 才用 `/v1/ready`。单节点 kind 不能改成 `required` 反亲和，本次没有修改 Compose、Helm 或集群。 + +## P6 Loader + +- [x] 使用已准备的 LAW Twitter-2010 固定百万点子集完成导入。图 `law_twitter_1m`，namespace `hg-closure-top-mmapfix-111`，Loader 退出 0，1000000 点、2098771 边、失败 0。证据 `evidence/loader-law-twitter-1m.json`。 +- [ ] 核对失败重试、计数、ID、方向、邻接和重启。2026-09-25 修复前五轮失败数 4、1、1、0、0。白名单提交 `08dd6f4f1` 后，overlay Server 上两轮各 320 次扫描失败都是 0。不是完整镜像重建，仍不勾选。2026-09-25 03:39 完整 e109 namespace 导入成功，重启前 32 样本通过；Store 重建后的完整 Server 复测因旧 IP `10.244.0.128` 超时，不能勾选。标准和 Topling 后来的 32/0/9 都发生在 overlay `sha256:3a129d448516`。缺口清单是 `evidence/e109-loader-restart-gap.json`。失败和重试为 0；重启前 32 个样本与 9 个自环通过。Ready 后立即的大扫描曾 UNAVAILABLE；稳定后 `54148543` IN 为 142 且前 20 匹配。混合扫描间歇 `Panic`。Store 崩溃恢复后又出现 4 次和 2 次，第三轮为 0；失败点立刻重试成功。仍未勾选。证据 `evidence/loader-law-twitter-1m-in142-settled.json`、`evidence/loader-scan-panic-sweeps.json`。 2026-09-25 标准 e109 用未修改 Server `sha256:5b9f40a9d1fc` 的临时 Pod 导入新图 `law_twitter_1m_std_fullsha`：Loader 退出 0，57.254 秒，1000000 点、2098771 边、失败 0。重启前 32 个样本 0 不一致，9 个自环都在。Store 没有重启，JNI 仍是 `/tmp/librocksdbjni*.so` 的 `8b8fb2ed…6dff`。这不覆盖 Store IP 变化，也不勾选。证据 `evidence/loader-law-twitter-1m-std-e109-fullserver.json`。 2026-09-25 未修改 Helm Server `closure-e109012a0`（import digest `sha256:d0c5346b760f`）上重读固定子集，没有重启 Store。标准图 `law_twitter_1m_std_fullsha` 和 Topling 图 `law_twitter_1m` 都是 32 个样本 0 不一致、9 个自环通过，Gremlin 计数 `1000000:2098771`。标准 JNI `8b8fb2ed…6dff`，Topling JNI `c25ff6e6…dd38`。这不覆盖 Store IP 变化，不勾选。证据 `evidence/loader-law-twitter-1m-std-e109-fullsha-helmserver.json`、`evidence/loader-law-twitter-1m-top-e109-fullsha-helmserver.json`。 +2026-09-25 标准 a35 `hg-closure-std-a35-333` 图 `law_a35std_1m`：Loader 退出 0，141.323 秒，1000000 点、2098771 边、失败 0，checksums_ok。没有计数、邻接或重启复测，不勾选。证据 `evidence/a35-std-333-loader.json`。 +同日复测：Gremlin 重启前后都是 `1000000:2098771`，32 个样本 0 不一致，9 个自环通过。Store-0 SIGTERM 后同一 UID 和 IP，12.252 秒 Ready，JNI 仍是标准 `8b8fb2ed…6dff`。没有 Store IP 变化，失败重试路径没有被触发，仍不勾选。证据 `evidence/a35-std-333-loader-verify.json`。 2026-09-25 删除 Store-1 后 IP 从 `10.244.0.68` 变为 `10.244.0.97`，12.768 秒 Ready，PVC 不变。Gremlin 仍是 `1000000:2098771`。样本 `56862681` 在新 IP 后仍匹配。JNI 仍是标准 `8b8fb2ed…6dff`。不是 32 个样本，也不是 Topling，仍不勾选。证据 `evidence/a35-std-333-store-ip.json`。 随后不再删除 Pod，重读 32 个样本 0 不一致，9 个自环通过。证据 `evidence/a35-std-333-store-ip-adjacency.json`。 +同日 Topling `hg-closure-top-a35-s3` 图 `law_a35top_1m`：Loader 退出 0，201.39 秒，1000000 点、2098771 边、失败 0。Gremlin 重启前后都是 `1000000:2098771`，32 个样本 0 不一致，9 个自环通过。Store-0 SIGTERM 后同一 UID 和 IP，76.611 秒 Ready，JNI `c25ff6e6…dd38`，上一容器有 1 次 `db not closed`。只有 1 个 PD，没有 Store IP 变化，仍不勾选。证据 `evidence/a35-top-s3-loader.json`。 2026-09-25 删除 Store-1 后 IP 从 `10.244.0.33` 变为 `10.244.0.98`，12.797 秒 Ready。Gremlin 仍是 `1000000:2098771`，随后 32 个样本 0 不一致，9 个自环通过。JNI `c25ff6e6…dd38`。仍是 1 个 PD，失败重试未触发，不勾选。证据 `evidence/a35-top-s3-store-ip.json`、`evidence/a35-top-s3-store-ip-adjacency.json`。 +- [ ] 全量导入仅在容量评估通过后附加执行，不阻塞固定子集结论。2026-09-25 评估：原始图 41652230 节点、1468365182 条弧，主机剩余约 1.1T，但全量约是已加载子集边数的 700 倍，不在当前 e109 namespace 执行。证据 `evidence/full-law-capacity.json`。 + +数据线索：41,652,230 个原始 ID 已检查；固定子集有 2,098,771 条边、9 个自环、0 条重复边。固定子集已在多个旧集群导入。标准 `law_a35std_1m` 和 Topling `law_a35top_1m` 已导入，并在 Store IP 变化后核对过计数、32 个邻接样本和 9 个自环。失败重试没有被触发。全量 LAW 仍未导入。 Topling 3+3+3 图 `law_a35top333_1m` 也已导入，Loader 退出 0，141.109 秒，计数 `1000000:2098771`，32 个样本 0 不一致，9 个自环通过。没有 Store IP 变化。证据 `evidence/a35-top-333-loader.json`。 2026-09-25 删除该集群 Store-1 后 IP 从 `10.244.0.93` 变为 `10.244.0.99`，12.883 秒 Ready。DNS 指向新 IP。全图计数超时，32 个样本中只有 `54148543` IN 失败，9 个自环通过。重试仍超时。随后复查 DNS 是 `10.244.0.99`，到新 IP 8500 的 TCP 成功，但 gRPC 计数仍连接旧地址 `10.244.0.93:8500`。不是 DNS 缓存。不勾选。证据 `evidence/a35-top-333-store-ip.json`、`evidence/a35-top-333-store-ip-settled.json`。 随后 `nwjsq` SIGTERM 后计数恢复为 `1000000:2098771`。未再重启的 `qkpz5` 也返回同一计数，所以不能写成必须重启才恢复，也不能写成已修复。证据 `evidence/a35-top-333-server-refresh.json`。仍不勾选。 + +## P7 Benchmark + +- [ ] 核心功能通过后，按同 SHA 和固定资源完成标准/Topling 至少 3 轮对照。 +- [ ] 保存原始结果和统计，不把未跑项目写成性能收益。 + +## 阻塞修复 + +- [ ] 测试暴露的可复现缺陷在 `toplingdb` 分支修复并附回归测试。channel refresh 仍未提交。07:40 第三轮审查 3 是 HIGH_SEVERITY=1：`Subchannel shutdown invoked`。当时改为拒绝该描述并使用 `shutdownNow`，JUnit 14 个通过。08:00 第三轮审查 2 另有 HIGH_SEVERITY=1：`getChannels` 会发布含 null 的数组。随后改为失败时不发布数组，JUnit 15 个通过。这些结论都早于 07:54 之后的最终差异，没有第 4 轮审查，所以不勾选、不提交，也不自动再开审查。证据 `evidence/build/channel-refresh-round3-3.md`、`evidence/build/channel-refresh-round3-2.md`、`evidence/build/channel-refresh-junit.txt`。 +- [ ] 行为修复完成 3 名独立审查和必要重审后再勾选。嵌套 WAL 修复已在 `457295ac8` 提交，e109 Topling 单机完整镜像已证明独立 WAL 回滚；审查结论尚未按当前 SHA 收口。 2026-09-25 三份只读审查里，审查 1 认为成功路径符合提交说明，审查 2 和 3 各有 1 个 P1：嵌套恢复在移动失败时会把快照后的 WAL 留在活动目录，分开的 WAL 目录发布失败时会丢掉 checkpoint tail。本地已先退役全部活动日志再发布 tail，目录改名失败时改为复制 tail，符号链接 WAL 不再被替换成普通目录。`RocksDBSessionsTest` 17 个通过、0 失败、0 跳过，其中新增 3 个回归。证据 `evidence/build/wal-review-1.md`、`wal-review-2.md`、`wal-review-3.md`、`rocksdb-sessions-junit.txt`。重审还没结束，不勾选、不提交。 第一轮重审的审查 1 和 2 都是 HIGH_SEVERITY=2：同名旧 WAL 会被当成 tail，短复制也会在删除源文件前被接受。审查 3 当时还在重连。随后改为同目录改名隔离旧日志，并用长度核对后的临时文件发布 tail；失败时保留 data 目录里的 checkpoint 源文件。第二次 `RocksDBSessionsTest` 19 个通过、0 失败、0 跳过，耗时 1.274 秒。第二轮重审未完成，仍不勾选、不提交。 随后按重审 2 和迟到的第一轮重审 3，补上字节比对、rename 失败时删除活动日志，以及 WAL 目录无法挪走时改为原地安装。`RocksDBSessionsTest` 20 个通过、0 失败、0 跳过，耗时 1.355 秒。这已超过三轮修复上限，最新差异没有新的三份审查，所以仍不勾选、不提交。 +- [ ] 保持 Store shutdown fail-closed，不为超时测试强行关库。2026-09-25 `GrpcShutdownBarrierTest` 与 `ScanShutdownTest` 共 11 个通过，证据 `evidence/build/store-shutdown-junit.txt`。这还不是整项完成。 2026-09-25 另跑 `ContextClosedListenerTest` 2 个通过，0.487 秒：关闭会等 worker 清理，回调未结束时不关库。证据 `evidence/build/context-closed-listener-junit.txt`。这仍不是线上超时演练。 2026-09-25 `hg-closure-top-a35-333` Store-0：最终 41 次 HTTP 200，没有非 200，74.674 秒 Ready，同一 UID 和 IP。日志有 closed gRPC 和 closing all rocksdb，随后仍有 `db not closed`，`still_waiting` 是 0，`forced_db_close` 是 false。证据 JSON 没有信号名、顶点 ID、请求 URL 或发信号瞬间的计数。证据 `evidence/a35-top-333-shutdown-inflight.json`。仍不是卡住 worker 的超时演练，不勾选。 + +## 明确不由本机构成完成 + +- macOS ARM/Intel 最终 SHA 的 Cypher CI。 +- 发行审批和公共仓库发布。 + +## 终态但未勾选 + +- HStore `snapshot_create` 终态是失败，不勾选 P4。标准 `hg-closure-std-a35-333` 图 `hugegraph` 的 http 是 500，message 是 `UnsupportedOperationException: createSnapshot`，server_image 是 `hugegraph/server:closure-a35ebeb17`，source_revision 是 `a35ebeb17faaa191d2360ad5fea02b8aeb04eb60`。证据 `evidence/a35-std-333-server-loss.json`。Topling `hg-closure-top-a35-333` 图 `hugegraph` 的 http 是 `500`，message 是 `createSnapshot`，exception 是 `class java.lang.UnsupportedOperationException`，server_image 是 `docker.io/hugegraph/server:closure-a35ebeb17`，source_revision 是 `a35ebeb17faaa191d2360ad5fea02b8aeb04eb60`。证据 `evidence/a35-top-333-snapshot.json`。 +- 网络分区终态是后置，不勾选。`kind_node_count` 是 1，`kubectl_node_count` 是 1。kind nodes stdout 是 `kind-control-plane`。kubectl nodes stdout 是 `kind-control-plane Ready control-plane 25d v1.37.0`。`crd_stdout_empty` 是 true，crds rc 是 0，crds stderr 是 `No resources found`。purpose 写了没有新增节点，也没有安装 CRD。证据 `evidence/a35-kind-partition-capability.json`。 +- 固定子集的失败重试终态是后置，不勾选 P6。标准 `hg-closure-std-a35-333` 图 `law_a35std_1m` 的 loader_exit 是 0，elapsed_seconds 是 141.323，source_revision 是 `a35ebeb17faaa191d2360ad5fea02b8aeb04eb60`。log_tail 含 `vertex insert failure : 0` 和 `edge insert failure : 0`。证据 `evidence/a35-std-333-loader.json`。Topling `hg-closure-top-a35-333` 图 `law_a35top333_1m` 的 loader_exit 是 0，elapsed_seconds 是 141.109，source_revision 是 `a35ebeb17faaa191d2360ad5fea02b8aeb04eb60`。log_tail 含 `vertex insert failure : 0` 和 `edge insert failure : 0`。证据 `evidence/a35-top-333-loader.json`。 +- P7 终态是未开始，不勾选，也不写成性能收益。`benchmark_name_count` 是 0，`benchmark_names` 是 `[]`。purpose 写了没有运行 benchmark。`standard_snapshot_http` 是 500，`standard_snapshot_message` 是 `UnsupportedOperationException: createSnapshot`。`topling_snapshot_http` 是 `500`，`topling_snapshot_message` 是 `createSnapshot`。证据 `evidence/a35-p7-not-started.json`。 +- HA Compose 与 Helm 的配置对齐终态是不勾选。证据 `evidence/ha-compose-helm-alignment.md` 写了：端口、副本数、Store `/v1/health`、Server `/versions` 和默认 RocksDB provider 是对齐的。PD readiness、启动依赖、反亲和、资源限制、provider marker、Hubble 和凭据来源不一致。因此这项不能勾成已经对齐。同文件写了没有修改 Compose、Helm 或正在运行的集群，以及因此 P5 的这一项继续不勾选。 +- Store 关闭证据终态是不勾选，也不是卡住 worker 的超时演练。`ok_200` 是 41，`non_200` 是 `[]`，`previous.still_waiting` 是 0，`previous.db_not_closed` 是 1，`previous.closed_grpc` 是 1，`previous.closing` 是 1，`forced_db_close` 是 false。before 与 after 的 uid 都是 `434437bd-f2fa-4ead-8eed-fc7b590968ce`，ip 都是 `10.244.0.89`，pvc 都是 `store-data-hg-closure-top-a35-333-hugegraph-store-0`。before `restarts` 是 1，after `restarts` 是 2，`ready_seconds` 是 74.674。`first_attempt_note` 是 `The first shell loop issued no requests because sh rejected SECONDS.`。sample 含 `closed gRPC callbacks, scan and TTL workers`、`closing all rocksdb....` 和 `db not closed`。证据 `evidence/a35-top-333-shutdown-inflight.json`。 +- 全量 LAW 终态是后置，不勾选，也不阻塞固定子集。`raw.nodes` 是 41652230，`raw.arcs` 是 1468365182。`subset.vertices` 是 1000000，`subset.edges` 是 2098771。`edge_scale_vs_subset` 是 699.6。`host_filesystem.avail` 是 `1.1T`，`store_pvc_request` 是 `50Gi`。`decision` 是 `Do not start the full import on the current e109 namespaces. The compressed source and host free space exist, but the full arc count is about 700 times the loaded subset and would share the live validation stores. This item does not block the fixed-subset conclusion.`。证据 `evidence/full-law-capacity.json`。 diff --git a/.specs/hugegraph-server/ToplingDB/design.md b/.specs/hugegraph-server/ToplingDB/design.md new file mode 100644 index 0000000000..38685cbbcd --- /dev/null +++ b/.specs/hugegraph-server/ToplingDB/design.md @@ -0,0 +1,159 @@ +# Design of ToplingDB Easy Migrate + +## Overview + +ToplingDB is an optional RocksDB-compatible runtime for HugeGraph. HugeGraph +Server, PD, and Store keep their existing RocksDB Java API calls. The +integration does not add a Java provider SPI and does not create or manage a +`SidePluginRepo`. + +Easy Migrate is activated before the JVM starts: + +1. The installation layer prepares the selected JAR, native library, and Web + resources. +2. The startup layer selects `rocksdb` or `topling` from the component + configuration. +3. For `topling`, the startup layer exports + `TOPLINGDB_EASY_MIGRATE_CONF`, `LD_LIBRARY_PATH`, and `LD_PRELOAD`. +4. The JVM opens and closes RocksDB through the existing Java API. +5. The Topling native hook imports the YAML options and tracks the normal + DB/column-family lifecycle. + +## Goals + +- Keep the standard RocksDB Java API and the existing HugeGraph storage + lifecycle. +- Make ToplingDB an explicit, startup-time selection. +- Apply component-specific YAML configuration through Easy Migrate. +- Preserve a strict standard RocksDB fallback when ToplingDB is not selected. +- Keep Server, PD, and Store shutdown behavior uniform. + +## Runtime Architecture + +```mermaid +sequenceDiagram + autonumber + participant User + participant Install as prepare-topling.sh + participant Start as component start script + participant Preload as preload-topling.sh + participant JVM as HugeGraph JVM + participant Hook as Topling native hook + participant Rocks as RocksDB Java API + + User->>Install: Prepare selected runtime + User->>Start: Start Server, PD, or Store + Start->>Preload: Select configured provider + alt provider is rocksdb or unset + Preload->>Preload: Keep standard RocksDB runtime + else provider is topling + Preload->>Preload: Validate prepared Topling runtime + Preload->>Preload: Export TOPLINGDB_EASY_MIGRATE_CONF + Preload->>Hook: Preload native library + end + Start->>JVM: Launch component + JVM->>Rocks: Normal RocksDB open/CF operations + Hook->>Hook: Apply YAML options and retain DB/CF state + JVM->>Rocks: Normal CF/DB close + Hook->>Hook: MaybeForgetCF / MaybeForgetDB +``` + +HugeGraph Java code must not: + +- define or load `RocksDBProviderLoader` or `ToplingRocksDBProvider`; +- reflectively construct a `SidePluginRepo`; +- call `SidePluginRepo.openDB()` or `closeAllDB()`; +- treat ToplingDB as a separate service process. + +## Component Boundaries + +| Component | RocksDB role | Easy Migrate configuration | +|---|---|---| +| Standalone Server | Opens its local RocksDB backend | `conf/toplingdb.yaml` | +| PD | Opens PD metadata RocksDB | `conf/rocksdb_pd.yaml` | +| Store | Opens Store data RocksDB | `conf/rocksdb_store.yaml` | +| HStore Server | Remote client; no local RocksDB | Not applicable | + +HStore does not make HugeGraph Server a local RocksDB owner. ToplingDB for a +distributed deployment applies to PD and Store, where RocksDB is actually +opened. + +## Configuration Contract + +`rocksdb.provider` is the explicit runtime selector: + +```properties +# Default +#rocksdb.provider=rocksdb + +# Optional ToplingDB runtime +rocksdb.provider=topling +``` + +When `topling` is selected, the startup script must export a readable, +component-specific YAML path: + +```bash +export TOPLINGDB_EASY_MIGRATE_CONF=/path/to/conf/toplingdb.yaml +``` + +The YAML keeps `DBOptions.default` as the global fallback for databases that do +not have a more specific mapping. `DBOptions.log` is a deliberate specialized +profile and must remain distinct. + +## Open and Close Lifecycle + +The Java lifecycle stays unchanged: + +```text +RocksDB.open(...) / openReadOnly(...) + -> create, open, drop, and close column families + -> ColumnFamilyHandle.close() + -> RocksDB.close() +``` + +With Easy Migrate enabled, the native hook observes this lifecycle, applies the +matching options, retains native DB/CF state while it is in use, and runs +`MaybeForgetCF` and `MaybeForgetDB` during normal close. Java must not add a +second ownership path. + +The supported shutdown sequence is: + +```text +SIGTERM + -> HugeGraph/PD/Store graceful shutdown + -> normal CF and DB close + -> native MaybeForgetCF and MaybeForgetDB + -> JVM exit +``` + +Store already has a bounded shutdown wait. A timeout means the ordinary Store +shutdown did not finish within that boundary; it is not a reason to call +Topling-specific cleanup APIs. General thread-exit fixes are outside this +integration. + +## Data Compatibility and Rollback + +Runtime selection is not a data migration or rollback. The provided ToplingDB +configuration may enable Topling-specific WAL, SST, memtable, or compression +behavior. After ToplingDB writes data, changing only the JAR or provider is not +a safe rollback. + +Before migration, create a complete RocksDB-consistent checkpoint or backup. +To roll back, stop all writers, restore the complete pre-migration snapshot +into an empty data directory, restore the matching standard RocksDB runtime, +and validate the database before accepting traffic. + +## Verification + +ToplingDB DB/CF functional tests must run with +`TOPLINGDB_EASY_MIGRATE_CONF` set to a readable component configuration. +Unsetting the variable is allowed only for standard RocksDB tests or an ABI +diagnostic that does not open a database. + +The focused runtime test verifies: + +- the selected JAR and native library mapping; +- Easy Migrate configuration presence for ToplingDB; +- DB create, write, close, and reopen; +- CF create, drop, recreate, close, and reopen. diff --git a/.specs/hugegraph-server/ToplingDB/requirements.md b/.specs/hugegraph-server/ToplingDB/requirements.md new file mode 100644 index 0000000000..3c43a6c333 --- /dev/null +++ b/.specs/hugegraph-server/ToplingDB/requirements.md @@ -0,0 +1,48 @@ +# Requirements of ToplingDB Easy Migrate + +## Runtime Compatibility + +HugeGraph must support standard RocksDB by default and an explicitly selected +ToplingDB runtime without changing the RocksDB Java API used by Server, PD, or +Store. + +Acceptance criteria: + +- ToplingDB is selected only by component configuration. +- Missing or unreadable Easy Migrate configuration fails ToplingDB startup. +- Standard RocksDB does not require an Easy Migrate configuration. +- No HugeGraph Java code creates or manages a `SidePluginRepo`. + +## External Configuration + +The startup layer must set `TOPLINGDB_EASY_MIGRATE_CONF` to the configuration +for the component that owns RocksDB. The native hook must apply the YAML +options to normal RocksDB open and column-family operations. + +`DBOptions.default` remains the fallback for unmatched databases. +`DBOptions.log` remains a separate specialized mapping. + +## Component Ownership + +Standalone Server, PD, and Store may own local RocksDB instances. A Server +using the HStore backend is a remote client and must not be described as +opening a local RocksDB instance. + +## Graceful Shutdown + +SIGTERM must enter the existing HugeGraph, PD, or Store graceful shutdown, +which closes column families and databases through the normal RocksDB API. +Easy Migrate then releases native tracking through `MaybeForgetCF` and +`MaybeForgetDB`. + +No operation or test may require `SidePluginRepo.closeAllDB()`. The existing +bounded Store shutdown timeout remains a general lifecycle boundary, not a +ToplingDB-specific cleanup mechanism. + +## Verification + +- Standard RocksDB DB/CF tests run without Easy Migrate. +- ToplingDB ABI diagnostics may inspect only JAR/native availability and + mappings. +- Every ToplingDB DB/CF functional test runs with a readable + `TOPLINGDB_EASY_MIGRATE_CONF`. diff --git a/.specs/hugegraph-server/ToplingDB/task.md b/.specs/hugegraph-server/ToplingDB/task.md new file mode 100644 index 0000000000..757d4f88e1 --- /dev/null +++ b/.specs/hugegraph-server/ToplingDB/task.md @@ -0,0 +1,32 @@ +# Tasks of ToplingDB Easy Migrate + +## Runtime Integration + +- [x] Keep Server, PD, and Store on the existing RocksDB Java API. +- [x] Prepare the optional ToplingDB JAR, native library, and Web resources + before service startup. +- [x] Select ToplingDB explicitly from component configuration. +- [x] Export a readable component-specific + `TOPLINGDB_EASY_MIGRATE_CONF` before the JVM starts. +- [x] Let the native Easy Migrate hook apply options and track DB/CF lifecycle. +- [x] Keep `DBOptions.default` as the global fallback and retain the dedicated + `DBOptions.log` profile. + +## Component Integration + +- [x] Use `conf/toplingdb.yaml` for standalone Server. +- [x] Use `conf/rocksdb_pd.yaml` for PD. +- [x] Use `conf/rocksdb_store.yaml` for Store. +- [x] Keep HStore-backed Server as a remote client without local RocksDB + ownership. + +## Lifecycle and Testing + +- [x] Use normal RocksDB Java close paths during graceful shutdown. +- [x] Rely on native `MaybeForgetCF` and `MaybeForgetDB`; do not add Java-side + repository cleanup. +- [x] Verify JAR/native mapping without constructing a plugin repository. +- [x] Run every ToplingDB DB/CF functional test with Easy Migrate configuration + enabled. +- [ ] Keep general Store shutdown timeout and unrelated thread-exit fixes in + their existing workstream. diff --git a/README.md b/README.md index 0e48ced340..93e9aab17e 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,10 @@ --- -**Quick Navigation:** [Architecture](#architecture) • [Quick Start](#quick-start) • [Module Map](#module-map) • [Ecosystem](#ecosystem) • [For Contributors](#for-contributors) • [Community](#community) +**Quick Navigation:** [Architecture](#architecture) • [Quick Start](#quick-start) • +[ToplingDB](docs/toplingdb/README.md) • [Module Map](#module-map) • +[Ecosystem](#ecosystem) • [For Contributors](#for-contributors) • +[Community](#community) --- @@ -31,7 +34,12 @@ achieved through the powerful [Gremlin](https://tinkerpop.apache.org/gremlin.htm - **Schema Metadata Management**: VertexLabel, EdgeLabel, PropertyKey, and IndexLabel - **Multi-type Indexes**: Exact query, range query, and complex conditions combination query -- **Plug-in Backend Store Framework**: RocksDB powers standalone deployments and HStore powers distributed clusters. See the [backend evolution guide](hugegraph-server/README.md#backend-evolution-and-compatibility) for compatibility details. +- **Plug-in Backend Store Framework**: RocksDB powers standalone deployments + and HStore powers distributed clusters. ToplingDB is an optional + RocksDB-compatible runtime; see the + [ToplingDB documentation](docs/toplingdb/README.md). See the + [backend evolution guide](hugegraph-server/README.md#backend-evolution-and-compatibility) + for compatibility details. - **Big Data Integration**: Seamless integration with `Flink`/`Spark`/`HDFS` - **Complete Graph Ecosystem**: In/out-memory Graph Computing + Graph Visualization & Tools + Graph Learning & AI - **Dual Query Language Support**: [Gremlin](https://tinkerpop.apache.org/gremlin.html) (via [Apache TinkerPop 3](https://tinkerpop.apache.org/)) and [Cypher](https://en.wikipedia.org/wiki/Cypher_(query_language)) (OpenCypher) diff --git a/docker/README.md b/docker/README.md index a9d43dcb1f..c00051a244 100644 --- a/docker/README.md +++ b/docker/README.md @@ -18,6 +18,15 @@ cd docker Standalone uses `hugegraph/hugegraph:${HUGEGRAPH_VERSION:-latest}`. The HStore topologies use the matching `hugegraph/pd`, `hugegraph/store`, and `hugegraph/server` tags. Hubble is selected independently with `${HUBBLE_IMAGE:-hugegraph/hubble:latest}`. +ToplingDB is selected by injecting the documented image, provider, data-root, +marker, volume, and pull-policy variables in the Developers section. Its +published images target Linux x86_64 (`linux/amd64`); on macOS, use Docker +Desktop or OrbStack container mode and set `--platform linux/amd64` when the +engine does not select it automatically. Native macOS execution, including +Intel, is outside this ToplingDB matrix. Standalone selects Topling for the +Server, while HStore selects it only for PD and Store. The HStore Server +remains the standard image and never loads a local ToplingDB runtime. + ### Create the authentication environment Create `.env` once. Replace `replace-with-your-password` with an administrator password that you choose; the command generates and persists a random 32-byte JWT secret. For this simple single-quoted format, do not use a password that contains a single quote or newline. @@ -206,6 +215,147 @@ HUGEGRAPH_VERSION=1.7.0 \ docker compose -f docker-compose-hstore.yml up -d ``` +--- + +## Environment Variable Reference + +Configuration is injected via environment variables. The old `docker/configs/application-pd*.yml` and `docker/configs/application-store*.yml` files are no longer used. + +The generic Compose files pass explicit `rocksdb` defaults to each local +RocksDB owner. Selecting a `:topling` image therefore requires setting its +provider, data-root, marker, and provider-specific volume together; changing +only the image tag is intentionally not a provider switch. + +### PD Environment Variables + +| Variable | Required | Default | Maps To (`application.yml`) | Description | +|----------|----------|---------|-----------------------------|-------------| +| `HG_PD_GRPC_HOST` | Yes | — | `grpc.host` | This node's hostname/IP for gRPC | +| `HG_PD_RAFT_ADDRESS` | Yes | — | `raft.address` | This node's Raft address (e.g. `pd0:8610`) | +| `HG_PD_RAFT_PEERS_LIST` | Yes | — | `raft.peers-list` | All PD peers (e.g. `pd0:8610,pd1:8610,pd2:8610`) | +| `HG_PD_INITIAL_STORE_LIST` | Yes | — | `pd.initial-store-list` | Expected stores (e.g. `store0:8500,store1:8500,store2:8500`) | +| `HG_PD_GRPC_PORT` | No | `8686` | `grpc.port` | gRPC server port | +| `HG_PD_REST_PORT` | No | `8620` | `server.port` | REST API port | +| `HG_PD_ROCKSDB_PROVIDER` | No | `rocksdb` | `rocksdb.provider` | `rocksdb` or `topling`; set `topling` explicitly with a Topling image | +| `HG_PD_DATA_PATH` | No | Provider-specific path | `pd.data-path` | `/hugegraph-pd/pd_data` for RocksDB or `/hugegraph-pd/topling-pd-data` for ToplingDB | +| `HG_PD_ENFORCE_PROVIDER_MARKER` | No | `false` | Startup safety gate | Rejects an unmarked non-empty data path when `true`; enabled by Topling images | +| `HG_PD_INITIAL_STORE_COUNT` | No | `1` | `pd.initial-store-count` | Min stores for cluster availability | + +**Deprecated aliases** (still work but log a warning): + +| Deprecated | Use Instead | +|------------|-------------| +| `GRPC_HOST` | `HG_PD_GRPC_HOST` | +| `RAFT_ADDRESS` | `HG_PD_RAFT_ADDRESS` | +| `RAFT_PEERS` | `HG_PD_RAFT_PEERS_LIST` | +| `PD_INITIAL_STORE_LIST` | `HG_PD_INITIAL_STORE_LIST` | + +### Store Environment Variables + +| Variable | Required | Default | Maps To (`application.yml`) | Description | +|----------|----------|---------|-----------------------------|-------------| +| `HG_STORE_PD_ADDRESS` | Yes | — | `pdserver.address` | PD gRPC addresses (e.g. `pd0:8686,pd1:8686,pd2:8686`) | +| `HG_STORE_GRPC_HOST` | Yes | — | `grpc.host` | This node's hostname (e.g. `store0`) | +| `HG_STORE_RAFT_ADDRESS` | Yes | — | `raft.address` | This node's Raft address (e.g. `store0:8510`) | +| `HG_STORE_GRPC_PORT` | No | `8500` | `grpc.port` | gRPC server port | +| `HG_STORE_REST_PORT` | No | `8520` | `server.port` | REST API port | +| `HG_STORE_ROCKSDB_PROVIDER` | No | `rocksdb` | `rocksdb.provider` | `rocksdb` or `topling`; set `topling` explicitly with a Topling image | +| `HG_STORE_DATA_PATH` | No | Provider-specific path | `app.data-path` | `/hugegraph-store/storage` for RocksDB or `/hugegraph-store/topling-storage` for ToplingDB | +| `HG_STORE_ENFORCE_PROVIDER_MARKER` | No | `false` | Startup safety gate | Rejects an unmarked non-empty data path when `true`; enabled by Topling images | + +**Deprecated aliases** (still work but log a warning): + +| Deprecated | Use Instead | +|------------|-------------| +| `PD_ADDRESS` | `HG_STORE_PD_ADDRESS` | +| `GRPC_HOST` | `HG_STORE_GRPC_HOST` | +| `RAFT_ADDRESS` | `HG_STORE_RAFT_ADDRESS` | + +### Server Environment Variables + +| Variable | Required | Default | Maps To | Description | +|----------|----------|---------|-----------------------------|-------------| +| `HG_SERVER_BACKEND` | Yes | — | `backend` in `hugegraph.properties` | Storage backend (e.g. `hstore`) | +| `HG_SERVER_ROCKSDB_PROVIDER` | No | `rocksdb` | `rocksdb.provider` | RocksDB JNI provider (`rocksdb` or `topling`); set `topling` explicitly with a Topling image | +| `HG_SERVER_DATA_PATH` | No | Provider-specific image path | `rocksdb.data_path`, `rocksdb.wal_path` | RocksDB data root; standard uses `/hugegraph-server/rocksdb-data`, Topling uses `/hugegraph-server/topling-data` | +| `HG_SERVER_ENFORCE_PROVIDER_MARKER` | No | `false` | Startup safety gate | Rejects an unmarked non-empty data root when `true`; enabled by the Topling image | +| `HG_SERVER_PD_PEERS` | Yes | — | `pd.peers` | PD cluster addresses (e.g. `pd0:8686,pd1:8686,pd2:8686`) | +| `HG_SERVER_CLUSTER` | No | — | `cluster` in `rest-server.properties` | PD discovery application name; single-node Compose uses `hg` to match Hubble | +| `HG_SERVER_USE_PD` | No | — | `usePD` in `rest-server.properties` | Enables Server PD registration and discovery | +| `HG_SERVER_REST_URL` | No | — | `restserver.url` | Address registered with PD and used by clients | +| `HG_SERVER_MIN_FREE_MEMORY` | No | — | `restserver.min_free_memory` | Minimum free-memory guard in MB; local Compose uses `0` | +| `HG_SERVER_AUTH_TOKEN_SECRET` | No | generated in auth mode | `auth.token_secret` | Shared JWT secret for REST and embedded Gremlin authentication; explicit values must be at least 32 bytes | +| `STORE_REST` | No | — | Used by `wait-partition.sh` | Store REST endpoint for partition verification (e.g. `store0:8520`) | +| `PASSWORD` | No | — | Enables auth and sets `auth.admin_pa` | Initial administrator password; disabled init-store does not read it from stdin, but the entrypoint still applies it to the PD bootstrap path | +| `HG_SERVER_INIT_STORE_ENABLED` | No | `true` | `init_store.enabled` in `rest-server.properties` | Set `false` in PD/HStore deployments so init-store skips local backend and admin initialization | + +> **The built-in authenticator with `HG_SERVER_INIT_STORE_ENABLED=false` requires `usePD=true` and an HStore-backed `auth.graph_store`, unless `auth.remote_url` delegates auth elsewhere.** With init-store skipped, the server creates the built-in admin in PD metadata, and only an HStore auth graph uses the PD-backed auth manager that can read that account. init-store exits non-zero when the combination is unusable, rather than leaving a server nobody can log in to. A custom `auth.authenticator` is exempt because it manages its own identities. +> +> For a local RocksDB backend, init-store writes its completion marker below the provider data root at `.hugegraph-state/init_complete`. The marker therefore survives container recreation with the data volume. A standard RocksDB deployment migrates the legacy `docker/init_complete` marker once. HStore keeps the existing container-local marker behavior. A skipped run records nothing, so a later re-enable can still initialize. +> +> The entrypoint maps **`PASSWORD` to `auth.admin_pa`** before init-store runs. A disabled init-store does not read the password from standard input, but the PD startup path uses the explicit `auth.admin_pa` value when it first creates the administrator. Changing it later does not rotate an existing password. + +The single-node Compose files also accept these deployment-level overrides: + +| Variable | Default | Description | +|----------|---------|-------------| +| `HUGEGRAPH_SERVER_IMAGE` | `hugegraph/hugegraph:` (standalone); `hugegraph/server:` (HStore) | Server image reference | +| `HUGEGRAPH_SERVER_PULL_POLICY` | `missing` (`build` for dev) | Server pull policy | +| `HUBBLE_IMAGE` | `hugegraph/hubble:latest` | Complete Hubble image reference | +| `HUBBLE_PULL_POLICY` | `missing` | Hubble pull policy | +| `HUBBLE_PUBLISH_HOST` | `127.0.0.1` | Hubble host bind address; remote access requires an HTTPS reverse proxy | +| `HUGEGRAPH_ADMIN_PASSWORD` | required (`docker/.env`) | Initial admin password; no public default is provided | +| `HUGEGRAPH_AUTH_TOKEN_SECRET` | generated | JWT signing secret; explicit values must be at least 32 bytes | + +When authentication is enabled and no token secret is supplied, the Server entrypoint generates a random secret and writes it to both authentication configurations. The value is reused on container restart while the container filesystem is preserved. To preserve tokens across container recreation, generate a compatible secret once and add it to the mode-600 `docker/.env`: + +```bash +( + set -euo pipefail + cd docker + secret_pattern='^[[:space:]]*(export[[:space:]]+)?HUGEGRAPH_AUTH_TOKEN_SECRET[[:space:]]*=' + secret_count="$(grep -Ec "${secret_pattern}" .env || true)" + case "${secret_count}" in + 0) + command -v openssl >/dev/null 2>&1 + token_secret="$(openssl rand -hex 32)" + if (( ${#token_secret} != 64 )); then + echo "Failed to generate a 64-character token secret" >&2 + exit 1 + fi + printf "HUGEGRAPH_AUTH_TOKEN_SECRET='%s'\n" "${token_secret}" >> .env + unset token_secret + echo "Generated HUGEGRAPH_AUTH_TOKEN_SECRET" + ;; + 1) + token_secret="$(sed -nE "s/${secret_pattern}'([^']*)'[[:space:]]*$/\\2/p" .env)" + if (( ${#token_secret} < 32 )); then + echo "Existing token secret must contain at least 32 bytes" >&2 + exit 1 + fi + unset token_secret + echo "HUGEGRAPH_AUTH_TOKEN_SECRET already exists; reusing it" + ;; + *) + echo "Duplicate HUGEGRAPH_AUTH_TOKEN_SECRET entries; repair .env" >&2 + exit 1 + ;; + esac + chmod 600 .env +) +``` + +The entrypoint rejects shorter explicit values before changing either Server configuration file. + +**Deprecated aliases** (still work but log a warning): + +| Deprecated | Use Instead | +|------------|-------------| +| `BACKEND` | `HG_SERVER_BACKEND` | +| `PD_PEERS` | `HG_SERVER_PD_PEERS` | + +--- + Select Hubble independently: ```bash @@ -273,6 +423,75 @@ HUBBLE_PULL_POLICY=never \ docker compose -f docker-compose-hstore.yml up -d --wait ``` +### ToplingDB variants + +Build the Linux x86_64 Topling images from the shared Bake graph: + +```bash +RUNTIME_VARIANT=topling IMAGE_TAG=topling \ + docker buildx bake --file docker/bake.hcl +``` + +Run standalone HugeGraph with its isolated Topling data volume: + +```bash +HUGEGRAPH_SERVER_IMAGE=hugegraph/hugegraph:topling \ +HUGEGRAPH_SERVER_PULL_POLICY=always \ +HG_SERVER_ROCKSDB_PROVIDER=topling \ +HG_SERVER_DATA_PATH=/hugegraph-server/topling-data \ +HG_SERVER_ENFORCE_PROVIDER_MARKER=true \ +HUGEGRAPH_SERVER_VOLUME=server-topling-data \ +docker compose -f docker-compose.yml \ + up -d --wait +``` + +The same parameters work with a published image or a locally built image. The +standard `server-data` volume is not used; Topling data is stored in the +provider-specific `server-topling-data` volume. + +Run the minimal distributed 1+1+1 topology from source with Topling PD and +Store runtimes. The HStore Server remains standard: + +```bash +HUGEGRAPH_PD_IMAGE=local/hugegraph-pd:topling \ +HUGEGRAPH_PD_PULL_POLICY=never \ +HUGEGRAPH_PD_BUILD_TARGET=topling \ +HUGEGRAPH_PD_VOLUME=pd-topling-data \ +HG_PD_ROCKSDB_PROVIDER=topling \ +HG_PD_DATA_PATH=/hugegraph-pd/topling-pd-data \ +HG_PD_ENFORCE_PROVIDER_MARKER=true \ +HUGEGRAPH_STORE_IMAGE=local/hugegraph-store:topling \ +HUGEGRAPH_STORE_PULL_POLICY=never \ +HUGEGRAPH_STORE_BUILD_TARGET=topling \ +HUGEGRAPH_STORE_VOLUME=store-topling-data \ +HG_STORE_ROCKSDB_PROVIDER=topling \ +HG_STORE_DATA_PATH=/hugegraph-store/topling-storage \ +HG_STORE_ENFORCE_PROVIDER_MARKER=true \ +HUGEGRAPH_SERVER_IMAGE=hugegraph/server:dev \ +HUGEGRAPH_SERVER_PULL_POLICY=build \ +docker compose -f docker-compose-hstore.yml -f docker-compose.dev.yml \ + up -d --build --wait +``` + +For published images, replace the two local image values and `never` policies +with `hugegraph/{pd,store}:topling` and `always`. Set +`HUGEGRAPH_SERVER_IMAGE=hugegraph/server:topling` and +`HUGEGRAPH_SERVER_PULL_POLICY=always` for the matching HStore Server. The +standard `pd-data` and `store-data` volumes are not mounted. + +The HA reference uses the same parameters with +`docker-compose-3pd-3store-3server.yml`; set +`HUGEGRAPH_PD0_VOLUME`, `HUGEGRAPH_PD1_VOLUME`, `HUGEGRAPH_PD2_VOLUME`, +`HUGEGRAPH_STORE0_VOLUME`, `HUGEGRAPH_STORE1_VOLUME`, and +`HUGEGRAPH_STORE2_VOLUME` to their `*-topling-data` names. + +The generic Compose files inject image, provider, data-root, and volume +parameters; no Topling-specific Compose file is maintained. Each Topling owner +uses a provider-specific named volume and marker. Do not reuse a standard +RocksDB volume with a Topling image. See +[`docs/toplingdb/README.md`](../docs/toplingdb/README.md) for distribution +configuration, provider markers, restart checks, and troubleshooting. + ### Image build arguments and cache refresh Run image builds from the **repository root**. Direct Dockerfile builds and Bake use the same defaults: build the Server, PD, and Store distributions plus their dependencies (`-pl ... -am`), and reuse the OS package layer across source changes. diff --git a/docker/bake.hcl b/docker/bake.hcl index 2fc1cf4626..0713ba0325 100644 --- a/docker/bake.hcl +++ b/docker/bake.hcl @@ -31,6 +31,10 @@ variable "SOURCE_REVISION" { default = "local" } +variable "SOURCE_URL" { + default = "https://github.com/apache/hugegraph" +} + variable "IMAGE_TAG" { default = "local" } @@ -43,6 +47,14 @@ variable "EXPORT_CACHE" { default = false } +variable "RUNTIME_VARIANT" { + default = "standard" + validation { + condition = contains(["standard", "topling"], RUNTIME_VARIANT) + error_message = "RUNTIME_VARIANT must be standard or topling" + } +} + target "_common" { context = "." args = { @@ -50,8 +62,11 @@ target "_common" { MAVEN_PROJECTS = MAVEN_PROJECTS RUNTIME_DEPS_EPOCH = RUNTIME_DEPS_EPOCH SOURCE_REVISION = SOURCE_REVISION + SOURCE_REPOSITORY = SOURCE_URL } - platforms = [ + platforms = RUNTIME_VARIANT == "topling" ? [ + "linux/amd64", + ] : [ "linux/amd64", "linux/arm64", ] @@ -77,6 +92,7 @@ target "build-cache" { target "pd" { inherits = ["_common"] dockerfile = "hugegraph-pd/Dockerfile" + target = RUNTIME_VARIANT tags = ["hugegraph/pd:${IMAGE_TAG}"] output = ["type=docker"] cache-from = [ @@ -91,6 +107,7 @@ target "pd" { target "store" { inherits = ["_common"] dockerfile = "hugegraph-store/Dockerfile" + target = RUNTIME_VARIANT tags = ["hugegraph/store:${IMAGE_TAG}"] output = ["type=docker"] cache-from = [ @@ -119,6 +136,7 @@ target "server-hstore" { target "server-standalone" { inherits = ["_common"] dockerfile = "hugegraph-server/Dockerfile" + target = RUNTIME_VARIANT tags = ["hugegraph/hugegraph:${IMAGE_TAG}"] output = ["type=docker"] cache-from = [ diff --git a/docker/docker-compose-3pd-3store-3server.yml b/docker/docker-compose-3pd-3store-3server.yml index 999f2126fb..f12e1e628e 100644 --- a/docker/docker-compose-3pd-3store-3server.yml +++ b/docker/docker-compose-3pd-3store-3server.yml @@ -23,17 +23,23 @@ networks: volumes: hg-pd0-data: + hg-pd0-topling-data: hg-pd1-data: + hg-pd1-topling-data: hg-pd2-data: + hg-pd2-topling-data: hg-store0-data: + hg-store0-topling-data: hg-store1-data: + hg-store1-topling-data: hg-store2-data: + hg-store2-topling-data: hubble-data: # ── Shared service defaults ────────────────────────────────────────── x-pd-common: &pd-common - image: hugegraph/pd:${HUGEGRAPH_VERSION:-latest} - pull_policy: missing + image: ${HUGEGRAPH_PD_IMAGE:-hugegraph/pd:${HUGEGRAPH_VERSION:-latest}} + pull_policy: ${HUGEGRAPH_PD_PULL_POLICY:-${HUGEGRAPH_PULL_POLICY:-missing}} restart: unless-stopped networks: [hg-net] healthcheck: @@ -44,8 +50,8 @@ x-pd-common: &pd-common start_period: 120s x-store-common: &store-common - image: hugegraph/store:${HUGEGRAPH_VERSION:-latest} - pull_policy: missing + image: ${HUGEGRAPH_STORE_IMAGE:-hugegraph/store:${HUGEGRAPH_VERSION:-latest}} + pull_policy: ${HUGEGRAPH_STORE_PULL_POLICY:-${HUGEGRAPH_PULL_POLICY:-missing}} restart: unless-stopped networks: [hg-net] depends_on: @@ -77,8 +83,8 @@ x-server-environment: &server-environment PD_AUTH_PASSWORD: ${HG_PD_AUTH_SECRET_KEY:?set HG_PD_AUTH_SECRET_KEY in .env; see docker/README.md} x-server-common: &server-common - image: hugegraph/server:${HUGEGRAPH_VERSION:-latest} - pull_policy: missing + image: ${HUGEGRAPH_SERVER_IMAGE:-hugegraph/server:${HUGEGRAPH_VERSION:-latest}} + pull_policy: ${HUGEGRAPH_SERVER_PULL_POLICY:-missing} restart: unless-stopped networks: [hg-net] depends_on: @@ -110,12 +116,14 @@ services: HG_PD_RAFT_ADDRESS: pd0:8610 HG_PD_RAFT_PEERS_LIST: pd0:8610,pd1:8610,pd2:8610 HG_PD_INITIAL_STORE_LIST: store0:8500,store1:8500,store2:8500 - HG_PD_DATA_PATH: /hugegraph-pd/pd_data + HG_PD_DATA_PATH: ${HG_PD_DATA_PATH:-/hugegraph-pd/pd_data} + HG_PD_ROCKSDB_PROVIDER: ${HG_PD_ROCKSDB_PROVIDER:-rocksdb} + HG_PD_ENFORCE_PROVIDER_MARKER: ${HG_PD_ENFORCE_PROVIDER_MARKER:-false} HG_PD_INITIAL_STORE_COUNT: 3 HG_PD_AUTH_SECRET_KEY: ${HG_PD_AUTH_SECRET_KEY:?set HG_PD_AUTH_SECRET_KEY in .env; see docker/README.md} ports: ["8620:8620", "8686:8686"] volumes: - - hg-pd0-data:/hugegraph-pd/pd_data + - ${HUGEGRAPH_PD0_VOLUME:-hg-pd0-data}:${HG_PD_DATA_PATH:-/hugegraph-pd/pd_data} pd1: <<: *pd-common @@ -129,12 +137,14 @@ services: HG_PD_RAFT_ADDRESS: pd1:8610 HG_PD_RAFT_PEERS_LIST: pd0:8610,pd1:8610,pd2:8610 HG_PD_INITIAL_STORE_LIST: store0:8500,store1:8500,store2:8500 - HG_PD_DATA_PATH: /hugegraph-pd/pd_data + HG_PD_DATA_PATH: ${HG_PD_DATA_PATH:-/hugegraph-pd/pd_data} + HG_PD_ROCKSDB_PROVIDER: ${HG_PD_ROCKSDB_PROVIDER:-rocksdb} + HG_PD_ENFORCE_PROVIDER_MARKER: ${HG_PD_ENFORCE_PROVIDER_MARKER:-false} HG_PD_INITIAL_STORE_COUNT: 3 HG_PD_AUTH_SECRET_KEY: ${HG_PD_AUTH_SECRET_KEY:?set HG_PD_AUTH_SECRET_KEY in .env; see docker/README.md} ports: ["8621:8620", "8687:8686"] volumes: - - hg-pd1-data:/hugegraph-pd/pd_data + - ${HUGEGRAPH_PD1_VOLUME:-hg-pd1-data}:${HG_PD_DATA_PATH:-/hugegraph-pd/pd_data} pd2: <<: *pd-common @@ -148,12 +158,14 @@ services: HG_PD_RAFT_ADDRESS: pd2:8610 HG_PD_RAFT_PEERS_LIST: pd0:8610,pd1:8610,pd2:8610 HG_PD_INITIAL_STORE_LIST: store0:8500,store1:8500,store2:8500 - HG_PD_DATA_PATH: /hugegraph-pd/pd_data + HG_PD_DATA_PATH: ${HG_PD_DATA_PATH:-/hugegraph-pd/pd_data} + HG_PD_ROCKSDB_PROVIDER: ${HG_PD_ROCKSDB_PROVIDER:-rocksdb} + HG_PD_ENFORCE_PROVIDER_MARKER: ${HG_PD_ENFORCE_PROVIDER_MARKER:-false} HG_PD_INITIAL_STORE_COUNT: 3 HG_PD_AUTH_SECRET_KEY: ${HG_PD_AUTH_SECRET_KEY:?set HG_PD_AUTH_SECRET_KEY in .env; see docker/README.md} ports: ["8622:8620", "8688:8686"] volumes: - - hg-pd2-data:/hugegraph-pd/pd_data + - ${HUGEGRAPH_PD2_VOLUME:-hg-pd2-data}:${HG_PD_DATA_PATH:-/hugegraph-pd/pd_data} # --- Store cluster (3 nodes) --- store0: @@ -166,10 +178,12 @@ services: HG_STORE_GRPC_PORT: "8500" HG_STORE_REST_PORT: "8520" HG_STORE_RAFT_ADDRESS: store0:8510 - HG_STORE_DATA_PATH: /hugegraph-store/storage + HG_STORE_DATA_PATH: ${HG_STORE_DATA_PATH:-/hugegraph-store/storage} + HG_STORE_ROCKSDB_PROVIDER: ${HG_STORE_ROCKSDB_PROVIDER:-rocksdb} + HG_STORE_ENFORCE_PROVIDER_MARKER: ${HG_STORE_ENFORCE_PROVIDER_MARKER:-false} ports: ["8500:8500", "8510:8510", "8520:8520"] volumes: - - hg-store0-data:/hugegraph-store/storage + - ${HUGEGRAPH_STORE0_VOLUME:-hg-store0-data}:${HG_STORE_DATA_PATH:-/hugegraph-store/storage} store1: <<: *store-common @@ -181,10 +195,12 @@ services: HG_STORE_GRPC_PORT: "8500" HG_STORE_REST_PORT: "8520" HG_STORE_RAFT_ADDRESS: store1:8510 - HG_STORE_DATA_PATH: /hugegraph-store/storage + HG_STORE_DATA_PATH: ${HG_STORE_DATA_PATH:-/hugegraph-store/storage} + HG_STORE_ROCKSDB_PROVIDER: ${HG_STORE_ROCKSDB_PROVIDER:-rocksdb} + HG_STORE_ENFORCE_PROVIDER_MARKER: ${HG_STORE_ENFORCE_PROVIDER_MARKER:-false} ports: ["8501:8500", "8511:8510", "8521:8520"] volumes: - - hg-store1-data:/hugegraph-store/storage + - ${HUGEGRAPH_STORE1_VOLUME:-hg-store1-data}:${HG_STORE_DATA_PATH:-/hugegraph-store/storage} store2: <<: *store-common @@ -196,10 +212,12 @@ services: HG_STORE_GRPC_PORT: "8500" HG_STORE_REST_PORT: "8520" HG_STORE_RAFT_ADDRESS: store2:8510 - HG_STORE_DATA_PATH: /hugegraph-store/storage + HG_STORE_DATA_PATH: ${HG_STORE_DATA_PATH:-/hugegraph-store/storage} + HG_STORE_ROCKSDB_PROVIDER: ${HG_STORE_ROCKSDB_PROVIDER:-rocksdb} + HG_STORE_ENFORCE_PROVIDER_MARKER: ${HG_STORE_ENFORCE_PROVIDER_MARKER:-false} ports: ["8502:8500", "8512:8510", "8522:8520"] volumes: - - hg-store2-data:/hugegraph-store/storage + - ${HUGEGRAPH_STORE2_VOLUME:-hg-store2-data}:${HG_STORE_DATA_PATH:-/hugegraph-store/storage} # --- Server cluster (3 nodes) --- server0: diff --git a/docker/docker-compose-hstore.yml b/docker/docker-compose-hstore.yml index 0fcc3697f6..948354ff2f 100644 --- a/docker/docker-compose-hstore.yml +++ b/docker/docker-compose-hstore.yml @@ -22,13 +22,15 @@ networks: volumes: pd-data: + pd-topling-data: store-data: + store-topling-data: hubble-data: services: pd: - image: hugegraph/pd:${HUGEGRAPH_VERSION:-latest} - pull_policy: ${HUGEGRAPH_PULL_POLICY:-missing} + image: ${HUGEGRAPH_PD_IMAGE:-hugegraph/pd:${HUGEGRAPH_VERSION:-latest}} + pull_policy: ${HUGEGRAPH_PD_PULL_POLICY:-${HUGEGRAPH_PULL_POLICY:-missing}} restart: unless-stopped networks: [hg-net] environment: @@ -38,12 +40,14 @@ services: HG_PD_RAFT_ADDRESS: pd:8610 HG_PD_RAFT_PEERS_LIST: pd:8610 HG_PD_INITIAL_STORE_LIST: store:8500 - HG_PD_DATA_PATH: /hugegraph-pd/pd_data + HG_PD_DATA_PATH: ${HG_PD_DATA_PATH:-/hugegraph-pd/pd_data} + HG_PD_ROCKSDB_PROVIDER: ${HG_PD_ROCKSDB_PROVIDER:-rocksdb} + HG_PD_ENFORCE_PROVIDER_MARKER: ${HG_PD_ENFORCE_PROVIDER_MARKER:-false} HG_PD_AUTH_SECRET_KEY: ${HG_PD_AUTH_SECRET_KEY:?set HG_PD_AUTH_SECRET_KEY in .env; see docker/README.md} ports: - "8620:8620" volumes: - - pd-data:/hugegraph-pd/pd_data + - ${HUGEGRAPH_PD_VOLUME:-pd-data}:${HG_PD_DATA_PATH:-/hugegraph-pd/pd_data} healthcheck: test: ["CMD-SHELL", "curl -fsS http://localhost:8620/v1/health >/dev/null"] interval: 10s @@ -52,8 +56,8 @@ services: start_period: 30s store: - image: hugegraph/store:${HUGEGRAPH_VERSION:-latest} - pull_policy: ${HUGEGRAPH_PULL_POLICY:-missing} + image: ${HUGEGRAPH_STORE_IMAGE:-hugegraph/store:${HUGEGRAPH_VERSION:-latest}} + pull_policy: ${HUGEGRAPH_STORE_PULL_POLICY:-${HUGEGRAPH_PULL_POLICY:-missing}} restart: unless-stopped networks: [hg-net] depends_on: @@ -65,11 +69,13 @@ services: HG_STORE_GRPC_PORT: "8500" HG_STORE_REST_PORT: "8520" HG_STORE_RAFT_ADDRESS: store:8510 - HG_STORE_DATA_PATH: /hugegraph-store/storage + HG_STORE_DATA_PATH: ${HG_STORE_DATA_PATH:-/hugegraph-store/storage} + HG_STORE_ROCKSDB_PROVIDER: ${HG_STORE_ROCKSDB_PROVIDER:-rocksdb} + HG_STORE_ENFORCE_PROVIDER_MARKER: ${HG_STORE_ENFORCE_PROVIDER_MARKER:-false} ports: - "8520:8520" volumes: - - store-data:/hugegraph-store/storage + - ${HUGEGRAPH_STORE_VOLUME:-store-data}:${HG_STORE_DATA_PATH:-/hugegraph-store/storage} healthcheck: test: ["CMD-SHELL", "curl -fsS http://localhost:8520/v1/health >/dev/null"] interval: 10s @@ -78,8 +84,8 @@ services: start_period: 60s server: - image: hugegraph/server:${HUGEGRAPH_VERSION:-latest} - pull_policy: ${HUGEGRAPH_PULL_POLICY:-missing} + image: ${HUGEGRAPH_SERVER_IMAGE:-hugegraph/server:${HUGEGRAPH_VERSION:-latest}} + pull_policy: ${HUGEGRAPH_SERVER_PULL_POLICY:-missing} restart: unless-stopped networks: [hg-net] depends_on: diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 4da9c32e19..3de4a7d8ee 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -17,26 +17,28 @@ services: pd: - image: hugegraph/pd:dev - pull_policy: build + image: ${HUGEGRAPH_PD_IMAGE:-hugegraph/pd:dev} + pull_policy: ${HUGEGRAPH_PD_PULL_POLICY:-build} build: context: .. dockerfile: hugegraph-pd/Dockerfile + target: ${HUGEGRAPH_PD_BUILD_TARGET:-standard} healthcheck: start_period: 20s store: - image: hugegraph/store:dev - pull_policy: build + image: ${HUGEGRAPH_STORE_IMAGE:-hugegraph/store:dev} + pull_policy: ${HUGEGRAPH_STORE_PULL_POLICY:-build} build: context: .. dockerfile: hugegraph-store/Dockerfile + target: ${HUGEGRAPH_STORE_BUILD_TARGET:-standard} healthcheck: start_period: 30s server: - image: hugegraph/server:dev - pull_policy: build + image: ${HUGEGRAPH_SERVER_IMAGE:-hugegraph/server:dev} + pull_policy: ${HUGEGRAPH_SERVER_PULL_POLICY:-build} build: context: .. dockerfile: hugegraph-server/Dockerfile-hstore diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d85cf5b1c6..171e2e9835 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -22,18 +22,22 @@ networks: volumes: server-data: + server-topling-data: hubble-data: services: server: - image: hugegraph/hugegraph:${HUGEGRAPH_VERSION:-latest} - pull_policy: ${HUGEGRAPH_PULL_POLICY:-missing} + image: ${HUGEGRAPH_SERVER_IMAGE:-hugegraph/hugegraph:${HUGEGRAPH_VERSION:-latest}} + pull_policy: ${HUGEGRAPH_SERVER_PULL_POLICY:-${HUGEGRAPH_PULL_POLICY:-missing}} restart: unless-stopped networks: [hg-net] environment: PASSWORD: ${HUGEGRAPH_ADMIN_PASSWORD:-} HG_SERVER_AUTH_TOKEN_SECRET: ${HUGEGRAPH_AUTH_TOKEN_SECRET:-} HG_SERVER_MIN_FREE_MEMORY: "0" + HG_SERVER_ROCKSDB_PROVIDER: ${HG_SERVER_ROCKSDB_PROVIDER:-rocksdb} + HG_SERVER_DATA_PATH: ${HG_SERVER_DATA_PATH:-/hugegraph-server/rocksdb-data} + HG_SERVER_ENFORCE_PROVIDER_MARKER: ${HG_SERVER_ENFORCE_PROVIDER_MARKER:-false} # Unset-only default: a host value overrides it, an absent one # renders the entrypoint default. ":-" would turn an empty host # value into 120 silently, which the entrypoint rejects on purpose. @@ -41,7 +45,7 @@ services: ports: - "8080:8080" volumes: - - server-data:/hugegraph-server/rocksdb-data + - ${HUGEGRAPH_SERVER_VOLUME:-server-data}:${HG_SERVER_DATA_PATH:-/hugegraph-server/rocksdb-data} healthcheck: test: ["CMD-SHELL", "curl -fsS http://localhost:8080/versions >/dev/null"] interval: 10s diff --git a/docker/test-compose.sh b/docker/test-compose.sh index 54418d413a..2dc3e2854a 100644 --- a/docker/test-compose.sh +++ b/docker/test-compose.sh @@ -52,6 +52,66 @@ compose_auth() { docker compose "$@" } +topling_env() { + env HG_PD_AUTH_SECRET_KEY="${PD_SECRET}" \ + HUGEGRAPH_VERSION="${VERSION}" \ + HUBBLE_IMAGE="${RENDER_HUBBLE_IMAGE}" \ + HUGEGRAPH_ADMIN_PASSWORD="${PASSWORD}" \ + HUGEGRAPH_AUTH_TOKEN_SECRET="${SECRET}" \ + HUGEGRAPH_PD_IMAGE="local/hugegraph-pd:topling" \ + HUGEGRAPH_PD_PULL_POLICY="never" \ + HUGEGRAPH_PD_BUILD_TARGET="topling" \ + HUGEGRAPH_PD_VOLUME="pd-topling-data" \ + HUGEGRAPH_PD0_VOLUME="hg-pd0-topling-data" \ + HUGEGRAPH_PD1_VOLUME="hg-pd1-topling-data" \ + HUGEGRAPH_PD2_VOLUME="hg-pd2-topling-data" \ + HG_PD_ROCKSDB_PROVIDER="topling" \ + HG_PD_DATA_PATH="/hugegraph-pd/topling-pd-data" \ + HG_PD_ENFORCE_PROVIDER_MARKER="true" \ + HUGEGRAPH_STORE_IMAGE="local/hugegraph-store:topling" \ + HUGEGRAPH_STORE_PULL_POLICY="never" \ + HUGEGRAPH_STORE_BUILD_TARGET="topling" \ + HUGEGRAPH_STORE_VOLUME="store-topling-data" \ + HUGEGRAPH_STORE0_VOLUME="hg-store0-topling-data" \ + HUGEGRAPH_STORE1_VOLUME="hg-store1-topling-data" \ + HUGEGRAPH_STORE2_VOLUME="hg-store2-topling-data" \ + HG_STORE_ROCKSDB_PROVIDER="topling" \ + HG_STORE_DATA_PATH="/hugegraph-store/topling-storage" \ + HG_STORE_ENFORCE_PROVIDER_MARKER="true" \ + "$@" +} + +render_topling_standalone() { + local output="$1" + shift + topling_env \ + env HUGEGRAPH_SERVER_IMAGE="local/hugegraph:topling" \ + HUGEGRAPH_SERVER_PULL_POLICY="never" \ + HUGEGRAPH_SERVER_VOLUME="server-topling-data" \ + HG_SERVER_ROCKSDB_PROVIDER="topling" \ + HG_SERVER_DATA_PATH="/hugegraph-server/topling-data" \ + HG_SERVER_ENFORCE_PROVIDER_MARKER="true" \ + docker compose "$@" config --format json > "${output}" +} + +render_topling_hstore() { + local output="$1" + shift + topling_env \ + env HUGEGRAPH_SERVER_IMAGE="hugegraph/server:${VERSION}" \ + HUGEGRAPH_SERVER_PULL_POLICY="missing" \ + docker compose "$@" config --format json > "${output}" +} + +render_topling_ha() { + local output="$1" + shift + topling_env \ + env HUGEGRAPH_SERVER_IMAGE="hugegraph/server:${VERSION}" \ + HUGEGRAPH_SERVER_PULL_POLICY="missing" \ + docker compose "$@" config --format json > "${output}" +} + render() { local output="$1" shift @@ -313,6 +373,60 @@ assert_dev_override() { ' "${override}" >/dev/null } +assert_topling_standalone() { + local rendered="$1" + jq -e ' + .name == "hugegraph-standalone" and + .services.server.image == "local/hugegraph:topling" and + .services.server.pull_policy == "never" and + .services.server.environment.HG_SERVER_ROCKSDB_PROVIDER == "topling" and + .services.server.environment.HG_SERVER_ENFORCE_PROVIDER_MARKER == "true" and + .services.server.environment.HG_SERVER_DATA_PATH == "/hugegraph-server/topling-data" and + (.services.server.volumes | length == 1) and + .services.server.volumes[0].source == "server-topling-data" and + .services.server.volumes[0].target == "/hugegraph-server/topling-data" + ' "${rendered}" >/dev/null +} + +assert_topling_hstore() { + local rendered="$1" + jq -e ' + .name == "hugegraph-hstore" and + .services.pd.image == "local/hugegraph-pd:topling" and + .services.store.image == "local/hugegraph-store:topling" and + .services.pd.pull_policy == "never" and + .services.store.pull_policy == "never" and + .services.pd.environment.HG_PD_ROCKSDB_PROVIDER == "topling" and + .services.store.environment.HG_STORE_ROCKSDB_PROVIDER == "topling" and + (.services.pd.volumes | length == 1) and + (.services.store.volumes | length == 1) and + .services.pd.volumes[0].source == "pd-topling-data" and + .services.store.volumes[0].source == "store-topling-data" and + .services.server.environment.HG_SERVER_BACKEND == "hstore" and + (.services.server.environment.HG_SERVER_ROCKSDB_PROVIDER == null) + ' "${rendered}" >/dev/null +} + +assert_topling_ha() { + local rendered="$1" + jq -e ' + .name == "hugegraph-3x3" and + ([.services.pd0, .services.pd1, .services.pd2] | + all(.image == "local/hugegraph-pd:topling" and + .pull_policy == "never" and + .environment.HG_PD_ROCKSDB_PROVIDER == "topling" and + (.volumes | length == 1))) and + ([.services.store0, .services.store1, .services.store2] | + all(.image == "local/hugegraph-store:topling" and + .pull_policy == "never" and + .environment.HG_STORE_ROCKSDB_PROVIDER == "topling" and + (.volumes | length == 1))) and + ([.services.server0, .services.server1, .services.server2] | + all(.environment.HG_SERVER_BACKEND == "hstore" and + (.environment.HG_SERVER_ROCKSDB_PROVIDER == null))) + ' "${rendered}" >/dev/null +} + cleanup() { if [[ -n "${ACTIVE_PROJECT}" ]]; then compose_active down -v --remove-orphans >/dev/null 2>&1 || true @@ -400,12 +514,22 @@ run_render() { -f "${DOCKER_DIR}/docker-compose.dev.yml" render "${RENDER_DIR}/override.json" \ -f "${DOCKER_DIR}/docker-compose.dev.yml" + render_topling_standalone "${RENDER_DIR}/topling-standalone.json" \ + -f "${DOCKER_DIR}/docker-compose.yml" + render_topling_hstore "${RENDER_DIR}/topling-hstore.json" \ + -f "${DOCKER_DIR}/docker-compose-hstore.yml" \ + -f "${DOCKER_DIR}/docker-compose.dev.yml" + render_topling_ha "${RENDER_DIR}/topling-ha.json" \ + -f "${DOCKER_DIR}/docker-compose-3pd-3store-3server.yml" assert_standalone "${RENDER_DIR}/standalone.json" assert_hstore "${RENDER_DIR}/hstore.json" assert_ha "${RENDER_DIR}/ha.json" assert_dev_override "${RENDER_DIR}/dev.json" \ "${RENDER_DIR}/override.json" + assert_topling_standalone "${RENDER_DIR}/topling-standalone.json" + assert_topling_hstore "${RENDER_DIR}/topling-hstore.json" + assert_topling_ha "${RENDER_DIR}/topling-ha.json" # An absent host value renders the entrypoint default in every topology and # a host value overrides it, so the container budget can be lowered to meet diff --git a/docs/toplingdb/README.md b/docs/toplingdb/README.md new file mode 100644 index 0000000000..1b5458d0a0 --- /dev/null +++ b/docs/toplingdb/README.md @@ -0,0 +1,31 @@ +# ToplingDB Documentation + +ToplingDB is an optional RocksDB-compatible runtime for HugeGraph. Select it +when a HugeGraph process owns a local RocksDB database: + +| Deployment | Process that selects ToplingDB | +|---|---| +| Standalone | HugeGraph Server | +| Distributed HStore | PD and Store | +| HStore-backed Server | None; it is a remote client | + +Start with one of these paths: + +- [Quickstart](toplingdb-quickstart.md) explains how to build a Topling + distribution, select the provider, keep its data separate, start the + component, and return to standard RocksDB safely. +- [Developer guide](toplingdb-development.md) covers the source layout, + distribution contract, tests, CI, Docker status, and native runtime work. + +Use the reference guides when you need more detail: + +- [Architecture and configuration](toplingdb.md) +- [HStore integration](toplingdb-hstore-integration.md) +- [Operations](toplingdb-operations.md) +- [Troubleshooting](toplingdb-troubleshooting.md) +- [Security](toplingdb-security.md) + +ToplingDB selection happens only at process startup. Changing +`rocksdb.provider` does not migrate or convert an existing data directory. +Standard RocksDB and ToplingDB must use separate data directories unless you +restore a verified, compatible snapshot into an empty directory. diff --git a/docs/toplingdb/toplingdb-development.md b/docs/toplingdb/toplingdb-development.md new file mode 100644 index 0000000000..8ee50cb65b --- /dev/null +++ b/docs/toplingdb/toplingdb-development.md @@ -0,0 +1,239 @@ +# ToplingDB Developer Guide + +This guide describes the Topling integration maintained in this repository. +Read the [quickstart](toplingdb-quickstart.md) first if you only need to run it. + +The runtime support target is native Linux x86_64 and Linux containers on +macOS (Docker Desktop or OrbStack). Native macOS execution, including Intel, +is deliberately outside the ToplingDB matrix; do not treat that CI job as a +Topling image or container failure. + +## Runtime Contract + +HugeGraph keeps the RocksDB Java API. Topling Easy Migrate supplies the native +runtime and configuration at process startup. + +```text +component config + -> preload-topling.sh validates rocksdb or topling + -> component-local JAR joins CLASSPATH + -> component-local native library joins LD_PRELOAD + -> Easy Migrate YAML is exported + -> existing RocksDB Java calls open and close DB/CF handles +``` + +The supported providers are exactly `rocksdb` and `topling`. Duplicate +configuration values must agree. Missing Topling files or dependencies stop +startup. PD and Store never search a neighboring Server distribution. + +## Source Layout + +| Area | Path | +|---|---| +| Shared helper source | `hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh` | +| Package-local preparation | `hugegraph-server/hugegraph-dist/src/assembly/static/bin/prepare-topling.sh` | +| Startup selection | `hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh` | +| Checked-in Topling JAR | `hugegraph-server/hugegraph-dist/src/assembly/static/lib/topling/` | +| Standalone Easy Migrate YAML | `hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb.yaml` | +| PD Easy Migrate YAML | `hugegraph-pd/hg-pd-dist/src/assembly/static/conf/rocksdb_pd.yaml` | +| Store Easy Migrate YAML | `hugegraph-store/hg-store-dist/src/assembly/static/conf/rocksdb_store.yaml` | +| Topling distribution generator | `install-dist/scripts/build-topling-distribution.sh` | +| Provider data marker | `hugegraph-server/hugegraph-dist/src/assembly/static/bin/verify-rocksdb-provider.sh` | +| Docker build graph | `docker/bake.hcl` | +| Focused shell tests | `hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-*.sh` | + +The PD and Store assembly descriptors copy the canonical helper scripts into +their own `bin/` directories. Change the canonical Server copy, then verify all +three assembled copies are identical. + +## Build Standard and Topling Distributions + +Use Linux x86_64. The distribution generator also requires `rsync`, `unzip`, +`tar`, and `sha256sum`. + +```bash +VERSION=$(mvn help:evaluate \ + -Dexpression=project.version -q -DforceStdout) + +mvn clean package \ + -pl hugegraph-server/hugegraph-dist,hugegraph-pd/hg-pd-dist,hugegraph-store/hg-store-dist \ + -am -Dmaven.test.skip=true -Dmaven.javadoc.skip=true -ntp + +for component in server pd store; do + install-dist/scripts/build-topling-distribution.sh \ + "$component" "$VERSION" +done +``` + +The standard distributions contain the selection helpers but no +`lib/topling/rocksdbjni*.jar` or prepared Topling native library. Selecting +Topling from a standard distribution must fail. + +When a Topling distribution is switched back to `rocksdb`, the Server and +`init-store` launchers exclude the canonical `lib/topling/` subtree (including +symlink aliases) from their classpaths; the optional Topling JAR is only added +after the startup selector accepts `topling`. + +The Topling distributions add only build-time runtime files. The generator +excludes PID files, logs, data directories, and prepared runtime state from the +standard distribution before creating the Topling tarball. It writes +`lib/topling/runtime.properties` with the component, project version, JAR name, +and JAR SHA-256. + +That SHA-256 identifies the local input. It is not an official release +signature or a full source-to-binary provenance record. Issue +[#213](https://github.com/hugegraph/hugegraph/issues/213) tracks immutable +coordinates, toolchain and CPU baseline, licensing, SBOM, signatures, and +consumer verification. + +## Provider Configuration + +Server reads `rocksdb.provider` from graph `.properties` files: + +```properties +rocksdb.provider=topling +``` + +PD and Store read the provider under the root `rocksdb` YAML section: + +```yaml +rocksdb: + provider: topling +``` + +Keep only one effective value for a component. The loader rejects unknown or +conflicting values. The default is standard RocksDB when no provider is set. + +Every Topling functional test must use a readable Easy Migrate YAML. Removing +`TOPLINGDB_EASY_MIGRATE_CONF` reduces a test to Java ABI coverage and does not +exercise the Topling runtime. + +## Focused Tests + +Run the platform-independent selection and packaging tests first: + +```bash +TRAVIS_DIR=hugegraph-server/hugegraph-dist/src/assembly/travis + +"$TRAVIS_DIR/test-topling-runtime-selection.sh" +"$TRAVIS_DIR/test-topling-docker-entrypoints.sh" +"$TRAVIS_DIR/test-topling-runtime-packaging.sh" \ + "hugegraph-server/apache-hugegraph-server-$VERSION" \ + "hugegraph-pd/apache-hugegraph-pd-$VERSION" \ + "hugegraph-store/apache-hugegraph-store-$VERSION" +``` + +On Linux x86_64, validate each generated Topling distribution: + +```bash +"$TRAVIS_DIR/test-topling-distribution.sh" \ + server \ + "hugegraph-server/apache-hugegraph-server-$VERSION" \ + "hugegraph-server/apache-hugegraph-server-$VERSION-topling" + +"$TRAVIS_DIR/test-rocksdb-runtime.sh" \ + topling \ + "hugegraph-server/apache-hugegraph-server-$VERSION-topling" +``` + +Repeat the distribution and runtime checks for PD and Store. The CI jobs +`server-rocksdb-runtime` and `distributed-rocksdb-runtime` build a matrix across +standard RocksDB and ToplingDB. Standard runtime checks and both distribution +contracts are required. The synthetic Topling column-family lifecycle check is +reported as a non-blocking diagnostic for issue #212; image-backed service +lifecycle tests remain required. The jobs also contaminate the standard build +with known runtime-state fixtures and verify that the Topling directory and +tarball remain clean. + +Run repository checks before pushing: + +```bash +mvn editorconfig:check -ntp +mvn clean compile -Dmaven.javadoc.skip=true -ntp +git diff --check +``` + +Use ShellCheck on each changed shell script. Existing warnings in untouched +lines do not justify unrelated cleanup in a Topling change. + +## Docker Images + +Server, PD, and Store Dockerfiles have explicit `standard` and `topling` +targets. Build the complete Topling deployment set with Bake: + +```bash +RUNTIME_VARIANT=topling \ +IMAGE_TAG=topling \ +docker buildx bake --file docker/bake.hcl +``` + +The Topling variant is Linux x86_64 only. It builds +`hugegraph/hugegraph`, `hugegraph/pd`, and `hugegraph/store` from their +`topling` targets. `hugegraph/server` remains the standard HStore Server and +receives the same tag as the compatible deployment set. + +An HStore Server image remains free of a local Topling JAR and native library. +PD and Store own their local runtime. + +Published candidates carry `org.opencontainers.image.source`, +`org.opencontainers.image.revision`, and +`org.apache.hugegraph.rocksdb-runtime` labels. Verify the exact source commit +before accepting a mutable deployment tag: + +```bash +docker image inspect hugegraph/hugegraph:topling \ + --format '{{json .Config.Labels}}' +``` + +Run the generic standalone, minimal HStore, or 3+3+3 Compose file with +provider parameters injected through the environment. Local development can +use `local/hugegraph-{pd,store}:topling` with +`HUGEGRAPH_{PD,STORE}_BUILD_TARGET=topling`; published candidates use +`hugegraph/{pd,store}:topling` and `HUGEGRAPH_{PD,STORE}_PULL_POLICY=always`. +Set `HUGEGRAPH_SERVER_IMAGE=hugegraph/server:topling` and +`HUGEGRAPH_SERVER_PULL_POLICY=always` only when the matching HStore image is +intended. Keep the provider-specific volume names and data roots explicit. +The repository deliberately maintains one Compose topology per deployment +shape rather than separate Topling files. + +Every local RocksDB owner uses a provider-specific data root. The entrypoint +validates `.hugegraph-rocksdb-provider` before it mutates configuration or +starts Java. Topling rejects unmarked non-empty data. Standard RocksDB accepts +legacy unmarked data, but rejects a Topling marker. The helper rejects symlinked +path components and serializes marker initialization on a pinned directory +inode. + +## Native Runtime Changes + +When replacing `rocksdbjni*.jar`: + +1. Keep exactly one Topling JAR in the checked-in source directory. +2. Run `bin/prepare-topling.sh` in every generated component distribution. +3. Check `ldd library/librocksdbjni-linux64.so` for unresolved dependencies. +4. Run real DB and column-family create, close, reopen, restart, and persistence + tests with the component Easy Migrate YAML enabled. +5. Record the JAR SHA-256 and the exact source and toolchain evidence available. + +Issue [#212](https://github.com/hugegraph/hugegraph/issues/212) tracks native +DB/column-family lifecycle behavior. Real Server and 1+1+1 HStore tests must +cover CRUD, clean HugeGraph transaction close, container recreation, and +persistence. The current native runtime can still report a SidePluginRepo +bookkeeping warning after HugeGraph closes successfully. Keep that upstream +evidence separate from the provider-selection and data-isolation gates. ABI +loading or `ldd` output cannot establish service correctness. + +## Review Boundaries + +A complete Topling change must keep these properties: + +- standard artifacts have no Topling JAR or native library; +- Server, PD, and Store load only their component-local runtime; +- an HStore Server does not load Topling locally; +- standard and Topling data directories stay separate; +- invalid or incomplete Topling selection fails before service startup; +- shutdown uses the normal component lifecycle; +- documentation names the exact configuration file and verification command. + +Do not claim hot switching, automatic data migration, safe reuse of a +Topling-modified directory by standard RocksDB, or release provenance that the +artifact metadata does not prove. diff --git a/docs/toplingdb/toplingdb-hstore-integration.md b/docs/toplingdb/toplingdb-hstore-integration.md new file mode 100644 index 0000000000..be6b21cbce --- /dev/null +++ b/docs/toplingdb/toplingdb-hstore-integration.md @@ -0,0 +1,95 @@ +# ToplingDB and HStore Integration + +## Deployment Model + +ToplingDB and HStore solve different problems: + +- ToplingDB is an optional RocksDB-compatible native runtime. +- HStore is HugeGraph's distributed storage backend. + +In an HStore deployment, HugeGraph Server sends storage requests to Store. It +does not open a local RocksDB database. + +```text +HugeGraph Server + HStore client + | + v +PD cluster ---------------- Store cluster +local RocksDB metadata local RocksDB data +optional Topling runtime optional Topling runtime +``` + +ToplingDB therefore applies independently to PD and Store. Each process keeps +its existing RocksDB Java calls and loads its own native runtime and Easy +Migrate configuration. + +## Runtime Setup + +PD startup sets: + +```bash +TOPLINGDB_EASY_MIGRATE_CONF="$PD_HOME/conf/rocksdb_pd.yaml" +``` + +Store startup sets: + +```bash +TOPLINGDB_EASY_MIGRATE_CONF="$STORE_HOME/conf/rocksdb_store.yaml" +``` + +Both distributions carry the same `preload-topling.sh` helper and load only +their component-local Topling JAR and native library. PD and Store do not +discover or depend on a neighboring Server distribution. This is shared source +tooling, not a shared Java provider or shared DB owner. + +## Configuration + +Set the provider for each component that should use ToplingDB: + +```yaml +rocksdb: + provider: topling +``` + +PD and Store use distinct Easy Migrate YAML files and distinct HTTP ports. +Their `DBOptions.default` mappings remain the global fallbacks, while +`DBOptions.log` remains the specialized log-database profile. + +The Server's graph configuration does not enable a local ToplingDB instance +when the backend is HStore. + +## Open and Close Behavior + +PD and Store continue to use normal RocksDB Java APIs. The preloaded native hook +applies options and tracks DB/CF lifecycle. Neither component reflectively +creates a repository or calls an alternate `openDB`. + +Shutdown is also unchanged: + +```text +SIGTERM + -> PD/Store graceful shutdown + -> Java closes CF handles and RocksDB + -> native MaybeForgetCF and MaybeForgetDB + -> process exits +``` + +Store's existing shutdown timeout is a general process boundary. A timeout does +not justify calling `SidePluginRepo.closeAllDB()` and this integration does not +change the broader Store thread-exit behavior. + +## Validation + +Validate PD and Store separately: + +1. Confirm the component provider is `topling`. +2. Confirm the startup log reports the expected + `TOPLINGDB_EASY_MIGRATE_CONF`. +3. Confirm only the prepared component native library is mapped. +4. Run DB/CF create, reopen, drop, and recreate operations with Easy Migrate + enabled. +5. Stop the component with SIGTERM and verify normal process exit. + +An ABI-only check may inspect the JAR, Topling API marker, and native mapping +without opening a database. It is not a storage-functionality test. diff --git a/docs/toplingdb/toplingdb-operations.md b/docs/toplingdb/toplingdb-operations.md new file mode 100644 index 0000000000..39bb8bcf3f --- /dev/null +++ b/docs/toplingdb/toplingdb-operations.md @@ -0,0 +1,69 @@ +# ToplingDB Operations Guide + +## Preflight + +Before startup: + +1. Confirm the component configuration selects `topling`. +2. Run the installation step for that component. +3. Confirm its Easy Migrate YAML and prepared native library are readable. +4. Check that the configured HTTP port is available and restricted to a + trusted interface. +5. Create and verify a complete pre-migration RocksDB checkpoint or backup. + +The expected configuration files are: + +| Component | File | +|---|---| +| Standalone Server | `conf/toplingdb.yaml` | +| PD | `conf/rocksdb_pd.yaml` | +| Store | `conf/rocksdb_store.yaml` | + +## Monitoring + +Monitor read/write latency, compaction backlog, cache usage, WAL growth, disk +space, native memory, and process health. The sample configurations keep the +Topling HTTP endpoint disabled by default (`auto_start_http: false`). If it is +needed, enable it explicitly; it can show native state but has no +authentication and must not be exposed directly to untrusted networks. + +Use the component logs to confirm the selected configuration path and native +library. Do not infer ToplingDB activation from the JAR name alone. + +## Tuning + +Change one YAML setting group at a time and benchmark with a representative +workload. Preserve: + +- `DBOptions.default` as the global fallback; +- the dedicated `DBOptions.log` profile; +- component-specific HTTP ports; +- a memory budget that includes JVM heap, block cache, memtables, native + allocations, and background jobs. + +## Graceful Stop and Restart + +Use the normal stop script or SIGTERM. Do not stop a separate “ToplingDB +process”; none exists. + +```text +SIGTERM + -> HugeGraph/PD/Store graceful shutdown + -> normal CF/DB close + -> native MaybeForgetCF / MaybeForgetDB + -> JVM exit +``` + +Do not invoke `SidePluginRepo.closeAllDB()`. Store's stop script already has a +bounded wait. If it reports a timeout, collect thread dumps and component logs +and investigate the ordinary Store shutdown path before any restart. + +## Upgrade and Rollback + +Drain traffic and stop the component cleanly before changing the runtime or +YAML. Validate JAR/native compatibility and the YAML schema in staging first. + +A provider switch is not a data rollback. If the upgraded Topling runtime has +written data and rollback is required, stop all writers and restore the full +pre-upgrade snapshot into an empty data directory with the matching previous +runtime. diff --git a/docs/toplingdb/toplingdb-quickstart.md b/docs/toplingdb/toplingdb-quickstart.md new file mode 100644 index 0000000000..b969314238 --- /dev/null +++ b/docs/toplingdb/toplingdb-quickstart.md @@ -0,0 +1,335 @@ +# ToplingDB Quickstart + +This guide covers a source-built ToplingDB distribution on Linux x86_64. On +macOS, run the published Linux image through Docker Desktop or OrbStack +container mode. Native macOS execution (including Intel) is outside this +ToplingDB support matrix. Use a standard HugeGraph distribution when you want +standard RocksDB. + +## Choose the Correct Component + +| What you run | Topling package | Provider configuration | +|---|---|---| +| Standalone Server | `apache-hugegraph-server--topling` | `conf/graphs/hugegraph.properties` | +| PD | `apache-hugegraph-pd--topling` | `conf/application.yml` | +| Store | `apache-hugegraph-store--topling` | `conf/application-pd.yml` | +| HStore Server | Standard HStore Server | No local Topling setting | + +PD and Store select their providers independently. An HStore-backed Server does +not load a Topling JAR or native library. + +Set standalone data with `rocksdb.data_path` in +`conf/graphs/hugegraph.properties`, PD metadata with `pd.data-path` in +`conf/application.yml`, and Store data with `app.data-path` in +`conf/application-pd.yml`. + +## Prerequisites + +Topling distributions and images currently target Linux x86_64. The published +images use the `linux/amd64` platform, so macOS container mode may need +`--platform linux/amd64`; the native packages required by the bundled library +are installed in the images. The preparation script reports any unresolved +dependency and stops. + +## Docker Images + +The `topling` image tag names one compatible deployment set: + +| Image | Role | +|---|---| +| `hugegraph/hugegraph:topling` | Standalone Server with local ToplingDB | +| `hugegraph/pd:topling` | PD metadata on ToplingDB | +| `hugegraph/store:topling` | Store data on ToplingDB | +| `hugegraph/server:topling` | HStore Server without a local Topling runtime | + +Run the standalone image with its provider-specific data volume: + +```bash +docker volume create hugegraph-topling-data +docker run -d \ + --pull always \ + --platform linux/amd64 \ + --name hugegraph-topling \ + -p 8080:8080 \ + -v hugegraph-topling-data:/hugegraph-server/topling-data \ + hugegraph/hugegraph:topling + +curl --fail http://127.0.0.1:8080/versions +``` + +The same standalone image can be run with the generic repository Compose file. +Inject the provider and volume parameters so the standard volume cannot be +reused: + +```bash +export HUGEGRAPH_SERVER_IMAGE='hugegraph/hugegraph:topling' +export HUGEGRAPH_SERVER_PULL_POLICY='always' +export HG_SERVER_ROCKSDB_PROVIDER='topling' +export HG_SERVER_DATA_PATH='/hugegraph-server/topling-data' +export HG_SERVER_ENFORCE_PROVIDER_MARKER='true' +export HUGEGRAPH_SERVER_VOLUME='server-topling-data' +docker compose -f docker/docker-compose.yml \ + up -d --wait +``` + +Run a source-checkout 1+1+1 HStore stack with Topling PD and Store images. +The generic Compose files accept the same parameters for local builds or +published images; the HStore Server itself remains standard: + +```bash +export HUGEGRAPH_ADMIN_PASSWORD='replace-with-a-strong-password' +export HG_PD_AUTH_SECRET_KEY='replace-with-a-separate-strong-pd-secret' +export HUGEGRAPH_PD_IMAGE='hugegraph/pd:topling' +export HUGEGRAPH_PD_PULL_POLICY='always' +export HUGEGRAPH_PD_VOLUME='pd-topling-data' +export HG_PD_ROCKSDB_PROVIDER='topling' +export HG_PD_DATA_PATH='/hugegraph-pd/topling-pd-data' +export HG_PD_ENFORCE_PROVIDER_MARKER='true' +export HUGEGRAPH_STORE_IMAGE='hugegraph/store:topling' +export HUGEGRAPH_STORE_PULL_POLICY='always' +export HUGEGRAPH_STORE_VOLUME='store-topling-data' +export HG_STORE_ROCKSDB_PROVIDER='topling' +export HG_STORE_DATA_PATH='/hugegraph-store/topling-storage' +export HG_STORE_ENFORCE_PROVIDER_MARKER='true' +export HUGEGRAPH_SERVER_IMAGE='hugegraph/server:topling' +export HUGEGRAPH_SERVER_PULL_POLICY='always' + +docker compose \ + -f docker/docker-compose-hstore.yml \ + up -d --wait pd store server +``` + +For source-built PD and Store images, use `local/hugegraph-{pd,store}:topling`, +set both pull policies to `never`, add `HUGEGRAPH_{PD,STORE}_BUILD_TARGET=topling`, +and append `-f docker/docker-compose.dev.yml`. The generic files mount the +separate `pd-topling-data` and `store-topling-data` volumes. The HStore Server +does not load a local Topling library. + +For a published 3+3+3 stack, select all three deployment images explicitly: + +```bash +export HUGEGRAPH_ADMIN_PASSWORD='replace-with-a-strong-password' +export HG_PD_AUTH_SECRET_KEY='replace-with-a-separate-strong-pd-secret' +export HUGEGRAPH_PD_IMAGE=hugegraph/pd:topling +export HUGEGRAPH_PD_PULL_POLICY=always +export HG_PD_ROCKSDB_PROVIDER=topling +export HG_PD_DATA_PATH=/hugegraph-pd/topling-pd-data +export HG_PD_ENFORCE_PROVIDER_MARKER=true +export HUGEGRAPH_PD0_VOLUME=hg-pd0-topling-data +export HUGEGRAPH_PD1_VOLUME=hg-pd1-topling-data +export HUGEGRAPH_PD2_VOLUME=hg-pd2-topling-data +export HUGEGRAPH_STORE_IMAGE=hugegraph/store:topling +export HUGEGRAPH_STORE_PULL_POLICY=always +export HG_STORE_ROCKSDB_PROVIDER=topling +export HG_STORE_DATA_PATH=/hugegraph-store/topling-storage +export HG_STORE_ENFORCE_PROVIDER_MARKER=true +export HUGEGRAPH_STORE0_VOLUME=hg-store0-topling-data +export HUGEGRAPH_STORE1_VOLUME=hg-store1-topling-data +export HUGEGRAPH_STORE2_VOLUME=hg-store2-topling-data +export HUGEGRAPH_SERVER_IMAGE=hugegraph/server:topling +export HUGEGRAPH_SERVER_PULL_POLICY=always + +docker compose -f docker/docker-compose-3pd-3store-3server.yml \ + up -d --wait +``` + +Each PD and Store instance receives a distinct Topling volume. For local Bake +images, use the same variables with the tags produced by `docker/bake.hcl`. +No Topling-specific Compose file is required or maintained. + +The `topling` tag is mutable. The commands above force a pull so a cached image +cannot silently stand in for the current deployment set. Pin the registry +digest when an exact build must be reproduced, and verify the image source, +revision, and runtime labels during acceptance. + +## Build Distributions from Source + +Build on Linux x86_64 with Java 11+, Maven 3.5+, `rsync`, `unzip`, and `tar`. + +```bash +git clone https://github.com/hugegraph/hugegraph.git +cd hugegraph +git switch toplingdb + +VERSION=$(mvn help:evaluate \ + -Dexpression=project.version -q -DforceStdout) + +mvn clean package \ + -pl hugegraph-server/hugegraph-dist,hugegraph-pd/hg-pd-dist,hugegraph-store/hg-store-dist \ + -am -Dmaven.test.skip=true -Dmaven.javadoc.skip=true -ntp +``` + +The build creates standard distributions first. Generate only the Topling +distributions you need: + +```bash +install-dist/scripts/build-topling-distribution.sh server "$VERSION" +install-dist/scripts/build-topling-distribution.sh pd "$VERSION" +install-dist/scripts/build-topling-distribution.sh store "$VERSION" +``` + +Each command creates a directory and a matching `.tar.gz` file beside the +standard distribution. The Topling package contains: + +```text +bin/prepare-topling.sh +bin/preload-topling.sh +lib/topling/rocksdbjni*.jar +lib/topling/runtime.properties +library/librocksdbjni-linux64.so +``` + +The generator selects `provider=topling` and prepares the native runtime. Run +`bin/prepare-topling.sh` again after replacing the bundled Topling JAR. + +## Provider Data Markers + +Docker entrypoints validate `.hugegraph-rocksdb-provider` before starting a +JVM. The marker records the component and provider. A mismatched marker always +stops startup. Topling also rejects an unmarked, non-empty data directory. +The configured data root must already exist as a real directory; mount or +create it before startup. Symlinked path components are rejected. + +Standard RocksDB accepts an existing unmarked directory for backward +compatibility. New deployments should still use the image defaults: + +| Component | Standard data root | Topling data root | +|---|---|---| +| Server | `/hugegraph-server/rocksdb-data` | `/hugegraph-server/topling-data` | +| PD | `/hugegraph-pd/pd_data` | `/hugegraph-pd/topling-pd-data` | +| Store | `/hugegraph-store/storage` | `/hugegraph-store/topling-storage` | + +The marker prevents accidental reuse. It does not convert data between +providers. + +## Standalone Server + +Enter the generated Server distribution: + +```bash +cd "hugegraph-server/apache-hugegraph-server-$VERSION-topling" +``` + +Confirm the provider and assign a data directory that no standard RocksDB +process uses: + +```properties +# conf/graphs/hugegraph.properties +backend=rocksdb +rocksdb.provider=topling +rocksdb.data_path=/srv/hugegraph/topling/server +``` + +Initialize and start the Server: + +```bash +bin/init-store.sh +bin/start-hugegraph.sh +curl --fail http://127.0.0.1:8080/versions +``` + +The startup output must include the selected Easy Migrate configuration: + +```text +[preload-topling] TOPLINGDB_EASY_MIGRATE_CONF=.../conf/toplingdb.yaml +``` + +Stop with the normal HugeGraph script: + +```bash +bin/stop-hugegraph.sh +``` + +Restart the same Topling distribution and read previously written data before +you accept persistence for that deployment. + +## Distributed HStore + +Build one Topling distribution for PD and one for Store. Keep the HStore Server +on its normal HStore distribution. + +In every PD distribution, enable ToplingDB and use a provider-specific metadata +directory: + +```yaml +# conf/application.yml +rocksdb: + provider: topling + option-path: ./conf/rocksdb_pd.yaml + +pd: + data-path: /srv/hugegraph/topling/pd +``` + +In every Store distribution, enable ToplingDB: + +```yaml +# conf/application-pd.yml +rocksdb: + provider: topling +``` + +Set a provider-specific Store data directory in the Store service +configuration: + +```yaml +# conf/application-pd.yml +app: + data-path: /srv/hugegraph/topling/store +``` + +Configure the normal PD, Store, and HStore network addresses as described in +the [distributed deployment guide](../../hugegraph-store/docs/deployment-guide.md). +Start PD before Store, then start the HStore-backed Server: + +```bash +# Run from the corresponding distribution directory +bin/start-hugegraph-pd.sh +bin/start-hugegraph-store.sh +bin/start-hugegraph.sh +``` + +Check each PD and Store log for its own Easy Migrate path. PD must use +`conf/rocksdb_pd.yaml`; Store must use `conf/rocksdb_store.yaml`. The HStore +Server must not report a local Topling runtime. + +## Topling HTTP Monitor + +The sample Easy Migrate files bind their HTTP monitors to loopback, but set +`auto_start_http: false`, so the monitor is disabled by default: + +| Component | Configuration | Default address | +|---|---|---| +| Server | `conf/toplingdb.yaml` | `127.0.0.1:2011` | +| PD | `conf/rocksdb_pd.yaml` | `127.0.0.1:2012` | +| Store | `conf/rocksdb_store.yaml` | `127.0.0.1:2013` | + +The endpoint has no authentication. Keep the loopback binding. Enable it only +when you need it: + +```yaml +http: + auto_start_http: true +``` + +Leave `auto_start_http: false` to keep the monitor disabled. + +## Return to Standard RocksDB + +A provider change is not a data conversion. Use this procedure: + +1. Stop all writers and stop the component cleanly. +2. Keep the Topling data directory unchanged. +3. Restore a full pre-Topling snapshot into a new, empty standard RocksDB data + directory. +4. Start the standard distribution with `provider=rocksdb`. +5. Validate schema, reads, writes, restart, and persistence before serving + traffic. + +Do not point standard RocksDB at a directory that ToplingDB has modified. + +## Startup Failures + +Startup stops when the provider is invalid or the component-local JAR, native +library, Easy Migrate file, or system dependency is missing. Follow the exact +error and see the [troubleshooting guide](toplingdb-troubleshooting.md). diff --git a/docs/toplingdb/toplingdb-security.md b/docs/toplingdb/toplingdb-security.md new file mode 100644 index 0000000000..1f1811ab80 --- /dev/null +++ b/docs/toplingdb/toplingdb-security.md @@ -0,0 +1,90 @@ +# ToplingDB Security Hardening Guide + +This document provides best practices for securing a ToplingDB deployment. It +covers file permissions, network access control, firewall rules, and additional +hardening measures that reduce the production attack surface. + +--- + +## 1. File Permissions + +Keep installation files owned by a deployment account. Give the service +account write access only to data, log, and runtime-state directories: + +```bash +chown root:hugegraph "$HUGEGRAPH_HOME"/bin/*.sh +chmod 750 "$HUGEGRAPH_HOME"/bin/*.sh + +for config in toplingdb.yaml rocksdb_pd.yaml rocksdb_store.yaml; do + test -e "$HUGEGRAPH_HOME/conf/$config" || continue + chown root:hugegraph "$HUGEGRAPH_HOME/conf/$config" + chmod 640 "$HUGEGRAPH_HOME/conf/$config" +done + +install -d -o hugegraph -g hugegraph -m 750 \ + "$HUGEGRAPH_HOME"/logs \ + /srv/hugegraph/topling/data \ + /run/hugegraph +``` + +- Replace `root` with the actual deployment owner when packages are installed + by a non-root account. +- The service account can read scripts and configuration but cannot modify the + startup chain, JARs, native libraries, or Easy Migrate configuration. +- Adjust the data path for Server, PD, or Store. Do not recursively change + ownership of the whole installation directory. + +--- + +## 2. Network Access Control + +Restrict network exposure by binding services to localhost or specific +interfaces: + +```yaml +# Localhost-only access +http: + listening_ports: '127.0.0.1:2011' +``` + +- Avoid binding to `0.0.0.0` unless absolutely necessary. +- Use reverse proxies (e.g., Nginx) or VPN tunnels if remote access is required. + +--- + +## 3. Firewall Rules + +The Topling HTTP monitor is disabled by default. If it is explicitly enabled, +insert a dedicated chain before broader `ACCEPT` rules. Preserve established +connections and define equivalent IPv4 and IPv6 policies: + +```bash +iptables -N HG_TOPLING 2>/dev/null || true +iptables -F HG_TOPLING +iptables -A HG_TOPLING -s 192.168.1.0/24 -j ACCEPT +iptables -A HG_TOPLING -j DROP +iptables -C INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT || + iptables -I INPUT 1 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT +iptables -C INPUT -p tcp -m multiport --dports 2011,2012,2013 \ + -j HG_TOPLING || + iptables -I INPUT 2 -p tcp -m multiport --dports 2011,2012,2013 \ + -j HG_TOPLING + +ip6tables -N HG_TOPLING6 2>/dev/null || true +ip6tables -F HG_TOPLING6 +ip6tables -A HG_TOPLING6 -s 2001:db8:1234::/48 -j ACCEPT +ip6tables -A HG_TOPLING6 -j DROP +ip6tables -C INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT || + ip6tables -I INPUT 1 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT +ip6tables -C INPUT -p tcp -m multiport --dports 2011,2012,2013 \ + -j HG_TOPLING6 || + ip6tables -I INPUT 2 -p tcp -m multiport --dports 2011,2012,2013 \ + -j HG_TOPLING6 +``` + +- Replace both documentation prefixes with trusted production networks. +- Existing established connections remain allowed. Revoke them separately + through the host connection-tracking policy when immediate eviction is + required. +- Inspect the resulting rule order before applying it to a remote host. +- Consider using `firewalld` or `ufw` for simplified management. diff --git a/docs/toplingdb/toplingdb-troubleshooting.md b/docs/toplingdb/toplingdb-troubleshooting.md new file mode 100644 index 0000000000..d44bb5a14a --- /dev/null +++ b/docs/toplingdb/toplingdb-troubleshooting.md @@ -0,0 +1,89 @@ +# ToplingDB Troubleshooting + +## Startup Rejects the Easy Migrate Configuration + +Confirm that the startup log prints the expected component path: + +```text +[preload-topling] TOPLINGDB_EASY_MIGRATE_CONF=/path/to/config.yaml +``` + +Then check: + +- the file exists and is readable by the service user; +- the YAML uses valid indentation and supported Topling option names; +- `DBOptions.default` exists as the fallback; +- the dedicated `DBOptions.log` mapping is still present. + +Do not diagnose configuration loading by calling `SidePluginRepo` from Java. + +## Wrong or Missing Native Runtime + +Verify the prepared library and the process mapping: + +```bash +test -r /path/to/component/library/librocksdbjni-linux64.so +grep librocksdbjni /proc/$PID/maps +``` + +Only the selected component library should be mapped. A Topling class marker in +the JAR is useful for ABI diagnosis, but it does not prove that Easy Migrate +options were applied or that DB/CF operations work. + +## HTTP Port Conflict + +Check the component-specific `http.listening_ports` value and whether another +process already owns it: + +```bash +lsof -i : +``` + +The provided configurations use 2011 for Server, 2012 for PD, and 2013 for +Store. Confirm the actual value in the selected YAML. Components on one host +must not share a port. Keep the endpoint on a trusted interface because it has +no built-in authentication. + +## Database Lock Failure + +A lock error normally means another process owns the database or the service +user cannot access the data directory. + +1. Confirm there is only one owner for the database path. +2. Confirm directory permissions. +3. Confirm the component uses its own Easy Migrate configuration. +4. Inspect the component and RocksDB logs. + +Do not bypass the normal RocksDB open path and do not attempt a reflected +`openDB`. + +## Shutdown Timeout + +The expected lifecycle is: + +```text +SIGTERM -> graceful shutdown -> CF/DB close + -> native MaybeForgetCF/MaybeForgetDB -> exit +``` + +If Store exceeds its existing timeout: + +1. capture thread dumps before forcing termination; +2. inspect Store shutdown and background-thread logs; +3. confirm CF handles and RocksDB reached their normal close paths; +4. treat the result as a general Store lifecycle issue. + +Do not call `SidePluginRepo.closeAllDB()`. This integration does not provide a +second Java-side owner and does not expand the scope of general thread-exit +repairs. + +## DB/CF Test Gives a False Positive + +A ToplingDB test that unsets `TOPLINGDB_EASY_MIGRATE_CONF` before opening a +database proves only RocksDB Java ABI compatibility. It does not test ToplingDB +configuration or native lifecycle hooks. + +For a functional test, keep a readable component configuration in the +environment and exercise create, write, close, reopen, drop, and recreate +operations. Reserve no-configuration checks for ABI diagnostics that do not +open a database. diff --git a/docs/toplingdb/toplingdb.md b/docs/toplingdb/toplingdb.md new file mode 100644 index 0000000000..77e9904a0e --- /dev/null +++ b/docs/toplingdb/toplingdb.md @@ -0,0 +1,134 @@ +# ToplingDB Support and Configuration + +ToplingDB is an optional RocksDB-compatible runtime for HugeGraph. The +integration uses ToplingDB Easy Migrate: HugeGraph keeps its existing RocksDB +Java API, while a native hook applies ToplingDB configuration and tracks the +DB/column-family lifecycle. + +New users should start with the [ToplingDB quickstart](toplingdb-quickstart.md). +Contributors should use the +[ToplingDB developer guide](toplingdb-development.md). + +## Architecture + +```text +component configuration + -> component-local prepare-topling.sh prepares the runtime + -> start script selects rocksdb or topling + -> preload-topling.sh exports TOPLINGDB_EASY_MIGRATE_CONF + -> JVM uses the existing RocksDB Java API + -> native Easy Migrate hook applies options and tracks DB/CF close +``` + +There is no HugeGraph Java provider SPI, reflected `SidePluginRepo`, or +separate ToplingDB service process. + +## Component Scope + +| Component | Local RocksDB owner | Easy Migrate YAML | +|---|---:|---| +| Standalone HugeGraph Server | Yes | `conf/toplingdb.yaml` | +| HugeGraph PD | Yes | `conf/rocksdb_pd.yaml` | +| HugeGraph Store | Yes | `conf/rocksdb_store.yaml` | +| HugeGraph Server using HStore | No | Not applicable | + +An HStore-backed Server is a remote client. Configure ToplingDB on PD and Store, +not on that Server process. + +## Enable ToplingDB + +ToplingDB is opt-in. Use the Topling distribution or image for the component, +set the provider in its configuration, then prepare the bundled runtime before +starting the service. + +For Server: + +```properties +rocksdb.provider=topling +``` + +For PD or Store, set the corresponding `rocksdb.provider` value in its +application YAML. + +Each Server, PD, and Store distribution carries the same runtime helpers. A +Topling distribution additionally carries its component-local JAR under +`lib/topling/`. Prepare that runtime without a source checkout: + +```bash +bin/prepare-topling.sh +``` + +The standard distribution carries the helpers but no Topling JAR or native +library. Selecting `topling` from a standard distribution therefore fails +explicitly instead of falling back to RocksDB. + +The normal start script sources its own `bin/preload-topling.sh`. For ToplingDB +it validates the prepared files and exports: + +```bash +TOPLINGDB_EASY_MIGRATE_CONF=/absolute/path/to/component/config.yaml +LD_LIBRARY_PATH=/path/to/component/library:... +LD_PRELOAD=.../librocksdbjni-linux64.so +``` + +A missing JAR, native library, or readable configuration is a startup error. +HugeGraph does not silently change providers. + +## Easy Migrate Configuration + +The component YAML defines ToplingDB options and optional HTTP observability. +The provided configurations use: + +- `DBOptions.default` as the global fallback; +- `DBOptions.log` as a dedicated profile for the log database. + +Keep both mappings. The log profile is intentional and must not be folded into +the fallback. + +The embedded HTTP endpoint has no authentication. Keep it bound to a trusted +interface and restrict network access. + +## DB and Column-Family Lifecycle + +HugeGraph continues to call `RocksDB.open()`, column-family APIs, handle +`close()`, and `RocksDB.close()`. Easy Migrate observes these calls in native +code, applies the matching configuration, retains live DB/CF state, and releases +it through `MaybeForgetCF` and `MaybeForgetDB` on normal close. + +Do not create or close a `SidePluginRepo` from HugeGraph Java code. + +## Shutdown + +Use the normal component stop script or send SIGTERM: + +```text +SIGTERM + -> component graceful shutdown + -> normal CF/DB close + -> native MaybeForgetCF / MaybeForgetDB + -> JVM exit +``` + +Store has an existing bounded shutdown wait. If it times out, diagnose the +ordinary Store thread/lifecycle issue. Do not call a Topling-specific +`closeAllDB()` workaround. + +## Compatibility and Rollback + +ToplingDB-specific options may produce WAL, SST, or metadata that standard +RocksDB cannot safely reopen. Switching only `rocksdb.provider` or replacing a +JAR is not a rollback. + +Before enabling ToplingDB, create a complete RocksDB-consistent checkpoint or +backup. To roll back, stop writers and restore the full pre-migration snapshot +with the matching standard RocksDB runtime. + +## Related Guides + +- [Documentation index](README.md) +- [Quickstart](toplingdb-quickstart.md) +- [Developer guide](toplingdb-development.md) +- [ToplingDB and HStore integration](toplingdb-hstore-integration.md) +- [Operations guide](toplingdb-operations.md) +- [Troubleshooting](toplingdb-troubleshooting.md) +- [Security guide](toplingdb-security.md) diff --git a/hugegraph-cluster-test/README.md b/hugegraph-cluster-test/README.md new file mode 100644 index 0000000000..adafd07982 --- /dev/null +++ b/hugegraph-cluster-test/README.md @@ -0,0 +1,22 @@ +# Cluster integration tests + +The simple and multi cluster suites start isolated PD, Store and Server processes +using the locally packaged distributions. Build the distributions first, then run +`mvn test -pl hugegraph-cluster-test/hugegraph-clustertest-test -am -P simple-cluster-test` +or the `multi-cluster-test` profile. + +Each node has a five-minute startup deadline. An exited process fails immediately, +even if an old log contains a ready marker. Failure messages identify the node and +its startup log. Each launch archives an existing startup log beside the new +log under a unique `.previous-*.log` name so readiness only uses the current attempt. A startup failure stops the task's node processes and preserves +their directories for diagnosis; normal teardown removes test directories. +An interrupted startup wait preserves the thread interrupt status. Shutdown waits +up to 20 seconds for a graceful exit and then up to 10 seconds after force-kill. +Interruptions do not reset these deadlines or abandon the wait; the interrupt +status is restored afterward. A process surviving force-kill fails teardown, and +its directory is retained. Cleanup still attempts the remaining nodes. These checks bound +startup diagnostics; the suites must still pass their functional assertions. + +The generated Server metadata `pd.peers` uses the same randomized PD addresses +as graph storage. Store readiness waits for the completed application startup +marker, rather than Spring's initial starting message. diff --git a/hugegraph-cluster-test/hugegraph-clustertest-dist/src/assembly/static/conf/rest-server.properties.template b/hugegraph-cluster-test/hugegraph-clustertest-dist/src/assembly/static/conf/rest-server.properties.template index 43979ec120..5b7c770641 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-dist/src/assembly/static/conf/rest-server.properties.template +++ b/hugegraph-cluster-test/hugegraph-clustertest-dist/src/assembly/static/conf/rest-server.properties.template @@ -19,10 +19,13 @@ # could use '0.0.0.0' or specified (real)IP to expose external network access restserver.url=http://$REST_SERVER_ADDRESS$ # gremlin server url, need to be consistent with host and port in gremlin-server.yaml -#gremlinserver.url=http://$REST_SERVER_ADDRESS$ +gremlinserver.url=http://$GREMLIN_SERVER_ADDRESS$ graphs=./conf/graphs +# Server metadata and graph storage must use the same test PD cluster. +pd.peers=$PD_PEERS_LIST$ + # The maximum thread ratio for batch writing, only take effect if the batch.max_write_threads is 0 batch.max_write_ratio=80 batch.max_write_threads=0 diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/pom.xml b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/pom.xml index b59648304f..b261e0f5b1 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/pom.xml +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/pom.xml @@ -51,7 +51,8 @@ org.slf4j slf4j-api - 2.0.9 + + 1.7.25 compile diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ClusterConfig.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ClusterConfig.java index c71e4b07e1..6e1e9881f0 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ClusterConfig.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ClusterConfig.java @@ -59,6 +59,7 @@ public ClusterConfig(int pdCnt, int storeCnt, int serverCnt) { for (int i = 0; i < serverCnt; i++) { ServerConfig serverConfig = new ServerConfig(); + serverConfig.setPDPeersList(pdGrpcList); serverConfigs.add(serverConfig); GraphConfig graphConfig = new GraphConfig(); graphConfig.setPDPeersList(pdGrpcList); diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ServerConfig.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ServerConfig.java index 569a11dddf..bffc775a0c 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ServerConfig.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ServerConfig.java @@ -24,6 +24,7 @@ import static org.apache.hugegraph.ct.base.EnvUtil.getAvailablePort; import java.nio.file.Paths; +import java.util.List; import lombok.Getter; @@ -32,16 +33,23 @@ public class ServerConfig extends AbstractConfig { private final int rpcPort; private final int restPort; + private final int gremlinPort; public ServerConfig() { readTemplate(Paths.get(CONFIG_FILE_PATH + SERVER_TEMPLATE_FILE)); this.fileName = SERVER_PROPERTIES; this.rpcPort = getAvailablePort(); this.restPort = getAvailablePort(); + this.gremlinPort = getAvailablePort(); properties.put("REST_SERVER_ADDRESS", LOCALHOST + ":" + this.restPort); + properties.put("GREMLIN_SERVER_ADDRESS", LOCALHOST + ":" + this.gremlinPort); properties.put("RPC_PORT", String.valueOf(this.rpcPort)); } + public void setPDPeersList(List pdPeers) { + setProperty("PD_PEERS_LIST", String.join(",", pdPeers)); + } + public void setServerID(String serverID) { setProperty("SERVER_ID", serverID); } @@ -51,4 +59,3 @@ public void setRole(String role) { } } - diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java index 0c24860929..2e1aa07bba 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.hugegraph.ct.base.HGTestLogger; import org.apache.hugegraph.ct.config.ClusterConfig; @@ -28,6 +29,8 @@ import org.apache.hugegraph.ct.config.PDConfig; import org.apache.hugegraph.ct.config.ServerConfig; import org.apache.hugegraph.ct.config.StoreConfig; +import org.apache.hugegraph.ct.node.AbstractNodeWrapper; +import org.apache.hugegraph.ct.node.BaseNodeWrapper; import org.apache.hugegraph.ct.node.PDNodeWrapper; import org.apache.hugegraph.ct.node.ServerNodeWrapper; import org.apache.hugegraph.ct.node.StoreNodeWrapper; @@ -41,6 +44,8 @@ public abstract class AbstractEnv implements BaseEnv { private static final Logger LOG = HGTestLogger.ENV_LOG; + private boolean startupFailed; + protected ClusterConfig clusterConfig; protected List pdNodeWrappers; protected List serverNodeWrappers; @@ -71,9 +76,11 @@ protected void init(int pdCnt, int storeCnt, int serverCnt) { } for (int i = 0; i < serverCnt; i++) { - ServerNodeWrapper serverNodeWrapper = new ServerNodeWrapper(cluster_id, i); - serverNodeWrappers.add(serverNodeWrapper); ServerConfig serverConfig = clusterConfig.getServerConfig(i); + int gremlinPort = serverConfig.getGremlinPort(); + ServerNodeWrapper serverNodeWrapper = new ServerNodeWrapper(cluster_id, i, + gremlinPort); + serverNodeWrappers.add(serverNodeWrapper); serverConfig.setServerID(serverNodeWrapper.getID()); GraphConfig graphConfig = clusterConfig.getGraphConfig(i); if (i == 0) { @@ -87,47 +94,94 @@ protected void init(int pdCnt, int storeCnt, int serverCnt) { } public void startCluster() { - for (PDNodeWrapper pdNodeWrapper : pdNodeWrappers) { - pdNodeWrapper.start(); - while (!pdNodeWrapper.isStarted()) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + try { + for (PDNodeWrapper node : this.pdNodeWrappers) { + startNode(node); } - } - for (StoreNodeWrapper storeNodeWrapper : storeNodeWrappers) { - storeNodeWrapper.start(); - while (!storeNodeWrapper.isStarted()) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + for (StoreNodeWrapper node : this.storeNodeWrappers) { + startNode(node); + } + for (ServerNodeWrapper node : this.serverNodeWrappers) { + startNode(node); } + } catch (RuntimeException | Error failure) { + this.startupFailed = true; + try { + this.stopCluster(); + } catch (RuntimeException | Error cleanup) { + failure.addSuppressed(cleanup); + } + throw failure; } - for (ServerNodeWrapper serverNodeWrapper : serverNodeWrappers) { - serverNodeWrapper.start(); - while (!serverNodeWrapper.isStarted()) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - throw new RuntimeException(e); + } + + private static void startNode(AbstractNodeWrapper node) { + node.start(); + awaitStarted(node, TimeUnit.MINUTES.toMillis(5)); + } + + static void awaitStarted(BaseNodeWrapper node, long timeoutMillis) { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + try { + while (true) { + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException(); + } + if (!node.isAlive()) { + throw startupFailure(node, "process exited before becoming ready"); + } + if (node.isStarted()) { + return; + } + long remaining = deadline - System.nanoTime(); + if (remaining <= 0L) { + throw startupFailure(node, "startup timed out after " + timeoutMillis + " ms"); } + TimeUnit.NANOSECONDS.sleep(Math.min(remaining, TimeUnit.SECONDS.toNanos(1))); } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while starting " + node.getID() + + "; log: " + node.getLogPath(), e); } } + private static IllegalStateException startupFailure(BaseNodeWrapper node, String reason) { + return new IllegalStateException("Node " + node.getID() + " " + reason + + "; log: " + node.getLogPath()); + } + public void stopCluster() { - for (ServerNodeWrapper serverNodeWrapper : serverNodeWrappers) { - serverNodeWrapper.stop(); - } - for (StoreNodeWrapper storeNodeWrapper : storeNodeWrappers) { - storeNodeWrapper.stop(); + List nodes = new ArrayList<>(); + nodes.addAll(this.serverNodeWrappers); + nodes.addAll(this.storeNodeWrappers); + nodes.addAll(this.pdNodeWrappers); + stopNodes(nodes, !this.startupFailed); + } + + static void stopNodes(List nodes, boolean deleteData) { + boolean interrupted = Thread.interrupted(); + IllegalStateException failure = null; + try { + for (AbstractNodeWrapper node : nodes) { + try { + node.stop(deleteData); + } catch (RuntimeException | Error e) { + if (failure == null) { + failure = new IllegalStateException("Failed to stop cluster nodes"); + } + failure.addSuppressed(e); + } finally { + interrupted |= Thread.interrupted(); + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } } - for (PDNodeWrapper pdNodeWrapper : pdNodeWrappers) { - pdNodeWrapper.stop(); + if (failure != null) { + throw failure; } } diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/AbstractNodeWrapper.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/AbstractNodeWrapper.java index 8236bb1392..39d46e867e 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/AbstractNodeWrapper.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/AbstractNodeWrapper.java @@ -28,6 +28,7 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.List; import java.util.Scanner; @@ -158,29 +159,61 @@ public boolean isStarted() { } public void stop() { + this.stop(true); + } + + public void stop(boolean deleteData) { if (this.instance == null) { return; } this.instance.destroy(); + if (!waitForExit(this.instance, 20)) { + this.instance.destroyForcibly(); + if (!waitForExit(this.instance, 10)) { + throw new IllegalStateException("Node " + this.getID() + + " survived forced stop; log: " + this.getLogPath()); + } + } + if (deleteData) { + deleteDir(); + } + } + + private static boolean waitForExit(Process process, long seconds) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(seconds); + boolean interrupted = false; try { - if (!this.instance.waitFor(20, TimeUnit.SECONDS)) { - this.instance.destroyForcibly().waitFor(10, TimeUnit.SECONDS); + while (true) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0L) { + return !process.isAlive(); + } + try { + return process.waitFor(remaining, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + interrupted = true; + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.error("Waiting node to shutdown error.", e); } - deleteDir(); } public boolean isAlive() { - return this.instance.isAlive(); + return this.instance != null && this.instance.isAlive(); } protected ProcessBuilder runCmd(List startCmd, File stdoutFile) throws IOException { + if (stdoutFile.exists()) { + Path previous = Files.createTempFile(stdoutFile.toPath().toAbsolutePath().getParent(), + stdoutFile.getName() + ".previous-", ".log"); + Files.move(stdoutFile.toPath(), previous, StandardCopyOption.REPLACE_EXISTING); + } FileUtils.write(stdoutFile, String.join(" ", startCmd) + System.lineSeparator() + System.lineSeparator(), - StandardCharsets.UTF_8, true); + StandardCharsets.UTF_8, false); ProcessBuilder processBuilder = new ProcessBuilder(startCmd) .redirectOutput(ProcessBuilder.Redirect.appendTo(stdoutFile)) .redirectError(ProcessBuilder.Redirect.appendTo(stdoutFile)); diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/ServerNodeWrapper.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/ServerNodeWrapper.java index e16b96781e..4db3e1615b 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/ServerNodeWrapper.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/ServerNodeWrapper.java @@ -40,6 +40,8 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; @@ -49,19 +51,48 @@ public class ServerNodeWrapper extends AbstractNodeWrapper { private static List hgJars = loadHgJarsOnce(); - public ServerNodeWrapper(int clusterIndex, int index) { + public ServerNodeWrapper(int clusterIndex, int index, int gremlinPort) { super(clusterIndex, index); this.fileNames = new ArrayList<>( List.of(LOG4J_FILE, GREMLIN_SERVER_FILE, GREMLIN_DRIVER_SETTING_FILE, REMOTE_SETTING_FILE, REMOTE_OBJECTS_SETTING_FILE)); this.workPath = SERVER_LIB_PATH; createNodeDir(Paths.get(SERVER_TEMPLATE_PATH), getNodePath() + CONF_DIR + File.separator); + configureGremlinPort(gremlinPort); this.fileNames = new ArrayList<>(List.of(EMPTY_SAMPLE_GROOVY_FILE, EXAMPLE_GROOVY_FILE)); - this.startLine = "INFO: [HttpServer] Started."; createNodeDir(Paths.get(SERVER_PACKAGE_PATH), getNodePath()); createLogDir(); } + @Override + public boolean isStarted() { + try { + List lines = Files.readAllLines(Paths.get(this.getLogPath())); + boolean gremlinStarted = lines.stream().anyMatch( + line -> line.contains("Channel started at port")); + boolean restStarted = lines.stream().anyMatch( + line -> line.contains("RestServer started")); + return gremlinStarted && restStarted; + } catch (IOException ignored) { + return false; + } + } + + private void configureGremlinPort(int gremlinPort) { + Path path = Paths.get(getNodePath(), CONF_DIR, GREMLIN_SERVER_FILE); + try { + String config = Files.readString(path); + String updated = config.replace("#port: 8182", + "port: " + gremlinPort); + if (updated.equals(config)) { + throw new IllegalStateException("Missing default Gremlin port"); + } + Files.writeString(path, updated); + } catch (IOException e) { + throw new AssertionError("Failed to configure Gremlin port", e); + } + } + private static void addJarsToClasspath(File directory, List classpath) { if (directory.exists() && directory.isDirectory()) { File[] files = directory.listFiles((dir, name) -> name.endsWith(".jar")); diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/StoreNodeWrapper.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/StoreNodeWrapper.java index 1cb0f67eae..868f4037eb 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/StoreNodeWrapper.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/StoreNodeWrapper.java @@ -39,7 +39,7 @@ public StoreNodeWrapper() { super(); this.fileNames = new ArrayList<>(List.of(LOG4J_FILE)); this.workPath = STORE_LIB_PATH; - this.startLine = "o.a.h.s.n.StoreNodeApplication - Starting StoreNodeApplication"; + this.startLine = "StoreNodeApplication started."; createNodeDir(Paths.get(STORE_TEMPLATE_PATH), getNodePath() + CONF_DIR + File.separator); createLogDir(); } @@ -48,7 +48,7 @@ public StoreNodeWrapper(int clusterId, int index) { super(clusterId, index); this.fileNames = new ArrayList<>(List.of(LOG4J_FILE)); this.workPath = STORE_LIB_PATH; - this.startLine = "o.a.h.s.n.StoreNodeApplication - Starting StoreNodeApplication"; + this.startLine = "StoreNodeApplication started."; createNodeDir(Paths.get(STORE_TEMPLATE_PATH), getNodePath() + CONF_DIR + File.separator); createLogDir(); } diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/env/NodeStartupTest.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/env/NodeStartupTest.java new file mode 100644 index 0000000000..e7d82205c8 --- /dev/null +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/env/NodeStartupTest.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.ct.env; + +import java.util.Arrays; + +import org.apache.hugegraph.ct.node.AbstractNodeWrapper; +import org.apache.hugegraph.ct.node.BaseNodeWrapper; +import org.junit.Assert; +import org.junit.Test; + +public class NodeStartupTest { + + @Test + public void testLiveReadyNode() { + AbstractEnv.awaitStarted(new TestNode(true, true), 100); + } + + @Test + public void testExitedNodeCannotPassWithOldReadyMarker() { + IllegalStateException failure = Assert.assertThrows(IllegalStateException.class, + () -> AbstractEnv.awaitStarted(new TestNode(false, true), 100)); + Assert.assertTrue(failure.getMessage().contains("process exited")); + Assert.assertTrue(failure.getMessage().contains("test-node")); + Assert.assertTrue(failure.getMessage().contains("test-start.log")); + } + + @Test(timeout = 5000) + public void testMissingReadyMarkerTimesOut() { + IllegalStateException failure = Assert.assertThrows(IllegalStateException.class, + () -> AbstractEnv.awaitStarted(new TestNode(true, false), 10)); + Assert.assertTrue(failure.getMessage().contains("timed out")); + } + + @Test + public void testInterruptIsPreserved() { + Thread.currentThread().interrupt(); + try { + IllegalStateException failure = Assert.assertThrows(IllegalStateException.class, + () -> AbstractEnv.awaitStarted(new TestNode(true, false), 100)); + Assert.assertTrue(failure.getCause() instanceof InterruptedException); + Assert.assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + } + + @Test + public void testCleanupContinuesAfterFailureAndPreservesDirectories() { + StopNode first = new StopNode(true); + StopNode second = new StopNode(false); + IllegalStateException failure = Assert.assertThrows(IllegalStateException.class, + () -> AbstractEnv.stopNodes(Arrays.asList(first, second), false)); + Assert.assertEquals(1, failure.getSuppressed().length); + Assert.assertTrue(first.stopped); + Assert.assertTrue(second.stopped); + Assert.assertFalse(first.deleteData); + Assert.assertFalse(second.deleteData); + } + + @Test + public void testCleanupPreservesInterruptAfterStoppingEveryNode() { + StopNode first = new StopNode(false); + StopNode second = new StopNode(false); + Thread.currentThread().interrupt(); + try { + AbstractEnv.stopNodes(Arrays.asList(first, second), false); + Assert.assertTrue(first.stopped); + Assert.assertTrue(second.stopped); + Assert.assertFalse(first.interruptedAtStop); + Assert.assertFalse(second.interruptedAtStop); + Assert.assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + } + + private static class StopNode extends AbstractNodeWrapper { + + private final boolean fail; + private boolean stopped; + private boolean deleteData; + private boolean interruptedAtStop; + + StopNode(boolean fail) { + this.fail = fail; + } + + @Override + public void start() { + throw new UnsupportedOperationException(); + } + + @Override + public String getID() { + return "stop-node"; + } + + @Override + public void stop(boolean deleteData) { + this.stopped = true; + this.deleteData = deleteData; + this.interruptedAtStop = Thread.currentThread().isInterrupted(); + if (this.fail) { + throw new IllegalStateException("test stop failure"); + } + } + } + + private static class TestNode implements BaseNodeWrapper { + + private final boolean alive; + private final boolean ready; + + TestNode(boolean alive, boolean ready) { + this.alive = alive; + this.ready = ready; + } + + @Override + public void start() { + throw new UnsupportedOperationException(); + } + + @Override + public void stop() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isAlive() { + return this.alive; + } + + @Override + public boolean isStarted() { + return this.ready; + } + + @Override + public String getID() { + return "test-node"; + } + + @Override + public String getNodePath() { + return "test-node/"; + } + + @Override + public String getLogPath() { + return "test-node/test-start.log"; + } + + @Override + public void updateWorkPath(String path) { + throw new UnsupportedOperationException(); + } + + @Override + public void updateConfigPath(String path) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/node/NodeProcessTest.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/node/NodeProcessTest.java new file mode 100644 index 0000000000..14f9674192 --- /dev/null +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/node/NodeProcessTest.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.ct.node; + +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import org.junit.Assert; +import org.junit.Test; + +public class NodeProcessTest { + + @Test + public void testNewCommandArchivesOldReadyMarker() throws Exception { + Path directory = Files.createTempDirectory("hugegraph-node-log-"); + try { + ProcessNode node = new ProcessNode(null, directory); + node.startLine = "OLD_READY"; + node.configPath = directory.toString(); + Path log = directory.resolve("start.log"); + Files.writeString(log, "OLD_READY\n"); + node.runCmd(Collections.singletonList("test-command"), log.toFile()); + Assert.assertEquals("test-command", Files.readString(log).trim()); + try (Stream files = Files.list(directory)) { + Path archive = files.filter(path -> path.getFileName().toString() + .startsWith("start.log.previous-")) + .findFirst().orElseThrow(AssertionError::new); + Assert.assertEquals("OLD_READY", Files.readString(archive).trim()); + } + } finally { + try (Stream files = Files.list(directory)) { + for (Path file : (Iterable) files::iterator) { + Files.delete(file); + } + } + Files.delete(directory); + } + } + + @Test(timeout = 40000) + public void testForcedStopPreservesFailureDirectory() throws Exception { + checkStop(true, false, false); + } + + @Test(timeout = 20000) + public void testGracefulStopDeletesNormalDirectory() throws Exception { + checkStop(false, true, false); + } + + @Test(timeout = 40000) + public void testInterruptDuringStopStillReapsChild() throws Exception { + checkStop(true, false, true); + } + + @Test + public void testSurvivingForcedStopFailsAndRetainsDirectory() throws Exception { + Path directory = Files.createTempDirectory("hugegraph-node-survivor-"); + try { + ProcessNode node = new ProcessNode(new StubbornProcess(), directory); + IllegalStateException failure = Assert.assertThrows(IllegalStateException.class, + () -> node.stop(true)); + Assert.assertTrue(failure.getMessage().contains("survived forced stop")); + Assert.assertTrue(Files.exists(directory)); + } finally { + Files.delete(directory); + } + } + + private static void checkStop(boolean slowShutdown, boolean deleteData, + boolean interrupt) throws Exception { + Path directory = Files.createTempDirectory("hugegraph-node-stop-"); + Process process = null; + Thread interrupter = null; + try { + process = new ProcessBuilder( + Paths.get(System.getProperty("java.home"), "bin", "java").toString(), + "-cp", System.getProperty("java.class.path"), NodeProcessTest.class.getName(), + directory.toString(), Boolean.toString(slowShutdown)) + .redirectErrorStream(true).redirectOutput(directory.resolve("child.log").toFile()) + .start(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!Files.exists(directory.resolve("ready"))) { + Assert.assertTrue("Child exited before ready", process.isAlive()); + Assert.assertTrue("Child readiness timed out", System.nanoTime() < deadline); + Thread.sleep(10); + } + ProcessNode node = new ProcessNode(process, directory); + if (interrupt) { + Thread caller = Thread.currentThread(); + interrupter = new Thread(() -> { + try { + Thread.sleep(100); + caller.interrupt(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + interrupter.start(); + } + node.stop(deleteData); + Assert.assertEquals(interrupt, Thread.currentThread().isInterrupted()); + Thread.interrupted(); + Assert.assertFalse("Node process survived stop", process.isAlive()); + Assert.assertEquals(!deleteData, Files.exists(directory)); + if (!deleteData) { + Assert.assertTrue(Files.exists(directory.resolve("child.log"))); + } + } finally { + Thread.interrupted(); + if (interrupter != null) { + interrupter.interrupt(); + interrupter.join(1000); + } + if (process != null && process.isAlive()) { + process.destroyForcibly(); + process.waitFor(5, TimeUnit.SECONDS); + } + if (Files.exists(directory)) { + Files.deleteIfExists(directory.resolve("ready")); + Files.deleteIfExists(directory.resolve("child.log")); + Files.delete(directory); + } + } + } + + public static void main(String[] args) throws Exception { + if (Boolean.parseBoolean(args[1])) { + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + Thread.sleep(60000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + })); + } + Files.createFile(Paths.get(args[0], "ready")); + Thread.sleep(60000); + } + + private static class StubbornProcess extends Process { + + @Override + public OutputStream getOutputStream() { + return OutputStream.nullOutputStream(); + } + + @Override + public InputStream getInputStream() { + return InputStream.nullInputStream(); + } + + @Override + public InputStream getErrorStream() { + return InputStream.nullInputStream(); + } + + @Override + public int waitFor() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean waitFor(long timeout, TimeUnit unit) { + return false; + } + + @Override + public int exitValue() { + throw new IllegalThreadStateException("still alive"); + } + + @Override + public boolean isAlive() { + return true; + } + + @Override + public void destroy() { + } + + @Override + public Process destroyForcibly() { + return this; + } + } + + private static class ProcessNode extends AbstractNodeWrapper { + + private final Path directory; + + ProcessNode(Process process, Path directory) { + this.instance = process; + this.directory = directory; + } + + @Override + public void start() { + throw new UnsupportedOperationException(); + } + + @Override + public String getID() { + return "process-test"; + } + + @Override + public String getNodePath() { + return this.directory == null ? super.getNodePath() : this.directory.toString(); + } + } +} diff --git a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/LoggingBindingTest.java b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/LoggingBindingTest.java new file mode 100644 index 0000000000..1318a0a229 --- /dev/null +++ b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/LoggingBindingTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph; + +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.LoggerFactory; +import org.slf4j.helpers.NOPLoggerFactory; + +public class LoggingBindingTest { + + @Test + public void testClusterDiagnosticsHaveLoggingBackend() { + Assert.assertFalse("Cluster diagnostics must not use a no-op logger", + LoggerFactory.getILoggerFactory() instanceof NOPLoggerFactory); + Assert.assertTrue(LoggerFactory.getLogger(LoggingBindingTest.class).isErrorEnabled()); + } +} diff --git a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSuiteTest.java b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSuiteTest.java index 6e55cdd200..40d3be41fb 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSuiteTest.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSuiteTest.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.MultiClusterTest; +import org.apache.hugegraph.LoggingBindingTest; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -24,6 +25,7 @@ @RunWith(Suite.class) @Suite.SuiteClasses({ + LoggingBindingTest.class, MultiClusterDeployTest.class, MultiClusterFileTest.class, }) diff --git a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/BaseSimpleTest.java b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/BaseSimpleTest.java index f0f0c33461..d6df82bcfa 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/BaseSimpleTest.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/BaseSimpleTest.java @@ -74,7 +74,10 @@ public static void initEnv() { public static void clearEnv() throws InterruptedException { env.stopCluster(); Thread.sleep(2000); - client.close(); + if (client != null) { + client.close(); + client = null; + } } protected String execCmd(String[] cmds) throws IOException { diff --git a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterDeployTest.java b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterDeployTest.java index 267e186f9e..345ee487ef 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterDeployTest.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterDeployTest.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.List; +import java.util.Map; import org.apache.hugegraph.pd.client.PDClient; import org.apache.hugegraph.pd.client.PDConfig; @@ -26,6 +27,8 @@ import org.junit.Assert; import org.junit.Test; +import jakarta.ws.rs.core.Response; + public class SimpleClusterDeployTest extends BaseSimpleTest { @Test @@ -113,4 +116,10 @@ public void testServerNode() { + "\"properties\":[]\n" + "}", 202); } + + @Test + public void testGremlinEndpoint() { + Response response = client.get("gremlin", Map.of("gremlin", "1 + 1")); + assertResponseStatus(200, response); + } } diff --git a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterFileTest.java b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterFileTest.java index 1cae2bcdba..9e5637ebe6 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterFileTest.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterFileTest.java @@ -18,6 +18,10 @@ package org.apache.hugegraph.SimpleClusterTest; import java.io.File; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Properties; import org.junit.Assert; import org.junit.Test; @@ -45,4 +49,17 @@ public void checkServerNodeDir() { } } + @Test + public void checkServerMetadataPDPeers() throws Exception { + String expected = String.join(",", env.getPDGrpcAddrs()); + for (String nodeDir : env.getServerNodeDir()) { + Properties config = new Properties(); + try (Reader reader = Files.newBufferedReader( + Paths.get(nodeDir, "conf", "rest-server.properties"))) { + config.load(reader); + } + Assert.assertEquals(expected, config.getProperty("pd.peers")); + } + } + } diff --git a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterSuiteTest.java b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterSuiteTest.java index 7f24d8b46c..671e51deee 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterSuiteTest.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterSuiteTest.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.SimpleClusterTest; +import org.apache.hugegraph.LoggingBindingTest; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -24,6 +25,7 @@ @RunWith(Suite.class) @Suite.SuiteClasses({ + LoggingBindingTest.class, SimpleClusterDeployTest.class, SimpleClusterFileTest.class, }) diff --git a/hugegraph-commons/hugegraph-common/src/test/java/org/apache/hugegraph/unit/license/MachineInfoTest.java b/hugegraph-commons/hugegraph-common/src/test/java/org/apache/hugegraph/unit/license/MachineInfoTest.java index 9dc69cbdb9..4e33f99151 100644 --- a/hugegraph-commons/hugegraph-common/src/test/java/org/apache/hugegraph/unit/license/MachineInfoTest.java +++ b/hugegraph-commons/hugegraph-common/src/test/java/org/apache/hugegraph/unit/license/MachineInfoTest.java @@ -18,6 +18,7 @@ package org.apache.hugegraph.unit.license; import java.net.InetAddress; +import java.net.Inet6Address; import java.net.UnknownHostException; import java.util.List; import java.util.regex.Pattern; @@ -33,10 +34,6 @@ public class MachineInfoTest { "^(([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\.){3}" + "([01]?\\d\\d?|2[0-4]\\d|25[0-5])$" ); - private static final Pattern IPV6_PATTERN = Pattern.compile( - "^(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$" - ); - private static final Pattern MAC_PATTERN = Pattern.compile( "^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$" ); @@ -47,8 +44,7 @@ public class MachineInfoTest { public void testGetIpAddressList() { List ipAddressList = machineInfo.getIpAddress(); for (String ip : ipAddressList) { - Assert.assertTrue(IPV4_PATTERN.matcher(ip).matches() || - IPV6_PATTERN.matcher(ip).matches()); + Assert.assertTrue(isIpAddress(ip)); } Assert.assertEquals(ipAddressList, machineInfo.getIpAddress()); } @@ -67,8 +63,47 @@ public void testGetLocalAllInetAddress() { List addressList = machineInfo.getLocalAllInetAddress(); for (InetAddress address : addressList) { String ip = address.getHostAddress(); - Assert.assertTrue(IPV4_PATTERN.matcher(ip).matches() || - IPV6_PATTERN.matcher(ip).matches()); + Assert.assertTrue(isIpAddress(ip)); + } + } + + @Test + public void testScopedIpv6AddressValidation() throws UnknownHostException { + byte[] bytes = new byte[16]; + bytes[0] = 0x20; + bytes[1] = 0x01; + bytes[2] = 0x0d; + bytes[3] = (byte) 0xb8; + bytes[15] = 1; + Inet6Address scoped = Inet6Address.getByAddress(null, bytes, 7); + Assert.assertTrue(isIpAddress(scoped.getHostAddress())); + Inet6Address parsed = (Inet6Address) InetAddress.getByName(scoped.getHostAddress()); + Assert.assertArrayEquals(bytes, parsed.getAddress()); + Assert.assertEquals(7, parsed.getScopeId()); + Assert.assertTrue(isIpAddress("2001:db8::1")); + Assert.assertTrue(isIpAddress("192.0.2.1")); + } + + @Test + public void testRejectMalformedIpAddresses() { + for (String ip : new String[]{"localhost", "256.0.0.1", "2001:db8::gg", + "2001::db8::1", "localhost:8080"}) { + Assert.assertFalse(isIpAddress(ip)); + } + } + + private static boolean isIpAddress(String ip) { + if (IPV4_PATTERN.matcher(ip).matches()) { + return true; + } + // Only parse IPv6 literals here; never resolve a hostname through DNS. + if (!ip.contains(":")) { + return false; + } + try { + return InetAddress.getByName(ip) instanceof Inet6Address; + } catch (UnknownHostException e) { + return false; } } diff --git a/hugegraph-pd/Dockerfile b/hugegraph-pd/Dockerfile index 655860d407..077e77ffac 100644 --- a/hugegraph-pd/Dockerfile +++ b/hugegraph-pd/Dockerfile @@ -33,9 +33,9 @@ RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sh -am $MAVEN_ARGS -e -B -ntp -Dmaven.test.skip=true -Dmaven.javadoc.skip=true \ && rm ./hugegraph-server/*.tar.gz ./hugegraph-pd/*.tar.gz ./hugegraph-store/*.tar.gz -# 2nd stage: runtime env +# Shared runtime environment # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 -FROM eclipse-temurin:11-jre-jammy +FROM eclipse-temurin:11-jre-noble AS runtime LABEL maintainer="HugeGraph Docker Maintainers " @@ -56,21 +56,82 @@ RUN apt-get -q update \ dumb-init \ procps \ curl \ + util-linux \ vim \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -COPY --from=build /pkg/hugegraph-pd/apache-hugegraph-pd-*/ /hugegraph-pd/ - # 2. Init docker script COPY hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh . RUN chmod 755 ./docker-entrypoint.sh EXPOSE 8620 -VOLUME /hugegraph-pd HEALTHCHECK --interval=15s --timeout=10s --start-period=90s --retries=3 \ CMD curl -fsS http://localhost:8620/v1/health >/dev/null ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["./docker-entrypoint.sh"] + +# Topling runtime packages. The current native library requires glibc 2.38+. +FROM runtime AS topling-runtime + +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends --no-install-suggests \ + libaio1t64 \ + libbz2-1.0 \ + libjemalloc2 \ + liblz4-1 \ + libsnappy1v5 \ + liburing2 \ + zlib1g \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Build the formal component-local Topling distribution on Linux amd64. +FROM topling-runtime AS build-topling + +COPY --from=build /pkg/ /pkg/ +WORKDIR /pkg +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends --no-install-suggests \ + rsync \ + unzip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && PD_DIR=$(find /pkg/hugegraph-pd -maxdepth 1 -type d \ + -name 'apache-hugegraph-pd-*' ! -name '*-topling' \ + -print -quit) \ + && test -n "$PD_DIR" \ + && VERSION=${PD_DIR##*/apache-hugegraph-pd-} \ + && install-dist/scripts/build-topling-distribution.sh pd "$VERSION" + +# Topling image: contains both providers and selects ToplingDB by default. +# Build with: docker build --platform linux/amd64 --target topling ... +FROM topling-runtime AS topling + +ARG SOURCE_REPOSITORY=https://github.com/apache/hugegraph +ARG SOURCE_REVISION=local +LABEL org.opencontainers.image.source="${SOURCE_REPOSITORY}" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" + +LABEL org.apache.hugegraph.rocksdb-runtime="topling" +COPY --from=build-topling /pkg/hugegraph-pd/apache-hugegraph-pd-*-topling/ /hugegraph-pd/ +ENV HG_PD_ROCKSDB_PROVIDER="topling" \ + HG_PD_ENFORCE_PROVIDER_MARKER="true" +RUN mkdir -p /hugegraph-pd/pd_data /hugegraph-pd/topling-pd-data +VOLUME ["/hugegraph-pd/pd_data", "/hugegraph-pd/topling-pd-data"] + +# Keep the standard image as the final/default target for existing builds. +FROM runtime AS standard + +ARG SOURCE_REPOSITORY=https://github.com/apache/hugegraph +ARG SOURCE_REVISION=local +LABEL org.opencontainers.image.source="${SOURCE_REPOSITORY}" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" + +LABEL org.apache.hugegraph.rocksdb-runtime="standard" +COPY --from=build /pkg/hugegraph-pd/apache-hugegraph-pd-*/ /hugegraph-pd/ +ENV HG_PD_ROCKSDB_PROVIDER="rocksdb" +# Preserve the historical root volume for anonymous-volume upgrades. +VOLUME /hugegraph-pd diff --git a/hugegraph-pd/hg-pd-cli/pom.xml b/hugegraph-pd/hg-pd-cli/pom.xml index 4920174d76..98b5954278 100644 --- a/hugegraph-pd/hg-pd-cli/pom.xml +++ b/hugegraph-pd/hg-pd-cli/pom.xml @@ -49,7 +49,7 @@ com.alipay.sofa jraft-core - 1.3.13 + 1.3.14 org.rocksdb diff --git a/hugegraph-pd/hg-pd-core/pom.xml b/hugegraph-pd/hg-pd-core/pom.xml index e17570d592..48d43dbd41 100644 --- a/hugegraph-pd/hg-pd-core/pom.xml +++ b/hugegraph-pd/hg-pd-core/pom.xml @@ -38,7 +38,7 @@ com.alipay.sofa jraft-core - 1.3.13 + 1.3.14 org.rocksdb @@ -49,7 +49,7 @@ org.rocksdb rocksdbjni - 6.29.5 + ${rocksdb.version} org.apache.hugegraph diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/StoreNodeService.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/StoreNodeService.java index 3503d1ffc8..3479040d0c 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/StoreNodeService.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/StoreNodeService.java @@ -858,6 +858,10 @@ public synchronized void checkStoreStatus() { } catch (PDException e) { log.error("StoreNodeService updateClusterStatus exception {}", e); + // A failed read must not replace the previous state with the + // builder's initial Cluster_OK. REST refreshes a stale + // Cluster_Not_Ready from this method. + return; } this.clusterStats = builder.setTimestamp(System.currentTimeMillis()).build(); if (this.clusterStats.getState() != Metapb.ClusterState.Cluster_OK) { diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImpl.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImpl.java index bd2e7a9e22..6f7ea817b5 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImpl.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImpl.java @@ -67,7 +67,7 @@ public void init(PDConfig config) { final Lock writeLock = this.readWriteLock.writeLock(); writeLock.lock(); try { - this.dbPath = config.getDataPath() + "/rocksdb/"; + this.dbPath = config.getDataPath() + "/rocksdb"; File file = new File(this.dbPath); if (!file.exists()) { try { diff --git a/hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh b/hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh index 77af2af89d..630d5020ff 100755 --- a/hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh +++ b/hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh @@ -87,8 +87,30 @@ require_env "HG_PD_AUTH_SECRET_KEY" : "${HG_PD_GRPC_PORT:=8686}" : "${HG_PD_REST_PORT:=8620}" -: "${HG_PD_DATA_PATH:=/hugegraph-pd/pd_data}" : "${HG_PD_INITIAL_STORE_COUNT:=1}" +: "${HG_PD_ROCKSDB_PROVIDER:=rocksdb}" + +case "${HG_PD_ROCKSDB_PROVIDER}" in + rocksdb | topling) ;; + *) + log "ERROR: HG_PD_ROCKSDB_PROVIDER must be rocksdb or topling" + exit 2 + ;; +esac +if [[ -z "${HG_PD_DATA_PATH:-}" ]]; then + if [[ "${HG_PD_ROCKSDB_PROVIDER}" == "topling" ]]; then + HG_PD_DATA_PATH="$(pwd)/topling-pd-data" + else + HG_PD_DATA_PATH="$(pwd)/pd_data" + fi +fi +export TOPLINGDB_ROCKSDB_PROVIDER="${HG_PD_ROCKSDB_PROVIDER}" +./bin/verify-rocksdb-provider.sh \ + pd \ + "${HG_PD_ROCKSDB_PROVIDER}" \ + "${HG_PD_DATA_PATH}" \ + "${HG_PD_ENFORCE_PROVIDER_MARKER:-false}" + # Actuator endpoints reachable without a credential. Hardened by default; an # operator who needs /actuator/info or /actuator/loggers from this image opts # in deliberately instead of losing the endpoint. "*" is refused: /actuator/env @@ -119,6 +141,7 @@ SPRING_APPLICATION_JSON="$(cat < + + + + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh + + bin + common-topling.sh + 0755 + + + + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/bin/prepare-topling.sh + + bin + prepare-topling.sh + 0755 + + + + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh + + bin + preload-topling.sh + 0755 + + + + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/bin/verify-rocksdb-provider.sh + + bin + verify-rocksdb-provider.sh + 0755 + + + diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh index 1329df2271..f17c791888 100755 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh @@ -66,6 +66,13 @@ PID_FILE="$BIN/pid" ensure_path_writable "$LOGS" ensure_path_writable "$PLUGINS" +# preload rocksdb/toplingdb +if [ ! -r "$BIN/preload-topling.sh" ]; then + echo "Required RocksDB runtime selector not found: $BIN/preload-topling.sh" >&2 + exit 1 +fi +source "$BIN/preload-topling.sh" || exit 1 + # The maximum and minimum heap memory that service can use MAX_MEM=$((32 * 1024)) MIN_MEM=$((1 * 512)) @@ -83,7 +90,7 @@ fi # check jdk version JAVA_VERSION=$($JAVA -version 2>&1 | awk 'NR==1{gsub(/"/,""); print $3}' | awk -F'_' '{print $1}') -if [[ $? -ne 0 || $JAVA_VERSION < $EXPECT_JDK_VERSION ]]; then +if [[ $? -ne 0 || $JAVA_VERSION -lt $EXPECT_JDK_VERSION ]]; then echo "Please make sure that the JDK is installed and the version >= $EXPECT_JDK_VERSION" >> ${OUTPUT} exit 1 fi @@ -163,22 +170,34 @@ fi #if [ "${JMX_EXPORT_PORT}" != "" ] && [ ${JMX_EXPORT_PORT} -ne 0 ] ; then # JAVA_OPTIONS="${JAVA_OPTIONS} -javaagent:${LIB}/jmx_prometheus_javaagent-0.16.1.jar=${JMX_EXPORT_PORT}:${CONF}/jmx_exporter.yml" #fi -if [ $(ps -ef|grep -v grep| grep java|grep -cE ${CONF}) -ne 0 ]; then +if [ "$(ps -ef | grep -v grep | grep java | grep -cE "${CONF}")" -ne 0 ]; then echo "HugeGraphPDServer is already running..." exit 0 fi JVM_OPTIONS="-Dlog4j.configurationFile=${CONF}/log4j2.xml -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" +BOOT_JARS=("${LIB}"/hg-pd-service-*.jar) +if [ "${#BOOT_JARS[@]}" -ne 1 ] || [ ! -f "${BOOT_JARS[0]}" ]; then + echo "Expected exactly one HugeGraph PD executable JAR in ${LIB}" >> "${OUTPUT}" + exit 1 +fi +BOOT_JAR="${BOOT_JARS[0]}" +JAVA_MAIN=(-jar "$BOOT_JAR") +if [ -n "${TOPLING_RUNTIME_CLASSPATH:-}" ]; then + JAVA_MAIN=(-cp "${TOPLING_RUNTIME_CLASSPATH}:${BOOT_JAR}" \ + org.springframework.boot.loader.JarLauncher) +fi # Turn on security check if [[ $DAEMON == "true" ]]; then echo "Starting HugeGraphPDServer in daemon mode..." if [[ "${STDOUT_MODE:-false}" == "true" ]]; then - exec ${JAVA} -Dname="HugeGraphPD" ${JVM_OPTIONS} ${JAVA_OPTIONS} -jar \ - -Dspring.config.location=${CONF}/application.yml ${LIB}/hg-pd-service-*.jar & + exec ${JAVA} -Dname="HugeGraphPD" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + -Dspring.config.location=${CONF}/application.yml "${JAVA_MAIN[@]}" & else - exec ${JAVA} -Dname="HugeGraphPD" ${JVM_OPTIONS} ${JAVA_OPTIONS} -jar \ - -Dspring.config.location=${CONF}/application.yml ${LIB}/hg-pd-service-*.jar >> ${OUTPUT} 2>&1 & + exec ${JAVA} -Dname="HugeGraphPD" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + -Dspring.config.location=${CONF}/application.yml "${JAVA_MAIN[@]}" \ + >> ${OUTPUT} 2>&1 & fi PID="$!" # Write pid to file @@ -190,10 +209,11 @@ else echo "$$" > "$PID_FILE" echo "[+pid] $$" if [[ "${STDOUT_MODE:-false}" == "true" ]]; then - exec ${JAVA} -Dname="HugeGraphPD" ${JVM_OPTIONS} ${JAVA_OPTIONS} -jar \ - -Dspring.config.location=${CONF}/application.yml ${LIB}/hg-pd-service-*.jar + exec ${JAVA} -Dname="HugeGraphPD" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + -Dspring.config.location=${CONF}/application.yml "${JAVA_MAIN[@]}" else - exec ${JAVA} -Dname="HugeGraphPD" ${JVM_OPTIONS} ${JAVA_OPTIONS} -jar \ - -Dspring.config.location=${CONF}/application.yml ${LIB}/hg-pd-service-*.jar >> ${OUTPUT} 2>&1 + exec ${JAVA} -Dname="HugeGraphPD" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + -Dspring.config.location=${CONF}/application.yml "${JAVA_MAIN[@]}" \ + >> ${OUTPUT} 2>&1 fi fi diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh index b476d7935b..b42239d2b7 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh @@ -394,3 +394,21 @@ function kill_process_and_wait() { kill_process "$process_name" "$pid" wait_for_shutdown "$process_name" "$pid" "$timeout_s" } + +# Find HugeGraph server directory in parent path using prefix glob. +# Usage: find_hugegraph_server_dir "/path/to/parent" +# Returns: first matching directory path or empty string +function find_hugegraph_server_dir() { + local parent_dir="$1" + if [ -z "$parent_dir" ]; then + parent_dir="$(cd "${TOP:-$(pwd)}"/.. && pwd)" + fi + local found="" + for d in "$parent_dir"/apache-hugegraph-server*; do + if [ -d "$d" ]; then + found="$d" + break + fi + done + echo "$found" +} diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml index ada9791922..e1c4af7c8a 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml @@ -42,6 +42,10 @@ grpc: port: 8686 # The service address of grpc needs to be changed to the actual local IPv4 address when deploying. host: 127.0.0.1 +# rocksdb: +# provider: rocksdb +# option-path: ./conf/rocksdb_pd.yaml +# open-http: true server: # REST service port number diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/rocksdb_pd.yaml b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/rocksdb_pd.yaml new file mode 100644 index 0000000000..cc87950bbe --- /dev/null +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/rocksdb_pd.yaml @@ -0,0 +1,164 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# common parameters +http: + # normally parent path of db path + document_root: ./library/rocksdb_resource + listening_ports: '127.0.0.1:2012' + auto_start_http: false +setenv: + StrSimpleEnvNameNotOverwrite: StringValue + IntSimpleEnvNameNotOverwrite: 16384 + OverwriteThisEnv: + #comment: overwrite is default to false + overwrite: true + value: force overwrite this env by overwrite true +Cache: + lru_cache: + class: LRUCache + params: + capacity: 8G + num_shard_bits: -1 + strict_capacity_limit: false + high_pri_pool_ratio: 0.5 + use_adaptive_mutex: false + metadata_charge_policy: kFullChargeCacheMetadata +Statistics: + stat: + class: default + params: + discard_tickers: + - rocksdb.block.cache + - rocksdb.block.cachecompressed + - rocksdb.block + - rocksdb.memtable.payload.bytes.at.flush + - rocksdb.memtable.garbage.bytes.at.flush + - rocksdb.txn + - rocksdb.blobdb + - rocksdb.row.cache + - rocksdb.number.block + - rocksdb.bloom.filter + - rocksdb.persistent + - rocksdb.sim.block.cache + discard_histograms: + # comment: .... + - rocksdb.blobdb + - rocksdb.bytes.compressed + - rocksdb.bytes.decompressed + - rocksdb.num.index.and.filter.blocks.read.per.level + - rocksdb.num.data.blocks.read.per.level + - rocksdb.compression.times.nanos + - rocksdb.decompression.times.nanos + - rocksdb.read.block.get.micros + - rocksdb.write.raw.block.micros + # comment end of array + #stats_level: kAll + stats_level: kDisableAll +MemTableRepFactory: + cspp: + class: cspp + params: + mem_cap: 16G + use_vm: false + token_use_idle: true + chunk_size: 16K + convert_to_sst: kFileMmap + sync_sst_file: false + skiplist: + class: SkipList + params: + lookahead: 0 +TableFactory: + cspp_memtab_sst: + class: CSPPMemTabTable + params: # empty params + bb: + class: BlockBasedTable + params: + checksum: kCRC32c + block_size: 4K + block_restart_interval: 16 + index_block_restart_interval: 1 + metadata_block_size: 4K + enable_index_compression: true + block_cache: "${lru_cache}" + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst + block_cache_compressed: + persistent_cache: + filter_policy: + dispatch: + class: DispatcherTable + params: + default: bb + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst + level_writers: [ bb, bb, bb, bb, bb, bb ] +CFOptions: + default: + max_write_buffer_number: 6 + memtable_factory: "${cspp}" + write_buffer_size: 128M + # set target_file_size_base as small as 512K is to make many SST files, + # thus key prefix cache can present efficiency + target_file_size_base: 64M + target_file_size_multiplier: 1 + table_factory: dispatch + max_bytes_for_level_base: 512M + max_bytes_for_level_multiplier: 10 + level_compaction_dynamic_level_bytes: false + level0_slowdown_writes_trigger: 20 + level0_stop_writes_trigger: 36 + level0_file_num_compaction_trigger: 2 + merge_operator: uint64add # support merge + level_compaction_dynamic_file_size: true + optimize_filters_for_hits: true + allow_merge_memtables: true + min_write_buffer_number_to_merge: 2 + compression_per_level: + - kNoCompression + - kNoCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression +DBOptions: + log: + create_if_missing: true + create_missing_column_families: true + default: + create_if_missing: true + create_missing_column_families: false # this is important, must be false to hugegraph + max_background_compactions: -1 + max_subcompactions: 4 + max_level1_subcompactions: 0 + inplace_update_support: false + WAL_size_limit_MB: 0 + statistics: "${stat}" + max_manifest_file_size: 100M + max_background_jobs: 8 + # Java org.rocksdb.WriteBatch has no Topling mmap WAL. With + # memtable_as_log_index enabled, every batch write fails: + # "WriteBatch has no mmap wal". Keep the option false for this client. + # convert_to_sst is still Topling-specific. Do not open data written by + # this profile with the standard RocksDB provider; restore a snapshot + # taken before migration instead. + compaction_readahead_size: 0 + memtable_as_log_index: false diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/IndexAPI.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/IndexAPI.java index bee39f23ed..d196c21f7d 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/IndexAPI.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/IndexAPI.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.concurrent.ExecutionException; +import org.apache.hugegraph.pd.StoreNodeService; import org.apache.hugegraph.pd.common.PDException; import org.apache.hugegraph.pd.grpc.Metapb; import org.apache.hugegraph.pd.grpc.Pdpb; @@ -65,7 +66,7 @@ public BriefStatistics index() throws PDException, ExecutionException, Interrupt BriefStatistics statistics = new BriefStatistics(); statistics.leader = RaftEngine.getInstance().getLeaderGrpcAddress(); - statistics.state = pdService.getStoreNodeService().getClusterStats().getState().toString(); + statistics.state = this.clusterState(); // Use pdService (consistent with cluster()) rather than RaftEngine directly CallStreamObserverWrap membersResp = @@ -99,6 +100,19 @@ public BriefStatistics index() throws PDException, ExecutionException, Interrupt } + /** + * Recompute cluster state from the replicated store list. Registration and + * heartbeats update the cache only on the raft leader, so a follower can + * keep either the constructor value {@code Cluster_Not_Ready} or a stale + * {@code Cluster_OK} from an earlier term. {@code /v1/ready} stays the raft + * probe and does not call this method. + */ + String clusterState() { + StoreNodeService stores = this.pdService.getStoreNodeService(); + stores.checkStoreStatus(); + return stores.getClusterStats().getState().toString(); + } + /** * Get cluster statistics * Obtain various statistics about the cluster by calling related services, including node status, member list, storage information, graph information, etc., @@ -109,13 +123,13 @@ public BriefStatistics index() throws PDException, ExecutionException, Interrupt * @throws ExecutionException If an exception occurs during task execution, this exception is thrown * @throws PDException If an exception occurs while processing cluster statistics, such as service call failure or data processing errors, a PDException exception is thrown */ + @GetMapping(value = "/v1/cluster", produces = MediaType.APPLICATION_JSON_VALUE) @ResponseBody public RestApiResponse cluster() throws InterruptedException, ExecutionException { Statistics statistics = new Statistics(); try { - statistics.state = - String.valueOf(pdService.getStoreNodeService().getClusterStats().getState()); + statistics.state = this.clusterState(); String leaderGrpcAddress = RaftEngine.getInstance().getLeaderGrpcAddress(); CallStreamObserverWrap response = new CallStreamObserverWrap<>(); diff --git a/hugegraph-pd/hg-pd-test/README.md b/hugegraph-pd/hg-pd-test/README.md new file mode 100644 index 0000000000..81fc525d3c --- /dev/null +++ b/hugegraph-pd/hg-pd-test/README.md @@ -0,0 +1,15 @@ +# PD module tests + +Run common and core suites separately so native and Raft singletons do not carry +state between suite processes: + +```bash +mvn test -pl hugegraph-pd/hg-pd-test -am -P pd-common-test +mvn test -pl hugegraph-pd/hg-pd-test -am -P pd-core-test +``` + +The core fixture uses `tmp/pd-core-data` relative to the test module by default. +Override it with `-Dpd.test.data_path=/absolute/task-owned/path` when isolating a +validation run. The fixture deletes that directory before initialization; never +point it at an existing service's data. Client and REST suites also require the +PD service setup described in the repository CI workflow. diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreTestBase.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreTestBase.java index 9e7b03d98e..cb56beae2b 100644 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreTestBase.java +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreTestBase.java @@ -44,7 +44,8 @@ public class PDCoreTestBase { - private static final String DATA_PATH = "/tmp/pd_data"; + private static final String DATA_PATH = System.getProperty("pd.test.data_path", + "tmp/pd-core-data"); private static PDConfig pdConfig; private static StoreNodeService storeNodeService; private static PartitionService partitionService; diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/rest/IndexAPIClusterStateTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/rest/IndexAPIClusterStateTest.java new file mode 100644 index 0000000000..bc6cf234d8 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/rest/IndexAPIClusterStateTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.rest; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.apache.hugegraph.pd.StoreNodeService; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.pd.service.PDService; +import org.junit.Assert; +import org.junit.Test; + +public class IndexAPIClusterStateTest { + + @Test + public void testRestRecomputesBeforeReadingNotReady() { + StoreNodeService stores = mock(StoreNodeService.class); + when(stores.getClusterStats()).thenReturn(stats(Metapb.ClusterState.Cluster_Not_Ready)); + IndexAPI api = api(stores); + + Assert.assertEquals("Cluster_Not_Ready", api.clusterState()); + verify(stores, times(1)).checkStoreStatus(); + } + + @Test + public void testRestRecomputesBeforeReadingOk() { + StoreNodeService stores = mock(StoreNodeService.class); + when(stores.getClusterStats()).thenReturn(stats(Metapb.ClusterState.Cluster_OK)); + IndexAPI api = api(stores); + + Assert.assertEquals("Cluster_OK", api.clusterState()); + verify(stores, times(1)).checkStoreStatus(); + } + + private static IndexAPI api(StoreNodeService stores) { + PDService pdService = mock(PDService.class); + when(pdService.getStoreNodeService()).thenReturn(stores); + IndexAPI api = new IndexAPI(); + api.pdService = pdService; + return api; + } + + private static Metapb.ClusterStats stats(Metapb.ClusterState state) { + return Metapb.ClusterStats.newBuilder().setState(state).build(); + } +} diff --git a/hugegraph-server/Dockerfile b/hugegraph-server/Dockerfile index 67866974cf..4a543c649a 100644 --- a/hugegraph-server/Dockerfile +++ b/hugegraph-server/Dockerfile @@ -33,9 +33,9 @@ RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sh -am $MAVEN_ARGS -e -B -ntp -Dmaven.test.skip=true -Dmaven.javadoc.skip=true \ && rm ./hugegraph-server/*.tar.gz ./hugegraph-pd/*.tar.gz ./hugegraph-store/*.tar.gz -# 2nd stage: runtime env +# Shared runtime environment # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 -FROM eclipse-temurin:11-jre-jammy +FROM eclipse-temurin:11-jre-noble AS runtime LABEL maintainer="HugeGraph Docker Maintainers " @@ -49,7 +49,7 @@ WORKDIR /hugegraph-server/ # 1. Install runtime dependencies # Note: iproute2 provides `ss`, which the bin/util.sh port preflight needs. The -# jammy base image ships neither ss nor netstat, so without it the preflight is +# The base image ships neither ss nor netstat, so without it the preflight is # permanently inconclusive and a duplicate start is no longer caught. It # replaces lsof, which no shipped script calls any more. ARG RUNTIME_DEPS_EPOCH=1 @@ -59,24 +59,91 @@ RUN apt-get -q update \ procps \ curl \ iproute2 \ + util-linux \ vim \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -COPY --from=build /pkg/hugegraph-server/apache-hugegraph-server-*/ /hugegraph-server/ -RUN sed -i "s/^restserver.url.*$/restserver.url=http:\/\/0.0.0.0:8080/g" ./conf/rest-server.properties - # 2. Init docker script -COPY hugegraph-server/hugegraph-dist/docker/scripts/remote-connect.groovy ./scripts -COPY hugegraph-server/hugegraph-dist/docker/scripts/detect-storage.groovy ./scripts +COPY hugegraph-server/hugegraph-dist/docker/scripts/remote-connect.groovy /opt/hugegraph-docker/ +COPY hugegraph-server/hugegraph-dist/docker/scripts/detect-storage.groovy /opt/hugegraph-docker/ COPY hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh . RUN chmod 755 ./docker-entrypoint.sh EXPOSE 8080 -VOLUME /hugegraph-server HEALTHCHECK --interval=15s --timeout=10s --start-period=90s --retries=3 \ CMD curl -fsS http://localhost:8080/versions >/dev/null ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["./docker-entrypoint.sh"] + +# Topling runtime packages. The current native library requires glibc 2.38+. +FROM runtime AS topling-runtime + +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends --no-install-suggests \ + libaio1t64 \ + libbz2-1.0 \ + libjemalloc2 \ + liblz4-1 \ + libsnappy1v5 \ + liburing2 \ + zlib1g \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Build the formal Server Topling distribution on Linux amd64. +FROM topling-runtime AS build-topling + +COPY --from=build /pkg/ /pkg/ +WORKDIR /pkg +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends --no-install-suggests \ + rsync \ + unzip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && SERVER_DIR=$(find /pkg/hugegraph-server -maxdepth 1 -type d \ + -name 'apache-hugegraph-server-*' ! -name '*-topling' \ + -print -quit) \ + && test -n "$SERVER_DIR" \ + && VERSION=${SERVER_DIR##*/apache-hugegraph-server-} \ + && install-dist/scripts/build-topling-distribution.sh server "$VERSION" + +# ToplingDB image: contains both providers and selects ToplingDB by default. +# Build with: docker build --platform linux/amd64 --target topling ... +FROM topling-runtime AS topling + +ARG SOURCE_REPOSITORY=https://github.com/apache/hugegraph +ARG SOURCE_REVISION=local +LABEL org.opencontainers.image.source="${SOURCE_REPOSITORY}" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" + +LABEL org.apache.hugegraph.rocksdb-runtime="topling" +COPY --from=build-topling /pkg/hugegraph-server/apache-hugegraph-server-*-topling/ /hugegraph-server/ +RUN cp /opt/hugegraph-docker/*.groovy ./scripts/ \ + && sed -i "s/^restserver.url.*$/restserver.url=http:\/\/0.0.0.0:8080/g" \ + ./conf/rest-server.properties +ENV HG_SERVER_ROCKSDB_PROVIDER="topling" +ENV HG_SERVER_ENFORCE_PROVIDER_MARKER="true" +RUN mkdir -p /hugegraph-server/rocksdb-data /hugegraph-server/topling-data +VOLUME ["/hugegraph-server/rocksdb-data", "/hugegraph-server/topling-data"] + +# Keep the standard image as the final/default target for existing builds. +FROM runtime AS standard + +ARG SOURCE_REPOSITORY=https://github.com/apache/hugegraph +ARG SOURCE_REVISION=local +LABEL org.opencontainers.image.source="${SOURCE_REPOSITORY}" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" + +LABEL org.apache.hugegraph.rocksdb-runtime="standard" +COPY --from=build /pkg/hugegraph-server/apache-hugegraph-server-*/ /hugegraph-server/ +RUN cp /opt/hugegraph-docker/*.groovy ./scripts/ \ + && sed -i "s/^restserver.url.*$/restserver.url=http:\/\/0.0.0.0:8080/g" \ + ./conf/rest-server.properties +ENV HG_SERVER_ROCKSDB_PROVIDER="rocksdb" +# Preserve the historical root volume so existing anonymous-volume upgrades +# continue to expose their data. New named-volume examples use rocksdb-data. +VOLUME /hugegraph-server diff --git a/hugegraph-server/Dockerfile-hstore b/hugegraph-server/Dockerfile-hstore index e90979563b..8517621110 100644 --- a/hugegraph-server/Dockerfile-hstore +++ b/hugegraph-server/Dockerfile-hstore @@ -61,6 +61,12 @@ RUN apt-get -q update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +ARG SOURCE_REPOSITORY=https://github.com/apache/hugegraph +ARG SOURCE_REVISION=local +LABEL org.opencontainers.image.source="${SOURCE_REPOSITORY}" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" \ + org.apache.hugegraph.rocksdb-runtime="hstore" + COPY --from=build /pkg/hugegraph-server/apache-hugegraph-server-*/ /hugegraph-server/ # Configure the default HStore backend and REST listen address RUN cd /hugegraph-server/conf/graphs \ diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeFactoryAuthProxy.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeFactoryAuthProxy.java index 5ff2925e69..e44412d852 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeFactoryAuthProxy.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeFactoryAuthProxy.java @@ -185,7 +185,7 @@ private static void registerPrivateActions() { "this$0"); Reflection.registerFieldsToFilter(HugeGraphAuthProxy.Context.class, "ADMIN", "user"); Reflection.registerFieldsToFilter(HugeGraphAuthProxy.ContextTask.class, "runner", - "context"); + "cleanup", "context"); Reflection.registerFieldsToFilter(StandardHugeGraph.class, "LOG", "started", "closed", "mode", "variables", "name", "params", "configuration", "schemaEventHub", "graphEventHub", "indexEventHub", @@ -203,7 +203,8 @@ private static void registerPrivateActions() { "access$14", "access$15", "access$16", "access$17", "access$18", "serializer", "loadSchemaStore", "loadSystemStore", "loadGraphStore", "closeTx", - "analyzer", "serverInfoManager", "reloadRamtable", + "closeCurrentThreadTransaction", "analyzer", + "serverInfoManager", "reloadRamtable", "reloadRamtable", "access$19", "access$20", "access$21"); Reflection.registerFieldsToFilter( loadClass("org.apache.hugegraph.StandardHugeGraph$StandardHugeGraphParams"), @@ -298,8 +299,10 @@ private static void registerPrivateActions() { "autoCommit", "beforeRead", "afterWrite", "afterRead", "commitMutation2Backend", "checkOwnerThread", "doAction", "store", "reset"); - Reflection.registerFieldsToFilter(HugeFactory.class, "LOG", "NAME_REGEX", "graphs"); - Reflection.registerMethodsToFilter(HugeFactory.class, "lambda$0"); + Reflection.registerFieldsToFilter(HugeFactory.class, "LOG", "NAME_REGEX", "graphs", + "GRAPHS"); + Reflection.registerMethodsToFilter(HugeFactory.class, "lambda$0", + "closeCurrentThreadTransactions"); Reflection.registerFieldsToFilter(SchemaElement.class, "graph", "id", "name", "userdata", "status"); Reflection.registerFieldsToFilter(HugeVertex.class, "EMPTY_SET", "id", "label", "edges", diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java index 510b838437..ff5739812b 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java @@ -40,6 +40,7 @@ import javax.security.sasl.AuthenticationException; import org.apache.commons.configuration2.Configuration; +import org.apache.hugegraph.HugeFactory; import org.apache.hugegraph.HugeGraph; import org.apache.hugegraph.auth.HugeAuthenticator.RolePerm; import org.apache.hugegraph.auth.HugeAuthenticator.User; @@ -1296,11 +1297,17 @@ public User user() { static class ContextTask implements Runnable { private final Runnable runner; + private final Runnable cleanup; private final Context context; public ContextTask(Runnable runner) { + this(runner, HugeFactory::closeCurrentThreadTransactions); + } + + ContextTask(Runnable runner, Runnable cleanup) { this.context = getContext(); this.runner = runner; + this.cleanup = cleanup; } @Override @@ -1309,7 +1316,13 @@ public void run() { try { this.runner.run(); } finally { - resetContext(); + try { + this.cleanup.run(); + } catch (Throwable e) { + LOG.error("Failed to close Gremlin worker transactions", e); + } finally { + resetContext(); + } } } } diff --git a/hugegraph-server/hugegraph-core/pom.xml b/hugegraph-server/hugegraph-core/pom.xml index b2519633a9..70aafa3278 100644 --- a/hugegraph-server/hugegraph-core/pom.xml +++ b/hugegraph-server/hugegraph-core/pom.xml @@ -29,7 +29,7 @@ ${basedir}/.. - 1.3.11 + 1.3.14 0.7.4 5.12.1 1.8.1 diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeFactory.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeFactory.java index f258cb5c7b..e0b4906f39 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeFactory.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeFactory.java @@ -19,7 +19,10 @@ import java.io.File; import java.net.URL; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; @@ -104,11 +107,39 @@ public static HugeGraph open(URL url) { return open(getRemoteConfig(url)); } - public static void remove(HugeGraph graph) { + public static synchronized void remove(HugeGraph graph) { String spaceGraphName = graph.spaceGraphName(); GRAPHS.remove(spaceGraphName); } + public static void closeCurrentThreadTransactions() { + List graphs; + synchronized (HugeFactory.class) { + graphs = new ArrayList<>(GRAPHS.values()); + } + closeCurrentThreadTransactions(graphs); + } + + static void closeCurrentThreadTransactions( + Collection graphs) { + Throwable failure = null; + for (HugeGraph graph : graphs) { + try { + ((StandardHugeGraph) graph).closeCurrentThreadTransaction(); + } catch (Throwable e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } + } + if (failure != null) { + throw new HugeException("Failed to close current thread " + + "transactions", failure); + } + } + public static void checkGraphName(String name, String configFile) { E.checkArgument(SYS_GRAPH.equals(name) || name.matches(NAME_REGEX), "Invalid graph name '%s' in %s, " + diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java index 8f2ff95d01..f405469d60 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java @@ -425,6 +425,10 @@ private void closeTx() { } } + void closeCurrentThreadTransaction() { + this.closeTx(); + } + @Override public GraphMode mode() { return this.mode; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java index 6faace9671..1f9f314d28 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java @@ -168,7 +168,7 @@ public void updateVertexIndex(HugeVertex vertex, boolean removed) { return; } // Update index(only property, no edge) of a vertex - for (Id id : vertex.schemaLabel().indexLabels()) { + for (Id id : this.indexLabelIds(vertex.schemaLabel())) { this.updateIndex(id, vertex, removed); } } @@ -176,7 +176,7 @@ public void updateVertexIndex(HugeVertex vertex, boolean removed) { @Watched(prefix = "index") public void updateEdgeIndex(HugeEdge edge, boolean removed) { // Update index of an edge - for (Id id : edge.schemaLabel().indexLabels()) { + for (Id id : this.indexLabelIds(edge.schemaLabel())) { this.updateIndex(id, edge, removed); } @@ -857,6 +857,9 @@ private MatchedIndex collectMatchedIndex(SchemaLabel schemaLabel, private ConditionQuery constructSearchQuery(ConditionQuery query, MatchedIndex index) { ConditionQuery newQuery = query; + // Keep this incoming branch's non-search conditions. Expanding an EQ + // branch back to its root IN condition can return the same element + // once per branch when joint indexes fall back to filtering. Set indexFields = new HashSet<>(); // Convert has(key, text) to has(key, textContainsAny(word1, word2)) for (IndexLabel il : index.indexLabels()) { @@ -1557,13 +1560,27 @@ private static void validateIndexLabel(IndexLabel indexLabel) { indexLabel, indexLabel.status()); } - private static boolean hasNullableProp(HugeElement element, Id key) { - return element.schemaLabel().nullableKeys().contains(key); + private boolean hasNullableProp(HugeElement element, Id key) { + return this.currentSchemaLabel(element.schemaLabel()).nullableKeys().contains(key); } - private static Set relatedIndexLabels(HugeElement element) { + Set indexLabelIds(SchemaLabel label) { + return this.currentSchemaLabel(label).indexLabels(); + } + + private SchemaLabel currentSchemaLabel(SchemaLabel label) { + // Elements can outlive schema cache eviction. Resolve the current index + // membership rather than using the schema object retained by an element. + ISchemaTransaction schema = this.params().schemaTransaction(); + SchemaLabel current = label.type() == HugeType.VERTEX_LABEL ? + schema.getVertexLabel(label.id()) : schema.getEdgeLabel(label.id()); + E.checkArgument(current != null, "Not exist schema label with id '%s'", label.id()); + return current; + } + + private Set relatedIndexLabels(HugeElement element) { Set indexLabels = InsertionOrderUtil.newSet(); - Set indexLabelIds = element.schemaLabel().indexLabels(); + Set indexLabelIds = this.indexLabelIds(element.schemaLabel()); for (Id id : indexLabelIds) { IndexLabel indexLabel = element.graph().indexLabel(id); @@ -1870,7 +1887,7 @@ private long processSecondaryOrSearchIndexLeft(ConditionQuery query, // Delete unused index long count = 0; Set incorrectPkIds; - for (IndexLabel il : relatedIndexLabels(deletion)) { + for (IndexLabel il : this.tx.relatedIndexLabels(deletion)) { incorrectPkIds = incorrectPKs.keySet().stream() .map(PropertyKey::id) .collect(Collectors.toSet()); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java index 3999b3d0c2..8587a3a79f 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java @@ -621,7 +621,7 @@ public HugeVertex addVertex(HugeVertex vertex) { this.locksTable.lockReads(LockUtil.VERTEX_LABEL_DELETE, vertex.schemaLabel().id()); this.locksTable.lockReads(LockUtil.INDEX_LABEL_DELETE, - vertex.schemaLabel().indexLabels()); + this.indexTx.indexLabelIds(vertex.schemaLabel())); // Ensure vertex label still exists from vertex-construct to lock this.graph().vertexLabel(vertex.schemaLabel().id()); /* @@ -694,7 +694,7 @@ public void removeVertex(HugeVertex vertex) { // Override vertices in local `addedVertices` this.addedVertices.remove(vertex.id()); // Force load vertex to ensure all properties are loaded (refer to #2181) - if (!vertex.schemaLabel().indexLabels().isEmpty()) { + if (!this.indexTx.indexLabelIds(vertex.schemaLabel()).isEmpty()) { vertex.forceLoad(); } // Collect the removed vertex @@ -900,7 +900,7 @@ public HugeEdge addEdge(HugeEdge edge) { this.locksTable.lockReads(LockUtil.EDGE_LABEL_DELETE, edge.schemaLabel().id()); this.locksTable.lockReads(LockUtil.INDEX_LABEL_DELETE, - edge.schemaLabel().indexLabels()); + this.indexTx.indexLabelIds(edge.schemaLabel())); // Ensure edge label still exists from edge-construct to lock this.graph().edgeLabel(edge.schemaLabel().id()); /* @@ -1839,7 +1839,7 @@ private void lockForUpdateProperty(SchemaLabel schemaLabel, Id pkey = prop.propertyKey().id(); Set indexIds = new HashSet<>(); - for (Id il : schemaLabel.indexLabels()) { + for (Id il : this.indexTx.indexLabelIds(schemaLabel)) { if (graph().indexLabel(il).indexFields().contains(pkey)) { indexIds.add(il); } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/HugeSecurityManager.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/HugeSecurityManager.java index 204320d400..1f695c9107 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/HugeSecurityManager.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/HugeSecurityManager.java @@ -63,7 +63,11 @@ public class HugeSecurityManager extends SecurityManager { "line.separator", "file.separator", // Sofa - "java.specification.version" + "java.specification.version", + // gRPC Netty reads these when a transport starts. Denying them + // under the Gremlin sandbox panics the shared channel. + "io.grpc.netty.shaded.io.grpc.netty.useCustomAllocator", + "io.grpc.netty.shaded.io.netty.allocator.maxOrder" ); private static final Map> ASYNC_TASKS = ImmutableMap.of( diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/algorithm/OltpTraverser.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/algorithm/OltpTraverser.java index 44f9cf8692..e118f033de 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/algorithm/OltpTraverser.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/algorithm/OltpTraverser.java @@ -43,6 +43,7 @@ import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.structure.Element; import org.apache.tinkerpop.gremlin.structure.Property; +import org.apache.tinkerpop.gremlin.structure.Transaction; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.util.CloseableIterator; @@ -73,7 +74,10 @@ protected OltpTraverser(HugeGraph graph) { @Override public void close() { - // pass + Transaction tx = this.graph().tx(); + if (tx.isOpen()) { + tx.close(); + } } public static void destroy() { diff --git a/hugegraph-server/hugegraph-core/src/test/java/org/apache/hugegraph/security/HugeSecurityManagerTest.java b/hugegraph-server/hugegraph-core/src/test/java/org/apache/hugegraph/security/HugeSecurityManagerTest.java index 5b1e32e5d6..d60e9361c3 100644 --- a/hugegraph-server/hugegraph-core/src/test/java/org/apache/hugegraph/security/HugeSecurityManagerTest.java +++ b/hugegraph-server/hugegraph-core/src/test/java/org/apache/hugegraph/security/HugeSecurityManagerTest.java @@ -36,5 +36,9 @@ public void testWhiteSystemPropertiesExcludeRemovedBackends() throws Exception { Assert.assertFalse(properties.contains("socksProxyHost")); Assert.assertFalse(properties.contains("file.encoding")); Assert.assertTrue(properties.contains("java.specification.version")); + Assert.assertTrue(properties.contains( + "io.grpc.netty.shaded.io.grpc.netty.useCustomAllocator")); + Assert.assertTrue(properties.contains( + "io.grpc.netty.shaded.io.netty.allocator.maxOrder")); } } diff --git a/hugegraph-server/hugegraph-dist/docker/README.md b/hugegraph-server/hugegraph-dist/docker/README.md index aa76092b82..7de1360f5b 100644 --- a/hugegraph-server/hugegraph-dist/docker/README.md +++ b/hugegraph-server/hugegraph-dist/docker/README.md @@ -27,6 +27,30 @@ Use Docker to quickly start a standalone HugeGraph Server with RocksDB. - 8080:8080 ``` +### ToplingDB image + +The same Server Dockerfile has a `topling` target. It copies the checked-in +ToplingDB JAR into the Server distribution and prepares its native runtime at +image build time, so starting a container needs no GitHub token or download. + +```bash +docker build --platform linux/amd64 --target topling \ + -f hugegraph-server/Dockerfile \ + -t hugegraph/hugegraph:1.8.0-topling . +docker run -itd --name=graph -p 8080:8080 \ + hugegraph/hugegraph:1.8.0-topling +``` + +The `-topling` image selects ToplingDB by default. Set +`HG_SERVER_ROCKSDB_PROVIDER=rocksdb` to use the standard RocksDB provider from +the same image. ToplingDB currently supports only Linux x86-64. Do not switch +an existing data volume between providers without a supported migration. +Before changing providers, stop writes and take a verified snapshot of the +current volume. Start the target provider with a new empty volume, or restore a +snapshot that was created by that provider. A provider setting change is not a +data migration; never mount a ToplingDB data directory into standard RocksDB +for rollback. + ## 2. Create Sample Graph on Server Startup To preload sample data on startup, set `PRELOAD=true`. @@ -159,3 +183,11 @@ docker run -itd --name=graph -p 8080:8080 -e HG_SERVER_STARTUP_TIMEOUT_S=450 hug Raising it does not move the health check above. The images set `--interval=15s --start-period=90s --retries=3`, so a container given a longer startup budget is reported `unhealthy` around 135 seconds while the entrypoint is still legitimately waiting; raise it with `--health-start-period` on `docker run`. The Compose files replace those values with their own (`start_period: 60s`, `interval: 10s`, `retries: 30`, so roughly 360 seconds), and anything gated on `depends_on: condition: service_healthy`, Hubble included, waits on that budget rather than on this variable. Move the two together. + +### Offline store dump output + +After stopping a standalone RocksDB Server, `bin/dump-store.sh` uses the +packaged `conf/log4j2.xml` configuration to print the selected table and entries. +No external `JAVA_TOOL_OPTIONS` setting is required. If an operator changes the +logging configuration, keep INFO output enabled for +`org.apache.hugegraph.cmd.StoreDumper` to see the dump contents. diff --git a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh index 42137dc3b5..6184a3eddf 100755 --- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh +++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh @@ -22,7 +22,10 @@ TEST_HOME=$(mktemp -d "${TMPDIR:-/tmp}/hugegraph-entrypoint-test.XXXXXX") trap 'rm -rf "${TEST_HOME}"' EXIT mkdir -p "${TEST_HOME}/bin" "${TEST_HOME}/conf/graphs" "${TEST_HOME}/docker" +mkdir -p "${TEST_HOME}/rocksdb-data" cp "${SCRIPT_DIR}/docker-entrypoint.sh" "${TEST_HOME}/docker-entrypoint.sh" +cp "${SCRIPT_DIR}/../src/assembly/static/bin/verify-rocksdb-provider.sh" \ + "${TEST_HOME}/bin/verify-rocksdb-provider.sh" touch "${TEST_HOME}/docker/init_complete" cat > "${TEST_HOME}/conf/rest-server.properties" <<'EOF' @@ -39,6 +42,19 @@ cat > "${TEST_HOME}/bin/start-hugegraph.sh" <<'EOF' # than substring-matching a flattened "$*", where -t 1200 contains -t 120. printf '%s\n' "$@" > ./docker/start-hugegraph-argv printf 'called\n' >> ./docker/start-hugegraph-calls +if [[ "${START_TEST_PRE_PID_DELAY:-false}" == "true" ]]; then + touch ./docker/start-before-pid + sleep 300 +fi +if [[ "${START_TEST_CHILD:-false}" == "true" ]]; then + sleep 300 & + child_pid=$! + printf '%s\n' "$child_pid" > ./bin/pid + if [[ "${START_TEST_DELAY:-false}" == "true" ]]; then + trap 'kill -TERM "$child_pid" 2>/dev/null || true; wait "$child_pid" 2>/dev/null || true; exit 0' TERM + wait "$child_pid" + fi +fi exit 0 EOF cat > "${TEST_HOME}/bin/init-store.sh" <<'EOF' @@ -65,6 +81,7 @@ chmod +x "${TEST_HOME}/bin/"*.sh ( cd "${TEST_HOME}" HG_SERVER_BACKEND=hstore \ + HG_SERVER_ROCKSDB_PROVIDER=rocksdb \ HG_SERVER_PD_PEERS=pd:8686 \ HG_SERVER_CLUSTER=hg \ HG_SERVER_USE_PD=true \ @@ -76,6 +93,8 @@ chmod +x "${TEST_HOME}/bin/"*.sh [[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -eq 1 ]] grep -qx 'backend=hstore' "${TEST_HOME}/conf/graphs/hugegraph.properties" +grep -qx 'rocksdb.provider=rocksdb' \ + "${TEST_HOME}/conf/graphs/hugegraph.properties" grep -qx 'pd.peers=pd:8686' "${TEST_HOME}/conf/graphs/hugegraph.properties" grep -qx 'usePD=true' "${TEST_HOME}/conf/rest-server.properties" grep -qx 'pd.peers=pd:8686' "${TEST_HOME}/conf/rest-server.properties" @@ -89,6 +108,78 @@ grep -qx 'auth.token_secret=12345678901234567890123456789012' \ grep -qx 'auth.token_secret=12345678901234567890123456789012' \ "${TEST_HOME}/conf/graphs/hugegraph.properties" +cp "${TEST_HOME}/conf/graphs/hugegraph.properties" \ + "${TEST_HOME}/conf/graphs/hugegraph.properties.before-hstore-topling" +if ( + cd "${TEST_HOME}" + HG_SERVER_BACKEND=hstore \ + HG_SERVER_ROCKSDB_PROVIDER=topling \ + bash ./docker-entrypoint.sh +); then + echo "HStore Server unexpectedly selected a local Topling provider" >&2 + exit 1 +fi +cmp "${TEST_HOME}/conf/graphs/hugegraph.properties.before-hstore-topling" \ + "${TEST_HOME}/conf/graphs/hugegraph.properties" + +mkdir -p "${TEST_HOME}/lib/topling" "${TEST_HOME}/library" \ + "${TEST_HOME}/topling-data" +touch "${TEST_HOME}/lib/topling/rocksdbjni-topling.jar" +touch "${TEST_HOME}/library/librocksdbjni-linux64.so" +if ( + cd "${TEST_HOME}" + HG_SERVER_BACKEND=hstore \ + HG_SERVER_ROCKSDB_PROVIDER=rocksdb \ + bash ./docker-entrypoint.sh +); then + echo "HStore Server unexpectedly accepted a local Topling payload" >&2 + exit 1 +fi +rm -f "${TEST_HOME}/lib/topling/rocksdbjni-topling.jar" +rm -f "${TEST_HOME}/library/librocksdbjni-linux64.so" + +cp "${TEST_HOME}/conf/graphs/hugegraph.properties" \ + "${TEST_HOME}/conf/graphs/hugegraph.properties.before-invalid-provider" +if ( + cd "${TEST_HOME}" + HG_SERVER_ROCKSDB_PROVIDER=invalid bash ./docker-entrypoint.sh +); then + echo "invalid RocksDB provider unexpectedly succeeded" >&2 + exit 1 +fi +cmp "${TEST_HOME}/conf/graphs/hugegraph.properties.before-invalid-provider" \ + "${TEST_HOME}/conf/graphs/hugegraph.properties" + +( + cd "${TEST_HOME}" + HG_SERVER_BACKEND=rocksdb \ + HG_SERVER_ROCKSDB_PROVIDER=topling \ + HG_SERVER_ENFORCE_PROVIDER_MARKER=true \ + bash ./docker-entrypoint.sh +) +grep -qx "rocksdb.data_path=${TEST_HOME}/topling-data/data" \ + "${TEST_HOME}/conf/graphs/hugegraph.properties" +grep -qx "rocksdb.wal_path=${TEST_HOME}/topling-data/wal" \ + "${TEST_HOME}/conf/graphs/hugegraph.properties" +grep -Fqx 'component=server' \ + "${TEST_HOME}/topling-data/.hugegraph-rocksdb-provider" +grep -Fqx 'provider=topling' \ + "${TEST_HOME}/topling-data/.hugegraph-rocksdb-provider" +cp "${TEST_HOME}/conf/graphs/hugegraph.properties" \ + "${TEST_HOME}/conf/graphs/hugegraph.properties.before-provider-mismatch" +if ( + cd "${TEST_HOME}" + HG_SERVER_BACKEND=rocksdb \ + HG_SERVER_ROCKSDB_PROVIDER=rocksdb \ + HG_SERVER_DATA_PATH="${TEST_HOME}/topling-data" \ + bash ./docker-entrypoint.sh +); then + echo "provider-mismatched Server data path unexpectedly succeeded" >&2 + exit 1 +fi +cmp "${TEST_HOME}/conf/graphs/hugegraph.properties.before-provider-mismatch" \ + "${TEST_HOME}/conf/graphs/hugegraph.properties" + cp "${TEST_HOME}/conf/rest-server.properties" \ "${TEST_HOME}/conf/rest-server.properties.before-short-secret" cp "${TEST_HOME}/conf/graphs/hugegraph.properties" \ @@ -117,7 +208,7 @@ if ( echo "required authentication token secret unexpectedly succeeded" >&2 exit 1 fi -[[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -eq 1 ]] +[[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -eq 2 ]] [[ ! -e "${TEST_HOME}/docker/enable-auth-calls" ]] ( @@ -125,7 +216,7 @@ fi HG_SERVER_REQUIRE_AUTH_TOKEN_SECRET=true \ bash ./docker-entrypoint.sh ) -[[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -eq 2 ]] +[[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -eq 3 ]] [[ ! -e "${TEST_HOME}/docker/enable-auth-calls" ]] ( @@ -135,7 +226,7 @@ fi HG_SERVER_AUTH_TOKEN_SECRET=12345678901234567890123456789012 \ bash ./docker-entrypoint.sh ) -[[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -eq 3 ]] +[[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -eq 4 ]] [[ "$(wc -l < "${TEST_HOME}/docker/enable-auth-calls")" -eq 1 ]] sed -i '/^auth\.token_secret=/d' "${TEST_HOME}/conf/rest-server.properties" @@ -181,7 +272,7 @@ sed -i "s|^auth\\.token_secret=.*|auth.token_secret ${rest_secret}|" \ grep -qx "auth.token_secret=${rest_secret}" \ "${TEST_HOME}/conf/rest-server.properties" -[[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -eq 7 ]] +[[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -eq 8 ]] [[ "$(wc -l < "${TEST_HOME}/docker/enable-auth-calls")" -eq 5 ]] ( @@ -230,12 +321,118 @@ grep -Fqx 'auth.admin_pa=Strong\\Pass\ 9!' \ "${TEST_HOME}/conf/rest-server.properties" rm -f "${TEST_HOME}/docker/init_complete" +rm -f "${TEST_HOME}/rocksdb-data/.hugegraph-state/init_complete" ( cd "${TEST_HOME}" PASSWORD=-n bash ./docker-entrypoint.sh ) grep -Fqx -- '-n' "${TEST_HOME}/docker/init-store-password" +rm -f "${TEST_HOME}/bin/pid" +( + cd "${TEST_HOME}" + exec setsid env \ + START_TEST_CHILD=true \ + HG_SERVER_BACKEND=hstore \ + HG_SERVER_ROCKSDB_PROVIDER=rocksdb \ + bash ./docker-entrypoint.sh +) & +entrypoint_pid=$! +for _ in $(seq 1 10); do + [[ -s "${TEST_HOME}/bin/pid" ]] && break + sleep 1 +done +[[ -s "${TEST_HOME}/bin/pid" ]] +child_pid=$(<"${TEST_HOME}/bin/pid") +kill -TERM -- "-${entrypoint_pid}" +for _ in $(seq 1 15); do + ! kill -0 "${entrypoint_pid}" 2>/dev/null && break + sleep 1 +done +if kill -0 "${entrypoint_pid}" 2>/dev/null; then + kill -KILL -- "-${entrypoint_pid}" 2>/dev/null || true + wait "${entrypoint_pid}" 2>/dev/null || true + echo "Docker entrypoint did not finish SIGTERM handling" >&2 + exit 1 +fi +if ! wait "${entrypoint_pid}"; then + echo "Docker entrypoint did not exit cleanly after SIGTERM" >&2 + exit 1 +fi +if kill -0 "${child_pid}" 2>/dev/null; then + echo "Docker entrypoint left its Server child running" >&2 + exit 1 +fi + +rm -f "${TEST_HOME}/bin/pid" +( + cd "${TEST_HOME}" + exec setsid env \ + START_TEST_CHILD=true \ + START_TEST_DELAY=true \ + HG_SERVER_BACKEND=hstore \ + HG_SERVER_ROCKSDB_PROVIDER=rocksdb \ + bash ./docker-entrypoint.sh +) & +entrypoint_pid=$! +for _ in $(seq 1 10); do + [[ -s "${TEST_HOME}/bin/pid" ]] && break + sleep 1 +done +[[ -s "${TEST_HOME}/bin/pid" ]] +child_pid=$(<"${TEST_HOME}/bin/pid") +kill -TERM "${entrypoint_pid}" +for _ in $(seq 1 15); do + ! kill -0 "${entrypoint_pid}" 2>/dev/null && break + sleep 1 +done +if kill -0 "${entrypoint_pid}" 2>/dev/null; then + kill -KILL -- "-${entrypoint_pid}" 2>/dev/null || true + wait "${entrypoint_pid}" 2>/dev/null || true + echo "Docker entrypoint did not handle SIGTERM during startup" >&2 + exit 1 +fi +if ! wait "${entrypoint_pid}"; then + echo "Docker entrypoint did not exit cleanly during startup" >&2 + exit 1 +fi +if kill -0 "${child_pid}" 2>/dev/null; then + echo "Docker entrypoint left its startup Server child running" >&2 + exit 1 +fi + +printf '%s\n' '-999999' > "${TEST_HOME}/bin/pid" +rm -f "${TEST_HOME}/docker/start-before-pid" +( + cd "${TEST_HOME}" + exec setsid env \ + START_TEST_PRE_PID_DELAY=true \ + HG_SERVER_BACKEND=hstore \ + HG_SERVER_ROCKSDB_PROVIDER=rocksdb \ + bash ./docker-entrypoint.sh +) & +entrypoint_pid=$! +for _ in $(seq 1 10); do + [[ -e "${TEST_HOME}/docker/start-before-pid" ]] && break + sleep 1 +done +[[ -e "${TEST_HOME}/docker/start-before-pid" ]] +[[ ! -e "${TEST_HOME}/bin/pid" ]] +kill -TERM "${entrypoint_pid}" +for _ in $(seq 1 15); do + ! kill -0 "${entrypoint_pid}" 2>/dev/null && break + sleep 1 +done +if kill -0 "${entrypoint_pid}" 2>/dev/null; then + kill -KILL -- "-${entrypoint_pid}" 2>/dev/null || true + wait "${entrypoint_pid}" 2>/dev/null || true + echo "Docker entrypoint did not ignore a stale pid during startup" >&2 + exit 1 +fi +if ! wait "${entrypoint_pid}"; then + echo "Docker entrypoint did not exit cleanly with a stale pid" >&2 + exit 1 +fi # The value start-hugegraph.sh actually received for -t, read from the # recorded argument vector so that -t 1200 can never satisfy an assertion # that wants 120. diff --git a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh index b5ba2de34f..837d53dfc8 100755 --- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh +++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh @@ -55,8 +55,25 @@ set_prop_encoded() { key_re="^[[:space:]]*${esc_key}([[:space:]]*[:=]|[[:space:]]+|[[:space:]]*$)" if grep -qE "${key_re}" "${file}"; then - sed -ri "0,/${key_re}/!{/${key_re}/d;}" "${file}" - sed -ri "0,/${key_re}/s~${key_re}.*~${key}=${esc_val}~" "${file}" + if sed --version >/dev/null 2>&1; then + # GNU sed supports the 0,/regexp/ address used by the Linux + # images. Keep the first property and remove later duplicates. + sed -ri "0,/${key_re}/!{/${key_re}/d;}" "${file}" + sed -ri "0,/${key_re}/s~${key_re}.*~${key}=${esc_val}~" "${file}" + else + # BSD sed (macOS) has neither -r nor the GNU line-0 address. Find + # the matching lines explicitly, delete duplicates from the end, + # then replace the first line in place. + first_line=$(grep -nE "${key_re}" "${file}" | head -n 1 | cut -d: -f1) + duplicate_lines=$(grep -nE "${key_re}" "${file}" | + cut -d: -f1 | tail -n +2 | sort -rn) + while IFS= read -r duplicate; do + [[ -z "${duplicate}" ]] || \ + sed -E -i '' "${duplicate}d" "${file}" + done <<< "${duplicate_lines}" + sed -E -i '' "${first_line}s~${key_re}.*~${key}=${esc_val}~" \ + "${file}" + fi else printf '%s=%s\n' "$key" "$encoded_val" >> "${file}" fi @@ -90,6 +107,56 @@ migrate_env() { migrate_env "BACKEND" "HG_SERVER_BACKEND" migrate_env "PD_PEERS" "HG_SERVER_PD_PEERS" +ROCKSDB_PROVIDER="${HG_SERVER_ROCKSDB_PROVIDER:-rocksdb}" +case "${ROCKSDB_PROVIDER}" in + rocksdb | topling) ;; + *) log "ERROR: HG_SERVER_ROCKSDB_PROVIDER must be rocksdb or topling" + exit 1 ;; +esac + +REQUESTED_BACKEND="${HG_SERVER_BACKEND:-$(get_prop_encoded "backend" "${GRAPH_CONF}")}" +if [[ "${REQUESTED_BACKEND}" == "hstore" ]]; then + TOPLING_JAR=$(find ./lib -path '*/topling/rocksdbjni*.jar' \ + -print -quit 2>/dev/null || true) + if [[ "${ROCKSDB_PROVIDER}" == "topling" ]]; then + log "ERROR: an HStore Server cannot select a local ToplingDB runtime" + exit 1 + fi + if [[ -n "${TOPLING_JAR}" || + -e ./library/librocksdbjni-linux64.so ]]; then + log "ERROR: an HStore Server image must not contain a local" \ + "ToplingDB runtime" + exit 1 + fi +fi + +if [[ -n "${HG_SERVER_DATA_PATH:-}" ]]; then + ROCKSDB_DATA_ROOT="${HG_SERVER_DATA_PATH}" +elif [[ "${ROCKSDB_PROVIDER}" == "topling" ]]; then + ROCKSDB_DATA_ROOT="$(pwd)/topling-data" +else + ROCKSDB_DATA_ROOT="$(pwd)/rocksdb-data" +fi +ENFORCE_PROVIDER_MARKER="${HG_SERVER_ENFORCE_PROVIDER_MARKER:-false}" +case "${ENFORCE_PROVIDER_MARKER}" in + true | false) ;; + *) log "ERROR: HG_SERVER_ENFORCE_PROVIDER_MARKER must be true or false" + exit 1 ;; +esac +if [[ "${REQUESTED_BACKEND}" == "rocksdb" ]]; then + ./bin/verify-rocksdb-provider.sh server "${ROCKSDB_PROVIDER}" \ + "${ROCKSDB_DATA_ROOT}" "${ENFORCE_PROVIDER_MARKER}" + LEGACY_INIT_MARKER="${DOCKER_FOLDER}/${INIT_FLAG_FILE}" + DOCKER_FOLDER="${ROCKSDB_DATA_ROOT}/.hugegraph-state" + mkdir -p "${DOCKER_FOLDER}" + if [[ "${ROCKSDB_PROVIDER}" == "rocksdb" && + -f "${LEGACY_INIT_MARKER}" && + ! -e "${DOCKER_FOLDER}/${INIT_FLAG_FILE}" ]]; then + cp "${LEGACY_INIT_MARKER}" "${DOCKER_FOLDER}/${INIT_FLAG_FILE}" + log "migrated the legacy RocksDB initialization marker" + fi +fi + if [[ -n "${HG_SERVER_AUTH_TOKEN_SECRET:-}" ]]; then LC_ALL=C if (( ${#HG_SERVER_AUTH_TOKEN_SECRET} < 32 )); then @@ -147,6 +214,11 @@ fi # ── Map env → properties file ───────────────────────────────────────── [[ -n "${HG_SERVER_BACKEND:-}" ]] && set_prop "backend" "${HG_SERVER_BACKEND}" "${GRAPH_CONF}" +set_prop "rocksdb.provider" "${ROCKSDB_PROVIDER}" "${GRAPH_CONF}" +if [[ "${REQUESTED_BACKEND}" == "rocksdb" ]]; then + set_prop "rocksdb.data_path" "${ROCKSDB_DATA_ROOT}/data" "${GRAPH_CONF}" + set_prop "rocksdb.wal_path" "${ROCKSDB_DATA_ROOT}/wal" "${GRAPH_CONF}" +fi [[ -n "${HG_SERVER_PD_PEERS:-}" ]] && set_prop "pd.peers" "${HG_SERVER_PD_PEERS}" "${GRAPH_CONF}" [[ -n "${HG_SERVER_USE_PD:-}" ]] && \ set_prop "usePD" "${HG_SERVER_USE_PD}" "${REST_SERVER_CONF}" @@ -240,7 +312,56 @@ else ./bin/init-store.sh fi -./bin/start-hugegraph.sh -j "${JAVA_OPTS:-}" -t "${SERVER_STARTUP_TIMEOUT_S}" +PID_FILE="./bin/pid" +START_PID="" +PID="" + +read_server_pid() { + local candidate + + candidate=$(cat "$PID_FILE" 2>/dev/null || true) + if [[ "$candidate" =~ ^[1-9][0-9]*$ ]]; then + printf '%s' "$candidate" + fi +} + +# A restarted container can retain the previous process id in its writable +# layer. Only trust a pid file created by the startup launched below. +rm -f "$PID_FILE" + +# shellcheck disable=SC2329 # Invoked by the TERM/INT trap below. +shutdown_server() { + local server_pid="${PID:-}" + + if [[ -z "$server_pid" ]]; then + server_pid=$(read_server_pid) + fi + if [[ -n "$server_pid" ]]; then + kill -TERM -- "$server_pid" 2>/dev/null || true + fi + if [[ -n "${START_PID:-}" ]]; then + kill -TERM -- "$START_PID" 2>/dev/null || true + fi + while [[ -n "$server_pid" ]] && kill -0 "$server_pid" 2>/dev/null; do + # kill -0 remains true for an unreaped zombie. Do not keep the + # container alive after the JVM has already completed shutdown. + if [[ -r "/proc/$server_pid/stat" ]]; then + PROCESS_STATE=$(awk '{ print $3 }' "/proc/$server_pid/stat" \ + 2>/dev/null || true) + if [[ "$PROCESS_STATE" == "Z" ]]; then + break + fi + fi + sleep 1 + done + exit 0 +} +trap shutdown_server TERM INT + +./bin/start-hugegraph.sh -j "${JAVA_OPTS:-}" -t "${SERVER_STARTUP_TIMEOUT_S}" & +START_PID=$! +wait "$START_PID" +START_PID="" # Post-startup cluster stabilization check (hstore only — rocksdb has no partitions) ACTUAL_BACKEND=$(grep -E '^[[:space:]]*backend[[:space:]]*=' "${GRAPH_CONF}" | head -n 1 | sed 's/.*=//' | tr -d '[:space:]' || true) @@ -250,9 +371,8 @@ if [[ "${ACTUAL_BACKEND}" == "hstore" ]]; then ./bin/wait-partition.sh || log "WARN: partitions not assigned yet" fi -PID=$(cat ./bin/pid 2>/dev/null || true) +PID=$(read_server_pid) if [[ -n "$PID" ]]; then - trap 'kill -TERM "$PID" 2>/dev/null; while kill -0 "$PID" 2>/dev/null; do sleep 1; done; exit 0' TERM INT tail --pid="$PID" -f /dev/null exit 1 fi diff --git a/hugegraph-server/hugegraph-dist/pom.xml b/hugegraph-server/hugegraph-dist/pom.xml index c4575bc877..5c88a931f8 100644 --- a/hugegraph-server/hugegraph-dist/pom.xml +++ b/hugegraph-server/hugegraph-dist/pom.xml @@ -112,6 +112,9 @@ ${basedir}/src/assembly/static true + + lib/topling/** + ${basedir}/src/main/resources diff --git a/hugegraph-server/hugegraph-dist/src/assembly/descriptor/assembly.xml b/hugegraph-server/hugegraph-dist/src/assembly/descriptor/assembly.xml index b7f9ee0487..1b77565124 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/descriptor/assembly.xml +++ b/hugegraph-server/hugegraph-dist/src/assembly/descriptor/assembly.xml @@ -37,6 +37,7 @@ false README.txt + lib/topling/** diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh new file mode 100644 index 0000000000..c9bdab237c --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh @@ -0,0 +1,288 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -Eeuo pipefail +IFS=$'\n\t' +trap 'echo "[common-topling] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR + +GITHUB="https://github.com" + +function abs_path() { + local SOURCE + SOURCE="${BASH_SOURCE[0]}" + while [[ -h "$SOURCE" ]]; do + local DIR + DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + done + cd -P "$(dirname "$SOURCE")" && pwd +} + +function extract_so_with_jar() { + local jar_file="$1" + local dest_dir="$2" + local abs_jar_path + + if [ ! -f "$jar_file" ]; then + echo "'$jar_file' Not Exist" >&2 + return 1 + fi + + mkdir -p "$dest_dir" || { + echo "Cannot mkdir '$dest_dir'" >&2 + return 1 + } + + if command -v realpath >/dev/null 2>&1; then + abs_jar_path="$(realpath "$jar_file")" + else + abs_jar_path="$(readlink -f "$jar_file")" + fi + if ! command -v unzip >/dev/null 2>&1; then + echo "Error: 'unzip' command not found. Please install unzip." >&2 + return 1 + fi + unzip -j -o "$abs_jar_path" "*.so" -d "$dest_dir" > /dev/null 2>&1 || { + local code=$? + if [ $code -eq 11 ]; then + echo "Error: No .so files found in '$abs_jar_path' (unzip exit 11)" >&2 + else + echo "Error: unzip failed (exit $code) for '$abs_jar_path'" >&2 + fi + return $code + } +} + +function extract_html_css_from_jar() { + local jar_file="$1" + local dest_dir="$2" + local abs_jar_path + local resource_target="$dest_dir/rocksdb_resource" + + if [ ! -f "$jar_file" ]; then + echo "Error: JAR file '$jar_file' does not exist." >&2 + return 1 + fi + + mkdir -p "$resource_target" || { + echo "Error: Cannot create resource directory '$resource_target'." >&2 + return 1 + } + + if command -v realpath >/dev/null 2>&1; then + abs_jar_path="$(realpath "$jar_file")" + else + abs_jar_path="$(readlink -f "$jar_file")" + fi + if ! command -v unzip >/dev/null 2>&1; then + echo "Error: 'unzip' command not found. Please install unzip." >&2 + return 1 + fi + unzip -j -o "$abs_jar_path" "*.html" "*.css" -d "$resource_target" > /dev/null || { + local code=$? + if [ $code -eq 11 ]; then + echo "Notice: No .html or .css files found in '$jar_file'." >&2 + return 0 + else + echo "Error: unzip failed with exit code $code" >&2 + return $code + fi + } + +} + +function ensure_libaio_symlink() { + local dest_dir="$1" + # Keep the Ubuntu 24.04 compatibility link inside the component runtime. + # Installation must never require sudo or modify /usr/lib. + if [ -f /etc/os-release ]; then + . /etc/os-release + if [ "${ID:-}" = "ubuntu" ] && + command -v dpkg >/dev/null 2>&1 && + dpkg --compare-versions "${VERSION_ID:-0}" "ge" "24.04" && + [ ! -e /usr/lib/x86_64-linux-gnu/libaio.so.1 ] && + [ -e /usr/lib/x86_64-linux-gnu/libaio.so.1t64 ]; then + mkdir -p "$dest_dir" + ln -sfn /usr/lib/x86_64-linux-gnu/libaio.so.1t64 \ + "$dest_dir/libaio.so.1" + echo "Prepared component-local libaio.so.1 compatibility link" + fi + fi +} + +function download_and_verify() { + local url=$1 + local filepath=$2 + local expected_sha256=$3 + local actual_sha256 + + if [[ -f $filepath ]]; then + echo "File $filepath exists. Verifying SHA-256 checksum..." + actual_sha256=$(sha256sum "$filepath" | awk '{ print $1 }') + if [[ "$actual_sha256" != "$expected_sha256" ]]; then + echo "SHA-256 checksum verification failed for $filepath. Expected: $expected_sha256, but got: $actual_sha256" + echo "Deleting $filepath..." + rm -f "$filepath" + else + echo "SHA-256 checksum verification succeeded for $filepath." + return 0 + fi + fi + + echo "Downloading $filepath..." + if ! curl -fL --retry 2 --retry-delay 2 --retry-max-time 90 \ + --connect-timeout 10 --max-time 30 -o "$filepath" "$url"; then + echo "Failed to download $filepath" >&2 + rm -f "$filepath" + return 1 + fi + + actual_sha256=$(sha256sum "$filepath" | awk '{ print $1 }') + if [[ "$actual_sha256" != "$expected_sha256" ]]; then + echo "SHA-256 checksum verification failed for $filepath after download. Expected: $expected_sha256, but got: $actual_sha256" + rm -f "$filepath" + return 1 + fi + + return 0 +} + +function download_and_setup_jemalloc() { + local arch lib_file download_url expected_sha256 system_lib top + top=$1 + system_lib="" + + if [[ "${LD_PRELOAD:-}" == *"libjemalloc"* ]]; then + return 0 + fi + + # Prefer system-installed jemalloc if available + # Try ldconfig first to locate the shared object + if command -v ldconfig >/dev/null 2>&1; then + system_lib=$(ldconfig -p 2>/dev/null | awk '/jemalloc/{print $4}' | head -n1) + fi + # Fallback to common library paths if ldconfig is not available or found nothing + if [[ -z "$system_lib" ]]; then + for p in \ + /usr/lib/libjemalloc.so \ + /usr/lib/libjemalloc.so.2 \ + /usr/lib64/libjemalloc.so \ + /usr/lib64/libjemalloc.so.2 \ + /usr/local/lib/libjemalloc.so \ + /usr/local/lib/libjemalloc.so.2 \ + /usr/lib/x86_64-linux-gnu/libjemalloc.so \ + /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 \ + /usr/lib/aarch64-linux-gnu/libjemalloc.so \ + /usr/lib/aarch64-linux-gnu/libjemalloc.so.2; do + if [[ -f "$p" ]]; then + system_lib="$p" + break + fi + done + fi + + # If found, set LD_PRELOAD and return immediately + if [[ -n "$system_lib" ]]; then + export LD_PRELOAD="${system_lib}${LD_PRELOAD:+:$LD_PRELOAD}" + return 0 + fi + + # Detect system architecture + arch=$(uname -m) + + # System jemalloc not found, try to download the correct library for the architecture + # Checksums match apache/hugegraph-doc@567625c6ec66907fc60f1864146fbec91b5f6204. + if [[ $arch == "aarch64" || $arch == "arm64" ]]; then + lib_file="$top/bin/libjemalloc_aarch64.so" + download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc_aarch64.so" + expected_sha256="6b7e6099b6da798829c6ce6fcb55a787508841edd52446332a73300889dcd1dc" + elif [[ $arch == "x86_64" ]]; then + lib_file="$top/bin/libjemalloc.so" + download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc.so" + expected_sha256="53b25e8626e1605cbd8b60befb3431cabc1b8851a54285e0dda412796feab67d" + else + echo "Unsupported architecture: $arch" + return 1 + fi + + # Download and verify jemalloc library (fallback when system lib not found) + if download_and_verify "$download_url" "$lib_file" "$expected_sha256"; then + export LD_PRELOAD="${lib_file}${LD_PRELOAD:+:$LD_PRELOAD}" + else + echo "Failed to verify or download jemalloc for $arch, skipping" + return 1 + fi +} + +function require_topling_platform() { + local os_name machine_arch + + os_name="$(uname -s)" + machine_arch="$(uname -m)" + if [ "$os_name" != "Linux" ] || + [[ "$machine_arch" != "x86_64" ]]; then + printf 'Error: ToplingDB native runtime supports Linux x86_64 only; ' >&2 + printf 'current platform is %s/%s\n' "$os_name" "$machine_arch" >&2 + return 1 + fi +} + +function prepare_toplingdb() { + local lib_dir="$1" + local dest_dir="$2" + local top_override="${3:-}" + + require_topling_platform || return 1 + + local top + if [ -n "$top_override" ]; then + top="$top_override" + else + top="$(cd "$lib_dir"/../ && pwd)" || { + echo "Error: failed to resolve the ToplingDB installation directory" >&2 + return 1 + } + fi + + local jar_file + jar_file=$(ls -1 "$lib_dir"/rocksdbjni*.jar 2>/dev/null | sort -V | tail -n1 || true) + if [ -z "${jar_file:-}" ]; then + echo "Error: No rocksdbjni*.jar found under '$lib_dir'" >&2 + return 1 + fi + + ensure_libaio_symlink "$dest_dir" + if ! download_and_setup_jemalloc "$top"; then + echo "Warning: jemalloc is unavailable; continuing without it" >&2 + fi + extract_so_with_jar "$jar_file" "$dest_dir" + if ! extract_html_css_from_jar "$jar_file" "$dest_dir"; then + echo "Warning: failed to extract optional ToplingDB web resources; continuing" >&2 + fi + if [ -d "$dest_dir" ]; then + if [[ ":${LD_LIBRARY_PATH:-}:" != *":$dest_dir:"* ]]; then + export LD_LIBRARY_PATH="$dest_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + fi + + if [ -f "$dest_dir/librocksdbjni-linux64.so" ] && [[ ":${LD_PRELOAD:-}:" != *"librocksdbjni-linux64.so:"* ]]; then + export LD_PRELOAD="${LD_PRELOAD:+$LD_PRELOAD:}$dest_dir/librocksdbjni-linux64.so" + fi + else + echo "Warn: LD paths skipped, directory '$dest_dir' does not exist." >&2 + fi +} diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/dump-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/dump-store.sh index 14b6ac1f79..0d7ada9276 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/dump-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/dump-store.sh @@ -51,5 +51,5 @@ echo "Dumping HugeGraph Store($conf)..." dump_store_ext_jar_path=$LIB/hugegraph-dist-*.jar for i in $LIB/*.jar; do dump_store_ext_jar_path=$dump_store_ext_jar_path:$i; export dump_store_ext_jar_path; done -exec "$JAVA" -cp "$dump_store_ext_jar_path" \ +exec "$JAVA" "-Dlog4j.configurationFile=${CONF}/log4j2.xml" -cp "$dump_store_ext_jar_path" \ org.apache.hugegraph.cmd.StoreDumper "$@" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index caffedc482..80da94d0e9 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -70,17 +70,18 @@ MIN_JAVA_VERSION=11 MAX_SECURITY_JAVA_VERSION=23 # Add the slf4j-log4j12 binding -CP=$(find -L $LIB -name 'log4j-slf4j-impl*.jar' | sort | tr '\n' ':') +CP=$(find_standard_lib_jars "$LIB" 'log4j-slf4j-impl*.jar' | + sort | tr '\n' ':') # Add the jars in lib that start with "hugegraph" -CP="$CP":$(find -L $LIB -name 'hugegraph*.jar' | sort | tr '\n' ':') +CP="$CP":$(find_standard_lib_jars "$LIB" 'hugegraph*.jar' | + sort | tr '\n' ':') # Add the remaining jars in lib. -CP="$CP":$(find -L $LIB -name '*.jar' \ - \! -name 'hugegraph*' \ - \! -name 'log4j-slf4j-impl*.jar' | sort | tr '\n' ':') +CP="$CP":$(find_standard_lib_jars "$LIB" '*.jar' \ + 'hugegraph*' 'log4j-slf4j-impl*.jar' | sort | tr '\n' ':') # Add the jars in ext (at any subdirectory depth) -CP="$CP":$(find -L $EXT -name '*.jar' | sort | tr '\n' ':') +CP="$CP":$(find -L "$EXT" -name '*.jar' | sort | tr '\n' ':') # Add the jars in plugins (at any subdirectory depth), check "javaagent" related jars carefully -CP="$CP":$(find -L $PLUGINS -name '*.jar' | sort | tr '\n' ':') +CP="$CP":$(find -L "$PLUGINS" -name '*.jar' | sort | tr '\n' ':') # (Cygwin only) Use ; classpath separator and reformat paths for Windows ("C:\foo") [[ $(uname) = CYGWIN* ]] && CP="$(cygpath -p -w "$CP")" @@ -259,12 +260,12 @@ fi # Turn on security check if [[ "${STDOUT_MODE:-false}" == "true" ]]; then exec ${JAVA} -Dname="HugeGraphServer" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ - ${SECURITY_MANAGER_OPTION} -cp ${CLASSPATH}: \ + ${SECURITY_MANAGER_OPTION} -cp "${CLASSPATH}:" \ org.apache.hugegraph.bootstrap.HugeGraphServerBootstrap \ ${OPEN_SECURITY_CHECK} ${GREMLIN_SERVER_CONF} ${REST_SERVER_CONF} else exec ${JAVA} -Dname="HugeGraphServer" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ - ${SECURITY_MANAGER_OPTION} -cp ${CLASSPATH}: \ + ${SECURITY_MANAGER_OPTION} -cp "${CLASSPATH}:" \ org.apache.hugegraph.bootstrap.HugeGraphServerBootstrap \ ${OPEN_SECURITY_CHECK} ${GREMLIN_SERVER_CONF} ${REST_SERVER_CONF} \ >> ${LOGS}/hugegraph-server-stdout.log 2>&1 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh index 74ec0bb731..7fd804251f 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh @@ -48,13 +48,19 @@ cd "${TOP}" || exit DEFAULT_JAVA_OPTIONS="--add-exports=java.base/jdk.internal.reflect=ALL-UNNAMED" +source "$BIN/preload-topling.sh" + echo "Initializing HugeGraph Store..." # Build classpath with hugegraph*.jar first to avoid class loading conflicts -CP=$(find -L "${LIB}" -name 'hugegraph*.jar' | sort | tr '\n' ':') -CP="$CP":$(find -L "${LIB}" -name '*.jar' \! -name 'hugegraph*' | sort | tr '\n' ':') +CP=$(find_standard_lib_jars "${LIB}" 'hugegraph*.jar' | sort | tr '\n' ':') +if [ -n "${TOPLING_RUNTIME_CLASSPATH:-}" ]; then + CP="$TOPLING_RUNTIME_CLASSPATH:$CP" +fi +CP="$CP":$(find_standard_lib_jars "${LIB}" '*.jar' 'hugegraph*' | + sort | tr '\n' ':') CP="$CP":$(find -L "${PLUGINS}" -name '*.jar' | sort | tr '\n' ':') -$JAVA -cp $CP ${DEFAULT_JAVA_OPTIONS} \ +$JAVA -cp "$CP" ${DEFAULT_JAVA_OPTIONS} \ org.apache.hugegraph.cmd.InitStore "${CONF}"/rest-server.properties INIT_STORE_STATUS=$? if [[ ${INIT_STORE_STATUS} -ne 0 ]]; then diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh new file mode 100644 index 0000000000..ff8761bd0a --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh @@ -0,0 +1,209 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ORIG_SHELL_FLAGS="$-" +ORIG_PIPEFAIL="$(set -o | awk '$1 == "pipefail" { print $2 }')" +ORIG_ERR_TRAP="$(trap -p ERR)" +# Save original IFS to avoid leaking into parent shell when sourced +ORIG_IFS="${IFS}" +set -Eeuo pipefail +IFS=$'\n\t' +# Unified error capture for easy positioning +trap 'echo "[preload-topling] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR + +RUNTIME_BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPONENT_TOP="$(cd "$RUNTIME_BIN"/../ && pwd)" +COMPONENT_LIB="$COMPONENT_TOP/lib" +DEST_DIR="$COMPONENT_TOP/library" + +detect_rocksdb_provider() { + local conf_dir="$1" + local file value provider="" seen_provider=false + + while IFS= read -r value; do + if [ "$seen_provider" = false ]; then + provider="$value" + seen_provider=true + elif [ "$provider" != "$value" ]; then + echo "Error: conflicting rocksdb.provider values: $provider,$value" >&2 + return 1 + fi + done < <( + for file in "$conf_dir"/graphs/*.properties; do + [ -f "$file" ] || continue + awk ' + /^[[:space:]]*#/ { next } + /^[[:space:]]*rocksdb\.provider[[:space:]]*=/ { + value = $0 + sub(/^[^=]*=[[:space:]]*/, "", value) + sub(/[[:space:]]+#.*/, "", value) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + print value + } + ' "$file" + done + for file in "$conf_dir"/application*.yml; do + [ -f "$file" ] || continue + awk ' + /^[[:space:]]*#/ || /^[[:space:]]*$/ { next } + /^[^[:space:]#][^:]*:/ { + in_rocksdb = ($0 ~ /^rocksdb[[:space:]]*:/) + } + in_rocksdb && /^[[:space:]]+provider[[:space:]]*:/ { + value = $0 + sub(/^[^:]*:[[:space:]]*/, "", value) + sub(/[[:space:]]+#.*/, "", value) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + if (value ~ /^"[^"]*"$/ || value ~ /^'\''[^'\'']*'\''$/) { + value = substr(value, 2, length(value) - 2) + } + print value + } + ' "$file" + done + ) + + provider="${provider:-rocksdb}" + case "$provider" in + rocksdb | topling) + echo "$provider" + ;; + *) + echo "Error: invalid rocksdb.provider '$provider'; expected rocksdb or topling" >&2 + return 1 + ;; + esac +} + +PROVIDER="${TOPLINGDB_ROCKSDB_PROVIDER:-}" +if [ -n "$PROVIDER" ]; then + case "$PROVIDER" in + rocksdb | topling) ;; + *) + echo "Error: invalid TOPLINGDB_ROCKSDB_PROVIDER '$PROVIDER';" \ + "expected rocksdb or topling" >&2 + exit 1 + ;; + esac +else + PROVIDER=$(detect_rocksdb_provider "$COMPONENT_TOP/conf") || exit 1 +fi + +remove_path_entry() { + local value="${1:-}" + local remove="${2:-}" + local entry result="" + local path_ifs="$IFS" + IFS=: + for entry in $value; do + [ -n "$entry" ] && [ "$entry" != "$remove" ] || continue + result="${result:+$result:}$entry" + done + IFS="$path_ifs" + echo "$result" +} + +# A parent launcher may start multiple components from one shell. Remove only +# the runtime entry previously selected by this helper before selecting ours. +if [ -n "${TOPLING_ACTIVE_NATIVE:-}" ]; then + LD_PRELOAD=$(remove_path_entry "${LD_PRELOAD:-}" "$TOPLING_ACTIVE_NATIVE") + LD_LIBRARY_PATH=$(remove_path_entry "${LD_LIBRARY_PATH:-}" \ + "$(dirname "$TOPLING_ACTIVE_NATIVE")") + export LD_PRELOAD LD_LIBRARY_PATH +fi +if [ -n "${TOPLING_ACTIVE_JAR:-}" ]; then + CLASSPATH=$(remove_path_entry "${CLASSPATH:-}" "$TOPLING_ACTIVE_JAR") + export CLASSPATH +fi +unset TOPLING_ACTIVE_NATIVE TOPLING_ACTIVE_JAR TOPLING_RUNTIME_CLASSPATH + +if [ "$PROVIDER" = "topling" ]; then + # Runtime selection is read-only. Installation prepares all files beforehand. + if [ "$(uname -s)" != "Linux" ] || [ "$(uname -m)" != "x86_64" ]; then + echo "Error: ToplingDB runtime supports Linux x86_64 only" >&2 + exit 1 + fi + TOPLING_JAR=$(ls -1 "$COMPONENT_LIB"/topling/rocksdbjni*.jar 2>/dev/null | + sort -V | tail -1 || true) + if [ -z "$TOPLING_JAR" ]; then + echo "Error: no prepared ToplingDB JAR found in $COMPONENT_LIB/topling/" >&2 + exit 1 + fi + + CONF_FILE="${TOPLINGDB_EASY_MIGRATE_CONF:-}" + if [ -z "$CONF_FILE" ]; then + CONF_FILE="$COMPONENT_TOP/conf/toplingdb.yaml" + if [ ! -f "$CONF_FILE" ]; then + CONF_FILE="$COMPONENT_TOP/conf/rocksdb_store.yaml" + fi + if [ ! -f "$CONF_FILE" ]; then + CONF_FILE="$COMPONENT_TOP/conf/rocksdb_pd.yaml" + fi + fi + if [ ! -f "$CONF_FILE" ]; then + echo "Error: required ToplingDB Easy Migrate config not found: $CONF_FILE" >&2 + exit 1 + fi + if [ ! -r "$CONF_FILE" ]; then + echo "Error: ToplingDB Easy Migrate config is not readable: $CONF_FILE" >&2 + exit 1 + fi + export TOPLINGDB_EASY_MIGRATE_CONF="$CONF_FILE" + echo "[preload-topling] TOPLINGDB_EASY_MIGRATE_CONF=$CONF_FILE" + NATIVE_LIBRARY="$DEST_DIR/librocksdbjni-linux64.so" + if [ ! -r "$NATIVE_LIBRARY" ]; then + echo "Error: prepared ToplingDB native library not found: $NATIVE_LIBRARY" >&2 + echo " Run bin/prepare-topling.sh for this component before startup." >&2 + exit 1 + fi + export LD_LIBRARY_PATH="$DEST_DIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + if command -v ldd >/dev/null 2>&1 && + ldd "$NATIVE_LIBRARY" 2>/dev/null | grep -q 'not found'; then + echo "Error: ToplingDB native library has unresolved system dependencies" >&2 + echo " Install them during package/deployment preparation." >&2 + exit 1 + fi + + export LD_PRELOAD="${LD_PRELOAD:+$LD_PRELOAD:}$NATIVE_LIBRARY" + export TOPLING_ACTIVE_NATIVE="$NATIVE_LIBRARY" + export TOPLING_RUNTIME_CLASSPATH="$TOPLING_JAR" + export CLASSPATH="$TOPLING_JAR${CLASSPATH:+:$CLASSPATH}" + export TOPLING_ACTIVE_JAR="$TOPLING_JAR" +else + unset TOPLINGDB_EASY_MIGRATE_CONF + echo "[preload-topling] Component uses rocksdb provider" +fi + +unset -f detect_rocksdb_provider remove_path_entry + +# Restore original IFS +IFS="$ORIG_IFS" +if [ -n "$ORIG_ERR_TRAP" ]; then + eval "$ORIG_ERR_TRAP" +else + trap - ERR +fi +# Restore shell options to their state before this script was sourced +case "$ORIG_SHELL_FLAGS" in *e*) set -e ;; *) set +e ;; esac +case "$ORIG_SHELL_FLAGS" in *u*) set -u ;; *) set +u ;; esac +case "$ORIG_SHELL_FLAGS" in *E*) set -E ;; *) set +E ;; esac +if [ "$ORIG_PIPEFAIL" = "on" ]; then + set -o pipefail +else + set +o pipefail +fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/prepare-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/prepare-topling.sh new file mode 100755 index 0000000000..fcfe243e7c --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/prepare-topling.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TOP="$(cd "$BIN"/.. && pwd)" +TOPLING_LIB="$TOP/lib/topling" +RUNTIME_DIR="$TOP/library" + +if [ ! -r "$BIN/common-topling.sh" ]; then + echo "Error: ToplingDB preparation helper not found: $BIN/common-topling.sh" >&2 + exit 1 +fi + +source "$BIN/common-topling.sh" +prepare_toplingdb "$TOPLING_LIB" "$RUNTIME_DIR" "$TOP" + +echo "[prepare-topling] Runtime prepared under $RUNTIME_DIR" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh index 2c99238327..56986ee5a9 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh @@ -99,6 +99,8 @@ if [[ $PRELOAD == "true" ]]; then sed -i -e '/registerBackends/d; /serverStarted/d' "${SCRIPTS}/${EXAMPLE_SCRIPT}" fi +source "$BIN/preload-topling.sh" + if [[ $DAEMON == "true" ]]; then echo "Starting HugeGraphServer in daemon mode..." "${BIN}"/hugegraph-server.sh "${CONF}/${GREMLIN_SERVER_CONF}" "${CONF}"/rest-server.properties \ diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index 8ae46e88a4..99744a88af 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -24,6 +24,75 @@ function command_available() { return 1 } +# Resolve a path before comparing it with an excluded runtime directory. The +# fallback keeps the launcher usable on macOS hosts where `realpath` may not be +# installed, while the depth limit prevents malformed symlink loops from +# hanging startup. +function canonical_path() { + local path="$1" + local depth="${2:-0}" + local target + local dir + + (( depth < 40 )) || return 1 + if command_available "realpath"; then + realpath "$path" + return + fi + if [[ -L "$path" ]]; then + target=$(readlink "$path") || return 1 + if [[ "$target" = /* ]]; then + canonical_path "$target" "$((depth + 1))" + else + canonical_path "$(dirname "$path")/$target" "$((depth + 1))" + fi + return + fi + dir=$(cd -P "$(dirname "$path")" 2>/dev/null && pwd) || return 1 + printf '%s/%s\n' "$dir" "$(basename "$path")" +} + +# Print JAR paths below a component's lib directory except anything whose +# canonical target lives below its optional Topling subtree. `find -P` keeps +# directory aliases from being traversed, while direct file aliases are +# resolved and emitted canonically after the exclusion check. +function find_standard_lib_jars() { + local lib_dir="$1" + local pattern="$2" + local -a excluded_patterns=("${@:3}") + local top_dir="$lib_dir/topling" + local canonical_top + local jar + local canonical_jar + local name + local excluded + + if [[ -d "$top_dir" ]]; then + canonical_top=$(canonical_path "$top_dir") || canonical_top="$top_dir" + else + canonical_top="$top_dir" + fi + while IFS= read -r -d '' jar; do + name=$(basename "$jar") + for excluded in "${excluded_patterns[@]}"; do + # shellcheck disable=SC2254 # exclusion arguments are glob patterns + case "$name" in + $excluded) continue 2 ;; + esac + done + canonical_jar=$(canonical_path "$jar") || continue + [[ -f "$canonical_jar" ]] || continue + if [[ "$canonical_top" = "/" ]]; then + continue + fi + case "$canonical_jar" in + "$canonical_top" | "$canonical_top"/*) continue ;; + esac + printf '%s\n' "$canonical_jar" + done < <(find -P "$lib_dir" -name "$pattern" \ + ! -path "$top_dir/*" -print0) +} + function configure_riscv64_libatomic() { if [[ "$(uname -s)" != "Linux" || "$(uname -m)" != "riscv64" ]]; then return 0 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/verify-rocksdb-provider.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/verify-rocksdb-provider.sh new file mode 100755 index 0000000000..c59cef1254 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/verify-rocksdb-provider.sh @@ -0,0 +1,148 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +if [ "$#" -lt 3 ] || [ "$#" -gt 4 ]; then + echo "Usage: $0 [enforce]" >&2 + exit 2 +fi + +COMPONENT="$1" +PROVIDER="$2" +DATA_PATHS="$3" +ENFORCE="${4:-false}" +MARKER_NAME=".hugegraph-rocksdb-provider" + +fail() { + echo "Error: $*" >&2 + exit 1 +} + +case "$COMPONENT" in + server | pd | store) ;; + *) fail "invalid RocksDB component '$COMPONENT'" ;; +esac +case "$PROVIDER" in + rocksdb | topling) ;; + *) fail "invalid RocksDB provider '$PROVIDER'" ;; +esac +case "$ENFORCE" in + true | false) ;; + *) fail "provider marker enforcement must be true or false" ;; +esac + +EXPECTED_MARKER=$(printf '%s\n' \ + "format=1" \ + "component=$COMPONENT" \ + "provider=$PROVIDER") + +command -v flock >/dev/null 2>&1 || + fail "flock is required to verify RocksDB provider markers" + +verify_marker() { + local data_path="$1" + local pinned_path="$2" + local marker="$pinned_path/$MARKER_NAME" + local actual_marker + + if [ -L "$marker" ] || { [ -e "$marker" ] && [ ! -f "$marker" ]; }; then + fail "provider marker is not a regular file: $marker" + fi + if [ -f "$marker" ]; then + [ -r "$marker" ] || fail "provider marker is not readable: $marker" + actual_marker=$(<"$marker") + if [ "$actual_marker" != "$EXPECTED_MARKER" ]; then + fail "provider marker mismatch in $data_path; expected" \ + "$COMPONENT/$PROVIDER" + fi + return 0 + fi + + if [ "$PROVIDER" = "rocksdb" ] && [ "$ENFORCE" = "false" ]; then + echo "[provider-marker] legacy unmarked RocksDB path accepted: $data_path" + return 0 + fi + + if find -H "$pinned_path" -mindepth 1 -maxdepth 1 \ + ! -name lost+found -print -quit | grep -q .; then + fail "refusing unmarked non-empty data path: $data_path" + fi + + local temporary_marker + temporary_marker=$(mktemp "$pinned_path/.provider-marker.XXXXXX") + chmod 600 "$temporary_marker" + printf '%s\n' "$EXPECTED_MARKER" > "$temporary_marker" + if ! mv -n "$temporary_marker" "$marker"; then + rm -f "$temporary_marker" + fail "could not create provider marker: $marker" + fi + if [ -e "$temporary_marker" ]; then + rm -f "$temporary_marker" + fi + + actual_marker=$(<"$marker") + if [ "$actual_marker" != "$EXPECTED_MARKER" ]; then + fail "provider marker changed while initializing: $marker" + fi + echo "[provider-marker] initialized $COMPONENT/$PROVIDER at $data_path" +} + +IFS=',' read -r -a PATH_LIST <<<"$DATA_PATHS" +[ "${#PATH_LIST[@]}" -gt 0 ] || fail "no RocksDB data path configured" + +for raw_path in "${PATH_LIST[@]}"; do + data_path=$(printf '%s' "$raw_path" | + sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + [ -n "$data_path" ] || fail "empty RocksDB data path" + [[ "$data_path" == /* ]] || + fail "RocksDB data path must be absolute: $data_path" + [ "$data_path" != "/" ] || fail "RocksDB data path cannot be /" + + [ -d "$data_path" ] || + fail "RocksDB data path must be an existing directory: $data_path" + [ ! -L "$data_path" ] || + fail "RocksDB data path cannot be a symlink: $data_path" + + lexical_path=$(realpath -m -s "$data_path") + physical_path=$(realpath -m "$data_path") + [ "$lexical_path" = "$physical_path" ] || + fail "RocksDB data path contains a symlink: $data_path" + + exec {data_path_fd}<"$data_path" + flock -x "$data_path_fd" || + fail "could not lock RocksDB data path: $data_path" + pinned_path="/proc/self/fd/$data_path_fd" + [ -d "$pinned_path" ] || + fail "could not pin RocksDB data path: $data_path" + + configured_identity=$(stat -Lc '%d:%i' "$data_path") + pinned_identity=$(stat -Lc '%d:%i' "$pinned_path") + [ "$configured_identity" = "$pinned_identity" ] || + fail "RocksDB data path changed while locking: $data_path" + + verify_marker "$data_path" "$pinned_path" + + configured_identity=$(stat -Lc '%d:%i' "$data_path") + pinned_identity=$(stat -Lc '%d:%i' "$pinned_path") + [ "$configured_identity" = "$pinned_identity" ] || + fail "RocksDB data path changed while verifying: $data_path" + + flock -u "$data_path_fd" + exec {data_path_fd}<&- +done diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties index 26adfe0183..f424cdb1f8 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties @@ -43,6 +43,10 @@ search.text_analyzer_mode=INDEX # rocksdb backend config #rocksdb.data_path=/path/to/disk #rocksdb.wal_path=/path/to/disk +# To enable ToplingDB: uncomment the line below (requires ToplingDB JAR in lib/topling/) +# The startup script detects this setting and handles JAR swap + env var setup automatically. +# ToplingDB config: conf/toplingdb.yaml (auto-loaded via TOPLINGDB_EASY_MIGRATE_CONF env var) +#rocksdb.provider=topling # hbase backend config #hbase.hosts=localhost diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb.yaml new file mode 100644 index 0000000000..f7c902cf4e --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb.yaml @@ -0,0 +1,164 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# common parameters +http: + # normally parent path of db path + document_root: ./library/rocksdb_resource + listening_ports: '127.0.0.1:2011' + auto_start_http: false +setenv: + StrSimpleEnvNameNotOverwrite: StringValue + IntSimpleEnvNameNotOverwrite: 16384 + OverwriteThisEnv: + #comment: overwrite is default to false + overwrite: true + value: force overwrite this env by overwrite true +Cache: + lru_cache: + class: LRUCache + params: + capacity: 8G + num_shard_bits: -1 + strict_capacity_limit: false + high_pri_pool_ratio: 0.5 + use_adaptive_mutex: false + metadata_charge_policy: kFullChargeCacheMetadata +Statistics: + stat: + class: default + params: + discard_tickers: + - rocksdb.block.cache + - rocksdb.block.cachecompressed + - rocksdb.block + - rocksdb.memtable.payload.bytes.at.flush + - rocksdb.memtable.garbage.bytes.at.flush + - rocksdb.txn + - rocksdb.blobdb + - rocksdb.row.cache + - rocksdb.number.block + - rocksdb.bloom.filter + - rocksdb.persistent + - rocksdb.sim.block.cache + discard_histograms: + # comment: .... + - rocksdb.blobdb + - rocksdb.bytes.compressed + - rocksdb.bytes.decompressed + - rocksdb.num.index.and.filter.blocks.read.per.level + - rocksdb.num.data.blocks.read.per.level + - rocksdb.compression.times.nanos + - rocksdb.decompression.times.nanos + - rocksdb.read.block.get.micros + - rocksdb.write.raw.block.micros + # comment end of array + #stats_level: kAll + stats_level: kDisableAll +MemTableRepFactory: + cspp: + class: cspp + params: + mem_cap: 16G + use_vm: false + token_use_idle: true + chunk_size: 16K + convert_to_sst: kFileMmap + sync_sst_file: false + skiplist: + class: SkipList + params: + lookahead: 0 +TableFactory: + cspp_memtab_sst: + class: CSPPMemTabTable + params: # empty params + bb: + class: BlockBasedTable + params: + checksum: kCRC32c + block_size: 4K + block_restart_interval: 16 + index_block_restart_interval: 1 + metadata_block_size: 4K + enable_index_compression: true + block_cache: "${lru_cache}" + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst + block_cache_compressed: + persistent_cache: + filter_policy: + dispatch: + class: DispatcherTable + params: + default: bb + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst + level_writers: [ bb, bb, bb, bb, bb, bb ] +CFOptions: + default: + max_write_buffer_number: 6 + memtable_factory: "${cspp}" + write_buffer_size: 128M + # set target_file_size_base as small as 512K is to make many SST files, + # thus key prefix cache can present efficiency + target_file_size_base: 64M + target_file_size_multiplier: 1 + table_factory: dispatch + max_bytes_for_level_base: 512M + max_bytes_for_level_multiplier: 10 + level_compaction_dynamic_level_bytes: false + level0_slowdown_writes_trigger: 20 + level0_stop_writes_trigger: 36 + level0_file_num_compaction_trigger: 2 + merge_operator: uint64add # support merge + level_compaction_dynamic_file_size: true + optimize_filters_for_hits: true + allow_merge_memtables: true + min_write_buffer_number_to_merge: 2 + compression_per_level: + - kNoCompression + - kNoCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression +DBOptions: + log: + create_if_missing: true + create_missing_column_families: true + default: + create_if_missing: true + create_missing_column_families: false # this is important, must be false to hugegraph + max_background_compactions: -1 + max_subcompactions: 4 + max_level1_subcompactions: 0 + inplace_update_support: false + WAL_size_limit_MB: 0 + statistics: "${stat}" + max_manifest_file_size: 100M + max_background_jobs: 8 + # Java org.rocksdb.WriteBatch has no Topling mmap WAL. With + # memtable_as_log_index enabled, every batch write fails: + # "WriteBatch has no mmap wal". Keep the option false for this client. + # convert_to_sst is still Topling-specific. Do not open data written by + # this profile with the standard RocksDB provider; restore a snapshot + # taken before migration instead. + compaction_readahead_size: 0 + memtable_as_log_index: false diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/lib/topling/rocksdbjni-8.10.2-20260725.141011-1.jar b/hugegraph-server/hugegraph-dist/src/assembly/static/lib/topling/rocksdbjni-8.10.2-20260725.141011-1.jar new file mode 100644 index 0000000000..61c401bb01 Binary files /dev/null and b/hugegraph-server/hugegraph-dist/src/assembly/static/lib/topling/rocksdbjni-8.10.2-20260725.141011-1.jar differ diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/RocksDBRuntimeSmokeTest.java b/hugegraph-server/hugegraph-dist/src/assembly/travis/RocksDBRuntimeSmokeTest.java new file mode 100644 index 0000000000..f7c3be9837 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/RocksDBRuntimeSmokeTest.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.DBOptions; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksIterator; + +public final class RocksDBRuntimeSmokeTest { + + private static final byte[] CF = bytes("runtime-smoke"); + private static final byte[] KEY = bytes("key"); + private static final byte[] VALUE = bytes("value-before-restart"); + private static final byte[] RECREATED_VALUE = bytes("value-after-recreate"); + + public static void main(String[] args) throws Exception { + if (args.length != 3) { + throw new IllegalArgumentException( + "Usage: "); + } + + String provider = args[0]; + String dbPath = args[1]; + String expectedNativePath = args[2]; + verifyProvider(provider); + verifyEasyMigrateConfig(provider); + RocksDB.loadLibrary(); + verifyNativeLibrary(expectedNativePath); + + createAndWrite(dbPath); + reopenDropAndRecreate(dbPath); + System.out.printf("Runtime smoke test passed: provider=%s, version=%s%n", + provider, RocksDB.rocksdbVersion()); + } + + private static void verifyProvider(String provider) { + boolean hasToplingApi; + try { + Class.forName("org.rocksdb.SidePluginRepo", false, + RocksDBRuntimeSmokeTest.class.getClassLoader()); + hasToplingApi = true; + } catch (ClassNotFoundException ignored) { + hasToplingApi = false; + } + + if (!"rocksdb".equals(provider) && !"topling".equals(provider)) { + throw new IllegalArgumentException("Unsupported provider: " + provider); + } + if ("rocksdb".equals(provider) && hasToplingApi) { + throw new IllegalStateException("Standard provider loaded a Topling JAR"); + } + if ("topling".equals(provider) && !hasToplingApi) { + throw new IllegalStateException("Topling provider loaded no Topling API"); + } + } + + private static void verifyEasyMigrateConfig(String provider) { + String config = System.getenv("TOPLINGDB_EASY_MIGRATE_CONF"); + if ("rocksdb".equals(provider)) { + if (config != null && !config.isEmpty()) { + throw new IllegalStateException( + "Standard RocksDB test must not enable Easy Migrate"); + } + return; + } + if (config == null || config.isEmpty()) { + throw new IllegalStateException( + "ToplingDB functional test requires Easy Migrate config"); + } + if (!Files.isReadable(Paths.get(config))) { + throw new IllegalStateException( + "Easy Migrate config is not readable: " + config); + } + System.out.println("Verified Easy Migrate config: " + + Paths.get(config).toAbsolutePath().normalize()); + } + + private static void verifyNativeLibrary(String expectedNativePath) + throws Exception { + if ("none".equals(expectedNativePath)) { + return; + } + String maps = new String(Files.readAllBytes(Paths.get("/proc/self/maps")), + StandardCharsets.UTF_8); + String absolutePath = Paths.get(expectedNativePath).toAbsolutePath() + .normalize().toString(); + if (!maps.contains(absolutePath)) { + throw new IllegalStateException("Expected native library is not mapped: " + + absolutePath); + } + for (String line : maps.split("\\R")) { + if (line.contains("librocksdbjni") && !line.contains(absolutePath)) { + throw new IllegalStateException( + "An unexpected RocksDB JNI library is also mapped: " + line); + } + } + System.out.println("Verified native library: " + absolutePath); + } + + private static void createAndWrite(String dbPath) throws Exception { + try (Options options = new Options().setCreateIfMissing(true); + RocksDB db = RocksDB.open(options, dbPath); + ColumnFamilyOptions cfOptions = new ColumnFamilyOptions(); + ColumnFamilyHandle handle = db.createColumnFamily( + new ColumnFamilyDescriptor(CF, cfOptions))) { + db.put(handle, KEY, VALUE); + assertBytes(VALUE, db.get(handle, KEY), "initial read"); + try (RocksIterator iterator = db.newIterator(handle)) { + iterator.seekToFirst(); + if (!iterator.isValid()) { + throw new AssertionError("iterator returned no data"); + } + assertBytes(KEY, iterator.key(), "iterator key"); + assertBytes(VALUE, iterator.value(), "iterator value"); + } + } + } + + private static void reopenDropAndRecreate(String dbPath) throws Exception { + List descriptors = new ArrayList<>(); + try (Options options = new Options()) { + for (byte[] name : RocksDB.listColumnFamilies(options, dbPath)) { + descriptors.add(new ColumnFamilyDescriptor(name)); + } + } + + List handles = new ArrayList<>(); + RocksDB db = null; + try (DBOptions options = new DBOptions().setCreateIfMissing(false)) { + db = RocksDB.open(options, dbPath, descriptors, handles); + ColumnFamilyHandle smoke = findHandle(descriptors, handles, CF); + assertBytes(VALUE, db.get(smoke, KEY), "read after reopen"); + db.dropColumnFamily(smoke); + handles.remove(smoke); + smoke.close(); + + try (ColumnFamilyOptions cfOptions = new ColumnFamilyOptions(); + ColumnFamilyHandle recreated = db.createColumnFamily( + new ColumnFamilyDescriptor(CF, cfOptions))) { + db.put(recreated, KEY, RECREATED_VALUE); + assertBytes(RECREATED_VALUE, db.get(recreated, KEY), + "read after CF recreation"); + } + } finally { + for (ColumnFamilyHandle handle : handles) { + handle.close(); + } + if (db != null) { + db.close(); + } + } + } + + private static ColumnFamilyHandle findHandle( + List descriptors, + List handles, + byte[] name) { + for (int i = 0; i < descriptors.size(); i++) { + if (Arrays.equals(name, descriptors.get(i).getName())) { + return handles.get(i); + } + } + throw new IllegalStateException("Column family was not reopened"); + } + + private static void assertBytes(byte[] expected, byte[] actual, + String operation) { + if (!Arrays.equals(expected, actual)) { + throw new AssertionError(operation + " returned unexpected data"); + } + } + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + private RocksDBRuntimeSmokeTest() { + } +} diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-deps.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-deps.sh new file mode 100755 index 0000000000..c5719254a4 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-deps.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -ev + +if [ -f /etc/debian_version ]; then + sudo apt-get update && sudo apt-get install -y liburing-dev libaio-dev libjemalloc-dev +elif [ -f /etc/redhat-release ]; then + sudo yum install -y liburing-devel libaio-devel jemalloc-devel +fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh new file mode 100755 index 0000000000..384a7ab274 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh @@ -0,0 +1,197 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +if [ "$(uname -s)" != "Linux" ]; then + echo "[install-rocksdb] Skip native preload on non-Linux platform: $(uname -s)" + return 0 2>/dev/null || exit 0 +fi + +ORIG_SHELL_FLAGS="$-" +ORIG_PIPEFAIL="$(set -o | awk '$1 == "pipefail" { print $2 }')" +ORIG_ERR_TRAP="$(trap -p ERR)" +ORIG_EXIT_TRAP="$(trap -p EXIT)" +# Save original IFS to avoid leaking into parent shell when sourced +ORIG_IFS="${IFS}" +set -Eeuo pipefail +IFS=$'\n\t' + +install_rocksdb_restore_state() { + local exit_status=$? + + # Prevent recursive execution while restoring the caller's EXIT trap. + trap - EXIT + IFS="$ORIG_IFS" + if [ -n "$ORIG_ERR_TRAP" ]; then + eval "$ORIG_ERR_TRAP" + else + trap - ERR + fi + if [ -n "$ORIG_EXIT_TRAP" ]; then + eval "$ORIG_EXIT_TRAP" + else + trap - EXIT + fi + case "$ORIG_SHELL_FLAGS" in *e*) set -e ;; *) set +e ;; esac + case "$ORIG_SHELL_FLAGS" in *u*) set -u ;; *) set +u ;; esac + case "$ORIG_SHELL_FLAGS" in *E*) set -E ;; *) set +E ;; esac + if [ "$ORIG_PIPEFAIL" = "on" ]; then + set -o pipefail + else + set +o pipefail + fi + return "$exit_status" +} +trap install_rocksdb_restore_state EXIT + +# Unified error capture for easy positioning +trap 'echo "[install-rocksdb] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR + +VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) +COMPONENT="${1:-server}" +SERVER_VERSION_DIR="$(pwd)/hugegraph-server/apache-hugegraph-server-$VERSION" +TOPLING_SOURCE_LIB="$(pwd)/hugegraph-server/hugegraph-dist/src/assembly/static/lib/topling" + +case "$COMPONENT" in + server | hstore) + COMPONENT_VERSION_DIR="$SERVER_VERSION_DIR" + ;; + pd) + COMPONENT_VERSION_DIR="$(pwd)/hugegraph-pd/apache-hugegraph-pd-$VERSION" + ;; + store) + COMPONENT_VERSION_DIR="$(pwd)/hugegraph-store/apache-hugegraph-store-$VERSION" + ;; + *) + echo "Error: unsupported component '$COMPONENT' (expected server, pd, store, or hstore)" >&2 + exit 1 + ;; +esac +COMPONENT_BIN="$COMPONENT_VERSION_DIR/bin" +COMPONENT_LIB="$COMPONENT_VERSION_DIR/lib" + +if [ ! -d "$COMPONENT_VERSION_DIR" ]; then + echo "Error: component dir not found: $COMPONENT_VERSION_DIR" >&2 + exit 1 +fi + +detect_rocksdb_provider() { + local conf_dir="$1" + local file + local -a values=() + local -a unique_values=() + local value existing duplicate conflicts + + for file in "$conf_dir"/graphs/*.properties; do + [ -f "$file" ] || continue + while IFS= read -r value; do + values[${#values[@]}]="$value" + done < <( + awk ' + /^[[:space:]]*#/ { next } + /^[[:space:]]*rocksdb\.provider[[:space:]]*=/ { + value = $0 + sub(/^[^=]*=[[:space:]]*/, "", value) + sub(/[[:space:]]+#.*/, "", value) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + print value + } + ' "$file" + ) + done + for file in "$conf_dir"/application*.yml; do + [ -f "$file" ] || continue + while IFS= read -r value; do + values[${#values[@]}]="$value" + done < <( + awk ' + /^[[:space:]]*#/ || /^[[:space:]]*$/ { next } + /^[^[:space:]#][^:]*:/ { + in_rocksdb = ($0 ~ /^rocksdb[[:space:]]*:/) + } + in_rocksdb && /^[[:space:]]+provider[[:space:]]*:/ { + value = $0 + sub(/^[^:]*:[[:space:]]*/, "", value) + sub(/[[:space:]]+#.*/, "", value) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + if (value ~ /^"[^"]*"$/ || value ~ /^'\''[^'\'']*'\''$/) { + value = substr(value, 2, length(value) - 2) + } + print value + } + ' "$file" + ) + done + + for value in "${values[@]}"; do + duplicate=false + for existing in "${unique_values[@]}"; do + if [ "$existing" = "$value" ]; then + duplicate=true + break + fi + done + if [ "$duplicate" = false ]; then + unique_values+=("$value") + fi + done + + if [ "${#unique_values[@]}" -gt 1 ]; then + conflicts=$(IFS=,; echo "${unique_values[*]}") + echo "Error: conflicting rocksdb.provider values: $conflicts" >&2 + return 1 + fi + local provider="${unique_values[0]:-rocksdb}" + case "$provider" in + rocksdb | topling) + echo "$provider" + ;; + *) + echo "Error: invalid rocksdb.provider '$provider'; expected rocksdb or topling" >&2 + return 1 + ;; + esac +} + +PROVIDER=$(detect_rocksdb_provider "$COMPONENT_VERSION_DIR/conf") || exit 1 + +if [ "$PROVIDER" = "topling" ]; then + if [ ! -x "$COMPONENT_BIN/prepare-topling.sh" ]; then + echo "Error: prepare-topling.sh not found under: $COMPONENT_BIN" >&2 + exit 1 + fi + + TOPLING_JARS=("$TOPLING_SOURCE_LIB"/rocksdbjni*.jar) + if [ "${#TOPLING_JARS[@]}" -ne 1 ] || [ ! -f "${TOPLING_JARS[0]}" ]; then + echo "Error: expected exactly one ToplingDB JAR under: $TOPLING_SOURCE_LIB" >&2 + exit 1 + fi + mkdir -p "$COMPONENT_LIB/topling" + rm -f "$COMPONENT_LIB"/topling/rocksdbjni*.jar + cp "${TOPLING_JARS[0]}" "$COMPONENT_LIB/topling/" + "$COMPONENT_BIN/prepare-topling.sh" +else + echo "[install-rocksdb] $COMPONENT uses rocksdb provider (or unset)," \ + "skipping native preload" +fi + +unset -f detect_rocksdb_provider + +# A sourced script does not trigger EXIT on normal return, so restore explicitly. +# The EXIT trap above covers exit and errexit failure paths. +install_rocksdb_restore_state +unset -f install_rocksdb_restore_state diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh index 7ffa6b8685..089477cae9 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh @@ -67,6 +67,9 @@ JACOCO_PORT=36320 mvn package -Dmaven.test.skip=true -ntp +# install rocksdb +source "$TRAVIS_DIR/install-rocksdb.sh" server + if [[ ! -e "$SERVER_DIR/lib/ikanalyzer-2012_u6.jar" ]]; then download_to_dir "$SERVER_DIR/lib/" \ "https://raw.githubusercontent.com/apache/hugegraph-doc/ik_binary/dist/server/ikanalyzer-2012_u6.jar" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-core-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-core-test.sh index a95d2f0806..b1f6dbdf36 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-core-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-core-test.sh @@ -18,5 +18,9 @@ set -ev BACKEND=$1 +TRAVIS_DIR=$(dirname $0) + +# install rocksdb +source "$TRAVIS_DIR/install-rocksdb.sh" server mvn test -pl hugegraph-server/hugegraph-test -am -P core-test,$BACKEND diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-unit-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-unit-test.sh index 5fe9b476b3..1f35812522 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-unit-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-unit-test.sh @@ -19,6 +19,9 @@ set -ev BACKEND=$1 +# install rocksdb +source "$TRAVIS_DIR/install-rocksdb.sh" server + if [[ "$BACKEND" == "memory" ]]; then mvn test -pl hugegraph-server/hugegraph-test -am -P unit-test fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-pd-docker-entrypoint.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-pd-docker-entrypoint.sh index ed39b67492..5d28c711a8 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-pd-docker-entrypoint.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-pd-docker-entrypoint.sh @@ -35,8 +35,11 @@ FAIL=0 [[ -f "${ENTRYPOINT}" ]] || { echo "entrypoint not found at ${ENTRYPOINT}" >&2; exit 1; } command -v python3 >/dev/null || { echo "python3 required" >&2; exit 1; } -mkdir -p "${TMP_DIR}/bin" +mkdir -p "${TMP_DIR}/bin" "${TMP_DIR}/pd_data" cp "${ENTRYPOINT}" "${TMP_DIR}/docker-entrypoint.sh" +cp "$(dirname "${BASH_SOURCE[0]}")/../static/bin/verify-rocksdb-provider.sh" \ + "${TMP_DIR}/bin/verify-rocksdb-provider.sh" +chmod +x "${TMP_DIR}/bin/verify-rocksdb-provider.sh" # Stand in for the launcher: record the generated config instead of starting PD cat > "${TMP_DIR}/bin/start-hugegraph-pd.sh" <<'STUB' #!/usr/bin/env bash diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-rocksdb-runtime.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-rocksdb-runtime.sh new file mode 100755 index 0000000000..acb7c3321c --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-rocksdb-runtime.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +PROVIDER="${1:?Usage: $0 }" +COMPONENT_DIR="${2:?Usage: $0 }" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +TEST_ROOT=$(mktemp -d /tmp/hugegraph-rocksdb-runtime.XXXXXX) +cleanup() { + rm -rf "$TEST_ROOT" +} +trap cleanup EXIT + +case "$PROVIDER" in + rocksdb) + EXPECTED_NATIVE_PATH=none + ;; + topling) + EXPECTED_NATIVE_PATH="$COMPONENT_DIR/library/librocksdbjni-linux64.so" + if [ ! -f "$EXPECTED_NATIVE_PATH" ]; then + echo "Error: Topling native library was not installed for the component" >&2 + exit 1 + fi + ;; + *) + echo "Error: unsupported provider '$PROVIDER'" >&2 + exit 1 + ;; +esac + +if compgen -G "$COMPONENT_DIR/lib/rocksdbjni*.jar" >/dev/null; then + if [ "$PROVIDER" = "topling" ]; then + JAR="${TOPLING_RUNTIME_CLASSPATH:-}" + else + JAR=$(ls -1 "$COMPONENT_DIR"/lib/rocksdbjni*.jar 2>/dev/null | + sort -V | tail -1 || true) + fi +else + if [ "$PROVIDER" = "topling" ]; then + JAR=$(ls -1 "$COMPONENT_DIR"/lib/topling/rocksdbjni*.jar 2>/dev/null | + sort -V | tail -1 || true) + else + BOOT_JAR=$(ls -1 "$COMPONENT_DIR"/lib/*.jar 2>/dev/null | + sort -V | tail -1 || true) + NESTED_JAR=$(unzip -Z1 "$BOOT_JAR" 'BOOT-INF/lib/rocksdbjni*.jar' | + sort -V | tail -1 || true) + if [ -z "$NESTED_JAR" ]; then + echo "Error: no embedded rocksdbjni JAR found in $BOOT_JAR" >&2 + exit 1 + fi + JAR="$TEST_ROOT/rocksdbjni.jar" + unzip -p "$BOOT_JAR" "$NESTED_JAR" > "$JAR" + fi +fi +if [ -z "$JAR" ]; then + echo "Error: no rocksdbjni JAR found for provider '$PROVIDER'" >&2 + exit 1 +fi + +if [ "$PROVIDER" = "topling" ]; then + EASY_MIGRATE_CONF="${TOPLINGDB_EASY_MIGRATE_CONF:-}" + if [ -z "$EASY_MIGRATE_CONF" ]; then + for candidate in \ + "$COMPONENT_DIR/conf/toplingdb.yaml" \ + "$COMPONENT_DIR/conf/rocksdb_store.yaml" \ + "$COMPONENT_DIR/conf/rocksdb_pd.yaml"; do + if [ -f "$candidate" ]; then + EASY_MIGRATE_CONF="$candidate" + break + fi + done + fi + if [ -z "$EASY_MIGRATE_CONF" ] || [ ! -r "$EASY_MIGRATE_CONF" ]; then + echo "Error: readable ToplingDB Easy Migrate config is required" >&2 + exit 1 + fi + + NATIVE_DIR="$(dirname "$EXPECTED_NATIVE_PATH")" + TEST_LD_LIBRARY_PATH="$NATIVE_DIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + TEST_LD_PRELOAD="${LD_PRELOAD:-}" + case ":$TEST_LD_PRELOAD:" in + *":$EXPECTED_NATIVE_PATH:"*) ;; + *) TEST_LD_PRELOAD="$EXPECTED_NATIVE_PATH${TEST_LD_PRELOAD:+:$TEST_LD_PRELOAD}" ;; + esac + + TOPLINGDB_EASY_MIGRATE_CONF="$EASY_MIGRATE_CONF" \ + LD_LIBRARY_PATH="$TEST_LD_LIBRARY_PATH" \ + LD_PRELOAD="$TEST_LD_PRELOAD" \ + java -cp "$JAR" "$SCRIPT_DIR/RocksDBRuntimeSmokeTest.java" \ + "$PROVIDER" "$TEST_ROOT/db" "$EXPECTED_NATIVE_PATH" +else + env -u TOPLINGDB_EASY_MIGRATE_CONF \ + java -cp "$JAR" "$SCRIPT_DIR/RocksDBRuntimeSmokeTest.java" \ + "$PROVIDER" "$TEST_ROOT/db" "$EXPECTED_NATIVE_PATH" +fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-stop-hugegraph-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-stop-hugegraph-store.sh new file mode 100755 index 0000000000..5d05ab6ef2 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-stop-hugegraph-store.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Verify stop failure propagation and PID retention without signalling real processes. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../.." && pwd)" +SCRIPT="$ROOT/hugegraph-store/hg-store-dist/src/assembly/static/bin/stop-hugegraph-store.sh" +FIXTURE=$(mktemp -d) +trap 'rm -rf "$FIXTURE"' EXIT +mkdir -p "$FIXTURE/bin" +cp "$SCRIPT" "$FIXTURE/bin/stop-hugegraph-store.sh" +cat > "$FIXTURE/bin/util.sh" <<'UTIL' +kill_process_and_wait() { + [[ "$1" == HugeGraphStoreServer && "$2" == 12345 && "$3" == 30 ]] || return 99 + return "$STOP_RESULT" +} +UTIL +printf '12345\n' > "$FIXTURE/bin/pid" +if STOP_RESULT=1 bash "$FIXTURE/bin/stop-hugegraph-store.sh"; then + echo "Stop timeout incorrectly returned success" >&2 + exit 1 +fi +[[ "$(cat "$FIXTURE/bin/pid")" == 12345 ]] +STOP_RESULT=0 bash "$FIXTURE/bin/stop-hugegraph-store.sh" +[[ ! -e "$FIXTURE/bin/pid" ]] +STOP_RESULT=0 bash "$FIXTURE/bin/stop-hugegraph-store.sh" +echo "store-stop-contract-ok" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-distribution.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-distribution.sh new file mode 100755 index 0000000000..b909930be6 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-distribution.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +COMPONENT="$1" +STANDARD_DIR="$2" +TOPLING_DIR="$3" +TOPLING_TAR="$TOPLING_DIR.tar.gz" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +case "$COMPONENT" in + server) CONFIG_FILE="$TOPLING_DIR/conf/graphs/hugegraph.properties" ;; + pd) CONFIG_FILE="$TOPLING_DIR/conf/application.yml" ;; + store) CONFIG_FILE="$TOPLING_DIR/conf/application-pd.yml" ;; + *) fail "unsupported component: $COMPONENT" ;; +esac + +if [ "${TOPLING_EXPECT_DIRTY_STANDARD:-false}" = "true" ]; then + [ -e "$STANDARD_DIR/bin/pid" ] || + fail "standard distribution has no PID test fixture" + case "$COMPONENT" in + server) + [ -e "$STANDARD_DIR/logs/topling-ci-marker" ] || + fail "standard Server has no logs test fixture" + ;; + pd) + [ -L "$STANDARD_DIR/pd_data" ] || + fail "standard PD has no pd_data symlink test fixture" + ;; + store) + [ -e "$STANDARD_DIR/storage" ] || + fail "standard Store has no storage test fixture" + ;; + esac +fi + +for helper in common-topling.sh prepare-topling.sh preload-topling.sh \ + verify-rocksdb-provider.sh; do + [ -x "$STANDARD_DIR/bin/$helper" ] || + fail "standard distribution is missing helper: $helper" + [ -x "$TOPLING_DIR/bin/$helper" ] || + fail "Topling distribution is missing helper: $helper" +done + +if find "$STANDARD_DIR/lib" -path '*/topling/*' -print -quit | grep -q .; then + fail "standard distribution contains a Topling JAR" +fi +[ ! -e "$STANDARD_DIR/library/librocksdbjni-linux64.so" ] || + fail "standard distribution contains a Topling native library" + +TOPLING_JAR=$(find "$TOPLING_DIR/lib/topling" -maxdepth 1 \ + -name 'rocksdbjni*.jar' -print -quit) +[ -n "$TOPLING_JAR" ] || fail "Topling distribution contains no Topling JAR" +NATIVE_LIBRARY="$TOPLING_DIR/library/librocksdbjni-linux64.so" +[ -r "$NATIVE_LIBRARY" ] || fail "Topling distribution contains no native library" +[ -r "$TOPLING_DIR/lib/topling/runtime.properties" ] || + fail "Topling distribution contains no runtime marker" +grep -qx 'provider=topling' "$TOPLING_DIR/lib/topling/runtime.properties" || + fail "Topling runtime marker has the wrong provider" +grep -Eq '^[[:space:]]*(rocksdb\.provider=|provider:[[:space:]]*)topling([[:space:]]|$)' \ + "$CONFIG_FILE" || fail "Topling provider is not selected in $CONFIG_FILE" + +for runtime_path in bin/pid logs pd_data rocksdb-data storage; do + if [ -e "$TOPLING_DIR/$runtime_path" ] || + [ -L "$TOPLING_DIR/$runtime_path" ]; then + fail "runtime state leaked into Topling directory: $runtime_path" + fi +done + +[ -r "$TOPLING_TAR" ] || fail "Topling archive not found: $TOPLING_TAR" +ARCHIVE_LIST=$(tar -tzf "$TOPLING_TAR") +if grep -Eq '/(bin/pid|logs|pd_data|rocksdb-data|storage)(/|$)' \ + <<<"$ARCHIVE_LIST"; then + fail "runtime state leaked into Topling archive" +fi +grep -Eq '/lib/topling/runtime.properties$' <<<"$ARCHIVE_LIST" || + fail "Topling archive contains no runtime marker" + +if ldd "$NATIVE_LIBRARY" 2>/dev/null | grep -q 'not found'; then + ldd "$NATIVE_LIBRARY" >&2 || true + fail "Topling native library has unresolved dependencies" +fi + +source "$TOPLING_DIR/bin/preload-topling.sh" +[ "$TOPLING_RUNTIME_CLASSPATH" = "$TOPLING_JAR" ] || + fail "Topling classpath does not use the component-local JAR" +[ "$TOPLING_ACTIVE_NATIVE" = "$NATIVE_LIBRARY" ] || + fail "Topling preload does not use the component-local native library" + +echo "PASS: clean component-local Topling distribution: $TOPLING_DIR" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-docker-entrypoints.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-docker-entrypoints.sh new file mode 100755 index 0000000000..4c95008cc9 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-docker-entrypoints.sh @@ -0,0 +1,243 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR"/../../../../.. && pwd)" +TEST_ROOT="$(mktemp -d)" + +cleanup() { + rm -rf "$TEST_ROOT" +} +trap cleanup EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +prepare_fixture() { + local component="$1" + local entrypoint="$2" + local start_script="$3" + local fixture="$TEST_ROOT/$component" + + mkdir -p "$fixture/bin" + cp "$entrypoint" "$fixture/docker-entrypoint.sh" + cp "$PROJECT_ROOT/hugegraph-server/hugegraph-dist/src/assembly/static/bin/verify-rocksdb-provider.sh" \ + "$fixture/bin/verify-rocksdb-provider.sh" + # shellcheck disable=SC2016 + printf '%s\n' \ + '#!/bin/bash' \ + 'set -euo pipefail' \ + 'printf "%s\n" "$TOPLINGDB_ROCKSDB_PROVIDER" > "$ENTRYPOINT_CAPTURE.provider"' \ + 'printf "%s\n" "$SPRING_APPLICATION_JSON" > "$ENTRYPOINT_CAPTURE.json"' \ + > "$fixture/bin/$start_script" + chmod +x "$fixture/docker-entrypoint.sh" \ + "$fixture/bin/$start_script" \ + "$fixture/bin/verify-rocksdb-provider.sh" +} + +run_pd() { + local provider="$1" + local capture="$TEST_ROOT/pd-$provider" + local data_path="$TEST_ROOT/pd/pd_data" + + if [ "$provider" = "topling" ]; then + data_path="$TEST_ROOT/pd/topling-pd-data" + fi + mkdir -p "$data_path" + ( + cd "$TEST_ROOT/pd" + ENTRYPOINT_CAPTURE="$capture" \ + HG_PD_AUTH_SECRET_KEY=ci-topling-pd-secret \ + HG_PD_GRPC_HOST=pd \ + HG_PD_RAFT_ADDRESS=pd:8610 \ + HG_PD_RAFT_PEERS_LIST=pd:8610 \ + HG_PD_INITIAL_STORE_LIST=store:8500 \ + HG_PD_ROCKSDB_PROVIDER="$provider" \ + HG_PD_ENFORCE_PROVIDER_MARKER=true \ + ./docker-entrypoint.sh >/dev/null + ) + grep -qx "$provider" "$capture.provider" || + fail "PD preload override does not use $provider" + grep -Fq "\"rocksdb\": { \"provider\": \"$provider\" }" \ + "$capture.json" || + fail "PD Spring configuration does not use $provider" + grep -Fq "\"data-path\": \"$data_path\"" "$capture.json" || + fail "PD default data path does not follow $provider" + grep -qx "provider=$provider" \ + "$data_path/.hugegraph-rocksdb-provider" || + fail "PD data path has no $provider marker" +} + +run_store() { + local provider="$1" + local capture="$TEST_ROOT/store-$provider" + local data_path="$TEST_ROOT/store/storage" + + if [ "$provider" = "topling" ]; then + data_path="$TEST_ROOT/store/topling-storage" + fi + mkdir -p "$data_path" + ( + cd "$TEST_ROOT/store" + ENTRYPOINT_CAPTURE="$capture" \ + HG_STORE_PD_ADDRESS=pd:8686 \ + HG_STORE_GRPC_HOST=store \ + HG_STORE_RAFT_ADDRESS=store:8510 \ + HG_STORE_ROCKSDB_PROVIDER="$provider" \ + HG_STORE_ENFORCE_PROVIDER_MARKER=true \ + ./docker-entrypoint.sh >/dev/null + ) + grep -qx "$provider" "$capture.provider" || + fail "Store preload override does not use $provider" + grep -Fq "\"rocksdb\": { \"provider\": \"$provider\" }" \ + "$capture.json" || + fail "Store Spring configuration does not use $provider" + grep -Fq "\"data-path\": \"$data_path\"" "$capture.json" || + fail "Store default data path does not follow $provider" + grep -qx "provider=$provider" \ + "$data_path/.hugegraph-rocksdb-provider" || + fail "Store data path has no $provider marker" +} + +expect_invalid_provider() { + local component="$1" + local output + + if [ "$component" = "pd" ]; then + if output=$( + cd "$TEST_ROOT/pd" + HG_PD_AUTH_SECRET_KEY=ci-topling-pd-secret \ + HG_PD_GRPC_HOST=pd \ + HG_PD_RAFT_ADDRESS=pd:8610 \ + HG_PD_RAFT_PEERS_LIST=pd:8610 \ + HG_PD_INITIAL_STORE_LIST=store:8500 \ + HG_PD_ROCKSDB_PROVIDER=invalid \ + ./docker-entrypoint.sh 2>&1 + ); then + fail "PD accepted an invalid provider" + fi + grep -Fq "HG_PD_ROCKSDB_PROVIDER must be rocksdb or topling" \ + <<<"$output" || fail "PD invalid-provider error is not actionable" + else + if output=$( + cd "$TEST_ROOT/store" + HG_STORE_PD_ADDRESS=pd:8686 \ + HG_STORE_GRPC_HOST=store \ + HG_STORE_RAFT_ADDRESS=store:8510 \ + HG_STORE_ROCKSDB_PROVIDER=invalid \ + ./docker-entrypoint.sh 2>&1 + ); then + fail "Store accepted an invalid provider" + fi + grep -Fq "HG_STORE_ROCKSDB_PROVIDER must be rocksdb or topling" \ + <<<"$output" || fail "Store invalid-provider error is not actionable" + fi +} + +prepare_fixture \ + pd \ + "$PROJECT_ROOT/hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh" \ + start-hugegraph-pd.sh +prepare_fixture \ + store \ + "$PROJECT_ROOT/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh" \ + start-hugegraph-store.sh + +for provider in rocksdb topling; do + run_pd "$provider" + run_store "$provider" +done +expect_invalid_provider pd +expect_invalid_provider store + +SHARED_DATA="$TEST_ROOT/shared-data" +mkdir -p "$SHARED_DATA" +( + cd "$TEST_ROOT/pd" + ENTRYPOINT_CAPTURE="$TEST_ROOT/pd-shared-topling" \ + HG_PD_AUTH_SECRET_KEY=ci-topling-pd-secret \ + HG_PD_GRPC_HOST=pd \ + HG_PD_RAFT_ADDRESS=pd:8610 \ + HG_PD_RAFT_PEERS_LIST=pd:8610 \ + HG_PD_INITIAL_STORE_LIST=store:8500 \ + HG_PD_ROCKSDB_PROVIDER=topling \ + HG_PD_DATA_PATH="$SHARED_DATA" \ + ./docker-entrypoint.sh >/dev/null +) +if output=$( + cd "$TEST_ROOT/pd" + ENTRYPOINT_CAPTURE="$TEST_ROOT/pd-shared-rocksdb" \ + HG_PD_AUTH_SECRET_KEY=ci-topling-pd-secret \ + HG_PD_GRPC_HOST=pd \ + HG_PD_RAFT_ADDRESS=pd:8610 \ + HG_PD_RAFT_PEERS_LIST=pd:8610 \ + HG_PD_INITIAL_STORE_LIST=store:8500 \ + HG_PD_ROCKSDB_PROVIDER=rocksdb \ + HG_PD_DATA_PATH="$SHARED_DATA" \ + ./docker-entrypoint.sh 2>&1 +); then + fail "PD accepted a Topling data path for standard RocksDB" +fi +grep -Fq "provider marker mismatch" <<<"$output" || + fail "PD provider mismatch error is not actionable" +grep -qx "provider=topling" \ + "$SHARED_DATA/.hugegraph-rocksdb-provider" || + fail "provider mismatch changed the existing data marker" + +NONEMPTY_DATA="$TEST_ROOT/nonempty-data" +mkdir -p "$NONEMPTY_DATA" +touch "$NONEMPTY_DATA/existing.sst" +PROVIDER_HELPER="$PROJECT_ROOT/hugegraph-server/hugegraph-dist/src/assembly/static/bin/verify-rocksdb-provider.sh" +if "$PROVIDER_HELPER" \ + store topling "$NONEMPTY_DATA" false >/dev/null 2>&1; then + fail "ToplingDB accepted a non-empty unmarked data path" +fi +[ -f "$NONEMPTY_DATA/existing.sst" ] || + fail "failed provider validation removed existing data" +[ ! -e "$NONEMPTY_DATA/.hugegraph-rocksdb-provider" ] || + fail "failed provider validation created a marker" + +OUTSIDE_DATA="$TEST_ROOT/outside-data" +mkdir -p "$OUTSIDE_DATA" +ln -s "$OUTSIDE_DATA" "$TEST_ROOT/data-link" +if "$PROVIDER_HELPER" \ + server topling "$TEST_ROOT/data-link/new" true >/dev/null 2>&1; then + fail "ToplingDB accepted a data path with a symlink prefix" +fi +[ ! -e "$OUTSIDE_DATA/new" ] || + fail "provider validation created a directory through a symlink prefix" + +CONCURRENT_DATA="$TEST_ROOT/concurrent-data" +mkdir -p "$CONCURRENT_DATA" +"$PROVIDER_HELPER" server topling "$CONCURRENT_DATA" true >/dev/null & +FIRST_PID=$! +"$PROVIDER_HELPER" server topling "$CONCURRENT_DATA" true >/dev/null & +SECOND_PID=$! +wait "$FIRST_PID" || + fail "first concurrent provider-marker initialization failed" +wait "$SECOND_PID" || + fail "second concurrent provider-marker initialization failed" +grep -qx "provider=topling" \ + "$CONCURRENT_DATA/.hugegraph-rocksdb-provider" || + fail "concurrent provider-marker initialization wrote the wrong marker" + +echo "PASS: PD and Store Docker entrypoints keep Spring and preload providers aligned" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-runtime-packaging.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-runtime-packaging.sh new file mode 100755 index 0000000000..c020dd6503 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-runtime-packaging.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +COMPONENT_DIRS=("$1" "$2" "$3") +TEST_ROOT="$(mktemp -d)" +FAKE_BIN="$TEST_ROOT/fake-bin" + +mkdir -p "$FAKE_BIN" +printf '%s\n' \ + '#!/bin/sh' \ + 'case "${1:-}" in' \ + ' -s) echo Linux ;;' \ + ' -m) echo x86_64 ;;' \ + ' *) echo Linux ;;' \ + 'esac' > "$FAKE_BIN/uname" +printf '%s\n' '#!/bin/sh' 'echo "all dependencies resolved"' > "$FAKE_BIN/ldd" +chmod +x "$FAKE_BIN/uname" "$FAKE_BIN/ldd" + +cleanup() { + rm -rf "$TEST_ROOT" +} +trap cleanup EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +reference_checksum="" +for component_dir in "${COMPONENT_DIRS[@]}"; do + for helper in common-topling.sh prepare-topling.sh preload-topling.sh \ + verify-rocksdb-provider.sh; do + if [ ! -x "$component_dir/bin/$helper" ]; then + fail "missing executable helper: $component_dir/bin/$helper" + fi + done + + if find "$component_dir/lib" -path '*/topling/*' -print -quit | grep -q .; then + fail "standard artifact contains a Topling JAR: $component_dir" + fi + if [ -e "$component_dir/library/librocksdbjni-linux64.so" ]; then + fail "standard artifact contains a Topling native library: $component_dir" + fi + + env -u TOPLINGDB_EASY_MIGRATE_CONF PATH="$FAKE_BIN:$PATH" bash -c ' + source "$1/bin/preload-topling.sh" + test -z "${TOPLINGDB_EASY_MIGRATE_CONF:-}" + test -z "${TOPLING_RUNTIME_CLASSPATH:-}" + ' _ "$component_dir" + + checksum=$(sha256sum "$component_dir/bin/preload-topling.sh" | + awk '{ print $1 }') + if [ -z "$reference_checksum" ]; then + reference_checksum="$checksum" + elif [ "$checksum" != "$reference_checksum" ]; then + fail "packaged runtime selectors differ across components" + fi + + fixture="$TEST_ROOT/$(basename "$component_dir")" + cp -R "$component_dir" "$fixture" + if TOPLINGDB_ROCKSDB_PROVIDER=topling \ + PATH="$FAKE_BIN:$PATH" bash -c \ + 'source "$1/bin/preload-topling.sh"' _ "$fixture" \ + > "$fixture/preload.out" 2>&1; then + fail "standard artifact accepted Topling without a local runtime: $component_dir" + fi + if ! grep -Fq "no prepared ToplingDB JAR found in $fixture/lib/topling/" \ + "$fixture/preload.out"; then + sed -n '1,120p' "$fixture/preload.out" >&2 + fail "missing component-local runtime error: $component_dir" + fi + + echo "PASS: standard artifact is isolated and fail-fast: $component_dir" +done diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-runtime-selection.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-runtime-selection.sh new file mode 100755 index 0000000000..bee44c6aac --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-runtime-selection.sh @@ -0,0 +1,253 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DIST_ROOT="$(cd "$SCRIPT_DIR"/../../.. && pwd)" +PRELOAD_SOURCE="$DIST_ROOT/src/assembly/static/bin/preload-topling.sh" +TEST_ROOT="$(mktemp -d)" +COMPONENT_ROOT="$TEST_ROOT/component" +FAKE_BIN="$TEST_ROOT/fake-bin" + +cleanup() { + rm -rf "$TEST_ROOT" +} +trap cleanup EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +expect_failure() { + local name="$1" + local expected="$2" + shift 2 + local output + + if output=$("$@" 2>&1); then + fail "$name unexpectedly succeeded" + fi + if ! grep -Fq "$expected" <<<"$output"; then + echo "$output" >&2 + fail "$name did not report: $expected" + fi + echo "PASS: $name" +} + +reset_fixture() { + rm -rf "$COMPONENT_ROOT" "$FAKE_BIN" + mkdir -p "$COMPONENT_ROOT/bin" "$COMPONENT_ROOT/conf/graphs" \ + "$COMPONENT_ROOT/lib/topling" "$FAKE_BIN" + cp "$PRELOAD_SOURCE" "$COMPONENT_ROOT/bin/preload-topling.sh" + + printf '%s\n' \ + '#!/bin/sh' \ + 'case "${1:-}" in' \ + ' -s) echo Linux ;;' \ + ' -m) echo x86_64 ;;' \ + ' *) echo Linux ;;' \ + 'esac' > "$FAKE_BIN/uname" + printf '%s\n' '#!/bin/sh' 'echo "all dependencies resolved"' > "$FAKE_BIN/ldd" + chmod +x "$FAKE_BIN/uname" "$FAKE_BIN/ldd" +} + +source_preload() { + PATH="$FAKE_BIN:$PATH" bash -c 'source "$1"' _ \ + "$COMPONENT_ROOT/bin/preload-topling.sh" +} + +source_preload_override() { + TOPLINGDB_ROCKSDB_PROVIDER="$1" PATH="$FAKE_BIN:$PATH" \ + bash -c 'source "$1"' _ \ + "$COMPONENT_ROOT/bin/preload-topling.sh" +} + +reset_fixture +source_preload +echo "PASS: unset provider selects standard RocksDB without Topling runtime" + +printf '%s\n' 'rocksdb.provider=not-topling' \ + > "$COMPONENT_ROOT/conf/graphs/hugegraph.properties" +expect_failure "invalid provider" \ + "invalid rocksdb.provider 'not-topling'" \ + source_preload + +expect_failure "invalid provider override" \ + "invalid TOPLINGDB_ROCKSDB_PROVIDER 'not-topling'" \ + source_preload_override not-topling + +printf '%s\n' 'rocksdb.provider=rocksdb' \ + > "$COMPONENT_ROOT/conf/graphs/hugegraph.properties" +printf '%s\n' 'rocksdb.provider=topling' \ + > "$COMPONENT_ROOT/conf/graphs/second.properties" +expect_failure "conflicting providers" \ + "conflicting rocksdb.provider values" \ + source_preload + +rm -f "$COMPONENT_ROOT/conf/graphs/second.properties" +printf '%s\n' 'rocksdb.provider=topling' \ + > "$COMPONENT_ROOT/conf/graphs/hugegraph.properties" +mkdir -p "$TEST_ROOT/apache-hugegraph-server/lib/topling" +touch "$TEST_ROOT/apache-hugegraph-server/lib/topling/rocksdbjni-stray.jar" +expect_failure "component-local JAR is required" \ + "no prepared ToplingDB JAR found in $COMPONENT_ROOT/lib/topling/" \ + source_preload + +touch "$COMPONENT_ROOT/lib/topling/rocksdbjni-topling.jar" +expect_failure "Easy Migrate configuration is required" \ + "required ToplingDB Easy Migrate config not found" \ + source_preload + +printf '%s\n' 'http:' ' auto_start_http: false' \ + > "$COMPONENT_ROOT/conf/toplingdb.yaml" +expect_failure "component-local native library is required" \ + "prepared ToplingDB native library not found" \ + source_preload + +mkdir -p "$COMPONENT_ROOT/library" +touch "$COMPONENT_ROOT/library/librocksdbjni-linux64.so" +printf '%s\n' '#!/bin/sh' 'echo "libmissing.so => not found"' > "$FAKE_BIN/ldd" +chmod +x "$FAKE_BIN/ldd" +expect_failure "unresolved native dependency" \ + "native library has unresolved system dependencies" \ + source_preload + +printf '%s\n' '#!/bin/sh' 'echo "all dependencies resolved"' > "$FAKE_BIN/ldd" +chmod +x "$FAKE_BIN/ldd" +PATH="$FAKE_BIN:$PATH" bash -c ' + source "$1" + test "$TOPLING_RUNTIME_CLASSPATH" = "$2/lib/topling/rocksdbjni-topling.jar" + test "$TOPLINGDB_EASY_MIGRATE_CONF" = "$2/conf/toplingdb.yaml" + test "$TOPLING_ACTIVE_NATIVE" = "$2/library/librocksdbjni-linux64.so" +' _ "$COMPONENT_ROOT/bin/preload-topling.sh" "$COMPONENT_ROOT" +echo "PASS: valid component-local Topling runtime is selected" + +printf '%s\n' 'rocksdb.provider=rocksdb' \ + > "$COMPONENT_ROOT/conf/graphs/hugegraph.properties" +TOPLINGDB_ROCKSDB_PROVIDER=topling PATH="$FAKE_BIN:$PATH" bash -c ' + source "$1" + test "$TOPLING_RUNTIME_CLASSPATH" = "$2/lib/topling/rocksdbjni-topling.jar" +' _ "$COMPONENT_ROOT/bin/preload-topling.sh" "$COMPONENT_ROOT" +echo "PASS: provider override selects ToplingDB before JVM startup" + +printf '%s\n' 'rocksdb.provider=topling' \ + > "$COMPONENT_ROOT/conf/graphs/hugegraph.properties" +TOPLINGDB_ROCKSDB_PROVIDER=rocksdb PATH="$FAKE_BIN:$PATH" bash -c ' + source "$1" + test -z "${TOPLING_RUNTIME_CLASSPATH:-}" + test -z "${TOPLINGDB_EASY_MIGRATE_CONF:-}" +' _ "$COMPONENT_ROOT/bin/preload-topling.sh" +echo "PASS: provider override selects standard RocksDB without Topling preload" + +# A Topling distribution must also be able to select standard RocksDB without +# putting its optional runtime JAR on the HugeGraph Server classpath. Exercise +# the launcher with a fake JVM and inspect the actual -cp argument rather than +# relying on a source-text assertion. +SERVER_LAUNCHER_ROOT="$TEST_ROOT/server launcher" +SERVER_JAVA_CAPTURE="$TEST_ROOT/server-java-classpath" +mkdir -p "$SERVER_LAUNCHER_ROOT/bin" "$SERVER_LAUNCHER_ROOT/conf" \ + "$SERVER_LAUNCHER_ROOT/lib/topling" "$SERVER_LAUNCHER_ROOT/ext" \ + "$SERVER_LAUNCHER_ROOT/plugins" "$SERVER_LAUNCHER_ROOT/logs" +mkdir -p "$SERVER_LAUNCHER_ROOT/lib/hugegraph-shadow/classes" +mkdir -p "$SERVER_LAUNCHER_ROOT/lib/hugegraph-shadow.jar" +cp "$DIST_ROOT/src/assembly/static/bin/hugegraph-server.sh" \ + "$SERVER_LAUNCHER_ROOT/bin/hugegraph-server.sh" +cp "$DIST_ROOT/src/assembly/static/bin/init-store.sh" \ + "$SERVER_LAUNCHER_ROOT/bin/init-store.sh" +cp "$DIST_ROOT/src/assembly/static/bin/preload-topling.sh" \ + "$SERVER_LAUNCHER_ROOT/bin/preload-topling.sh" +cp "$DIST_ROOT/src/assembly/static/bin/util.sh" \ + "$SERVER_LAUNCHER_ROOT/bin/util.sh" +touch "$SERVER_LAUNCHER_ROOT/lib/hugegraph-server-bootstrap.jar" \ + "$SERVER_LAUNCHER_ROOT/lib/topling/log4j-slf4j-impl-topling.jar" \ + "$SERVER_LAUNCHER_ROOT/lib/topling/hugegraph-topling.jar" \ + "$SERVER_LAUNCHER_ROOT/lib/topling/rocksdbjni-topling.jar" +ln -s "$SERVER_LAUNCHER_ROOT/lib/topling/log4j-slf4j-impl-topling.jar" \ + "$SERVER_LAUNCHER_ROOT/lib/log4j-slf4j-impl-topling-alias.jar" +ln -s "$SERVER_LAUNCHER_ROOT/lib/topling/hugegraph-topling.jar" \ + "$SERVER_LAUNCHER_ROOT/lib/hugegraph-topling-alias.jar" +ln -s "$SERVER_LAUNCHER_ROOT/lib/topling" \ + "$SERVER_LAUNCHER_ROOT/lib/topling-alias" +printf '%s\n' \ + 'gremlinserver.url=http://127.0.0.1:43123' \ + 'restserver.url=http://127.0.0.1:43124' \ + > "$SERVER_LAUNCHER_ROOT/conf/rest-server.properties" +printf '%s\n' \ + '#!/bin/sh' \ + 'if [ "${1:-}" = "-server" ]; then shift; fi' \ + 'if [ "${1:-}" = "-version" ]; then' \ + ' echo '\''openjdk version "11.0.32"'\'' >&2' \ + ' exit 0' \ + 'fi' \ + 'while [ "$#" -gt 0 ]; do' \ + ' if [ "$1" = "-cp" ]; then' \ + ' printf "%s\\n" "$2" > "$JAVA_CAPTURE"' \ + ' exit 0' \ + ' fi' \ + ' shift' \ + 'done' \ + 'exit 0' \ + > "$FAKE_BIN/java" +chmod +x "$FAKE_BIN/java" "$SERVER_LAUNCHER_ROOT/bin/hugegraph-server.sh" \ + "$SERVER_LAUNCHER_ROOT/bin/init-store.sh" \ + "$SERVER_LAUNCHER_ROOT/bin/preload-topling.sh" +JAVA_HOME='' JAVA_OPTIONS=-Xmx64m STDOUT_MODE=true CLASSPATH='' \ + JAVA_CAPTURE="$SERVER_JAVA_CAPTURE" PATH="$FAKE_BIN:$PATH" \ + bash "$SERVER_LAUNCHER_ROOT/bin/hugegraph-server.sh" \ + "$SERVER_LAUNCHER_ROOT/conf/gremlin-server.yaml" \ + "$SERVER_LAUNCHER_ROOT/conf/rest-server.properties" true \ + >/dev/null 2>&1 +grep -Fq "$SERVER_LAUNCHER_ROOT/lib/hugegraph-server-bootstrap.jar" \ + "$SERVER_JAVA_CAPTURE" || + fail "Server launcher omitted the standard bootstrap JAR" +if grep -Fq "$SERVER_LAUNCHER_ROOT/lib/topling/" "$SERVER_JAVA_CAPTURE"; then + fail "standard provider launcher leaked the optional Topling classpath" +fi +for leaked in \ + "$SERVER_LAUNCHER_ROOT/lib/log4j-slf4j-impl-topling-alias.jar" \ + "$SERVER_LAUNCHER_ROOT/lib/hugegraph-topling-alias.jar" \ + "$SERVER_LAUNCHER_ROOT/lib/topling-alias/" \ + "$SERVER_LAUNCHER_ROOT/lib/hugegraph-shadow"; do + if grep -Fq "$leaked" "$SERVER_JAVA_CAPTURE"; then + fail "standard provider launcher followed a Topling symlink: $leaked" + fi +done +echo "PASS: standard provider launcher excludes Topling JARs" + +SERVER_INIT_CAPTURE="$TEST_ROOT/server-init-classpath" +JAVA_HOME='' CLASSPATH='' JAVA_CAPTURE="$SERVER_INIT_CAPTURE" \ + PATH="$FAKE_BIN:$PATH" bash "$SERVER_LAUNCHER_ROOT/bin/init-store.sh" \ + >/dev/null 2>&1 +grep -Fq "$SERVER_LAUNCHER_ROOT/lib/hugegraph-server-bootstrap.jar" \ + "$SERVER_INIT_CAPTURE" || + fail "init-store omitted the standard bootstrap JAR" +if grep -Fq "$SERVER_LAUNCHER_ROOT/lib/topling/" "$SERVER_INIT_CAPTURE"; then + fail "standard provider init-store leaked the optional Topling classpath" +fi +for leaked in \ + "$SERVER_LAUNCHER_ROOT/lib/log4j-slf4j-impl-topling-alias.jar" \ + "$SERVER_LAUNCHER_ROOT/lib/hugegraph-topling-alias.jar" \ + "$SERVER_LAUNCHER_ROOT/lib/topling-alias/" \ + "$SERVER_LAUNCHER_ROOT/lib/hugegraph-shadow"; do + if grep -Fq "$leaked" "$SERVER_INIT_CAPTURE"; then + fail "standard provider init-store followed a Topling symlink: $leaked" + fi +done +echo "PASS: standard provider init-store excludes Topling JARs" diff --git a/hugegraph-server/hugegraph-rocksdb/pom.xml b/hugegraph-server/hugegraph-rocksdb/pom.xml index 845cf40f9c..276f8163cd 100644 --- a/hugegraph-server/hugegraph-rocksdb/pom.xml +++ b/hugegraph-server/hugegraph-rocksdb/pom.xml @@ -37,7 +37,7 @@ org.rocksdb rocksdbjni - 8.10.2 + ${rocksdb.version} diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java index 17148e0944..0f0d18f1a3 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java @@ -61,6 +61,14 @@ public static synchronized RocksDBOptions instance() { "rocksdb-data/data" ); + public static final ConfigOption PROVIDER = + new ConfigOption<>( + "rocksdb.provider", + "The RocksDB-compatible runtime provider.", + allowValues("rocksdb", "topling"), + "rocksdb" + ); + public static final ConfigListOption DATA_DISKS = new ConfigListOption<>( "rocksdb.data_disks", diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBSessions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBSessions.java index 7d0576e369..c5f3867ef6 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBSessions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBSessions.java @@ -17,6 +17,9 @@ package org.apache.hugegraph.backend.store.rocksdb; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Set; @@ -59,6 +62,40 @@ public RocksDBSessions(HugeConfig config, String database, String store) { public abstract void forceCloseRocksDB(); + public boolean databaseOpened() { + return this.opened(); + } + + public void clearTables(Collection tables) { + Session session = (Session) this.useSession(); + try { + List>> ranges = + new ArrayList<>(); + for (String table : tables) { + Pair range = session.keyRange(table); + if (range != null) { + ranges.add(Pair.of(table, range)); + } + } + + for (Pair> entry : ranges) { + String table = entry.getLeft(); + byte[] first = entry.getRight().getLeft(); + byte[] last = entry.getRight().getRight(); + if (!Arrays.equals(first, last)) { + session.deleteRange(table, first, last); + } + session.delete(table, last); + } + session.commit(); + } catch (Throwable e) { + session.rollback(); + throw e; + } finally { + this.close(); + } + } + @Override public abstract Session session(); diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java index c1cc1c5075..e5c054cdc7 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java @@ -18,6 +18,8 @@ package org.apache.hugegraph.backend.store.rocksdb; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -83,23 +85,28 @@ public class RocksDBStdSessions extends RocksDBSessions { private final AtomicInteger refCount; public RocksDBStdSessions(HugeConfig config, String database, String store, - String dataPath, String walPath) throws RocksDBException { + String dataPath, String walPath) throws + RocksDBException { super(config, database, store); this.config = config; this.dataPath = dataPath; this.walPath = walPath; + this.rocksdb = RocksDBStdSessions.openRocksDB(config, dataPath, walPath); this.refCount = new AtomicInteger(1); } public RocksDBStdSessions(HugeConfig config, String database, String store, String dataPath, String walPath, - List cfNames) throws RocksDBException { + List cfNames) throws + RocksDBException { super(config, database, store); this.config = config; this.dataPath = dataPath; this.walPath = walPath; - this.rocksdb = RocksDBStdSessions.openRocksDB(config, cfNames, dataPath, walPath); + + this.rocksdb = + RocksDBStdSessions.openRocksDB(config, cfNames, dataPath, walPath); this.refCount = new AtomicInteger(1); this.ingestExternalFile(); @@ -275,6 +282,13 @@ public void resumeSnapshot(String snapshotPath) { // Move snapshot directory to origin data directory FileUtils.moveDirectory(snapshotDir, originDataDir); LOG.info("Move snapshot directory {} to {}", snapshotDir, originDataDir); + /* + * Checkpoint copies a trimmed WAL tail into the data directory. + * Recovery reads only a separate WAL directory, so install that + * tail there. The live directory is moved aside first; failing + * to delete the retired copy must not restore its logs. + */ + this.replaceSeparateWalDirectory(); // Reload rocksdb instance this.reloadRocksDB(); } catch (Exception e) { @@ -334,6 +348,144 @@ private void checkValid() { E.checkState(this.rocksdb.isOwningHandle(), "It seems RocksDB has been closed"); } + private void replaceSeparateWalDirectory() throws IOException { + if (this.walPath == null || this.walPath.isEmpty()) { + return; + } + File walDir = new File(this.walPath); + File dataDir = new File(this.dataPath); + String walCanonical = walDir.getCanonicalPath(); + String dataCanonical = dataDir.getCanonicalPath(); + if (walCanonical.equals(dataCanonical)) { + return; + } + if (!dataDir.isDirectory()) { + throw new IOException("Snapshot data directory is missing: " + + dataDir); + } + File[] dataChildren = dataDir.listFiles(); + if (dataChildren == null) { + throw new IOException("Cannot list snapshot directory " + dataDir); + } + List checkpointLogs = new ArrayList<>(); + for (File child : dataChildren) { + if (child.isFile() && isWalLogName(child.getName())) { + checkpointLogs.add(child); + } + } + if (dataCanonical.startsWith(walCanonical + File.separator)) { + this.replaceLogsWithoutMovingWal(walDir, checkpointLogs); + return; + } + String token = Long.toString(System.nanoTime()); + File retired = new File(walDir.getPath() + ".resume-aside-" + token); + File staging = new File(walDir.getPath() + ".resume-staging-" + token); + if (walDir.exists() && !walDir.renameTo(retired)) { + throw new IOException("Failed to move WAL directory " + walDir + + " aside to " + retired); + } + try { + FileUtils.forceMkdir(staging); + for (File log : checkpointLogs) { + FileUtils.copyFile(log, new File(staging, log.getName())); + } + if (walDir.exists() || !staging.renameTo(walDir)) { + throw new IOException("Failed to publish WAL directory " + + staging); + } + } catch (IOException e) { + this.deleteRetiredWal(staging); + throw e; + } + for (File log : checkpointLogs) { + if (!log.delete()) { + LOG.warn("Failed to remove checkpoint WAL {}", log); + } + } + LOG.info("Replaced separate WAL directory {} with {} checkpoint log(s)", + walDir, checkpointLogs.size()); + this.deleteRetiredWal(retired); + } + + + private void replaceLogsWithoutMovingWal(File walDir, + List checkpointLogs) + throws IOException { + String token = Long.toString(System.nanoTime()); + File retired = new File(walDir.getPath() + ".resume-aside-" + token); + File staging = new File(walDir.getPath() + ".resume-staging-" + token); + FileUtils.forceMkdir(staging); + try { + for (File log : checkpointLogs) { + FileUtils.copyFile(log, new File(staging, log.getName())); + } + File[] liveChildren = walDir.listFiles(); + if (liveChildren == null) { + throw new IOException("Cannot list WAL directory " + walDir); + } + FileUtils.forceMkdir(retired); + for (File child : liveChildren) { + if (!child.isFile() || !isWalLogName(child.getName())) { + continue; + } + File dest = new File(retired, child.getName()); + if (!child.renameTo(dest)) { + throw new IOException("Failed to retire WAL log " + child); + } + } + File[] staged = staging.listFiles(); + if (staged == null) { + throw new IOException("Cannot list staging WAL " + staging); + } + for (File stagedLog : staged) { + File dest = new File(walDir, stagedLog.getName()); + if (!stagedLog.renameTo(dest)) { + throw new IOException("Failed to publish WAL log " + + stagedLog); + } + } + } catch (IOException e) { + this.deleteRetiredWal(staging); + throw e; + } + for (File log : checkpointLogs) { + if (!log.delete()) { + LOG.warn("Failed to remove checkpoint WAL {}", log); + } + } + LOG.info("Replaced WAL logs in {} with {} checkpoint log(s)", + walDir, checkpointLogs.size()); + this.deleteRetiredWal(retired); + this.deleteRetiredWal(staging); + } + + private static boolean isWalLogName(String name) { + int dot = name.lastIndexOf('.'); + if (dot <= 0 || !".log".equals(name.substring(dot))) { + return false; + } + for (int i = 0; i < dot; i++) { + if (!Character.isDigit(name.charAt(i))) { + return false; + } + } + return true; + } + + private void deleteRetiredWal(File retired) { + try { + if (Files.isSymbolicLink(retired.toPath())) { + Files.delete(retired.toPath()); + return; + } + if (retired.exists()) { + FileUtils.deleteDirectory(retired); + } + } catch (IOException e) { + LOG.warn("Failed to delete retired WAL directory {}", retired, e); + } + } + private RocksDB rocksdb() { this.checkValid(); return this.rocksdb.rocksdb(); @@ -366,25 +518,25 @@ private void ingestExternalFile() throws RocksDBException { } private static OpenedRocksDB openRocksDB(HugeConfig config, String dataPath, - String walPath) throws RocksDBException { + String walPath) throws + RocksDBException { // Init options Options options = new Options(); RocksDBStdSessions.initOptions(config, options, options, options, options); options.setWalDir(walPath); SstFileManager sstFileManager = new SstFileManager(Env.getDefault()); options.setSstFileManager(sstFileManager); - /* - * Open RocksDB at the first time - * Don't merge old CFs, we expect a clear DB when using this one - */ + RocksDB rocksdb = RocksDB.open(options, dataPath); + Map cfs = new ConcurrentHashMap<>(); return new OpenedRocksDB(rocksdb, cfs, sstFileManager); } private static OpenedRocksDB openRocksDB(HugeConfig config, List cfNames, String dataPath, - String walPath) throws RocksDBException { + String walPath) throws + RocksDBException { // Old CFs should always be opened Set mergedCFs = RocksDBStdSessions.mergeOldCFs(dataPath, cfNames); @@ -407,9 +559,9 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, } SstFileManager sstFileManager = new SstFileManager(Env.getDefault()); options.setSstFileManager(sstFileManager); - // Open RocksDB with CFs List cfhs = new ArrayList<>(); + RocksDB rocksdb = RocksDB.open(options, dataPath, cfds, cfhs); E.checkState(cfhs.size() == cfs.size(), diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java index 3b6b54eadb..420eb8caf0 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java @@ -103,6 +103,7 @@ public abstract class RocksDBStore extends AbstractBackendStore(); this.dbs = new ConcurrentHashMap<>(); this.storeLock = new ReentrantReadWriteLock(); + this.toplingProvider = false; this.registerMetaHandlers(); } @@ -212,6 +214,8 @@ public synchronized void open(HugeConfig config) { E.checkNotNull(config, "config"); String graphStore = config.get(CoreOptions.STORE_GRAPH); this.isGraphStore = this.store.equals(graphStore); + this.toplingProvider = "topling".equals( + config.get(RocksDBOptions.PROVIDER)); this.dataPath = config.get(RocksDBOptions.DATA_PATH); if (this.sessions != null && !this.sessions.closed()) { @@ -372,7 +376,8 @@ protected RocksDBSessions open(HugeConfig config, String dataPath, protected RocksDBSessions openSessionPool(HugeConfig config, String dataPath, String walPath, - List tableNames) throws RocksDBException { + List tableNames) throws + RocksDBException { if (tableNames == null) { return new RocksDBStdSessions(config, this.database, this.store, dataPath, walPath); } else { @@ -417,7 +422,7 @@ public void close() { @Override public boolean opened() { this.checkDbOpened(); - return this.sessions.session().opened(); + return this.sessions.databaseOpened(); } @Override @@ -630,8 +635,12 @@ public synchronized void truncate() { try { this.checkOpened(); - this.clear(false); - this.init(); + if (this.toplingProvider) { + this.clearTables(); + } else { + this.clear(false); + this.init(); + } // Clear write-batch this.dbs.values().forEach(BackendSessionPool::forceResetSessions); LOG.debug("Store truncated: {}", this.store); @@ -640,6 +649,22 @@ public synchronized void truncate() { } } + private void clearTables() { + this.sessions.clearTables(this.tableNames()); + + Map tableDBMap = this.tableDBMapping(); + for (Map.Entry entry : + tableDBMap.entrySet()) { + Collection tables; + if (entry.getKey().equals(HugeType.OLAP.string())) { + tables = this.olapTables(); + } else { + tables = Collections.singletonList(entry.getKey()); + } + entry.getValue().clearTables(tables); + } + } + @Override public void beginTx() { Lock readLock = this.storeLock.readLock(); diff --git a/hugegraph-server/hugegraph-test/README.md b/hugegraph-server/hugegraph-test/README.md new file mode 100644 index 0000000000..5abe65c382 --- /dev/null +++ b/hugegraph-server/hugegraph-test/README.md @@ -0,0 +1,20 @@ +# Server integration tests + +Run API tests against a dedicated test graph service with the matching backend +profile, for example `mvn test -pl hugegraph-server/hugegraph-test -am -P api-test,hstore`. +The suite creates and removes schemas, data, users and graphspaces. Its default +graph is `DEFAULT/hugegraph`; use an isolated service and data volumes. + +`BaseApiTest` accepts the following optional environment variables so the same +suite can run through a port-forward against a Helm-managed test deployment: + +| Variable | Default | +| --- | --- | +| `HUGEGRAPH_TEST_SERVER_URL` | `http://127.0.0.1:8080` | +| `HUGEGRAPH_TEST_ADMIN_USERNAME` | `admin` | +| `HUGEGRAPH_TEST_ADMIN_PASSWORD` | Existing local test fixture password | + +Supply credentials through the test process environment. Do not place them in +Maven command-line arguments or save them to test evidence. The overrides affect +the administrator client; per-test users still use their own fixture credentials. +Existing local and CI invocations keep their defaults when variables are absent. diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/HugeFactoryTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/HugeFactoryTest.java new file mode 100644 index 0000000000..101ab77860 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/HugeFactoryTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph; + +import java.util.Arrays; + +import org.apache.hugegraph.testutil.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +public class HugeFactoryTest { + + @Test + public void testCleanupContinuesAfterGraphFailure() { + StandardHugeGraph failed = Mockito.mock(StandardHugeGraph.class); + StandardHugeGraph cleaned = Mockito.mock(StandardHugeGraph.class); + Mockito.doThrow(new HugeException("test")) + .when(failed).closeCurrentThreadTransaction(); + + Assert.assertThrows(HugeException.class, () -> { + HugeFactory.closeCurrentThreadTransactions( + Arrays.asList(failed, cleaned)); + }); + + Mockito.verify(failed).closeCurrentThreadTransaction(); + Mockito.verify(cleaned).closeCurrentThreadTransaction(); + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/BaseApiTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/BaseApiTest.java index abe4b6839c..2b5d9aa129 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/BaseApiTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/BaseApiTest.java @@ -59,13 +59,16 @@ public class BaseApiTest { - protected static final String BASE_URL = "http://127.0.0.1:8080"; + protected static final String BASE_URL = System.getenv().getOrDefault( + "HUGEGRAPH_TEST_SERVER_URL", "http://127.0.0.1:8080"); private static final String GRAPH = "hugegraph"; private static final String GRAPHSPACE = "DEFAULT"; protected static final String URL_PREFIX = "graphspaces/" + GRAPHSPACE + "/graphs/" + GRAPH; protected static final String TRAVERSERS_API = URL_PREFIX + "/traversers"; - private static final String USERNAME = "admin"; - private static final String PASSWORD = "pa"; + private static final String USERNAME = System.getenv().getOrDefault( + "HUGEGRAPH_TEST_ADMIN_USERNAME", "admin"); + private static final String PASSWORD = System.getenv().getOrDefault( + "HUGEGRAPH_TEST_ADMIN_PASSWORD", "pa"); private static final int NO_LIMIT = -1; private static final String SCHEMA_PKS = "/schema/propertykeys"; private static final String SCHEMA_VLS = "/schema/vertexlabels"; @@ -181,12 +184,21 @@ protected static void waitTaskStatus(int task, Set expectedStatus) { String.valueOf(task)); String content = assertResponseStatus(200, r); status = assertJsonContains(content, "task_status"); + if (expectedStatus.contains(status)) { + return; + } + // A success-only wait must not spin after the task has ended. + if (!expectedStatus.contains("failed") && + ("failed".equals(status) || "cancelled".equals(status))) { + Assert.fail(String.format("Task %s ended with status %s: %s", + task, status, content)); + } if (times++ > maxTimes) { Assert.fail(String.format("Failed to wait for task %s " + - "due to timeout", task)); + "due to timeout, last status %s", + task, status)); } - } - while (!expectedStatus.contains(status)); + } while (true); } protected static void initVertexLabel() { diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/GraphSpaceApiTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/GraphSpaceApiTest.java index 98240e76f5..c8ded372d0 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/GraphSpaceApiTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/GraphSpaceApiTest.java @@ -24,6 +24,7 @@ import org.apache.hugegraph.util.JsonUtil; import org.junit.Assert; import org.junit.Assume; +import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -51,6 +52,32 @@ public void removeSpaces() { this.client().delete(PATH, space); } } + this.clearRoleTestFixtures(); + } + + @After + public void clearRoleTestFixtures() { + if (!"hstore".equals(System.getProperty("backend"))) { + return; + } + String space = "default_role_auth_space"; + Response spacesResponse = this.client().get(PATH); + Map spaces = JsonUtil.fromJson( + assertResponseStatus(200, spacesResponse), Map.class); + if (((List) spaces.get("graphSpaces")).contains(space)) { + assertResponseStatus(204, this.client().delete(PATH, space)); + } + String usersPath = "graphspaces/DEFAULT/auth/users"; + Response usersResponse = this.client().get(usersPath, ImmutableMap.of("limit", -1)); + Map result = JsonUtil.fromJson( + assertResponseStatus(200, usersResponse), Map.class); + List ownedNames = List.of("default_role_analyst", "default_role_target", + "default_role_manager"); + for (Map user : (List>) result.get("users")) { + if (ownedNames.contains(user.get("user_name"))) { + assertResponseStatus(204, this.client().delete(usersPath, (String) user.get("id"))); + } + } } @Test @@ -505,49 +532,60 @@ public void testDefaultRoleMutationRequiresSpaceManager() { String manager = "default_role_manager"; createSpace(space, true); - RestClient analystClient = userClient(analyst); - userClient(target); - RestClient managerClient = spaceManagerClient(space, manager); - - String rolePath = String.format("graphspaces/%s/role", space); - String analystBody = String.format("{\"user\":\"%s\",\"role\":\"ANALYST\"}", - analyst); - Response r = this.client().post(rolePath, analystBody); - assertResponseStatus(201, r); - - String grantTargetBody = String.format("{\"user\":\"%s\",\"role\":\"ANALYST\"}", - target); - r = analystClient.post(rolePath, grantTargetBody); - assertResponseStatus(403, r); - - r = analystClient.get(rolePath, - ImmutableMap.of("user", target, - "role", "ANALYST")); - assertResponseStatus(403, r); - - r = analystClient.delete(rolePath, - ImmutableMap.of("user", analyst, - "role", "ANALYST")); - assertResponseStatus(403, r); - - r = managerClient.post(rolePath, grantTargetBody); - assertResponseStatus(201, r); - - r = managerClient.get(rolePath, - ImmutableMap.of("user", target, - "role", "ANALYST")); - String result = assertResponseStatus(200, r); - Assert.assertTrue(result.contains("true")); - - String spaceRoleBody = String.format("{\"user\":\"%s\",\"role\":\"SPACE\"}", - target); - r = managerClient.post(rolePath, spaceRoleBody); - assertResponseStatus(403, r); - - r = managerClient.delete(rolePath, - ImmutableMap.of("user", target, - "role", "ANALYST")); - assertResponseStatus(204, r); + RestClient analystClient = null; + RestClient managerClient = null; + try { + analystClient = userClient(analyst); + userClient(target).close(); + managerClient = spaceManagerClient(space, manager); + + String rolePath = String.format("graphspaces/%s/role", space); + String analystBody = String.format("{\"user\":\"%s\",\"role\":\"ANALYST\"}", + analyst); + Response r = this.client().post(rolePath, analystBody); + assertResponseStatus(201, r); + + String grantTargetBody = String.format("{\"user\":\"%s\",\"role\":\"ANALYST\"}", + target); + r = analystClient.post(rolePath, grantTargetBody); + assertResponseStatus(403, r); + + r = analystClient.get(rolePath, + ImmutableMap.of("user", target, + "role", "ANALYST")); + assertResponseStatus(403, r); + + r = analystClient.delete(rolePath, + ImmutableMap.of("user", analyst, + "role", "ANALYST")); + assertResponseStatus(403, r); + + r = managerClient.post(rolePath, grantTargetBody); + assertResponseStatus(201, r); + + r = managerClient.get(rolePath, + ImmutableMap.of("user", target, + "role", "ANALYST")); + String result = assertResponseStatus(200, r); + Assert.assertTrue(result.contains("true")); + + String spaceRoleBody = String.format("{\"user\":\"%s\",\"role\":\"SPACE\"}", + target); + r = managerClient.post(rolePath, spaceRoleBody); + assertResponseStatus(403, r); + + r = managerClient.delete(rolePath, + ImmutableMap.of("user", target, + "role", "ANALYST")); + assertResponseStatus(204, r); + } finally { + if (analystClient != null) { + analystClient.close(); + } + if (managerClient != null) { + managerClient.close(); + } + } } @Test diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/auth/ContextTaskTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/auth/ContextTaskTest.java new file mode 100644 index 0000000000..a5ef4ad8d7 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/auth/ContextTaskTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.auth; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hugegraph.auth.HugeGraphAuthProxy.Context; +import org.apache.hugegraph.auth.HugeGraphAuthProxy.ContextTask; +import org.apache.hugegraph.testutil.Assert; +import org.junit.After; +import org.junit.Test; + +public class ContextTaskTest { + + @After + public void teardown() { + HugeGraphAuthProxy.resetContext(); + } + + @Test + public void testCleanupRunsBeforeContextReset() { + Context context = Context.admin(); + AtomicBoolean runnerCalled = new AtomicBoolean(false); + AtomicBoolean cleanupCalled = new AtomicBoolean(false); + HugeGraphAuthProxy.setContext(context); + + ContextTask task = new ContextTask(() -> { + Assert.assertSame(context, HugeGraphAuthProxy.getContext()); + runnerCalled.set(true); + }, () -> { + Assert.assertSame(context, HugeGraphAuthProxy.getContext()); + cleanupCalled.set(true); + }); + HugeGraphAuthProxy.resetContext(); + + task.run(); + + Assert.assertTrue(runnerCalled.get()); + Assert.assertTrue(cleanupCalled.get()); + Assert.assertNull(HugeGraphAuthProxy.getContext()); + } + + @Test + public void testCleanupRunsWhenTaskFails() { + AtomicBoolean cleanupCalled = new AtomicBoolean(false); + ContextTask task = new ContextTask(() -> { + throw new IllegalStateException("test"); + }, () -> cleanupCalled.set(true)); + + Assert.assertThrows(IllegalStateException.class, task::run); + Assert.assertTrue(cleanupCalled.get()); + Assert.assertNull(HugeGraphAuthProxy.getContext()); + } + + @Test + public void testContextResetRunsWhenCleanupFails() { + Context context = Context.admin(); + HugeGraphAuthProxy.setContext(context); + ContextTask task = new ContextTask(() -> { + Assert.assertSame(context, HugeGraphAuthProxy.getContext()); + }, () -> { + throw new IllegalStateException("test"); + }); + HugeGraphAuthProxy.resetContext(); + + task.run(); + + Assert.assertNull(HugeGraphAuthProxy.getContext()); + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/IndexLabelCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/IndexLabelCoreTest.java index 24a905427c..6bcfafd161 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/IndexLabelCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/IndexLabelCoreTest.java @@ -22,6 +22,7 @@ import org.apache.hugegraph.HugeException; import org.apache.hugegraph.HugeGraph; +import org.apache.hugegraph.HugeGraphParams; import org.apache.hugegraph.exception.ExistedException; import org.apache.hugegraph.exception.NoIndexException; import org.apache.hugegraph.exception.NotFoundException; @@ -31,10 +32,12 @@ import org.apache.hugegraph.schema.Userdata; import org.apache.hugegraph.schema.VertexLabel; import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.testutil.Whitebox; import org.apache.hugegraph.type.HugeType; import org.apache.hugegraph.type.define.IndexType; import org.apache.hugegraph.type.define.WriteType; import org.apache.hugegraph.util.DateUtil; +import org.apache.hugegraph.util.Events; import org.apache.tinkerpop.gremlin.process.traversal.P; import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.structure.T; @@ -1297,6 +1300,81 @@ public void testRemoveIndexLabelOfVertex() { }); } + @Test + public void testUpdateCachedVertexAfterIndexRemoval() throws Exception { + super.initPropertyKeys(); + SchemaManager schema = graph().schema(); + schema.vertexLabel("person").properties("name", "city") + .primaryKeys("name").create(); + schema.indexLabel("personByCity").onV("person").secondary() + .by("city").create(); + Vertex original = graph().addVertex(T.label, "person", "name", "cache-test", "city", "old"); + graph().tx().commit(); + Vertex cached = graph().vertices(original.id()).next(); + graph().tx().commit(); + // Schema cache eviction must not make a retained vertex's index list authoritative. + HugeGraphParams params = Whitebox.getInternalState(graph(), "params"); + params.schemaEventHub().notify(Events.CACHE, "clear", null).get(); + + schema.indexLabel("personByCity").remove(); + Assert.assertThrows(NotFoundException.class, () -> schema.getIndexLabel("personByCity")); + cached.property("city", "new"); + graph().tx().commit(); + Assert.assertEquals("new", graph().vertices(original.id()).next().value("city")); + graph().tx().commit(); + schema.indexLabel("personByCity").onV("person").secondary().by("city").create(); + schema.propertyKey("extra").asText().create(); + schema.vertexLabel("person").properties("extra").nullableKeys("extra").append(); + schema.indexLabel("personByExtra").onV("person").secondary().by("extra").create(); + graph().taskScheduler().waitUntilAllTasksCompleted(30); + cached.property("city", "newer"); + graph().tx().commit(); + Assert.assertFalse(graph().traversal().V().has("city", "new").hasNext()); + Assert.assertEquals(original.id(), graph().traversal().V().has("city", "newer").next().id()); + graph().tx().commit(); + cached.remove(); + graph().tx().commit(); + Assert.assertFalse(graph().vertices(original.id()).hasNext()); + } + + @Test + public void testUpdateCachedEdgeAfterIndexRemoval() throws Exception { + super.initPropertyKeys(); + SchemaManager schema = graph().schema(); + schema.vertexLabel("author").properties("id", "name").primaryKeys("id").create(); + schema.vertexLabel("book").properties("name").primaryKeys("name").create(); + schema.edgeLabel("authored").singleTime().link("author", "book") + .properties("contribution").create(); + schema.indexLabel("authoredByContri").onE("authored").secondary() + .by("contribution").create(); + Vertex author = graph().addVertex(T.label, "author", "id", 1, "name", "author"); + Vertex book = graph().addVertex(T.label, "book", "name", "book"); + Edge original = author.addEdge("authored", book, "contribution", "old"); + graph().tx().commit(); + Edge cached = graph().edges(original.id()).next(); + graph().tx().commit(); + HugeGraphParams params = Whitebox.getInternalState(graph(), "params"); + params.schemaEventHub().notify(Events.CACHE, "clear", null).get(); + + schema.indexLabel("authoredByContri").remove(); + Assert.assertThrows(NotFoundException.class, () -> schema.getIndexLabel("authoredByContri")); + cached.property("contribution", "new"); + graph().tx().commit(); + Assert.assertEquals("new", graph().edges(original.id()).next().value("contribution")); + graph().tx().commit(); + schema.indexLabel("authoredByContri").onE("authored").secondary() + .by("contribution").create(); + graph().taskScheduler().waitUntilAllTasksCompleted(30); + cached.property("contribution", "newer"); + graph().tx().commit(); + Assert.assertFalse(graph().traversal().E().has("contribution", "new").hasNext()); + Assert.assertEquals(original.id(), graph().traversal().E().has("contribution", "newer").next().id()); + graph().tx().commit(); + cached.remove(); + graph().tx().commit(); + Assert.assertFalse(graph().edges(original.id()).hasNext()); + } + @Test public void testRemoveIndexLabelOfEdge() { super.initPropertyKeys(); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java index 335130ede5..a50b66c756 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java @@ -5480,10 +5480,35 @@ public void testQueryByJointIndexesWithSearchAndTwoRangeIndexesAndWithin() { "confirmType", 2, "type", 1, "kid", 2); graph.addVertex(T.label, "test", "name", "诚信文明", "confirmType", 3, "type", 1, "kid", 3); + graph.addVertex(T.label, "test", "name", "诚信", + "confirmType", 4, "type", 1, "kid", 4); this.assertQueryByJointIndexesWithSearchAndTwoRangeIndexesAndWithin(); this.commitTx(); this.assertQueryByJointIndexesWithSearchAndTwoRangeIndexesAndWithin(); + + // Every confirmType branch and the search index reach the threshold. + // Only type=0 is selective, independently of joint-index iteration order. + int kid = 5; + for (int confirmType : new int[]{0, 1, 3}) { + graph.addVertex(T.label, "test", "name", "诚信", "confirmType", confirmType, + "type", 1, "kid", kid++); + } + this.commitTx(); + Object tx = Whitebox.invoke(graph.getClass(), "graphTransaction", graph); + Object threshold = Whitebox.getInternalState(tx, "indexTx.indexIntersectThresh"); + try { + Whitebox.setInternalState(tx, "indexTx.indexIntersectThresh", 2); + List vertices = graph.traversal().V() + .has("type", 0) + .has("confirmType", P.within(0, 1, 3)) + .has("name", Text.contains("诚信")) + .toList(); + Assert.assertEquals(1, vertices.size()); + assertContains(vertices, T.label, "test", "kid", 0); + } finally { + Whitebox.setInternalState(tx, "indexTx.indexIntersectThresh", threshold); + } } private void assertQueryByJointIndexesWithSearchAndTwoRangeIndexesAndWithin() { diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/testutil/Utils.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/testutil/Utils.java index e081423b8e..fc8fc8a3f4 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/testutil/Utils.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/testutil/Utils.java @@ -38,6 +38,10 @@ public class Utils { public static final String CONF_PATH = "hugegraph.properties"; public static HugeGraph open() { + return HugeFactory.open(getConf()); + } + + private static String configPath() { String confPath = System.getProperty("config_path"); if (confPath == null || confPath.isEmpty()) { confPath = CONF_PATH; @@ -49,7 +53,7 @@ public static HugeGraph open() { // ignored Exception } - return HugeFactory.open(getLocalConfig(confPath)); + return confPath; } private static PropertiesConfiguration getLocalConfig(String path) { @@ -97,21 +101,7 @@ public static Date date(String rawDate) { } public static PropertiesConfiguration getConf() { - String confFile = Utils.class.getClassLoader() - .getResource(CONF_PATH).getPath(); - File file = new File(confFile); - E.checkArgument(file.exists() && file.isFile() && file.canRead(), - "Need to specify a readable config file rather than:" + - " %s", file.toString()); - - PropertiesConfiguration config; - try { - config = new Configurations().properties(file); - } catch (ConfigurationException e) { - throw new HugeException("Unable to load config file: %s", - e, confFile); - } - return config; + return getLocalConfig(configPath()); } public static void println(String message) { diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java index 0e010ae5f7..a12c3215f0 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -17,8 +17,10 @@ package org.apache.hugegraph.unit; +import org.apache.hugegraph.HugeFactoryTest; import org.apache.hugegraph.api.auth.GraphSpaceAuthPayloadTest; import org.apache.hugegraph.api.auth.GraphSpaceGroupAPITest; +import org.apache.hugegraph.auth.ContextTaskTest; import org.apache.hugegraph.auth.StandardAuthManagerV2Test; import org.apache.hugegraph.auth.WsAndHttpBasicAuthHandlerTest; import org.apache.hugegraph.backend.page.QueryListTest; @@ -61,6 +63,7 @@ import org.apache.hugegraph.unit.core.IdHolderTest; import org.apache.hugegraph.unit.core.LocksTableTest; import org.apache.hugegraph.unit.core.PageStateTest; +import org.apache.hugegraph.unit.core.ConfigPathTest; import org.apache.hugegraph.unit.core.QueryResultsTest; import org.apache.hugegraph.unit.core.QueryTest; import org.apache.hugegraph.unit.core.RangeTest; @@ -92,6 +95,7 @@ import org.apache.hugegraph.unit.serializer.TextBackendEntryTest; import org.apache.hugegraph.unit.serializer.TextSerializerTest; import org.apache.hugegraph.unit.store.RamIntObjectMapTest; +import org.apache.hugegraph.unit.traversal.OltpTraverserTest; import org.apache.hugegraph.unit.traversal.ShortestPathTraverserTest; import org.apache.hugegraph.unit.util.CompressUtilTest; import org.apache.hugegraph.unit.util.JsonUtilTest; @@ -121,6 +125,8 @@ GraphSpaceGroupAPITest.class, GraphSpaceAuthPayloadTest.class, StandardAuthManagerV2Test.class, + ContextTaskTest.class, + HugeFactoryTest.class, AuthMetaManagerTest.class, /* api space */ @@ -163,6 +169,7 @@ GraphIndexTransactionTest.class, GraphTransactionTest.class, QueryTest.class, + ConfigPathTest.class, QueryResultsTest.class, QueryListTest.class, RangeTest.class, @@ -184,6 +191,7 @@ RoleElectionStateMachineTest.class, HugeGraphAuthProxyTest.class, SchemaElementTest.class, + OltpTraverserTest.class, ShortestPathTraverserTest.class, /* cmd */ diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/ConfigPathTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/ConfigPathTest.java new file mode 100644 index 0000000000..87b482ac89 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/ConfigPathTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.unit.core; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.hugegraph.testutil.Utils; +import org.junit.Assert; +import org.junit.Test; + +public class ConfigPathTest { + + @Test + public void testExplicitConfigAlsoAppliesToGraphCopies() throws Exception { + String previous = System.getProperty("config_path"); + Path config = Files.createTempFile("hugegraph-config-path-", ".properties"); + try { + Files.write(config, "hbase.port=22181\nstore=isolated\n" + .getBytes(StandardCharsets.UTF_8)); + System.setProperty("config_path", config.toString()); + Assert.assertEquals(22181, Utils.getConf().getInt("hbase.port")); + Assert.assertEquals("isolated", Utils.getConf().getString("store")); + Utils.getConf().setProperty("store", "copy"); + Assert.assertEquals("isolated", Utils.getConf().getString("store")); + } finally { + if (previous == null) { + System.clearProperty("config_path"); + } else { + System.setProperty("config_path", previous); + } + Files.deleteIfExists(config); + } + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java index 5dbc96c5c6..2cf1bcdb68 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java @@ -19,6 +19,11 @@ import java.io.File; import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.io.FileUtils; import org.apache.hugegraph.backend.store.rocksdb.RocksDBMetrics; @@ -28,6 +33,7 @@ import org.apache.hugegraph.backend.store.rocksdbsst.RocksDBSstSessions; import org.apache.hugegraph.config.HugeConfig; import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.testutil.Whitebox; import org.apache.hugegraph.unit.FakeObjects; import org.junit.Test; import org.rocksdb.RocksDBException; @@ -37,6 +43,57 @@ public class RocksDBSessionsTest extends BaseRocksDBUnitTest { + @Test + public void testClearTablesWithoutRecreatingColumnFamilies() + throws RocksDBException, InterruptedException { + final String table2 = "test-table2"; + this.rocks.createTable(table2); + this.rocks.session().put(TABLE, getBytes("person:1"), + getBytes("James")); + this.rocks.session().put(table2, getBytes("person:2"), + getBytes("Tom")); + this.commit(); + + AtomicReference failure = new AtomicReference<>(); + Thread thread = new Thread(() -> { + try { + this.rocks.clearTables(ImmutableList.of(TABLE, table2)); + } catch (Throwable e) { + failure.set(e); + } + }); + thread.start(); + thread.join(); + + Assert.assertNull(failure.get()); + Assert.assertNull(this.rocks.session().get(TABLE, + getBytes("person:1"))); + Assert.assertNull(this.rocks.session().get(table2, + getBytes("person:2"))); + Assert.assertTrue(this.rocks.existsTable(TABLE)); + Assert.assertTrue(this.rocks.existsTable(table2)); + AtomicInteger sessionCount = + Whitebox.getInternalState(this.rocks, "sessionCount"); + Assert.assertEquals(1, sessionCount.get()); + } + + @Test + public void testDatabaseOpenedDoesNotCreateSession() throws RocksDBException { + HugeConfig config = FakeObjects.newConfig(); + String path = DB_PATH + "/opened"; + RocksDBSessions sessions = + new RocksDBStdSessions(config, "db", "store", path, path); + AtomicInteger sessionCount = + Whitebox.getInternalState(sessions, "sessionCount"); + + Assert.assertEquals(0, sessionCount.get()); + Assert.assertTrue(sessions.databaseOpened()); + Assert.assertEquals(0, sessionCount.get()); + + sessions.forceCloseRocksDB(); + Assert.assertFalse(sessions.databaseOpened()); + } + @Test public void testTable() throws RocksDBException { final String TABLE2 = "test-table2"; @@ -135,6 +192,278 @@ public void testSnapshot() throws RocksDBException, IOException { } } + @Test + public void testSnapshotWithSeparateWalDirectory() throws Exception { + String dataPath = DB_PATH + "/separate-data"; + String walPath = DB_PATH + "/separate-wal"; + String snapshotPath = SNAPSHOT_PATH + "/separate-rocks"; + FileUtils.deleteDirectory(FileUtils.getFile(dataPath)); + FileUtils.deleteDirectory(FileUtils.getFile(walPath)); + FileUtils.deleteDirectory(FileUtils.getFile(snapshotPath)); + HugeConfig config = FakeObjects.newConfig(); + RocksDBSessions sessions = new RocksDBStdSessions(config, "db", "store", + dataPath, walPath); + try { + sessions.createTable(TABLE); + sessions.session().put(TABLE, getBytes("person:1gname"), + getBytes("James")); + sessions.session().put(TABLE, getBytes("person:2gname"), + getBytes("Lisa")); + sessions.session().commit(); + + sessions.createSnapshot(snapshotPath); + + sessions.session().put(TABLE, getBytes("person:1gname"), + getBytes("James2")); + sessions.session().put(TABLE, getBytes("person:3gname"), + getBytes("After")); + sessions.session().commit(); + Assert.assertEquals("James2", getString(sessions.session().get( + TABLE, getBytes("person:1gname")))); + + sessions.resumeSnapshot(snapshotPath); + + Assert.assertEquals("James", getString(sessions.session().get( + TABLE, getBytes("person:1gname")))); + Assert.assertEquals("Lisa", getString(sessions.session().get( + TABLE, getBytes("person:2gname")))); + Assert.assertNull(sessions.session().get(TABLE, + getBytes("person:3gname"))); + Assert.assertEquals(0, readWalLogs(dataPath).size()); + assertNoResumeResidue(walPath); + } finally { + sessions.close(); + FileUtils.deleteDirectory(FileUtils.getFile(dataPath)); + FileUtils.deleteDirectory(FileUtils.getFile(walPath)); + FileUtils.deleteDirectory(FileUtils.getFile(walPath + ".resume-aside")); + File snapshotFile = FileUtils.getFile(SNAPSHOT_PATH); + if (snapshotFile.exists()) { + FileUtils.forceDelete(snapshotFile); + } + } + } + + + @Test + public void testResumeWhenDataDirectoryIsInsideWal() throws Exception { + String walPath = nestedRoot("parent-wal"); + String dataPath = walPath + "/nested-data"; + resumeNestedAndExpectSnapshot(dataPath, walPath); + } + + @Test + public void testResumeWhenWalDirectoryIsInsideData() throws Exception { + String dataPath = nestedRoot("parent-data"); + String walPath = dataPath + "/nested-wal"; + resumeNestedAndExpectSnapshot(dataPath, walPath); + } + + + private static String nestedRoot(String name) { + return System.getProperty("java.io.tmpdir") + "/nested-wal-closure/" + name; + } + + private void resumeNestedAndExpectSnapshot(String dataPath, String walPath) + throws Exception { + String snapshotPath = SNAPSHOT_PATH + "/nested-rocks"; + FileUtils.deleteDirectory(FileUtils.getFile(dataPath)); + FileUtils.deleteDirectory(FileUtils.getFile(walPath)); + FileUtils.forceMkdir(FileUtils.getFile(walPath).getParentFile()); + FileUtils.forceMkdir(FileUtils.getFile(dataPath).getParentFile()); + HugeConfig config = FakeObjects.newConfig(); + RocksDBSessions sessions = new RocksDBStdSessions(config, "db", "store", + dataPath, walPath); + try { + sessions.createTable(TABLE); + sessions.session().put(TABLE, getBytes("person:1gname"), + getBytes("James")); + sessions.session().commit(); + sessions.createSnapshot(snapshotPath); + sessions.session().put(TABLE, getBytes("person:1gname"), + getBytes("James2")); + sessions.session().commit(); + sessions.resumeSnapshot(snapshotPath); + Assert.assertEquals("James", getString(sessions.session().get( + TABLE, getBytes("person:1gname")))); + } finally { + sessions.close(); + FileUtils.deleteDirectory(FileUtils.getFile(dataPath)); + FileUtils.deleteDirectory(FileUtils.getFile(walPath)); + File snapshotFile = FileUtils.getFile(SNAPSHOT_PATH); + if (snapshotFile.exists()) { + FileUtils.forceDelete(snapshotFile); + } + } + } + + @Test + public void testSeparateWalInstallsCheckpointTail() throws Exception { + String dataPath = DB_PATH + "/tail-data"; + String walPath = DB_PATH + "/tail-wal"; + FileUtils.deleteDirectory(FileUtils.getFile(dataPath)); + FileUtils.deleteDirectory(FileUtils.getFile(walPath)); + FileUtils.deleteDirectory(FileUtils.getFile(walPath + ".resume-aside")); + HugeConfig config = FakeObjects.newConfig(); + RocksDBSessions sessions = new RocksDBStdSessions(config, "db", "store", + dataPath, walPath); + boolean open = true; + try { + sessions.createTable(TABLE); + sessions.close(); + open = false; + byte[] checkpoint = "checkpoint-tail".getBytes("UTF-8"); + byte[] live = "live-longer-tail".getBytes("UTF-8"); + FileUtils.forceMkdir(FileUtils.getFile(dataPath)); + FileUtils.forceMkdir(FileUtils.getFile(walPath)); + FileUtils.writeByteArrayToFile(new File(dataPath, "000123.log"), + checkpoint); + FileUtils.writeByteArrayToFile(new File(walPath, "000123.log"), live); + FileUtils.writeByteArrayToFile(new File(walPath, "000124.log"), + "later".getBytes("UTF-8")); + + Whitebox.invoke(RocksDBStdSessions.class, + "replaceSeparateWalDirectory", sessions); + + assertSameWalLogs(Collections.singletonMap("000123.log", checkpoint), + readWalLogs(walPath)); + Assert.assertFalse(new File(dataPath, "000123.log").exists()); + Assert.assertFalse(new File(walPath, "000124.log").exists()); + assertNoResumeResidue(walPath); + } finally { + if (open) { + sessions.close(); + } + FileUtils.deleteDirectory(FileUtils.getFile(dataPath)); + FileUtils.deleteDirectory(FileUtils.getFile(walPath)); + FileUtils.deleteDirectory(FileUtils.getFile(walPath + ".resume-aside")); + } + } + + + @Test + public void testDataInsideWalKeepsDataAndInstallsTail() throws Exception { + String walPath = nestedRoot("contain-wal"); + String dataPath = walPath + "/data"; + FileUtils.deleteDirectory(FileUtils.getFile(walPath)); + FileUtils.forceMkdir(FileUtils.getFile(dataPath).getParentFile()); + HugeConfig config = FakeObjects.newConfig(); + RocksDBSessions sessions = new RocksDBStdSessions(config, "db", "store", + dataPath, walPath); + boolean open = true; + try { + sessions.createTable(TABLE); + sessions.close(); + open = false; + byte[] checkpoint = "checkpoint-tail".getBytes("UTF-8"); + FileUtils.forceMkdir(FileUtils.getFile(dataPath)); + FileUtils.forceMkdir(FileUtils.getFile(walPath)); + FileUtils.writeByteArrayToFile(new File(dataPath, "000123.log"), + checkpoint); + FileUtils.writeByteArrayToFile(new File(walPath, "000123.log"), + "live-longer-tail".getBytes("UTF-8")); + FileUtils.writeByteArrayToFile(new File(walPath, "000124.log"), + "later".getBytes("UTF-8")); + File marker = new File(dataPath, "keep-data.txt"); + FileUtils.writeByteArrayToFile(marker, "kept".getBytes("UTF-8")); + + Whitebox.invoke(RocksDBStdSessions.class, + "replaceSeparateWalDirectory", sessions); + + assertSameWalLogs(Collections.singletonMap("000123.log", checkpoint), + readWalLogs(walPath)); + Assert.assertTrue(marker.isFile()); + Assert.assertFalse(new File(dataPath, "000123.log").exists()); + assertNoResumeResidue(walPath); + } finally { + if (open) { + sessions.close(); + } + FileUtils.deleteDirectory(FileUtils.getFile(walPath)); + } + } + + @Test + public void testWalInsideDataInstallsTail() throws Exception { + String dataPath = nestedRoot("inside-data"); + String walPath = dataPath + "/wal"; + FileUtils.deleteDirectory(FileUtils.getFile(dataPath)); + FileUtils.forceMkdir(FileUtils.getFile(walPath)); + HugeConfig config = FakeObjects.newConfig(); + RocksDBSessions sessions = new RocksDBStdSessions(config, "db", "store", + dataPath, walPath); + boolean open = true; + try { + sessions.createTable(TABLE); + sessions.close(); + open = false; + byte[] checkpoint = "checkpoint-tail".getBytes("UTF-8"); + FileUtils.writeByteArrayToFile(new File(dataPath, "000123.log"), + checkpoint); + FileUtils.writeByteArrayToFile(new File(walPath, "000124.log"), + "later".getBytes("UTF-8")); + + Whitebox.invoke(RocksDBStdSessions.class, + "replaceSeparateWalDirectory", sessions); + + assertSameWalLogs(Collections.singletonMap("000123.log", checkpoint), + readWalLogs(walPath)); + Assert.assertFalse(new File(dataPath, "000123.log").exists()); + Assert.assertFalse(new File(walPath, "000124.log").exists()); + assertNoResumeResidue(walPath); + } finally { + if (open) { + sessions.close(); + } + FileUtils.deleteDirectory(FileUtils.getFile(dataPath)); + } + } + + private static Map readWalLogs(String directory) throws IOException { + Map logs = new HashMap<>(); + File dir = FileUtils.getFile(directory); + File[] children = dir.listFiles(); + if (children == null) { + return logs; + } + for (File child : children) { + String name = child.getName(); + int dot = name.lastIndexOf('.'); + if (!child.isFile() || dot <= 0 || !".log".equals(name.substring(dot))) { + continue; + } + boolean digits = true; + for (int i = 0; i < dot; i++) { + if (!Character.isDigit(name.charAt(i))) { + digits = false; + break; + } + } + if (digits) { + logs.put(name, FileUtils.readFileToByteArray(child)); + } + } + return logs; + } + + private static void assertSameWalLogs(Map expected, + Map actual) { + Assert.assertEquals(expected.keySet(), actual.keySet()); + for (Map.Entry entry : expected.entrySet()) { + Assert.assertArrayEquals(entry.getValue(), actual.get(entry.getKey())); + } + } + + private static void assertNoResumeResidue(String walPath) { + File wal = FileUtils.getFile(walPath); + File[] children = wal.getParentFile().listFiles(); + Assert.assertNotNull(children); + for (File child : children) { + String name = child.getName(); + Assert.assertFalse(name.startsWith(wal.getName() + ".resume-aside-")); + Assert.assertFalse(name.startsWith(wal.getName() + ".resume-staging-")); + } + } + @Test public void testCopySessions() throws RocksDBException { Assert.assertFalse(this.rocks.closed()); @@ -190,7 +519,8 @@ public void testIngestSst() throws RocksDBException { Assert.assertFalse(sstSessions.existsTable(TABLE1)); Assert.assertFalse(sstSessions.existsTable(TABLE2)); - RocksDBSessions rocks = new RocksDBStdSessions(config, "db", "store", sstPath, sstPath); + RocksDBSessions rocks = + new RocksDBStdSessions(config, "db", "store", sstPath, sstPath); // Will ingest sst file of TABLE1 rocks.createTable(TABLE1); Assert.assertEquals(ImmutableList.of("1000"), diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/traversal/OltpTraverserTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/traversal/OltpTraverserTest.java new file mode 100644 index 0000000000..ba4683b469 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/traversal/OltpTraverserTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.unit.traversal; + +import org.apache.hugegraph.HugeGraph; +import org.apache.hugegraph.config.CoreOptions; +import org.apache.hugegraph.traversal.algorithm.OltpTraverser; +import org.apache.hugegraph.type.define.CollectionType; +import org.apache.hugegraph.unit.BaseUnitTest; +import org.apache.tinkerpop.gremlin.structure.Transaction; +import org.junit.After; +import org.junit.Test; +import org.mockito.Mockito; + +public class OltpTraverserTest extends BaseUnitTest { + + @After + public void teardown() { + OltpTraverser.destroy(); + } + + @Test + public void testCloseOpenTransaction() { + HugeGraph graph = graph(); + Transaction tx = graph.tx(); + Mockito.when(tx.isOpen()).thenReturn(true); + + new TestTraverser(graph).close(); + + Mockito.verify(tx).close(); + } + + @Test + public void testCloseClosedTransaction() { + HugeGraph graph = graph(); + Transaction tx = graph.tx(); + Mockito.when(tx.isOpen()).thenReturn(false); + + new TestTraverser(graph).close(); + + Mockito.verify(tx, Mockito.never()).close(); + } + + private static HugeGraph graph() { + HugeGraph graph = Mockito.mock(HugeGraph.class); + Transaction tx = Mockito.mock(Transaction.class); + Mockito.when(graph.tx()).thenReturn(tx); + Mockito.when(graph.option(CoreOptions.OLTP_COLLECTION_TYPE)) + .thenReturn(CollectionType.JCF); + Mockito.when(graph.option(CoreOptions.OLTP_CONCURRENT_THREADS)) + .thenReturn(1); + return graph; + } + + private static class TestTraverser extends OltpTraverser { + + TestTraverser(HugeGraph graph) { + super(graph); + } + } +} diff --git a/hugegraph-store/Dockerfile b/hugegraph-store/Dockerfile index 988b472b09..91ddbaad21 100644 --- a/hugegraph-store/Dockerfile +++ b/hugegraph-store/Dockerfile @@ -33,9 +33,9 @@ RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sh -am $MAVEN_ARGS -e -B -ntp -Dmaven.test.skip=true -Dmaven.javadoc.skip=true \ && rm ./hugegraph-server/*.tar.gz ./hugegraph-pd/*.tar.gz ./hugegraph-store/*.tar.gz -# 2nd stage: runtime env +# Shared runtime environment # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 -FROM eclipse-temurin:11-jre-jammy +FROM eclipse-temurin:11-jre-noble AS runtime LABEL maintainer="HugeGraph Docker Maintainers " @@ -56,21 +56,82 @@ RUN apt-get -q update \ dumb-init \ procps \ curl \ + util-linux \ vim \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -COPY --from=build /pkg/hugegraph-store/apache-hugegraph-store-*/ /hugegraph-store/ - # 2. Init docker script COPY hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh . RUN chmod 755 ./docker-entrypoint.sh EXPOSE 8520 -VOLUME /hugegraph-store HEALTHCHECK --interval=15s --timeout=10s --start-period=90s --retries=3 \ CMD curl -fsS http://localhost:8520/v1/health >/dev/null ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["./docker-entrypoint.sh"] + +# Topling runtime packages. The current native library requires glibc 2.38+. +FROM runtime AS topling-runtime + +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends --no-install-suggests \ + libaio1t64 \ + libbz2-1.0 \ + libjemalloc2 \ + liblz4-1 \ + libsnappy1v5 \ + liburing2 \ + zlib1g \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Build the formal component-local Topling distribution on Linux amd64. +FROM topling-runtime AS build-topling + +COPY --from=build /pkg/ /pkg/ +WORKDIR /pkg +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends --no-install-suggests \ + rsync \ + unzip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && STORE_DIR=$(find /pkg/hugegraph-store -maxdepth 1 -type d \ + -name 'apache-hugegraph-store-*' ! -name '*-topling' \ + -print -quit) \ + && test -n "$STORE_DIR" \ + && VERSION=${STORE_DIR##*/apache-hugegraph-store-} \ + && install-dist/scripts/build-topling-distribution.sh store "$VERSION" + +# Topling image: contains both providers and selects ToplingDB by default. +# Build with: docker build --platform linux/amd64 --target topling ... +FROM topling-runtime AS topling + +ARG SOURCE_REPOSITORY=https://github.com/apache/hugegraph +ARG SOURCE_REVISION=local +LABEL org.opencontainers.image.source="${SOURCE_REPOSITORY}" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" + +LABEL org.apache.hugegraph.rocksdb-runtime="topling" +COPY --from=build-topling /pkg/hugegraph-store/apache-hugegraph-store-*-topling/ /hugegraph-store/ +ENV HG_STORE_ROCKSDB_PROVIDER="topling" \ + HG_STORE_ENFORCE_PROVIDER_MARKER="true" +RUN mkdir -p /hugegraph-store/storage /hugegraph-store/topling-storage +VOLUME ["/hugegraph-store/storage", "/hugegraph-store/topling-storage"] + +# Keep the standard image as the final/default target for existing builds. +FROM runtime AS standard + +ARG SOURCE_REPOSITORY=https://github.com/apache/hugegraph +ARG SOURCE_REVISION=local +LABEL org.opencontainers.image.source="${SOURCE_REPOSITORY}" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" + +LABEL org.apache.hugegraph.rocksdb-runtime="standard" +COPY --from=build /pkg/hugegraph-store/apache-hugegraph-store-*/ /hugegraph-store/ +ENV HG_STORE_ROCKSDB_PROVIDER="rocksdb" +# Preserve the historical root volume for anonymous-volume upgrades. +VOLUME /hugegraph-store diff --git a/hugegraph-store/README.md b/hugegraph-store/README.md index f12df4427e..fbb5d8b651 100644 --- a/hugegraph-store/README.md +++ b/hugegraph-store/README.md @@ -386,6 +386,20 @@ For Docker and Kubernetes deployment details, see [Deployment Guide](docs/deploy --- +## Stopping a Store node + +For an unpacked distribution, run `bin/stop-hugegraph-store.sh`. It waits up to +30 seconds for process exit. A timeout returns a nonzero status and retains +`bin/pid` for diagnosis; it does not force-kill the process or remove its data. +Check the Store log and thread dump before taking further action. + +Spring owns shutdown: new RPCs are refused, active RPCs are cancelled, and context +close waits for their callbacks and scan/TTL workers to release resources. The +Store engine then stops and joins partition Raft services before releasing +databases. A stuck callback keeps shutdown pending rather than allowing its +database to close underneath it. Do not add a separate JVM hook that closes +those databases concurrently. + ## Documentation Comprehensive documentation for HugeGraph Store: diff --git a/hugegraph-store/hg-store-core/pom.xml b/hugegraph-store/hg-store-core/pom.xml index 0ecf723280..4eed6d8e47 100644 --- a/hugegraph-store/hg-store-core/pom.xml +++ b/hugegraph-store/hg-store-core/pom.xml @@ -55,7 +55,7 @@ com.alipay.sofa jraft-core - 1.3.13 + ${jraft.version} org.rocksdb diff --git a/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh b/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh index 8ea9022a33..73a2a3b03a 100755 --- a/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh +++ b/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh @@ -53,11 +53,33 @@ require_env "HG_STORE_RAFT_ADDRESS" # ── Defaults ────────────────────────────────────────────────────────── : "${HG_STORE_GRPC_PORT:=8500}" : "${HG_STORE_REST_PORT:=8520}" -: "${HG_STORE_DATA_PATH:=/hugegraph-store/storage}" +: "${HG_STORE_ROCKSDB_PROVIDER:=rocksdb}" + +case "${HG_STORE_ROCKSDB_PROVIDER}" in + rocksdb | topling) ;; + *) + log "ERROR: HG_STORE_ROCKSDB_PROVIDER must be rocksdb or topling" + exit 2 + ;; +esac +if [[ -z "${HG_STORE_DATA_PATH:-}" ]]; then + if [[ "${HG_STORE_ROCKSDB_PROVIDER}" == "topling" ]]; then + HG_STORE_DATA_PATH="$(pwd)/topling-storage" + else + HG_STORE_DATA_PATH="$(pwd)/storage" + fi +fi +export TOPLINGDB_ROCKSDB_PROVIDER="${HG_STORE_ROCKSDB_PROVIDER}" +./bin/verify-rocksdb-provider.sh \ + store \ + "${HG_STORE_ROCKSDB_PROVIDER}" \ + "${HG_STORE_DATA_PATH}" \ + "${HG_STORE_ENFORCE_PROVIDER_MARKER:-false}" # ── Build SPRING_APPLICATION_JSON ───────────────────────────────────── SPRING_APPLICATION_JSON="$(cat < + + + + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh + + bin + common-topling.sh + 0755 + + + + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/bin/prepare-topling.sh + + bin + prepare-topling.sh + 0755 + + + + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh + + bin + preload-topling.sh + 0755 + + + + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/bin/verify-rocksdb-provider.sh + + bin + verify-rocksdb-provider.sh + 0755 + + + diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index 88165e47b6..c17138ea4c 100755 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -64,6 +64,13 @@ else echo "Unsupported architecture: $arch" fi +# preload rocksdb/toplingdb +if [ ! -r "$BIN/preload-topling.sh" ]; then + echo "Required RocksDB runtime selector not found: $BIN/preload-topling.sh" >&2 + exit 1 +fi +source "$BIN/preload-topling.sh" || exit 1 + ##pd/store max user processes, ulimit -u # Reduce the maximum number of processes that can be opened by a normal dev/user export PROC_LIMITN=1024 @@ -136,7 +143,7 @@ fi # check jdk version JAVA_VERSION=$($JAVA -version 2>&1 | awk 'NR==1{gsub(/"/,""); print $3}' | awk -F'_' '{print $1}') -if [[ $? -ne 0 || $JAVA_VERSION < $EXPECT_JDK_VERSION ]]; then +if [[ $? -ne 0 || $JAVA_VERSION -lt $EXPECT_JDK_VERSION ]]; then echo "Please make sure that the JDK is installed and the version >= $EXPECT_JDK_VERSION" >> ${OUTPUT} exit 1 fi @@ -225,18 +232,28 @@ if [ "$(ps -ef | grep -v grep | grep java | grep -cE "${CONF}")" -ne 0 ]; then fi echo "Starting HG-StoreServer..." +BOOT_JARS=("${LIB}"/hg-store-node-*.jar) +if [ "${#BOOT_JARS[@]}" -ne 1 ] || [ ! -f "${BOOT_JARS[0]}" ]; then + echo "Expected exactly one HugeGraph Store executable JAR in ${LIB}" >> "${OUTPUT}" + exit 1 +fi +BOOT_JAR="${BOOT_JARS[0]}" +JAVA_MAIN=(-jar "$BOOT_JAR") +if [ -n "${TOPLING_RUNTIME_CLASSPATH:-}" ]; then + JAVA_MAIN=(-cp "${TOPLING_RUNTIME_CLASSPATH}:${BOOT_JAR}" \ + org.springframework.boot.loader.JarLauncher) +fi # Turn on security check if [[ $DAEMON == "true" ]]; then echo "Starting HugeGraphStoreServer in daemon mode..." if [[ "${STDOUT_MODE:-false}" == "true" ]]; then - exec ${JAVA} -Dname="HugeGraphStore" ${JVM_OPTIONS} ${JAVA_OPTIONS} -jar \ - -Dspring.config.location=${CONF}/application.yml \ - ${LIB}/hg-store-node-*.jar & + exec ${JAVA} -Dname="HugeGraphStore" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + -Dspring.config.location=${CONF}/application.yml "${JAVA_MAIN[@]}" & else - exec ${JAVA} -Dname="HugeGraphStore" ${JVM_OPTIONS} ${JAVA_OPTIONS} -jar \ - -Dspring.config.location=${CONF}/application.yml \ - ${LIB}/hg-store-node-*.jar >> ${OUTPUT} 2>&1 & + exec ${JAVA} -Dname="HugeGraphStore" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + -Dspring.config.location=${CONF}/application.yml "${JAVA_MAIN[@]}" \ + >> ${OUTPUT} 2>&1 & fi PID="$!" # Write pid to file @@ -248,12 +265,11 @@ else echo "$$" > "$PID_FILE" echo "[+pid] $$" if [[ "${STDOUT_MODE:-false}" == "true" ]]; then - exec ${JAVA} -Dname="HugeGraphStore" ${JVM_OPTIONS} ${JAVA_OPTIONS} -jar \ - -Dspring.config.location=${CONF}/application.yml \ - ${LIB}/hg-store-node-*.jar + exec ${JAVA} -Dname="HugeGraphStore" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + -Dspring.config.location=${CONF}/application.yml "${JAVA_MAIN[@]}" else - exec ${JAVA} -Dname="HugeGraphStore" ${JVM_OPTIONS} ${JAVA_OPTIONS} -jar \ - -Dspring.config.location=${CONF}/application.yml \ - ${LIB}/hg-store-node-*.jar >> ${OUTPUT} 2>&1 + exec ${JAVA} -Dname="HugeGraphStore" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + -Dspring.config.location=${CONF}/application.yml "${JAVA_MAIN[@]}" \ + >> ${OUTPUT} 2>&1 fi fi diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/stop-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/stop-hugegraph-store.sh index 8f898df7da..e9f503b02a 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/stop-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/stop-hugegraph-store.sh @@ -41,8 +41,8 @@ if [ ! -f ${PID_FILE} ]; then fi PID=`cat $PID_FILE` -kill_process_and_wait "HugeGraphStoreServer" "$PID" "$SERVER_SHUTDOWN_TIMEOUT_S" - -if [ $? -eq 0 ]; then +if kill_process_and_wait "HugeGraphStoreServer" "$PID" "$SERVER_SHUTDOWN_TIMEOUT_S"; then rm "$PID_FILE" +else + exit 1 fi diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh index e45249a04b..579cc12c49 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh @@ -434,3 +434,21 @@ function kill_process_and_wait() { kill_process "$process_name" "$pid" wait_for_shutdown "$process_name" "$pid" "$timeout_s" } + +# Find HugeGraph server directory in parent path using prefix glob. +# Usage: find_hugegraph_server_dir "/path/to/parent" +# Returns: first matching directory path or empty string +function find_hugegraph_server_dir() { + local parent_dir="$1" + if [ -z "$parent_dir" ]; then + parent_dir="$(cd "${TOP:-$(pwd)}"/.. && pwd)" + fi + local found="" + for d in "$parent_dir"/apache-hugegraph-server*; do + if [ -d "$d" ]; then + found="$d" + break + fi + done + echo "$found" +} diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml index 0315c4b4fe..2d667afdb1 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml +++ b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml @@ -32,3 +32,6 @@ rocksdb: write_buffer_size: 32000000 # For each rocksdb, the number of memtables reaches this value for writing to disk. min_write_buffer_number_to_merge: 16 + # To enable ToplingDB: set provider to 'topling' (requires the shared ToplingDB runtime) + # ToplingDB config is loaded from conf/rocksdb_store.yaml + # provider: topling diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/conf/rocksdb_store.yaml b/hugegraph-store/hg-store-dist/src/assembly/static/conf/rocksdb_store.yaml new file mode 100644 index 0000000000..e60d866953 --- /dev/null +++ b/hugegraph-store/hg-store-dist/src/assembly/static/conf/rocksdb_store.yaml @@ -0,0 +1,164 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# common parameters +http: + # normally parent path of db path + document_root: ./library/rocksdb_resource + listening_ports: '127.0.0.1:2013' + auto_start_http: false +setenv: + StrSimpleEnvNameNotOverwrite: StringValue + IntSimpleEnvNameNotOverwrite: 16384 + OverwriteThisEnv: + #comment: overwrite is default to false + overwrite: true + value: force overwrite this env by overwrite true +Cache: + lru_cache: + class: LRUCache + params: + capacity: 8G + num_shard_bits: -1 + strict_capacity_limit: false + high_pri_pool_ratio: 0.5 + use_adaptive_mutex: false + metadata_charge_policy: kFullChargeCacheMetadata +Statistics: + stat: + class: default + params: + discard_tickers: + - rocksdb.block.cache + - rocksdb.block.cachecompressed + - rocksdb.block + - rocksdb.memtable.payload.bytes.at.flush + - rocksdb.memtable.garbage.bytes.at.flush + - rocksdb.txn + - rocksdb.blobdb + - rocksdb.row.cache + - rocksdb.number.block + - rocksdb.bloom.filter + - rocksdb.persistent + - rocksdb.sim.block.cache + discard_histograms: + # comment: .... + - rocksdb.blobdb + - rocksdb.bytes.compressed + - rocksdb.bytes.decompressed + - rocksdb.num.index.and.filter.blocks.read.per.level + - rocksdb.num.data.blocks.read.per.level + - rocksdb.compression.times.nanos + - rocksdb.decompression.times.nanos + - rocksdb.read.block.get.micros + - rocksdb.write.raw.block.micros + # comment end of array + #stats_level: kAll + stats_level: kDisableAll +MemTableRepFactory: + cspp: + class: cspp + params: + mem_cap: 16G + use_vm: false + token_use_idle: true + chunk_size: 16K + convert_to_sst: kFileMmap + sync_sst_file: false + skiplist: + class: SkipList + params: + lookahead: 0 +TableFactory: + cspp_memtab_sst: + class: CSPPMemTabTable + params: # empty params + bb: + class: BlockBasedTable + params: + checksum: kCRC32c + block_size: 4K + block_restart_interval: 16 + index_block_restart_interval: 1 + metadata_block_size: 4K + enable_index_compression: true + block_cache: "${lru_cache}" + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst + block_cache_compressed: + persistent_cache: + filter_policy: + dispatch: + class: DispatcherTable + params: + default: bb + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst + level_writers: [ bb, bb, bb, bb, bb, bb ] +CFOptions: + default: + max_write_buffer_number: 6 + memtable_factory: "${cspp}" + write_buffer_size: 128M + # set target_file_size_base as small as 512K is to make many SST files, + # thus key prefix cache can present efficiency + target_file_size_base: 64M + target_file_size_multiplier: 1 + table_factory: dispatch + max_bytes_for_level_base: 512M + max_bytes_for_level_multiplier: 10 + level_compaction_dynamic_level_bytes: false + level0_slowdown_writes_trigger: 20 + level0_stop_writes_trigger: 36 + level0_file_num_compaction_trigger: 2 + merge_operator: uint64add # support merge + level_compaction_dynamic_file_size: true + optimize_filters_for_hits: true + allow_merge_memtables: true + min_write_buffer_number_to_merge: 2 + compression_per_level: + - kNoCompression + - kNoCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression +DBOptions: + log: + create_if_missing: true + create_missing_column_families: true + default: + create_if_missing: true + create_missing_column_families: false # this is important, must be false to hugegraph + max_background_compactions: -1 + max_subcompactions: 4 + max_level1_subcompactions: 0 + inplace_update_support: false + WAL_size_limit_MB: 0 + statistics: "${stat}" + max_manifest_file_size: 100M + max_background_jobs: 8 + # Java org.rocksdb.WriteBatch has no Topling mmap WAL. With + # memtable_as_log_index enabled, every batch write fails: + # "WriteBatch has no mmap wal". Keep the option false for this client. + # convert_to_sst is still Topling-specific. Do not open data written by + # this profile with the standard RocksDB provider; restore a snapshot + # taken before migration instead. + compaction_readahead_size: 0 + memtable_as_log_index: false diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/StoreNodeApplication.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/StoreNodeApplication.java index c793ed96f2..4027659c7f 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/StoreNodeApplication.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/StoreNodeApplication.java @@ -17,7 +17,6 @@ package org.apache.hugegraph.store.node; -import org.apache.hugegraph.store.node.listener.ContextClosedListener; import org.apache.hugegraph.store.node.listener.PdConfigureListener; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -31,9 +30,6 @@ @SpringBootApplication public class StoreNodeApplication { - //TODO Is this OK? - private final AppShutdownHook shutdownHook = new AppShutdownHook(Thread.currentThread()); - public static void main(String[] args) { start(); } @@ -55,9 +51,7 @@ public static void start() { } SpringApplication application = new SpringApplication(StoreNodeApplication.class); PdConfigureListener listener = new PdConfigureListener(); - ContextClosedListener closedListener = new ContextClosedListener(); application.addListeners(listener); - application.addListeners(closedListener); ConfigurableApplicationContext context = application.run(); listener.setContext(context); System.out.println("StoreNodeApplication started."); diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcShutdownBarrier.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcShutdownBarrier.java new file mode 100644 index 0000000000..a9a6a9f95a --- /dev/null +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcShutdownBarrier.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.grpc; + +import java.util.HashSet; +import java.util.Set; + +import org.lognet.springboot.grpc.GRpcGlobalInterceptor; +import org.springframework.stereotype.Component; + +import io.grpc.ForwardingServerCallListener; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; +import lombok.extern.slf4j.Slf4j; + +/** Waits for terminal application callbacks, not just transport termination. */ +@Slf4j +@Component +@GRpcGlobalInterceptor +public class GrpcShutdownBarrier implements ServerInterceptor { + + private final Set calls = new HashSet<>(); + private boolean closing; + + public synchronized void stopAcceptingCalls() { + this.closing = true; + } + + @Override + public ServerCall.Listener interceptCall( + ServerCall call, Metadata headers, + ServerCallHandler next) { + Object token = new Object(); + synchronized (this) { + if (this.closing) { + call.close(Status.UNAVAILABLE.withDescription("Store is stopping"), new Metadata()); + return new ServerCall.Listener() { }; + } + this.calls.add(token); + } + try { + return new ForwardingServerCallListener.SimpleForwardingServerCallListener( + next.startCall(call, headers)) { + @Override + public void onCancel() { + try { + super.onCancel(); + } finally { + finished(token); + } + } + + @Override + public void onComplete() { + try { + super.onComplete(); + } finally { + finished(token); + } + } + }; + } catch (RuntimeException | Error e) { + finished(token); + throw e; + } + } + + private synchronized void finished(Object token) { + this.calls.remove(token); + this.notifyAll(); + } + + public synchronized void awaitCallbacks() { + boolean interrupted = false; + try { + while (!this.calls.isEmpty()) { + try { + this.wait(5000); + if (!this.calls.isEmpty()) { + log.warn("Waiting for {} RPC callbacks before closing databases", this.calls.size()); + } + } catch (InterruptedException e) { + interrupted = true; + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } +} diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStreamImpl.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStreamImpl.java index 7d01fa3db4..f920cccb45 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStreamImpl.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStreamImpl.java @@ -17,7 +17,9 @@ package org.apache.hugegraph.store.node.grpc; +import java.util.Set; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadPoolExecutor; import org.apache.hugegraph.store.grpc.state.ScanState; @@ -31,6 +33,8 @@ import org.lognet.springboot.grpc.GRpcService; import org.springframework.beans.factory.annotation.Autowired; +import io.grpc.Context; +import io.grpc.Status; import io.grpc.stub.StreamObserver; import lombok.extern.slf4j.Slf4j; @@ -47,6 +51,53 @@ public class HgStoreStreamImpl extends HgStoreStreamGrpc.HgStoreStreamImplBase { private AppConfig appConfig; private HgStoreWrapperEx wrapper; private ThreadPoolExecutor executor; + private boolean closing; + private final Set scans = ConcurrentHashMap.newKeySet(); + + /** Close admission before cancelling the gRPC server. */ + public synchronized void stopAcceptingScans() { + this.closing = true; + } + + /** Cancel streams, then drain queued cleanup tasks. Never discard the queue. */ + public void shutdownScans() { + stopAcceptingScans(); + for (Runnable cancel : this.scans.toArray(new Runnable[0])) { + cancel.run(); + } + ThreadPoolExecutor current = getRealExecutor(); + if (current != null) { + current.shutdown(); + } + } + + private synchronized void checkAcceptingScans() { + if (this.closing) { + throw Status.UNAVAILABLE.withDescription("Store scans are stopping") + .asRuntimeException(); + } + } + + private StreamObserver register(StreamObserver observer) { + Context context = Context.current(); + Runnable cancel = new Runnable() { + @Override + public synchronized void run() { + if (scans.contains(this)) { + try { + observer.onError(Status.CANCELLED.asRuntimeException()); + } catch (RuntimeException e) { + log.warn("Failed to cancel scan", e); + } finally { + scans.remove(this); + } + } + } + }; + this.scans.add(cancel); + context.addListener(ignored -> cancel.run(), Runnable::run); + return observer; + } private HgStoreWrapperEx getWrapper() { if (this.wrapper == null) { @@ -60,20 +111,16 @@ private HgStoreWrapperEx getWrapper() { return this.wrapper; } - public ThreadPoolExecutor getRealExecutor() { - return executor; + public synchronized ThreadPoolExecutor getRealExecutor() { + return this.executor; } - public ThreadPoolExecutor getExecutor() { + public synchronized ThreadPoolExecutor getExecutor() { + checkAcceptingScans(); if (this.executor == null) { - synchronized (this) { - if (this.executor == null) { - AppConfig.ThreadPoolScan scan = this.appConfig.getThreadPoolScan(); - this.executor = - HgExecutorUtil.createExecutor("hg-scan", scan.getCore(), scan.getMax(), - scan.getQueue()); - } - } + AppConfig.ThreadPoolScan scan = this.appConfig.getThreadPoolScan(); + this.executor = HgExecutorUtil.createExecutor("hg-scan", scan.getCore(), + scan.getMax(), scan.getQueue()); } return this.executor; } @@ -95,27 +142,32 @@ public ScanState getState() { } @Override - public StreamObserver scan(StreamObserver response) { - return ScanStreamResponse.of(response, getWrapper(), getExecutor(), appConfig); + public synchronized StreamObserver scan(StreamObserver response) { + checkAcceptingScans(); + return register(ScanStreamResponse.of(response, getWrapper(), getExecutor(), appConfig)); } @Override public void scanOneShot(ScanStreamReq request, StreamObserver response) { + checkAcceptingScans(); ScanOneShotResponse.scanOneShot(request, response, getWrapper()); } @Override - public StreamObserver scanBatch(StreamObserver response) { - return ScanBatchResponse3.of(response, getWrapper(), getExecutor()); + public synchronized StreamObserver scanBatch(StreamObserver response) { + checkAcceptingScans(); + return register(ScanBatchResponse3.of(response, getWrapper(), getExecutor())); } @Override - public StreamObserver scanBatch2(StreamObserver response) { - return ScanBatchResponseFactory.of(response, getWrapper(), getExecutor()); + public synchronized StreamObserver scanBatch2(StreamObserver response) { + checkAcceptingScans(); + return register(ScanBatchResponseFactory.of(response, getWrapper(), getExecutor())); } @Override public void scanBatchOneShot(ScanStreamBatchReq request, StreamObserver response) { + checkAcceptingScans(); ScanBatchOneShotResponse.scanOneShot(request, response, getWrapper()); } } diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ParallelScanIterator.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ParallelScanIterator.java index 56ce1f45f3..b164837a82 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ParallelScanIterator.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ParallelScanIterator.java @@ -88,7 +88,12 @@ private ParallelScanIterator(Supplier> iter this.maxInQueue = maxWorkThreads * 2; // Edge sorted requires a larger queue queue = new LinkedBlockingQueue<>(maxInQueue * 2); - createScanner(); + try { + createScanner(); + } catch (RuntimeException e) { + close(); + throw e; + } } public static ParallelScanIterator of( @@ -144,18 +149,21 @@ public List next() { return t; } + void requestStop() { + this.finished = true; + } + @Override public void close() { - finished = true; + requestStop(); + List pending; synchronized (scanners) { - scanners.forEach(scanner -> { - scanner.close(); - }); + pending = new ArrayList<>(scanners); } + // Never hold a registry lock while waiting for a scanner's iterator lock. + pending.forEach(KVScanner::close); synchronized (pauseScanners) { - pauseScanners.forEach(s -> { - s.close(); - }); + pauseScanners.clear(); } queue.clear(); } @@ -179,10 +187,15 @@ private void createScanner() { */ private void wakeUpScanner() { synchronized (pauseScanners) { - if (!pauseScanners.isEmpty()) { + if (!finished && !pauseScanners.isEmpty()) { KVScanner scanner = pauseScanners.poll(); if (scanner != null) { - executor.execute(() -> scanner.scanKV()); + try { + executor.execute(() -> scanner.scanKV()); + } catch (java.util.concurrent.RejectedExecutionException e) { + scanner.close(); + throw e; + } } } } @@ -195,7 +208,11 @@ private void wakeUpScanner() { */ private void suspendScanner(KVScanner scanner) { synchronized (pauseScanners) { - pauseScanners.add(scanner); + if (!finished) { + pauseScanners.add(scanner); + } else { + scanner.close(); + } } } @@ -218,30 +235,43 @@ private void quitScanner(KVScanner scanner) { */ private boolean putData(List data) { try { - this.queue.put(data); + while (!finished && !this.queue.offer(data, 100, TimeUnit.MILLISECONDS)) { + // Cancellation must release a producer whose client stopped consuming. + } } catch (InterruptedException e) { log.error("exception ", e); this.finished = true; return false; } - return this.queue.size() < maxInQueue; + return !finished && this.queue.size() < maxInQueue; } private boolean putData(List data, boolean hasNext) { + boolean locked = false; try { - queueLock.lock(); - this.queue.put(data); + while (!finished && !(locked = queueLock.tryLock(100, TimeUnit.MILLISECONDS))) { + // An ordered producer may be waiting behind a cancelled scanner. + } + if (!locked) { + return false; + } + while (!finished && !this.queue.offer(data, 100, TimeUnit.MILLISECONDS)) { + // Recheck cancellation while the output queue is full. + } } catch (InterruptedException e) { - log.error("exception ", e); + Thread.currentThread().interrupt(); this.finished = true; return false; } finally { - if (!hasNext) { + if (locked && finished) { + while (queueLock.isHeldByCurrentThread()) { + queueLock.unlock(); + } + } else if (locked && !hasNext) { queueLock.unlock(); } } - // Data not ended, thread continues to execute - return hasNext || this.queue.size() < maxInQueue; + return !finished && (hasNext || this.queue.size() < maxInQueue); } private synchronized KVPair getIterator() { @@ -328,12 +358,12 @@ public void scanKV() { iteratorLock.lock(); try { long entriesSize = 0, bodySize = 0; - while (canNext && !closed) { + while (canNext && !closed && !finished) { iterator = this.getIterator(); if (iterator == null) { break; } - while (iterator.hasNext() && entriesSize < batchSize && + while (!closed && !finished && iterator.hasNext() && entriesSize < batchSize && bodySize < maxBodySize && counter < limit && !closed) { KV kv = KV.of(iterator.next()); @@ -366,8 +396,14 @@ public void scanKV() { } catch (Exception e) { log.error("exception {}", e); } finally { + // putData(..., true) intentionally keeps this lock across batches to + // serialize ordered output. The scanner must always release it before + // suspending or quitting, including when the next iterator is empty. + while (queueLock.isHeldByCurrentThread()) { + queueLock.unlock(); + } iteratorLock.unlock(); - if (iterator != null && counter < limit && !closed) { + if (iterator != null && counter < limit && !closed && !finished) { suspendScanner(this); } else { quitScanner(this); @@ -381,6 +417,7 @@ public void close() { try { if (iterator != null) { iterator.close(); + iterator = null; } } finally { iteratorLock.unlock(); diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchOneShotResponse.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchOneShotResponse.java index 1234a5b3de..2caa1ecd56 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchOneShotResponse.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchOneShotResponse.java @@ -30,6 +30,7 @@ import com.google.protobuf.ByteString; +import io.grpc.Context; import io.grpc.Status; import io.grpc.stub.StreamObserver; import lombok.extern.slf4j.Slf4j; @@ -54,6 +55,9 @@ public static void scanOneShot(ScanStreamBatchReq request, String graph = request.getHeader().getGraph(); ScanQueryRequest queryRequest = request.getQueryRequest(); + if (Context.current().isCancelled()) { + return; + } ScanIterator iterator = getIterator(graph, queryRequest, wrapper); KvPageRes.Builder resBuilder = KvPageRes.newBuilder(); @@ -70,7 +74,8 @@ public static void scanOneShot(ScanStreamBatchReq request, int count = 0; try { - while (iterator.hasNext()) { + while (!Context.current().isCancelled() && + !Thread.currentThread().isInterrupted() && iterator.hasNext()) { if (++count > limit) { break; @@ -86,6 +91,9 @@ public static void scanOneShot(ScanStreamBatchReq request, } + if (Context.current().isCancelled()) { + return; + } responseObserver.onNext(resBuilder.build()); responseObserver.onCompleted(); diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse.java index f4485e1527..c9971e0948 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse.java @@ -20,7 +20,9 @@ import static org.apache.hugegraph.store.node.grpc.ScanUtil.getParallelIterator; import java.util.List; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -59,7 +61,7 @@ public class ScanBatchResponse implements StreamObserver { private final Object stateLock = new Object(); private final Lock iteratorLock = new ReentrantLock(); // Currently traversing iterator - private ScanIterator iterator; + private volatile ScanIterator iterator; // Next send sequence number private volatile int seqNo; // Client consumed sequence number @@ -72,6 +74,7 @@ public class ScanBatchResponse implements StreamObserver { // Last read data time private long activeTime; private volatile State state; + private final AtomicBoolean cancelled = new AtomicBoolean(); public ScanBatchResponse(StreamObserver response, HgStoreWrapperEx wrapper, ThreadPoolExecutor executor) { @@ -92,25 +95,30 @@ public ScanBatchResponse(StreamObserver response, HgStoreWrapperEx wra */ @Override public void onNext(ScanStreamBatchReq request) { + if (this.cancelled.get()) { + return; + } switch (request.getQueryCase()) { case QUERY_REQUEST: // query conditions - executor.execute(() -> { + submit(() -> { startQuery(request.getHeader().getGraph(), request.getQueryRequest()); }); break; case RECEIPT_REQUEST: // Message asynchronous response this.clientSeqNo = request.getReceiptRequest().getTimes(); if (seqNo - clientSeqNo < maxInFlightCount) { + boolean send = false; synchronized (stateLock) { if (state == State.IDLE) { state = State.DOING; - executor.execute(() -> { - sendEntries(); - }); - } else if (state == State.DONE) { - sendNoDataEntries(); + send = true; } } + if (send) { + submit(this::sendEntries); + } else if (state == State.DONE) { + sendNoDataEntries(); + } } break; case CANCEL_REQUEST: // close stream @@ -125,7 +133,7 @@ public void onNext(ScanStreamBatchReq request) { @Override public void onError(Throwable t) { log.error("onError ", t); - closeQuery(); + closeQuery(t); } @Override @@ -138,18 +146,43 @@ public void onCompleted() { * * @param request */ + private void submit(Runnable task) { + if (this.cancelled.get()) { + return; + } + try { + this.executor.execute(task); + } catch (RejectedExecutionException e) { + closeQuery(e); + } + } + private void startQuery(String graphName, ScanQueryRequest request) { - this.query = request; - this.limit = request.getLimit(); - this.count = 0; - this.iterator = getParallelIterator(graphName, request, this.wrapper, executor); - synchronized (stateLock) { - if (state == State.IDLE) { + this.iteratorLock.lock(); + try { + if (this.cancelled.get() || this.iterator != null) { + return; + } + this.query = request; + this.limit = request.getLimit(); + this.count = 0; + this.iterator = getParallelIterator(graphName, request, this.wrapper, executor); + if (this.cancelled.get()) { + closeIter(); + return; + } + synchronized (stateLock) { + if (state != State.IDLE) { + return; + } state = State.DOING; - executor.execute(() -> { - sendEntries(); - }); } + submit(this::sendEntries); + } catch (RuntimeException e) { + closeQuery(e); + log.warn("Failed to start scan", e); + } finally { + this.iteratorLock.unlock(); } } @@ -157,10 +190,25 @@ private void startQuery(String graphName, ScanQueryRequest request) { * Generate iterator */ private void closeQuery() { + closeQuery(null); + } + + private void closeQuery(Throwable error) { + if (!this.cancelled.compareAndSet(false, true)) { + return; + } setStateDone(); + ScanIterator current = this.iterator; + if (current instanceof ParallelScanIterator) { + ((ParallelScanIterator) current).requestStop(); + } try { closeIter(); - this.sender.onCompleted(); + if (error == null) { + this.sender.onCompleted(); + } else { + this.sender.onError(error); + } } catch (Exception e) { log.error("exception ", e); } @@ -169,13 +217,16 @@ private void closeQuery() { } private void closeIter() { + this.iteratorLock.lock(); try { if (this.iterator != null) { this.iterator.close(); this.iterator = null; } } catch (Exception e) { - + log.warn("Failed to close batch scan iterator", e); + } finally { + this.iteratorLock.unlock(); } } @@ -209,7 +260,7 @@ private void sendEntries() { this.sender.onNext(dataBuilder.build()); this.activeTime = System.currentTimeMillis(); } - if (!iterator.hasNext() || this.count >= limit || state == State.DONE) { + if (state == State.DONE || this.count >= limit || !iterator.hasNext()) { closeIter(); this.sender.onNext(KvStream.newBuilder().setOver(true).build()); setStateDone(); @@ -219,14 +270,7 @@ private void sendEntries() { } catch (Throwable e) { if (this.state != State.DONE) { log.error(" send data exception: ", e); - setStateIdle(); - if (this.sender != null) { - try { - this.sender.onError(e); - } catch (Exception ex) { - log.warn("Error when call sender.onError {}", e.getMessage()); - } - } + closeQuery(e); } } finally { iteratorLock.unlock(); diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse3.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse3.java index d0c52b372e..8710612bde 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse3.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse3.java @@ -160,16 +160,19 @@ private static class OrderManager { OrderState state = OrderState.NEW; OrderWorker worker; OrderDeliverer deliverer; + boolean cancelled; synchronized void deal(OrderWorker worker, OrderDeliverer deliverer) { if (log.isDebugEnabled()) { log.debug("Receiving query request."); } - if (this.state == OrderState.NEW) { + if (!this.cancelled && this.state == OrderState.NEW) { this.worker = worker; this.deliverer = deliverer; this.worker.hereWeGo(); this.state = OrderState.WORKING; + } else { + worker.iterator.close(); } } @@ -177,7 +180,9 @@ synchronized void receipt(int receiptTimes) { if (log.isDebugEnabled()) { log.debug("Receiving receipt request."); } - this.worker.setReceipt(receiptTimes); + if (this.worker != null) { + this.worker.setReceipt(receiptTimes); + } } synchronized void finished() { @@ -188,6 +193,7 @@ synchronized void finished() { } synchronized void breakdown() { + this.cancelled = true; if (this.worker != null) { this.worker.breakdown(); } @@ -297,7 +303,12 @@ void hereWeGo() { return; } - executor.execute(() -> working()); + try { + executor.execute(this::working); + } catch (java.util.concurrent.RejectedExecutionException e) { + this.iterator.close(); + throw e; + } Thread.yield(); } @@ -338,7 +349,8 @@ private void working() { Kv.Builder kvBuilder = Kv.newBuilder(); long packageCount = 0; - while (iterator.hasNext()) { + while (!this.breakdown.get() && !Thread.currentThread().isInterrupted() && + iterator.hasNext()) { if (++this.counter > limit) { this.completeFlag.set(true); break; @@ -353,7 +365,7 @@ private void working() { deliverer.deliver(dataBuilder, curTimes.incrementAndGet(), false); Thread.yield(); - if (!this.checkContinue()) { + if (!this.breakdown.get() && !this.checkContinue()) { long start = System.currentTimeMillis(); iterator.wait( HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS * diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanOneShotResponse.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanOneShotResponse.java index 5b77a4449a..257b715687 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanOneShotResponse.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanOneShotResponse.java @@ -27,6 +27,7 @@ import com.google.protobuf.ByteString; +import io.grpc.Context; import io.grpc.Status; import io.grpc.stub.StreamObserver; import lombok.extern.slf4j.Slf4j; @@ -57,12 +58,16 @@ public static void scanOneShot(ScanStreamReq request, responseObserver.onError(HgGrpc.toErr("limit<=0, please to invoke stream scan.")); return; } + if (Context.current().isCancelled()) { + return; + } ScanIterator iterator = ScanUtil.getIterator(request, wrapper); int count = 0; try { - while (iterator.hasNext()) { + while (!Context.current().isCancelled() && + !Thread.currentThread().isInterrupted() && iterator.hasNext()) { if (++count > limit) { break; @@ -78,6 +83,9 @@ public static void scanOneShot(ScanStreamReq request, } + if (Context.current().isCancelled()) { + return; + } responseObserver.onNext( resBuilder.setVersion(ScanUtil.responseVersion(request)) .build()); diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanStreamResponse.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanStreamResponse.java index e572862823..7e051fd48d 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanStreamResponse.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanStreamResponse.java @@ -61,6 +61,22 @@ public class ScanStreamResponse implements StreamObserver { private final int waitTime; private final HgChannel channel; private ScanIterator iterator; + private final Object cancellationLock = new Object(); + private Thread worker; + private Thread receiver; + + private void cancel() { + this.isStop.set(true); + this.channel.close(); + synchronized (this.cancellationLock) { + if (this.worker != null) { + this.worker.interrupt(); + } + if (this.receiver != null) { + this.receiver.interrupt(); + } + } + } private long limit = 0; private int times = 0; private long pageSize = 0; @@ -91,6 +107,9 @@ public static ScanStreamResponse of(StreamObserver responseObserver, @Override public void onNext(ScanStreamReq request) { + if (this.isStop.get()) { + return; + } try { this.responseVersion = Math.max( this.responseVersion, ScanUtil.responseVersion(request)); @@ -106,7 +125,7 @@ public void onNext(ScanStreamReq request) { @Override public void onError(Throwable t) { - this.isStop.set(true); + cancel(); this.finishServer(); log.warn("onError from client [ graph: {} , table: {}]; Reason: {}]", graph, table, t.getMessage()); @@ -114,7 +133,7 @@ public void onError(Throwable t) { @Override public void onCompleted() { - this.isStop.set(true); + cancel(); this.finishServer(); } @@ -123,7 +142,6 @@ private void initIterator(ScanStreamReq request) { if (this.isStarted.getAndSet(true)) { return; } - this.iterator = getIterator(request, this.wrapper); this.graph = request.getHeader().getGraph(); this.table = request.getTable(); this.limit = request.getLimit(); @@ -143,7 +161,15 @@ private void initIterator(ScanStreamReq request) { Kv.Builder kvBuilder = Kv.newBuilder(); int pageCount = 0; try { - while (iterator.hasNext()) { + synchronized (this.cancellationLock) { + if (this.isStop.get()) { + return; + } + this.worker = Thread.currentThread(); + } + this.iterator = getIterator(request, this.wrapper); + while (!this.isStop.get() && !Thread.currentThread().isInterrupted() && + iterator.hasNext()) { if (limit > 0 && ++this.total > limit) { break; } @@ -172,10 +198,16 @@ private void initIterator(ScanStreamReq request) { responseObserver.onError(ex); } finally { try { - this.iterator.close(); + if (this.iterator != null) { + this.iterator.close(); + } this.channel.close(); } catch (Exception e) { - + log.warn("Failed to close scan iterator", e); + } finally { + synchronized (this.cancellationLock) { + this.worker = null; + } } } @@ -184,12 +216,7 @@ private void initIterator(ScanStreamReq request) { } catch (Exception e) { StatusRuntimeException ex = HgGrpc.toErr(Status.INTERNAL, null, e); responseObserver.onError(ex); - try { - this.iterator.close(); - this.channel.close(); - } catch (Exception exception) { - - } + this.channel.close(); } /*** Scanning loop end ***/ @@ -205,8 +232,7 @@ private Kv toKv(Kv.Builder kvBuilder, RocksDBSession.BackendColumn col, } private void close() { - this.isStop.set(true); - this.channel.close(); + cancel(); if (!this.finishFlag.get()) { responseObserver.onNext(KvPageRes.newBuilder() .addAllData(Collections.EMPTY_LIST) @@ -225,6 +251,12 @@ private void next(ScanStreamReq request) { KvPageRes.Builder resBuilder; try { + synchronized (this.cancellationLock) { + if (this.isStop.get()) { + return; + } + this.receiver = Thread.currentThread(); + } resBuilder = this.channel.receive(); times++; } catch (Exception e) { @@ -232,6 +264,10 @@ private void next(ScanStreamReq request) { log.error(msg, e); responseObserver.onError(HgGrpc.toErr(msg + e.getMessage())); return; + } finally { + synchronized (this.cancellationLock) { + this.receiver = null; + } } boolean isOver = false; if (resBuilder == null || resBuilder.getDataList() == null || diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/ContextClosedListener.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/ContextClosedListener.java index df8d084e22..f5188b23e3 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/ContextClosedListener.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/ContextClosedListener.java @@ -17,106 +17,112 @@ package org.apache.hugegraph.store.node.listener; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadPoolExecutor; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; -import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.node.grpc.GrpcShutdownBarrier; import org.apache.hugegraph.store.node.grpc.HgStoreStreamImpl; import org.apache.hugegraph.store.node.task.TTLCleaner; +import org.lognet.springboot.grpc.context.GRpcServerInitializedEvent; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationListener; import org.springframework.context.event.ContextClosedEvent; +import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; -import com.alipay.sofa.jraft.Status; -import com.alipay.sofa.jraft.entity.PeerId; - +import io.grpc.Server; import lombok.extern.slf4j.Slf4j; @Slf4j @Service public class ContextClosedListener implements ApplicationListener { + private final List grpcServers = new CopyOnWriteArrayList<>(); + @Autowired HgStoreStreamImpl storeStream; @Autowired TTLCleaner cleaner; + @Autowired + GrpcShutdownBarrier grpcBarrier; + + @EventListener + public void onServerInitialized(GRpcServerInitializedEvent event) { + this.grpcServers.add(event.getServer()); + } @Override public void onApplicationEvent(ContextClosedEvent event) { + // Spring invokes HgStoreNodeService.destroy() after this event. Raft and + // its databases must remain available until local workers have stopped. + this.grpcBarrier.stopAcceptingCalls(); + if (storeStream != null) { + storeStream.stopAcceptingScans(); + } + this.grpcServers.forEach(Server::shutdownNow); + if (cleaner != null) { + // The scheduler can create the worker pool while a job is starting. + stopAndWait(cleaner.getScheduler(), "TTL scheduler"); + stopAndWait(cleaner.getExecutor(), "TTL workers"); + } + if (storeStream != null) { + // Cancelled queued scans must run their finally blocks to release iterators. + storeStream.shutdownScans(); + awaitWorkers(storeStream.getRealExecutor(), "scan workers"); + } + boolean interrupted = false; try { - try { - transferLeaders(); - - synchronized (ContextClosedListener.class) { - ContextClosedListener.class.wait(60 * 1000); - } - - transferLeaders(); - - synchronized (ContextClosedListener.class) { - ContextClosedListener.class.wait(30 * 1000); - } - } catch (Exception e) { - log.info("shutdown hook: ", e); - } - - log.info("closing scan threads...."); - if (storeStream != null) { - ThreadPoolExecutor executor = storeStream.getRealExecutor(); - if (executor != null) { + for (Server server : this.grpcServers) { + while (!server.isTerminated()) { try { - executor.shutdownNow(); - } catch (Exception e) { + if (!server.awaitTermination(5, TimeUnit.SECONDS)) { + log.warn("Still waiting for gRPC callbacks before closing databases"); + } + } catch (InterruptedException e) { + interrupted = true; } } } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + this.grpcBarrier.awaitCallbacks(); + log.info("closed gRPC callbacks, scan and TTL workers"); + } - if (cleaner != null) { - ThreadPoolExecutor cleanerExecutor = cleaner.getExecutor(); - if (cleanerExecutor != null) { - try { - cleanerExecutor.shutdownNow(); - } catch (Exception e) { - - } - } - ScheduledExecutorService scheduler = cleaner.getScheduler(); - if (scheduler != null) { - try { - scheduler.shutdownNow(); - } catch (Exception e) { + private static void stopAndWait(ExecutorService executor, String name) { + if (executor == null) { + return; + } + executor.shutdownNow(); + awaitWorkers(executor, name); + } + private static void awaitWorkers(ExecutorService executor, String name) { + if (executor == null) { + return; + } + boolean interrupted = false; + try { + while (!executor.isTerminated()) { + try { + if (!executor.awaitTermination(5, TimeUnit.SECONDS)) { + log.warn("Still waiting for {} to stop before closing databases", name); } + } catch (InterruptedException e) { + // An interrupted shutdown thread must not close databases underneath + // a worker that still owns a native iterator. + interrupted = true; } } - } catch (Exception e) { - log.error("ContextClosedListener: ", e); } finally { - log.info("closed scan threads"); - } - } - - private void transferLeaders() { - try { - HgStoreEngine.getInstance().getLeaderPartition() - .forEach(leader -> { - try { - Status status = - leader.getRaftNode().transferLeadershipTo(PeerId.ANY_PEER); - log.info("partition {} transfer leader status: {}", - leader.getGroupId(), status); - } catch (Exception e) { - log.info("partition {} transfer leader error: ", - leader.getGroupId(), e); - } - }); - HgStoreEngine.getInstance().getPartitionEngines().forEach( - ((integer, partitionEngine) -> partitionEngine.getRaftNode() - .shutdown()) - ); - } catch (Exception e) { - log.error("transfer leader failed: " + e.getMessage()); + if (interrupted) { + Thread.currentThread().interrupt(); + } } } } diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java index 94bdc4c6bc..ceecc43399 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java @@ -54,12 +54,12 @@ public final class RocksDBMetricsConst { TickerType.BLOCK_CACHE_INDEX_HIT, // Index cache hits. TickerType.BLOCK_CACHE_INDEX_ADD, // Index blocks added to cache. TickerType.BLOCK_CACHE_INDEX_BYTES_INSERT, // Bytes inserted into index cache. - TickerType.BLOCK_CACHE_INDEX_BYTES_EVICT, // Bytes evicted from index cache. + // BLOCK_CACHE_INDEX_BYTES_EVICT removed in RocksDB 8.x TickerType.BLOCK_CACHE_FILTER_MISS, // Filter cache misses. TickerType.BLOCK_CACHE_FILTER_HIT, // Filter cache hits. TickerType.BLOCK_CACHE_FILTER_ADD, // Filter blocks added to cache. TickerType.BLOCK_CACHE_FILTER_BYTES_INSERT, // Bytes inserted in filter cache. - TickerType.BLOCK_CACHE_FILTER_BYTES_EVICT, // Bytes evicted from filter cache. + // BLOCK_CACHE_FILTER_BYTES_EVICT removed in RocksDB 8.x TickerType.BLOCK_CACHE_DATA_MISS, // Data cache misses. TickerType.BLOCK_CACHE_DATA_HIT, // Data cache hits. TickerType.BLOCK_CACHE_DATA_ADD, // Data blocks added to cache. @@ -95,31 +95,31 @@ public final class RocksDBMetricsConst { TickerType.NUMBER_DB_NEXT_FOUND, // Number of successful next operations. TickerType.NUMBER_DB_PREV_FOUND, // Number of successful previous operations. TickerType.ITER_BYTES_READ, // Bytes read by iterators. - TickerType.NO_FILE_CLOSES, // Number of file close operations. + // NO_FILE_CLOSES removed in RocksDB 8.x TickerType.NO_FILE_OPENS, // Number of file open operations. TickerType.NO_FILE_ERRORS, // Number of file errors. TickerType.STALL_MICROS, // Time spent in a stall micro. TickerType.DB_MUTEX_WAIT_MICROS, // Time spent waiting on a mutex. - TickerType.RATE_LIMIT_DELAY_MILLIS, // Rate limiting delay in milliseconds. - TickerType.NO_ITERATORS, // Number of iterators created. + // RATE_LIMIT_DELAY_MILLIS removed in RocksDB 8.x + // NO_ITERATORS removed in RocksDB 8.x TickerType.NUMBER_MULTIGET_BYTES_READ, // Bytes read by multi-get operations. TickerType.NUMBER_MULTIGET_KEYS_READ, // Keys read in multi-get operations. TickerType.NUMBER_MULTIGET_CALLS, // Number of multi-get operations. - TickerType.NUMBER_FILTERED_DELETES, // Number of deletes filtered. + // NUMBER_FILTERED_DELETES removed in RocksDB 8.x TickerType.NUMBER_MERGE_FAILURES, // Number of merge failures. TickerType.BLOOM_FILTER_PREFIX_CHECKED, // Number of prefix bloom filter checks. TickerType.BLOOM_FILTER_PREFIX_USEFUL, // Number of useful prefix bloom filter checks. TickerType.NUMBER_OF_RESEEKS_IN_ITERATION, // Number of reseeks in iteration. TickerType.GET_UPDATES_SINCE_CALLS, // Number of get updates since calls. - TickerType.BLOCK_CACHE_COMPRESSED_MISS, // Misses in compressed block cache. - TickerType.BLOCK_CACHE_COMPRESSED_HIT, // Hits in compressed block cache. - TickerType.BLOCK_CACHE_COMPRESSED_ADD, // Compressed blocks added to cache. - TickerType.BLOCK_CACHE_COMPRESSED_ADD_FAILURES, // Failures adding compressed blocks. + // BLOCK_CACHE_COMPRESSED_MISS removed in RocksDB 8.x + // BLOCK_CACHE_COMPRESSED_HIT removed in RocksDB 8.x + // BLOCK_CACHE_COMPRESSED_ADD removed in RocksDB 8.x + // BLOCK_CACHE_COMPRESSED_ADD_FAILURES removed in RocksDB 8.x TickerType.WAL_FILE_SYNCED, // Number of synced WAL files. TickerType.WAL_FILE_BYTES, // Bytes written to WAL files. TickerType.WRITE_DONE_BY_SELF, // Writes completed by self. TickerType.WRITE_DONE_BY_OTHER, // Writes completed by others. - TickerType.WRITE_TIMEDOUT, // Number of write timeouts. + // WRITE_TIMEDOUT removed in RocksDB 8.x TickerType.WRITE_WITH_WAL, // Writes involving WAL. TickerType.COMPACT_READ_BYTES, // Bytes read during compaction. TickerType.COMPACT_WRITE_BYTES, // Bytes written during compaction. @@ -167,12 +167,10 @@ public final class RocksDBMetricsConst { // Time spent reading blocks during compaction. HistogramType.READ_BLOCK_GET_MICROS, // Time spent reading blocks during get. HistogramType.WRITE_RAW_BLOCK_MICROS, // Time spent writing raw blocks. - HistogramType.STALL_L0_SLOWDOWN_COUNT, // Count of stalls due to L0 slowdown. - HistogramType.STALL_MEMTABLE_COMPACTION_COUNT, - // Count of stalls due to memtable compaction. - HistogramType.STALL_L0_NUM_FILES_COUNT, // Count of stalls due to number of files at L0. - HistogramType.HARD_RATE_LIMIT_DELAY_COUNT, // Count of delays due to hard rate limits. - HistogramType.SOFT_RATE_LIMIT_DELAY_COUNT, // Count of delays due to soft rate limits. + // STALL_L0_SLOWDOWN_COUNT removed in RocksDB 8.x + // Note: The following constants were removed in RocksDB 8.10.2: + // STALL_MEMTABLE_COMPACTION_COUNT, STALL_L0_NUM_FILES_COUNT, + // HARD_RATE_LIMIT_DELAY_COUNT, SOFT_RATE_LIMIT_DELAY_COUNT HistogramType.NUM_FILES_IN_SINGLE_COMPACTION, // Number of files in a single compaction. HistogramType.DB_SEEK, // Latency of database seek operations. HistogramType.WRITE_STALL, // Time spent in write stalls. diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/task/TTLCleaner.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/task/TTLCleaner.java index 6836fe20b7..ebcdcc45b2 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/task/TTLCleaner.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/task/TTLCleaner.java @@ -299,7 +299,7 @@ private Runnable getTask( LinkedList all = new LinkedList<>(); AtomicBoolean state = new AtomicBoolean(true); AtomicLong partitionCounter = pc.get(id); - while (filter.hasNext() && state.get()) { + while (!Thread.currentThread().isInterrupted() && state.get() && filter.hasNext()) { RocksDBSession.BackendColumn current = filter.next(); byte[] realKey = Arrays.copyOfRange(current.name, 0, current.name.length - Short.BYTES); @@ -313,7 +313,7 @@ private Runnable getTask( all = new LinkedList<>(); } } - if (all.size() > 0 && state.get()) { + if (!Thread.currentThread().isInterrupted() && all.size() > 0 && state.get()) { submitter.submitClean(id, graph, table, all, state, tableCounter, partitionCounter); } diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/config/ToplingProfileConfigTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/config/ToplingProfileConfigTest.java new file mode 100644 index 0000000000..082f2379cd --- /dev/null +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/config/ToplingProfileConfigTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.config; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.Test; + +public class ToplingProfileConfigTest { + + @Test + public void testJavaWriteBatchProfileDisablesMmapWalIndex() throws IOException { + assertMemtableAsLogIndexFalse(repoFile( + "hugegraph-store/hg-store-dist/src/assembly/static/conf/rocksdb_store.yaml")); + assertMemtableAsLogIndexFalse(repoFile( + "hugegraph-pd/hg-pd-dist/src/assembly/static/conf/rocksdb_pd.yaml")); + assertMemtableAsLogIndexFalse(repoFile( + "hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb.yaml")); + } + + private static void assertMemtableAsLogIndexFalse(Path file) throws IOException { + List lines = Files.readAllLines(file); + int matches = 0; + for (String line : lines) { + String trimmed = line.trim(); + if (trimmed.startsWith("#") || !trimmed.startsWith("memtable_as_log_index:")) { + continue; + } + matches++; + assertEquals(file.toString(), "false", + trimmed.substring("memtable_as_log_index:".length()).trim()); + } + assertTrue(file.toString(), matches == 1); + } + + private static Path repoFile(String relative) { + Path module = Path.of("").toAbsolutePath(); + Path repo = module.getParent().getParent(); + Path file = repo.resolve(relative); + assertTrue(file.toString(), Files.isRegularFile(file)); + return file; + } +} diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/service/ContextClosedListenerTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/service/ContextClosedListenerTest.java new file mode 100644 index 0000000000..452e2c7b51 --- /dev/null +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/service/ContextClosedListenerTest.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.service; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.FutureTask; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hugegraph.store.node.grpc.GrpcShutdownBarrier; +import org.apache.hugegraph.store.node.grpc.HgStoreStreamImpl; +import org.apache.hugegraph.store.node.listener.ContextClosedListener; +import org.apache.hugegraph.store.node.task.TTLCleaner; +import org.junit.Test; +import org.lognet.springboot.grpc.context.GRpcServerInitializedEvent; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +import io.grpc.Server; + +public class ContextClosedListenerTest { + + @Test(timeout = 5000) + public void testContextCloseStopsActiveWorkersWithoutPartitionDelay() throws Exception { + ThreadPoolExecutor scan = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + ThreadPoolExecutor ttl = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); + CountDownLatch started = new CountDownLatch(3); + CountDownLatch interrupted = new CountDownLatch(3); + CountDownLatch cleanupStarted = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch cleanupAllowed = new CountDownLatch(1); + Runnable work = () -> { + started.countDown(); + try { + release.await(); + } catch (InterruptedException e) { + interrupted.countDown(); + cleanupStarted.countDown(); + boolean done = false; + while (!done) { + try { + cleanupAllowed.await(); + done = true; + } catch (InterruptedException ignored) { + // Model resource cleanup which must finish before DB destruction. + } + } + Thread.currentThread().interrupt(); + } + }; + AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + try { + HgStoreStreamImpl stream = mock(HgStoreStreamImpl.class); + TTLCleaner cleaner = mock(TTLCleaner.class); + when(stream.getRealExecutor()).thenReturn(scan); + doAnswer(invocation -> { + scan.shutdownNow(); + return null; + }).when(stream).shutdownScans(); + when(cleaner.getExecutor()).thenReturn(ttl); + when(cleaner.getScheduler()).thenReturn(scheduler); + context.getBeanFactory().registerSingleton("storeStream", stream); + context.getBeanFactory().registerSingleton("cleaner", cleaner); + context.register(ContextClosedListener.class, GrpcShutdownBarrier.class); + context.refresh(); + scan.execute(work); + ttl.execute(work); + scheduler.execute(work); + assertTrue("all shutdown participants must be active", started.await(1, TimeUnit.SECONDS)); + + FutureTask closing = new FutureTask<>(() -> { + context.close(); + return null; + }); + Thread closeThread = new Thread(closing, "test-context-close"); + closeThread.setDaemon(true); + closeThread.start(); + // Scheduler shutdown must not let DB destruction overtake resource cleanup. + assertTrue(cleanupStarted.await(1, TimeUnit.SECONDS)); + assertFalse("context close must wait for worker cleanup", closing.isDone()); + cleanupAllowed.countDown(); + closing.get(2, TimeUnit.SECONDS); + assertTrue("close must interrupt active workers", interrupted.await(1, TimeUnit.SECONDS)); + assertTrue(scan.awaitTermination(1, TimeUnit.SECONDS)); + assertTrue(ttl.awaitTermination(1, TimeUnit.SECONDS)); + assertTrue(scheduler.awaitTermination(1, TimeUnit.SECONDS)); + } finally { + cleanupAllowed.countDown(); + release.countDown(); + scan.shutdownNow(); + ttl.shutdownNow(); + scheduler.shutdownNow(); + context.close(); + } + } + + @Test(timeout = 5000) + public void testGrpcCallbacksFinishBeforeBeanDestructionEvenWhenInterrupted() throws Exception { + CountDownLatch awaitingCallbacks = new CountDownLatch(1); + CountDownLatch releaseCallbacks = new CountDownLatch(1); + AtomicBoolean terminated = new AtomicBoolean(); + AtomicBoolean databaseClosed = new AtomicBoolean(); + Server server = mock(Server.class); + when(server.shutdownNow()).thenReturn(server); + when(server.isTerminated()).thenAnswer(invocation -> terminated.get()); + when(server.awaitTermination(5, TimeUnit.SECONDS)).thenAnswer(invocation -> { + awaitingCallbacks.countDown(); + releaseCallbacks.await(); + terminated.set(true); + return true; + }); + AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + try { + context.getBeanFactory().registerSingleton("storeStream", mock(HgStoreStreamImpl.class)); + context.getBeanFactory().registerSingleton("cleaner", mock(TTLCleaner.class)); + context.getDefaultListableBeanFactory().registerDisposableBean("database", () -> databaseClosed.set(true)); + context.register(ContextClosedListener.class, GrpcShutdownBarrier.class); + context.refresh(); + context.publishEvent(new GRpcServerInitializedEvent(context, server)); + FutureTask closing = new FutureTask<>(() -> { + context.close(); + return Thread.currentThread().isInterrupted(); + }); + Thread closeThread = new Thread(closing, "test-grpc-context-close"); + closeThread.setDaemon(true); + closeThread.start(); + assertTrue(awaitingCallbacks.await(1, TimeUnit.SECONDS)); + closeThread.interrupt(); + assertFalse("DB must remain open while callbacks own its resources", databaseClosed.get()); + releaseCallbacks.countDown(); + assertTrue("shutdown must preserve interruption", closing.get(2, TimeUnit.SECONDS)); + assertTrue(terminated.get()); + assertTrue(databaseClosed.get()); + } finally { + releaseCallbacks.countDown(); + context.close(); + } + } +} diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/service/GrpcShutdownBarrierTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/service/GrpcShutdownBarrierTest.java new file mode 100644 index 0000000000..c32bcb0393 --- /dev/null +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/service/GrpcShutdownBarrierTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.service; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hugegraph.store.node.grpc.GrpcShutdownBarrier; +import org.junit.Test; + +import io.grpc.CallOptions; +import io.grpc.ManagedChannel; +import io.grpc.MethodDescriptor; +import io.grpc.Server; +import io.grpc.ServerServiceDefinition; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import io.grpc.stub.ClientCalls; +import io.grpc.stub.ServerCalls; +import io.grpc.stub.StreamObserver; + +public class GrpcShutdownBarrierTest { + + @Test(timeout = 7000) + public void testTransportTerminationDoesNotReleaseActiveCallback() throws Exception { + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicBoolean finished = new AtomicBoolean(); + ExecutorService executor = Executors.newSingleThreadExecutor(); + MethodDescriptor.Marshaller marshaller = new MethodDescriptor.Marshaller() { + @Override + public InputStream stream(byte[] value) { + return new ByteArrayInputStream(value); + } + @Override + public byte[] parse(InputStream stream) { + return new byte[0]; + } + }; + MethodDescriptor method = MethodDescriptor.newBuilder() + .setType(MethodDescriptor.MethodType.UNARY).setFullMethodName("probe/hold") + .setRequestMarshaller(marshaller).setResponseMarshaller(marshaller).build(); + String name = InProcessServerBuilder.generateName(); + GrpcShutdownBarrier barrier = new GrpcShutdownBarrier(); + Server server = InProcessServerBuilder.forName(name).executor(executor).intercept(barrier) + .addService(ServerServiceDefinition.builder("probe").addMethod(method, + ServerCalls.asyncUnaryCall((request, response) -> { + started.countDown(); + try { + release.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + finished.set(true); + } + response.onNext(new byte[0]); + response.onCompleted(); + })).build()).build().start(); + ManagedChannel channel = InProcessChannelBuilder.forName(name).directExecutor().build(); + try { + ClientCalls.asyncUnaryCall(channel.newCall(method, CallOptions.DEFAULT), new byte[0], + new StreamObserver() { + @Override + public void onNext(byte[] value) { + } + @Override + public void onError(Throwable error) { + } + @Override + public void onCompleted() { + } + }); + assertTrue("handler must start", started.await(2, TimeUnit.SECONDS)); + barrier.stopAcceptingCalls(); + server.shutdownNow(); + assertTrue(server.awaitTermination(1, TimeUnit.SECONDS)); + assertFalse("transport termination must not be mistaken for callback exit", finished.get()); + CountDownLatch waiting = new CountDownLatch(1); + FutureTask drained = new FutureTask<>(() -> { + waiting.countDown(); + barrier.awaitCallbacks(); + return null; + }); + Thread waiter = new Thread(drained, "test-rpc-drain"); + waiter.setDaemon(true); + waiter.start(); + assertTrue(waiting.await(1, TimeUnit.SECONDS)); + try { + drained.get(100, TimeUnit.MILLISECONDS); + fail("barrier passed while a handler still owns resources"); + } catch (TimeoutException expected) { + // The active handler must prevent database destruction. + } + release.countDown(); + drained.get(2, TimeUnit.SECONDS); + assertTrue(finished.get()); + } finally { + release.countDown(); + channel.shutdownNow(); + server.shutdownNow(); + executor.shutdown(); + assertTrue("cleanup must finish", executor.awaitTermination(2, TimeUnit.SECONDS)); + } + } +} diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/service/ScanShutdownTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/service/ScanShutdownTest.java new file mode 100644 index 0000000000..ade1cf9913 --- /dev/null +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/service/ScanShutdownTest.java @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.service; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.FutureTask; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.hugegraph.pd.common.KVPair; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.grpc.common.ScanMethod; +import org.apache.hugegraph.store.grpc.common.ScanOrderType; +import org.apache.hugegraph.store.grpc.stream.KvPageRes; +import org.apache.hugegraph.store.grpc.stream.KvStream; +import org.apache.hugegraph.store.grpc.stream.ScanStreamReq; +import org.apache.hugegraph.store.grpc.stream.ScanStreamBatchReq; +import org.apache.hugegraph.store.grpc.stream.ScanQueryRequest; +import org.apache.hugegraph.store.node.AppConfig; +import org.apache.hugegraph.store.node.grpc.HgStoreStreamImpl; +import org.apache.hugegraph.store.node.grpc.HgStoreWrapperEx; +import org.apache.hugegraph.store.node.grpc.ScanBatchResponse; +import org.apache.hugegraph.store.node.grpc.ParallelScanIterator; +import org.apache.hugegraph.store.node.grpc.QueryCondition; +import org.apache.hugegraph.store.node.grpc.ScanOneShotResponse; +import org.apache.hugegraph.store.node.grpc.ScanStreamResponse; +import org.junit.Test; + +import io.grpc.Context; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; + +public class ScanShutdownTest { + + @Test + public void testClosingPreventsLazyExecutorAndStateCreation() { + HgStoreStreamImpl service = new HgStoreStreamImpl(); + service.stopAcceptingScans(); + assertUnavailable(service::getExecutor); + assertUnavailable(service::getState); + assertUnavailable(() -> service.scan(mock(StreamObserver.class))); + service.shutdownScans(); + assertNull(service.getRealExecutor()); + } + + @Test(timeout = 5000) + public void testQueuedStreamCancellationDrainsWithoutOpeningIterator() throws Exception { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch occupied = new CountDownLatch(1); + executor.execute(() -> { + occupied.countDown(); + try { + release.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + HgStoreWrapperEx wrapper = mock(HgStoreWrapperEx.class); + AppConfig config = mock(AppConfig.class); + when(config.getServerWaitTime()).thenReturn(60); + StreamObserver output = mock(StreamObserver.class); + ScanStreamResponse response = ScanStreamResponse.of(output, wrapper, executor, config); + FutureTask request = new FutureTask<>(() -> { + response.onNext(ScanStreamReq.newBuilder().setMethod(ScanMethod.ALL) + .setPageSize(1).setLimit(10).build()); + return null; + }); + Thread caller = new Thread(request, "scan-shutdown-test"); + try { + assertTrue(occupied.await(1, TimeUnit.SECONDS)); + caller.start(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); + while (executor.getQueue().isEmpty() && System.nanoTime() < deadline) { + Thread.yield(); + } + assertEquals(1, executor.getQueue().size()); + response.onError(Status.CANCELLED.asRuntimeException()); + executor.shutdown(); + release.countDown(); + request.get(1, TimeUnit.SECONDS); + assertTrue(executor.awaitTermination(1, TimeUnit.SECONDS)); + verifyNoInteractions(wrapper); + assertEquals(2L, executor.getCompletedTaskCount()); + } finally { + response.onCompleted(); + release.countDown(); + executor.shutdownNow(); + caller.join(1000); + } + } + + @Test(timeout = 5000) + public void testOneShotObservesContextCancellationAndClosesIterator() { + HgStoreWrapperEx wrapper = mock(HgStoreWrapperEx.class); + ScanIterator iterator = mock(ScanIterator.class); + StreamObserver output = mock(StreamObserver.class); + Context.CancellableContext context = Context.current().withCancellation(); + when(wrapper.scanAll(anyString(), anyString(), any(byte[].class))).thenReturn(iterator); + when(iterator.hasNext()).thenAnswer(invocation -> { + context.cancel(null); + return true; + }); + when(iterator.next()).thenReturn(RocksDBSession.BackendColumn.of(new byte[4], new byte[0])); + when(iterator.position()).thenReturn(new byte[4]); + try { + context.run(() -> ScanOneShotResponse.scanOneShot( + ScanStreamReq.newBuilder().setMethod(ScanMethod.ALL).setLimit(100).build(), + output, wrapper)); + verify(iterator).close(); + verifyNoInteractions(output); + } finally { + context.cancel(null); + } + } + + @Test + public void testPausedBatchClosesIteratorAndRejectsLaterQuery() throws Exception { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + HgStoreWrapperEx wrapper = mock(HgStoreWrapperEx.class); + ScanIterator iterator = mock(ScanIterator.class); + StreamObserver output = mock(StreamObserver.class); + ScanBatchResponse response = new ScanBatchResponse(output, wrapper, executor); + Field field = ScanBatchResponse.class.getDeclaredField("iterator"); + field.setAccessible(true); + field.set(response, iterator); + try { + response.onCompleted(); + response.onNext(ScanStreamBatchReq.newBuilder() + .setQueryRequest(ScanQueryRequest.getDefaultInstance()) + .build()); + response.onCompleted(); + verify(iterator).close(); + verifyNoInteractions(wrapper); + assertNull(field.get(response)); + assertFalse(executor.isShutdown()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testRejectedBatchReportsFailureWithoutSuccessfulCompletion() { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + executor.shutdown(); + StreamObserver output = mock(StreamObserver.class); + ScanBatchResponse response = new ScanBatchResponse(output, mock(HgStoreWrapperEx.class), executor); + response.onNext(ScanStreamBatchReq.newBuilder() + .setQueryRequest(ScanQueryRequest.getDefaultInstance()).build()); + verify(output).onError(any(Throwable.class)); + verify(output, never()).onCompleted(); + } + + @Test + public void testBatchIteratorFailureClosesResourcesWithoutSuccessfulCompletion() throws Exception { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + StreamObserver output = mock(StreamObserver.class); + ScanIterator iterator = mock(ScanIterator.class); + IllegalStateException failure = new IllegalStateException("iterator failed"); + when(iterator.hasNext()).thenThrow(failure); + ScanBatchResponse response = new ScanBatchResponse(output, mock(HgStoreWrapperEx.class), executor); + Field field = ScanBatchResponse.class.getDeclaredField("iterator"); + field.setAccessible(true); + field.set(response, iterator); + Method send = ScanBatchResponse.class.getDeclaredMethod("sendEntries"); + send.setAccessible(true); + try { + send.invoke(response); + verify(iterator).close(); + verify(output).onError(failure); + verify(output, never()).onCompleted(); + } finally { + response.onCompleted(); + executor.shutdownNow(); + } + } + + @Test(timeout = 5000) + public void testActiveStreamCancellationReleasesBlockedProducer() throws Exception { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + HgStoreWrapperEx wrapper = mock(HgStoreWrapperEx.class); + ScanIterator iterator = mock(ScanIterator.class); + when(wrapper.scanAll(anyString(), anyString(), any(byte[].class))).thenReturn(iterator); + when(iterator.hasNext()).thenReturn(true); + when(iterator.next()).thenReturn(RocksDBSession.BackendColumn.of(new byte[4], new byte[0])); + when(iterator.position()).thenReturn(new byte[4]); + AppConfig config = mock(AppConfig.class); + when(config.getServerWaitTime()).thenReturn(60); + StreamObserver output = mock(StreamObserver.class); + ScanStreamResponse response = ScanStreamResponse.of(output, wrapper, executor, config); + try { + // Consume just one page, then leave the producer waiting for its consumer. + response.onNext(ScanStreamReq.newBuilder().setMethod(ScanMethod.ALL) + .setPageSize(1).setLimit(1000).build()); + response.onError(Status.CANCELLED.asRuntimeException()); + executor.shutdown(); + assertTrue("cancel must release a producer without its 60-second timeout", + executor.awaitTermination(1, TimeUnit.SECONDS)); + verify(iterator).close(); + } finally { + response.onCompleted(); + executor.shutdownNow(); + } + } + + @Test(timeout = 5000) + public void testParallelPausedScannerReleasesIterator() throws Exception { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + ScanIterator source = mock(ScanIterator.class); + when(source.hasNext()).thenReturn(true); + when(source.next()).thenReturn(RocksDBSession.BackendColumn.of(new byte[4], new byte[0])); + ParallelScanIterator scan = ParallelScanIterator.of( + () -> new KVPair<>(mock(QueryCondition.class), source), () -> Long.MAX_VALUE, + ScanQueryRequest.getDefaultInstance(), executor); + try { + // One worker fills its output allowance and pauses while retaining its iterator. + executor.submit(() -> { }).get(2, TimeUnit.SECONDS); + verify(source, atLeastOnce()).next(); + verify(source, never()).close(); + scan.close(); + scan.close(); + verify(source).close(); + executor.shutdown(); + assertTrue(executor.awaitTermination(1, TimeUnit.SECONDS)); + } finally { + scan.close(); + executor.shutdownNow(); + } + } + + @Test(timeout = 5000) + public void testParallelOrderedScannerReleasesQueueLockOnEmptyIterator() throws Exception { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + ScanIterator first = mock(ScanIterator.class); + ScanIterator empty = mock(ScanIterator.class); + when(first.hasNext()).thenReturn(true); + when(first.next()).thenReturn(RocksDBSession.BackendColumn.of(new byte[4], new byte[0])); + when(empty.hasNext()).thenReturn(false); + + AtomicInteger supplies = new AtomicInteger(); + Field bodySize = ParallelScanIterator.class.getDeclaredField("maxBodySize"); + bodySize.setAccessible(true); + int originalBodySize = bodySize.getInt(null); + bodySize.setInt(null, 2); + ParallelScanIterator scan = null; + try { + scan = ParallelScanIterator.of( + () -> { + int supply = supplies.incrementAndGet(); + if (supply == 1) { + return new KVPair<>(mock(QueryCondition.class), first); + } + if (supply == 2) { + return new KVPair<>(mock(QueryCondition.class), empty); + } + return new KVPair<>(mock(QueryCondition.class), null); + }, + () -> 1L, + ScanQueryRequest.newBuilder().setOrderType(ScanOrderType.ORDER_WITHIN_VERTEX).build(), + executor); + + // Wait for the scanner task to finish before checking the lock owner. + executor.submit(() -> { }).get(2, TimeUnit.SECONDS); + Field queueLock = ParallelScanIterator.class.getDeclaredField("queueLock"); + queueLock.setAccessible(true); + assertFalse(((ReentrantLock) queueLock.get(scan)).isLocked()); + } finally { + if (scan != null) { + scan.close(); + } + bodySize.setInt(null, originalBodySize); + executor.shutdownNow(); + } + } + + @Test(timeout = 5000) + public void testParallelOrderedProducerCancelsWithFullOutputQueue() throws Exception { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + ScanIterator source = mock(ScanIterator.class); + Field bodySize = ParallelScanIterator.class.getDeclaredField("maxBodySize"); + bodySize.setAccessible(true); + byte[] value = new byte[bodySize.getInt(null)]; + CountDownLatch full = new CountDownLatch(1); + AtomicInteger rows = new AtomicInteger(); + when(source.hasNext()).thenReturn(true); + when(source.next()).thenAnswer(invocation -> { + if (rows.incrementAndGet() == 5) { + full.countDown(); + } + return RocksDBSession.BackendColumn.of(new byte[4], value); + }); + ParallelScanIterator scan = ParallelScanIterator.of( + () -> new KVPair<>(mock(QueryCondition.class), source), () -> Long.MAX_VALUE, + ScanQueryRequest.newBuilder().setOrderType(ScanOrderType.ORDER_WITHIN_VERTEX).build(), + executor); + try { + assertTrue("four queued batches must fill the single-scanner queue", + full.await(2, TimeUnit.SECONDS)); + scan.close(); + executor.shutdown(); + assertTrue("cancellation must release the blocked ordered producer", + executor.awaitTermination(1, TimeUnit.SECONDS)); + verify(source).close(); + } finally { + scan.close(); + executor.shutdownNow(); + } + } + + private static void assertUnavailable(Runnable action) { + try { + action.run(); + fail("Scan admission must be closed"); + } catch (StatusRuntimeException e) { + assertEquals(Status.Code.UNAVAILABLE, e.getStatus().getCode()); + } + } +} diff --git a/hugegraph-store/hg-store-rocksdb/pom.xml b/hugegraph-store/hg-store-rocksdb/pom.xml index cd9cf28c6d..232aecffa1 100644 --- a/hugegraph-store/hg-store-rocksdb/pom.xml +++ b/hugegraph-store/hg-store-rocksdb/pom.xml @@ -57,7 +57,7 @@ org.rocksdb rocksdbjni - 7.7.3 + ${rocksdb.version} org.projectlombok diff --git a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java index 7fcd07f3b8..374778d9e2 100644 --- a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java +++ b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java @@ -56,7 +56,6 @@ public class RocksDBOptions extends OptionHolder { disallowEmpty(), false ); - // public static final ConfigListOption DATA_DISKS = // new ConfigListOption<>( // "rocksdb.data_disks", diff --git a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java index f4e7605a7f..bfd3240ba7 100644 --- a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java +++ b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java @@ -450,8 +450,9 @@ private void openRocksDB(String dbDataPath, long version) { new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions)); } List columnFamilyHandleList = new ArrayList<>(); - this.rocksDB = RocksDB.open(dbOptions, dbPath, columnFamilyDescriptorList, - columnFamilyHandleList); + this.rocksDB = + RocksDB.open(dbOptions, dbPath, columnFamilyDescriptorList, + columnFamilyHandleList); Asserts.isTrue(columnFamilyHandleList.size() > 0, "must have column family"); for (ColumnFamilyHandle handle : columnFamilyHandleList) { diff --git a/hugegraph-store/hg-store-test/pom.xml b/hugegraph-store/hg-store-test/pom.xml index cb9433e903..063f21bb6f 100644 --- a/hugegraph-store/hg-store-test/pom.xml +++ b/hugegraph-store/hg-store-test/pom.xml @@ -201,7 +201,7 @@ com.alipay.sofa jraft-core - 1.3.9 + ${jraft.version} org.rocksdb diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/HgStoreStateMachineTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/HgStoreStateMachineTest.java index 752a17ea59..9ec5750bb9 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/HgStoreStateMachineTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/HgStoreStateMachineTest.java @@ -130,6 +130,21 @@ public void testOnApply() { final java.util.Iterator iterator = tasks.iterator(); Task task; + @Override + public void setAutoCommitPerLog(boolean autoCommit) { + throw new UnsupportedOperationException("Test iterator does not manage commits"); + } + + @Override + public boolean commit() { + throw new UnsupportedOperationException("Test iterator does not manage commits"); + } + + @Override + public void commitAndSnapshotSync(Closure done) { + throw new UnsupportedOperationException("Test iterator does not manage snapshots"); + } + @Override public ByteBuffer getData() { return task.getData(); diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/snapshot/HgSnapshotHandlerTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/snapshot/HgSnapshotHandlerTest.java index e32f1df498..8c6c360edd 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/snapshot/HgSnapshotHandlerTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/snapshot/HgSnapshotHandlerTest.java @@ -32,6 +32,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -340,16 +341,22 @@ public void testDbCompactionReleasesPathLockWhenInterruptedWaitingForRangeLock() throws InterruptedException { BusinessHandler businessHandler = getStoreEngine().getBusinessHandler(); int partitionId = 5; - createPartitionEngine(partitionId); + PartitionEngine engine = createPartitionEngine(partitionId); + File snapshotDirectory = new File(engine.getOptions().getRaftSnapShotPath(), "snapshot"); + File[] initialSnapshots = snapshotDirectory.listFiles( + (dir, name) -> name.startsWith("snapshot_")); + assertTrue("this partition must not have a snapshot before compaction", + initialSnapshots == null || initialSnapshots.length == 0); long originalWaitMillis = BusinessHandlerImpl.getCompactionRangeLockWaitMillis(); // Long enough that the worker thread is still parked in tryLock() when interrupted, // rather than racing a real timeout. BusinessHandlerImpl.setCompactionRangeLockWaitMillis(60_000); + boolean rangeLockHeld = false; try { // Simulate a snapshot save that is still in progress, forcing dbCompaction() onto // the tryLock(wait) path rather than acquiring the range lock immediately. - assertTrue("snapshot save must reserve the range lock", - businessHandler.tryLockCompactionRange(partitionId)); + rangeLockHeld = businessHandler.tryLockCompactionRange(partitionId); + assertTrue("snapshot save must reserve the range lock", rangeLockHeld); businessHandler.dbCompaction("graph0", partitionId); @@ -403,23 +410,33 @@ public void testDbCompactionReleasesPathLockWhenInterruptedWaitingForRangeLock() // dbCompaction() call's own worker thread, which acquires and releases it itself - // do not touch compactionRangeLock again after this point. businessHandler.unlockCompactionRange(partitionId); + rangeLockHeld = false; businessHandler.dbCompaction("graph0", partitionId); - // Compaction on the test's near-empty RocksDB completes almost immediately, so the - // transient "doing" state cannot be reliably observed here - poll for the - // terminal compactionDone state instead. What this proves is that dbCompaction() - // was able to acquire lock(path) at all: before the fix, the leaked path lock - // would have made this call hang on lock(path) until the 6-hour timeout instead - // of ever reaching compactionDone. - long start = System.currentTimeMillis(); - while (businessHandler.getState(partitionId).get() != BusinessHandler.compactionDone && - System.currentTimeMillis() - start < 5000) { + // compactionDone is transient: the following Raft snapshot resets it. + // A newly published snapshot proves the second task passed both locks + // and compacted the DB; the released path lock proves its callback finished. + // Allow for doSnapshotSync's bounded 5-second wait before saving. + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(15); + boolean completed = false; + while (System.nanoTime() < deadline) { + File[] snapshots = snapshotDirectory.listFiles((dir, name) -> + name.startsWith("snapshot_") && + new File(new File(dir, name), "__raft_snapshot_meta").isFile()); + completed = snapshots != null && snapshots.length > 0 && + businessHandler.getPathLockState(path).get() == + BusinessHandler.compactionCanStart; + if (completed) { + break; + } Thread.sleep(50); } - assertEquals("second dbCompaction() must complete, proving the path lock was " + - "released rather than leaked by the interrupted task", - BusinessHandler.compactionDone, businessHandler.getState(partitionId).get()); + assertTrue("second dbCompaction() must publish a snapshot and release the " + + "path lock after the interrupted task", completed); } finally { + if (rangeLockHeld) { + businessHandler.unlockCompactionRange(partitionId); + } BusinessHandlerImpl.setCompactionRangeLockWaitMillis(originalWaitMillis); } } diff --git a/hugegraph-store/pom.xml b/hugegraph-store/pom.xml index a8212b70db..3a987f2f9f 100644 --- a/hugegraph-store/pom.xml +++ b/hugegraph-store/pom.xml @@ -44,6 +44,7 @@ + 1.3.14 2.15.0 apache-${release.name}-store-${project.version} diff --git a/install-dist/release-docs/LICENSE b/install-dist/release-docs/LICENSE index 06a31044eb..fdbff8fd3d 100644 --- a/install-dist/release-docs/LICENSE +++ b/install-dist/release-docs/LICENSE @@ -481,6 +481,12 @@ non-Apache-2.0 license text is required for a bundled component. https://central.sonatype.com/artifact/net.java.dev.jna/jna/5.12.1 -> Apache 2.0 https://central.sonatype.com/artifact/net.java.dev.jna/jna/5.5.0 -> Apache 2.0 https://central.sonatype.com/artifact/net.java.dev.jna/jna/5.7.0 -> Apache 2.0 + https://central.sonatype.com/artifact/com.github.jnr/jnr-ffi/2.1.7 -> Apache 2.0 + https://central.sonatype.com/artifact/joda-time/joda-time/2.10.8 -> Apache 2.0 + https://central.sonatype.com/artifact/com.alipay.sofa/jraft-core/1.3.14 -> Apache 2.0 + https://central.sonatype.com/artifact/org.skyscreamer/jsonassert/1.5.0 -> Apache 2.0 + https://central.sonatype.com/artifact/com.jayway.jsonpath/json-path/2.5.0 -> Apache 2.0 + https://central.sonatype.com/artifact/com.googlecode.json-simple/json-simple/1.1 -> Apache 2.0 https://central.sonatype.com/artifact/net.jodah/failsafe/2.4.1 -> Apache 2.0 https://central.sonatype.com/artifact/net.minidev/accessors-smart/1.2 -> Apache 2.0 https://central.sonatype.com/artifact/net.minidev/json-smart/2.3 -> Apache 2.0 @@ -655,10 +661,48 @@ non-Apache-2.0 license text is required for a bundled component. https://central.sonatype.com/artifact/org.powermock/powermock-module-junit4-rule/2.0.0-RC.3 -> Apache 2.0 https://central.sonatype.com/artifact/org.powermock/powermock-module-junit4/2.0.0-RC.3 -> Apache 2.0 https://central.sonatype.com/artifact/org.powermock/powermock-reflect/2.0.0-RC.3 -> Apache 2.0 + https://central.sonatype.com/artifact/com.google.api.grpc/proto-google-common-protos/1.17.0 -> Apache 2.0 + https://central.sonatype.com/artifact/com.google.api.grpc/proto-google-common-protos/2.0.1 -> Apache 2.0 + https://central.sonatype.com/artifact/io.protostuff/protostuff-api/1.6.0 -> Apache 2.0 + https://central.sonatype.com/artifact/io.protostuff/protostuff-collectionschema/1.6.0 -> Apache 2.0 + https://central.sonatype.com/artifact/io.protostuff/protostuff-core/1.6.0 -> Apache 2.0 + https://central.sonatype.com/artifact/io.protostuff/protostuff-runtime/1.6.0 -> Apache 2.0 + https://central.sonatype.com/artifact/com.addthis.metrics/reporter-config3/3.0.3 -> Apache 2.0 + https://central.sonatype.com/artifact/com.addthis.metrics/reporter-config-base/3.0.3 -> Apache 2.0 + https://central.sonatype.com/artifact/org.opencypher/rewriting-9.0/9.0.20190305 -> Apache 2.0 + https://central.sonatype.com/artifact/org.rocksdb/rocksdbjni/8.10.2 -> Apache 2.0 + https://github.com/hugegraph/toplingdb/packages/3151853 -> Apache 2.0 + https://central.sonatype.com/artifact/org.fusesource/sigar/1.6.4 -> Apache 2.0 + https://central.sonatype.com/artifact/io.prometheus/simpleclient/0.10.0 -> Apache 2.0 + https://central.sonatype.com/artifact/io.prometheus/simpleclient_common/0.10.0 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-agent/0.22 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-cli/0.14 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-cli/0.22 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-core/0.14 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-core/0.22 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-hflame/0.22 -> Apache 2.0 + https://central.sonatype.com/artifact/org.perfkit.sjk.parsers/sjk-jfr5/0.5 -> Apache 2.0 + https://central.sonatype.com/artifact/org.perfkit.sjk.parsers/sjk-jfr6/0.7 -> Apache 2.0 + https://central.sonatype.com/artifact/org.perfkit.sjk.parsers/sjk-jfr-standalone/0.7 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-json/0.14 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-json/0.22 -> Apache 2.0 + https://central.sonatype.com/artifact/org.perfkit.sjk.parsers/sjk-nps/0.9 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-stacktrace/0.14 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-stacktrace/0.22 -> Apache 2.0 + https://central.sonatype.com/artifact/org.yaml/snakeyaml/1.18 -> Apache 2.0 + https://central.sonatype.com/artifact/org.yaml/snakeyaml/1.26 -> Apache 2.0 + https://central.sonatype.com/artifact/org.yaml/snakeyaml/1.27 -> Apache 2.0 + https://central.sonatype.com/artifact/org.yaml/snakeyaml/1.28 -> Apache 2.0 + https://central.sonatype.com/artifact/org.yaml/snakeyaml/2.2 -> Apache 2.0 + https://central.sonatype.com/artifact/org.xerial.snappy/snappy-java/1.1.2.6 -> Apache 2.0 + https://central.sonatype.com/artifact/com.alipay.sofa.common/sofa-common-tools/1.0.12 -> Apache 2.0 + https://central.sonatype.com/artifact/com.alipay.sofa/sofa-rpc-all/5.7.6 -> Apache 2.0 + https://central.sonatype.com/artifact/org.springframework/spring-aop/5.3.20 -> Apache 2.0 + https://central.sonatype.com/artifact/org.springframework/spring-beans/5.3.20 -> Apache 2.0 + https://central.sonatype.com/artifact/org.springframework.boot/spring-boot/2.5.14 -> Apache 2.0 + https://central.sonatype.com/artifact/org.springframework.boot/spring-boot-actuator/2.5.14 -> Apache 2.0 https://central.sonatype.com/artifact/org.roaringbitmap/RoaringBitmap/0.9.38 -> Apache 2.0 https://central.sonatype.com/artifact/org.roaringbitmap/shims/0.9.38 -> Apache 2.0 - https://central.sonatype.com/artifact/org.rocksdb/rocksdbjni/6.29.5 -> Apache 2.0 - https://central.sonatype.com/artifact/org.rocksdb/rocksdbjni/7.7.3 -> Apache 2.0 https://central.sonatype.com/artifact/org.rocksdb/rocksdbjni/8.10.2 -> Apache 2.0 https://central.sonatype.com/artifact/org.skyscreamer/jsonassert/1.5.0 -> Apache 2.0 https://central.sonatype.com/artifact/org.springframework.boot/spring-boot-actuator-autoconfigure/2.5.14 -> Apache 2.0 diff --git a/install-dist/release-docs/licenses/LICENSE-rocksdbjni-8.10.2-SNAPSHOT.txt b/install-dist/release-docs/licenses/LICENSE-rocksdbjni-8.10.2-SNAPSHOT.txt new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/install-dist/release-docs/licenses/LICENSE-rocksdbjni-8.10.2-SNAPSHOT.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/install-dist/scripts/build-topling-distribution.sh b/install-dist/scripts/build-topling-distribution.sh new file mode 100755 index 0000000000..8d8d2b4e06 --- /dev/null +++ b/install-dist/scripts/build-topling-distribution.sh @@ -0,0 +1,180 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR"/../.. && pwd)" +COMPONENT="${1:-}" +VERSION="${2:-}" + +if [ -z "$COMPONENT" ]; then + echo "Usage: $0 [version]" >&2 + exit 1 +fi +if [ "$(uname -s)" != "Linux" ] || [ "$(uname -m)" != "x86_64" ]; then + echo "Error: Topling distributions can only be prepared on Linux x86_64" >&2 + exit 1 +fi +if [ -z "$VERSION" ]; then + VERSION=$(mvn -f "$PROJECT_ROOT/pom.xml" \ + help:evaluate -Dexpression=project.version -q -DforceStdout) +fi +if [[ ! "$VERSION" =~ ^[0-9A-Za-z][0-9A-Za-z._-]*$ ]]; then + echo "Error: unsafe project version: $VERSION" >&2 + exit 1 +fi + +case "$COMPONENT" in + server) + COMPONENT_PARENT="$PROJECT_ROOT/hugegraph-server" + STANDARD_NAME="apache-hugegraph-server-$VERSION" + CONFIG_FILE="conf/graphs/hugegraph.properties" + ;; + pd) + COMPONENT_PARENT="$PROJECT_ROOT/hugegraph-pd" + STANDARD_NAME="apache-hugegraph-pd-$VERSION" + CONFIG_FILE="conf/application.yml" + ;; + store) + COMPONENT_PARENT="$PROJECT_ROOT/hugegraph-store" + STANDARD_NAME="apache-hugegraph-store-$VERSION" + CONFIG_FILE="conf/application-pd.yml" + ;; + *) + echo "Error: unsupported component '$COMPONENT'" >&2 + exit 1 + ;; +esac + +STANDARD_DIR="$COMPONENT_PARENT/$STANDARD_NAME" +TOPLING_NAME="$STANDARD_NAME-topling" +TOPLING_DIR="$COMPONENT_PARENT/$TOPLING_NAME" +TOPLING_TAR="$COMPONENT_PARENT/$TOPLING_NAME.tar.gz" +TOPLING_SOURCE_LIB="$PROJECT_ROOT/hugegraph-server/hugegraph-dist/src/assembly/static/lib/topling" +TOPLING_JARS=("$TOPLING_SOURCE_LIB"/rocksdbjni*.jar) + +if [ ! -d "$STANDARD_DIR" ]; then + echo "Error: standard distribution not found: $STANDARD_DIR" >&2 + exit 1 +fi +if ! command -v rsync >/dev/null 2>&1; then + echo "Error: rsync is required to build a clean Topling distribution" >&2 + exit 1 +fi +if [ "${#TOPLING_JARS[@]}" -ne 1 ] || [ ! -f "${TOPLING_JARS[0]}" ]; then + echo "Error: expected exactly one ToplingDB JAR under: $TOPLING_SOURCE_LIB" >&2 + exit 1 +fi + +STAGING_ROOT="$(mktemp -d "$COMPONENT_PARENT/.topling-dist.XXXXXX")" +cleanup() { + rm -rf "$STAGING_ROOT" +} +trap cleanup EXIT +STAGING_DIR="$STAGING_ROOT/$TOPLING_NAME" + +mkdir -p "$STAGING_DIR" +rsync --archive \ + --exclude='/bin/pid' \ + --exclude='/library' \ + --exclude='/logs' \ + --exclude='/pd_data' \ + --exclude='/rocksdb-data' \ + --exclude='/storage' \ + "$STANDARD_DIR/" "$STAGING_DIR/" + +for runtime_path in \ + bin/pid \ + library \ + logs \ + pd_data \ + rocksdb-data \ + storage; do + if [ -e "$STAGING_DIR/$runtime_path" ] || + [ -L "$STAGING_DIR/$runtime_path" ]; then + echo "Error: runtime state leaked into staging: $runtime_path" >&2 + exit 1 + fi +done + +mkdir -p "$STAGING_DIR/lib/topling" +cp "${TOPLING_JARS[0]}" "$STAGING_DIR/lib/topling/" + +case "$COMPONENT" in + server) + sed -i '/^[[:space:]#]*rocksdb\.provider[[:space:]]*=/d' \ + "$STAGING_DIR/$CONFIG_FILE" + printf '\nrocksdb.provider=topling\n' >> "$STAGING_DIR/$CONFIG_FILE" + ;; + pd) + sed -i \ + -e 's/^# rocksdb:/rocksdb:/' \ + -e 's/^# provider: rocksdb/ provider: topling/' \ + "$STAGING_DIR/$CONFIG_FILE" + ;; + store) + sed -i 's/^ # provider: topling/ provider: topling/' \ + "$STAGING_DIR/$CONFIG_FILE" + ;; +esac + +if ! grep -Eq '^[[:space:]]*(rocksdb\.provider=|provider:[[:space:]]*)topling([[:space:]]|$)' \ + "$STAGING_DIR/$CONFIG_FILE"; then + echo "Error: failed to select ToplingDB in $STAGING_DIR/$CONFIG_FILE" >&2 + exit 1 +fi + +"$STAGING_DIR/bin/prepare-topling.sh" + +NATIVE_LIBRARY="$STAGING_DIR/library/librocksdbjni-linux64.so" +if [ ! -r "$NATIVE_LIBRARY" ]; then + echo "Error: prepared native library not found: $NATIVE_LIBRARY" >&2 + exit 1 +fi +if command -v ldd >/dev/null 2>&1 && + ldd "$NATIVE_LIBRARY" 2>/dev/null | grep -q 'not found'; then + echo "Error: prepared ToplingDB native library has unresolved dependencies" >&2 + ldd "$NATIVE_LIBRARY" >&2 || true + exit 1 +fi + +JAR_SHA256=$(sha256sum "${TOPLING_JARS[0]}" | awk '{ print $1 }') +printf '%s\n' \ + "provider=topling" \ + "component=$COMPONENT" \ + "version=$VERSION" \ + "jar=$(basename "${TOPLING_JARS[0]}")" \ + "jar.sha256=$JAR_SHA256" \ + > "$STAGING_DIR/lib/topling/runtime.properties" + +tar -czf "$STAGING_ROOT/$TOPLING_NAME.tar.gz" \ + -C "$STAGING_ROOT" "$TOPLING_NAME" + +if [ -e "$TOPLING_DIR" ] && + [[ "$TOPLING_DIR" != "$COMPONENT_PARENT"/apache-hugegraph-*-topling ]]; then + echo "Error: refusing to replace unexpected path: $TOPLING_DIR" >&2 + exit 1 +fi +rm -rf "$TOPLING_DIR" +rm -f "$TOPLING_TAR" +mv "$STAGING_DIR" "$TOPLING_DIR" +mv "$STAGING_ROOT/$TOPLING_NAME.tar.gz" "$TOPLING_TAR" + +echo "[build-topling-distribution] Directory: $TOPLING_DIR" +echo "[build-topling-distribution] Archive: $TOPLING_TAR" diff --git a/install-dist/scripts/dependency/known-dependencies.txt b/install-dist/scripts/dependency/known-dependencies.txt index 157da1c064..43cbedd763 100644 --- a/install-dist/scripts/dependency/known-dependencies.txt +++ b/install-dist/scripts/dependency/known-dependencies.txt @@ -282,8 +282,7 @@ jna-5.5.0.jar jna-5.7.0.jar joda-time-2.10.8.jar joni-2.2.1.jar -jraft-core-1.3.11.jar -jraft-core-1.3.13.jar +jraft-core-1.3.14.jar jraft-core-1.3.9.jar jsonassert-1.5.0.jar json-path-2.5.0.jar @@ -465,8 +464,6 @@ protostuff-core-1.6.0.jar protostuff-runtime-1.6.0.jar rewriting-9.0-9.0.20190305.jar RoaringBitmap-0.9.38.jar -rocksdbjni-6.29.5.jar -rocksdbjni-7.7.3.jar rocksdbjni-8.10.2.jar scala-java8-compat_2.12-0.8.0.jar scala-library-2.12.7.jar diff --git a/pom.xml b/pom.xml index 3581d0346f..cdb87fbe85 100644 --- a/pom.xml +++ b/pom.xml @@ -95,6 +95,7 @@ UTF-8 bash 1.5.0 + 8.10.2 @@ -200,6 +201,8 @@ .gitattributes .serena/** + + .goal-task/** **/*.iml **/*.iws @@ -218,6 +221,9 @@ **/src/main/java/org/apache/hugegraph/pd/grpc/** **/src/main/java/org/apache/hugegraph/store/grpc/** + + **/library/*.html + **/library/*.css true