From c5c6829b73d8a7d017fdea893a056a92309d32c8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:39:44 +0800 Subject: [PATCH 01/17] Add Pipe logical backup sink and restore tooling --- distribution/src/assembly/cli.xml | 2 + iotdb-client/cli/pom.xml | 5 + .../apache/iotdb/cli/i18n/CliMessages.java | 57 + .../apache/iotdb/cli/i18n/CliMessages.java | 57 + .../tool/pipe/PipeLogicalBackupTool.java | 817 ++++++++++++ .../tool/pipe/PipeLogicalBackupToolTest.java | 166 +++ .../iotdb/db/i18n/DataNodePipeMessages.java | 11 + .../iotdb/db/i18n/DataNodePipeMessages.java | 11 + .../PipeDataRegionSinkConstructor.java | 7 + .../PipeSchemaRegionSinkConstructor.java | 7 + .../logicalbackup/LogicalBackupSink.java | 591 +++++++++ iotdb-core/node-commons/pom.xml | 4 + .../commons/i18n/LogicalBackupMessages.java | 151 +++ .../commons/i18n/LogicalBackupMessages.java | 151 +++ .../plugin/builtin/BuiltinPipePlugin.java | 4 + .../sink/logicalbackup/LogicalBackupSink.java | 24 + .../config/constant/PipeSinkConstant.java | 49 + .../LogicalBackupArchiveReader.java | 481 +++++++ .../logicalbackup/LogicalBackupFormat.java | 62 + .../logicalbackup/LogicalBackupManifest.java | 68 + .../logicalbackup/LogicalBackupRecord.java | 136 ++ .../LogicalBackupRecordType.java | 54 + .../LogicalBackupSegmentReader.java | 541 ++++++++ .../logicalbackup/LogicalBackupWriter.java | 1162 +++++++++++++++++ .../LogicalBackupArchiveReaderTest.java | 272 ++++ .../LogicalBackupWriterTest.java | 588 +++++++++ scripts/tools/export-pipe-logical-backup.sh | 21 + scripts/tools/import-pipe-logical-backup.sh | 21 + scripts/tools/pipe-logical-backup.sh | 43 + .../windows/export-pipe-logical-backup.bat | 21 + .../windows/import-pipe-logical-backup.bat | 21 + scripts/tools/windows/pipe-logical-backup.bat | 40 + 32 files changed, 5645 insertions(+) create mode 100644 iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java create mode 100644 iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java create mode 100644 iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java create mode 100644 iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/sink/logicalbackup/LogicalBackupSink.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupFormat.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecord.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecordType.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSegmentReader.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java create mode 100644 scripts/tools/export-pipe-logical-backup.sh create mode 100644 scripts/tools/import-pipe-logical-backup.sh create mode 100644 scripts/tools/pipe-logical-backup.sh create mode 100644 scripts/tools/windows/export-pipe-logical-backup.bat create mode 100644 scripts/tools/windows/import-pipe-logical-backup.bat create mode 100644 scripts/tools/windows/pipe-logical-backup.bat diff --git a/distribution/src/assembly/cli.xml b/distribution/src/assembly/cli.xml index 6f4eed7f7c259..7e60322b49870 100644 --- a/distribution/src/assembly/cli.xml +++ b/distribution/src/assembly/cli.xml @@ -54,9 +54,11 @@ *data.* *schema.* *tsfile.* + *logical-backup* **/*data.* **/*schema.* **/*tsfile.* + **/*logical-backup* 0755 diff --git a/iotdb-client/cli/pom.xml b/iotdb-client/cli/pom.xml index 5de247a084a09..c9824ef4aec11 100644 --- a/iotdb-client/cli/pom.xml +++ b/iotdb-client/cli/pom.xml @@ -124,6 +124,10 @@ commons-cli commons-cli + + com.google.code.gson + gson + org.jline jline @@ -204,6 +208,7 @@ org.apache.iotdb:iotdb-server org/apache/iotdb/db/utils/* + org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferDataNodeHandshakeV2Req.class diff --git a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java index 0329812f2d4d3..77888e65768a0 100644 --- a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java @@ -96,4 +96,61 @@ private CliMessages() {} public static final String LOG_INPUT_TIME_FORMAT_ARG_NOT_SUPPORTED_00172A7B = "Input time format {} is not supported, "; public static final String LOG_PLEASE_INPUT_LIKE_YYYY_MM_DD_HH_MM_SS_SSS_9318BFC7 = "please input like yyyy-MM-dd\\ HH:mm:ss.SSS or yyyy-MM-dd'T'HH:mm:ss.SSS%n"; + // Pipe logical backup tool + public static final String EXCEPTION_LOGICAL_BACKUP_COMMAND_FAILED_ARG_9973B0C0 = + "Logical backup command failed: %s"; + public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619 = + "Unknown logical backup command: %s"; + public static final String EXCEPTION_OUTPUT_IS_REQUIRED_FOR_LOGICAL_BACKUP_EXPORT_603340B1 = + "--output is required for logical backup export"; + public static final String LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0 = + "Stream %s: %d records, %d committed event groups"; + public static final String + LOG_LOGICAL_BACKUP_VERIFIED_ARG_STREAMS_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_5210360B = + "Logical backup verified: %d streams, %d records, %d committed event groups"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994 = + "Logical backup export target must not be inside source"; + public static final String LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280 = + "Logical backup exported from %s to %s"; + public static final String LOG_DRY_RUN_COMPLETED_NO_DATA_WAS_WRITTEN_38AE244B = + "Dry run completed; no data was written"; + public static final String + LOG_LOGICAL_BACKUP_IMPORT_COMPLETED_ARG_EVENT_GROUPS_CHECKPOINT_ARG_16F6A72D = + "Logical backup import completed: %d event groups, checkpoint %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA = + "Logical backup option --%s is required"; + public static final String + EXCEPTION_LOGICAL_BACKUP_PASSWORD_ENVIRONMENT_VARIABLE_IS_NOT_SET_616738A2 = + "Logical backup password environment variable is not set"; + public static final String EXCEPTION_LOGICAL_BACKUP_HANDSHAKE_FAILED_ARG_7CDD4697 = + "Logical backup handshake failed: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11 = + "Logical backup request type %d failed: %s"; + public static final String + LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E = + "Use --input to specify the input. Export also requires --output; import requires --host and --port. Use --password-stdin or --password-env to avoid command-line passwords."; + public static final String + LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_RESTORE_STATS_BFF9FDC2 = + "pipe-logical-backup "; + public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 = + "Logical backup archive entry is unsafe: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 = + "Logical backup archive exceeds safety limit"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0 = + "Logical backup export source must be a directory or manifest: %s"; + public static final String + EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9 = + "Specify exactly one of --password-stdin and --password-env for logical backup import"; + public static final String EXCEPTION_NO_PASSWORD_WAS_READ_FROM_STANDARD_INPUT_6294AB8E = + "No password was read from standard input"; + public static final String + EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_DOES_NOT_MATCH_THE_SOURCE_OR_TARGET_ARG_B978184D = + "Logical backup checkpoint does not match the source or target: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C = + "Logical backup checkpoint is invalid: %s"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1 = + "Unsupported logical backup export format: %s"; + } diff --git a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java index 61a14d94d5884..530b504b29ce0 100644 --- a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java @@ -92,4 +92,61 @@ private CliMessages() {} public static final String LOG_INPUT_TIME_FORMAT_ARG_NOT_SUPPORTED_00172A7B = "不支持输入时间格式 {}，"; public static final String LOG_PLEASE_INPUT_LIKE_YYYY_MM_DD_HH_MM_SS_SSS_9318BFC7 = "请输入类似 yyyy-MM-dd\\ HH:mm:ss.SSS 或 yyyy-MM-dd'T'HH:mm:ss.SSS 的格式%n"; + // Pipe 逻辑备份工具 + public static final String EXCEPTION_LOGICAL_BACKUP_COMMAND_FAILED_ARG_9973B0C0 = + "逻辑备份命令执行失败：%s"; + public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619 = + "未知的逻辑备份命令：%s"; + public static final String EXCEPTION_OUTPUT_IS_REQUIRED_FOR_LOGICAL_BACKUP_EXPORT_603340B1 = + "逻辑备份导出必须指定 --output"; + public static final String LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0 = + "流 %s：%d 条记录，%d 个已提交事件组"; + public static final String + LOG_LOGICAL_BACKUP_VERIFIED_ARG_STREAMS_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_5210360B = + "逻辑备份校验通过：%d 个流，%d 条记录，%d 个已提交事件组"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994 = + "逻辑备份导出目标不能位于源目录内"; + public static final String LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280 = + "逻辑备份已从 %s 导出到 %s"; + public static final String LOG_DRY_RUN_COMPLETED_NO_DATA_WAS_WRITTEN_38AE244B = + "试运行完成，未写入数据"; + public static final String + LOG_LOGICAL_BACKUP_IMPORT_COMPLETED_ARG_EVENT_GROUPS_CHECKPOINT_ARG_16F6A72D = + "逻辑备份导入完成：%d 个事件组，checkpoint 为 %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA = + "逻辑备份选项 --%s 为必填项"; + public static final String + EXCEPTION_LOGICAL_BACKUP_PASSWORD_ENVIRONMENT_VARIABLE_IS_NOT_SET_616738A2 = + "未设置逻辑备份密码环境变量"; + public static final String EXCEPTION_LOGICAL_BACKUP_HANDSHAKE_FAILED_ARG_7CDD4697 = + "逻辑备份握手失败：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11 = + "逻辑备份请求类型 %d 执行失败：%s"; + public static final String + LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E = + "使用 --input 指定输入。导出还需要 --output；导入需要 --host 和 --port。请使用 --password-stdin 或 --password-env，避免密码出现在命令行中。"; + public static final String + LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_RESTORE_STATS_BFF9FDC2 = + "pipe-logical-backup "; + public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 = + "逻辑备份归档项路径不安全：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 = + "逻辑备份归档超过安全限制"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0 = + "逻辑备份导出源必须是目录或 manifest：%s"; + public static final String + EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9 = + "逻辑备份导入必须且只能指定 --password-stdin 或 --password-env 中的一项"; + public static final String EXCEPTION_NO_PASSWORD_WAS_READ_FROM_STANDARD_INPUT_6294AB8E = + "未能从标准输入读取密码"; + public static final String + EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_DOES_NOT_MATCH_THE_SOURCE_OR_TARGET_ARG_B978184D = + "逻辑备份 checkpoint 与源备份或目标实例不匹配：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C = + "逻辑备份 checkpoint 无效：%s"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1 = + "不支持的逻辑备份导出格式：%s"; + } diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java new file mode 100644 index 0000000000000..5d80243f3adf5 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java @@ -0,0 +1,817 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.pipe; + +import org.apache.iotdb.cli.i18n.CliMessages; +import org.apache.iotdb.commons.client.property.ThriftClientProperty; +import org.apache.iotdb.commons.pipe.sink.client.IoTDBSyncClient; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.EventGroup; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupFormat; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecord; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.common.PipeTransferHandshakeConstant; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferDataNodeHandshakeV2Req; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonParseException; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.thrift.TException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +/** Command line inspector, verifier, archiver and importer for Pipe logical backups. */ +public final class PipeLogicalBackupTool { + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + private static final String COMMAND_INSPECT = "inspect"; + private static final String COMMAND_VERIFY = "verify"; + private static final String COMMAND_EXPORT = "export"; + private static final String COMMAND_IMPORT = "import"; + private static final String COMMAND_RESTORE = "restore"; + private static final String COMMAND_STATS = "stats"; + private static final String OPTION_INPUT = "input"; + private static final String OPTION_OUTPUT = "output"; + private static final String OPTION_RESUME = "resume"; + private static final String OPTION_FORMAT = "format"; + private static final String OPTION_DEEP = "deep"; + // Compatibility aliases for the initial command-line prototype. + private static final String OPTION_SOURCE = "source"; + private static final String OPTION_TARGET = "target"; + private static final String OPTION_HOST = "host"; + private static final String OPTION_PORT = "port"; + private static final String OPTION_USER = "user"; + private static final String OPTION_PASSWORD_STDIN = "password-stdin"; + private static final String OPTION_PASSWORD_ENV = "password-env"; + private static final String OPTION_CHECKPOINT = "checkpoint"; + private static final String OPTION_DRY_RUN = "dry-run"; + private static final String OPTION_ALLOW_INCOMPLETE = "allow-incomplete"; + private static final String OPTION_JSON = "json"; + private static final int MAX_ARCHIVE_ENTRIES = 1_000_000; + private static final long MAX_ARCHIVE_BYTES = 4L * 1024 * 1024 * 1024 * 1024; + private static final String CHECKPOINT_FORMAT_NAME = + "iotdb-pipe-logical-backup-import-checkpoint"; + private static final String CHECKPOINT_FORMAT_VERSION = "1.0"; + + private PipeLogicalBackupTool() {} + + public static void main(final String[] args) { + int exitCode = 0; + try { + exitCode = run(args); + } catch (final Exception e) { + System.err.println( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_COMMAND_FAILED_ARG_9973B0C0, e.getMessage())); + exitCode = 1; + } + if (exitCode != 0) { + System.exit(exitCode); + } + } + + static int run(final String[] args) throws Exception { + if (args.length == 0 || "--help".equals(args[0]) || "-h".equals(args[0])) { + printUsage(); + return 0; + } + final String command = args[0].toLowerCase(java.util.Locale.ROOT); + final CommandLine line = parse(command, java.util.Arrays.copyOfRange(args, 1, args.length)); + switch (command) { + case COMMAND_INSPECT: + return inspect(line); + case COMMAND_VERIFY: + return verify(line); + case COMMAND_EXPORT: + return export(line); + case COMMAND_IMPORT: + case COMMAND_RESTORE: + return importBackup(line); + case COMMAND_STATS: + return inspect(line); + default: + throw new ParseException( + String.format( + CliMessages.EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619, command)); + } + } + + private static CommandLine parse(final String command, final String[] args) + throws ParseException { + final Options options = new Options(); + addOption(options, OPTION_INPUT, true, false); + addOption(options, OPTION_OUTPUT, true, false); + addOption(options, OPTION_RESUME, true, false); + addOption(options, OPTION_FORMAT, true, false); + addOption(options, OPTION_DEEP, false, false); + addOption(options, OPTION_SOURCE, true, false); + addOption(options, OPTION_TARGET, true, false); + addOption(options, OPTION_HOST, true, false); + addOption(options, OPTION_PORT, true, false); + addOption(options, OPTION_USER, true, false); + addOption(options, OPTION_PASSWORD_STDIN, false, false); + addOption(options, OPTION_PASSWORD_ENV, true, false); + addOption(options, OPTION_CHECKPOINT, true, false); + addOption(options, OPTION_DRY_RUN, false, false); + addOption(options, OPTION_ALLOW_INCOMPLETE, false, false); + addOption(options, OPTION_JSON, false, false); + final CommandLine line = new DefaultParser().parse(options, args); + final String input = optionValue(line, OPTION_INPUT, OPTION_SOURCE); + if (input == null || input.isBlank()) { + throw new ParseException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, OPTION_INPUT)); + } + final String output = optionValue(line, OPTION_OUTPUT, OPTION_TARGET); + if (COMMAND_EXPORT.equals(command) && (output == null || output.isBlank())) { + throw new ParseException( + CliMessages.EXCEPTION_OUTPUT_IS_REQUIRED_FOR_LOGICAL_BACKUP_EXPORT_603340B1); + } + final String format = line.getOptionValue(OPTION_FORMAT, "binary"); + if (COMMAND_EXPORT.equals(command) && !"binary".equalsIgnoreCase(format)) { + throw new ParseException( + String.format( + CliMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1, format)); + } + return line; + } + + private static void addOption( + final Options options, final String name, final boolean hasArg, final boolean required) { + options.addOption(Option.builder().longOpt(name).hasArg(hasArg).required(required).build()); + } + + private static int inspect(final CommandLine line) throws IOException { + final List streams = read(line); + if (line.hasOption(OPTION_JSON)) { + final List> output = new ArrayList<>(); + for (final BackupStream stream : streams) { + final Map item = new HashMap<>(); + item.put("manifest", stream.getManifest()); + item.put("recordCount", stream.getRecords().size()); + item.put("eventGroupCount", stream.getEventGroups().size()); + output.add(item); + } + System.out.println(GSON.toJson(output)); + return 0; + } + for (final BackupStream stream : streams) { + System.out.println( + String.format( + CliMessages.LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0, + stream.getManifest().streamId, + stream.getRecords().size(), + stream.getEventGroups().size())); + } + return 0; + } + + private static int verify(final CommandLine line) throws IOException { + final List streams = read(line); + long recordCount = 0; + long eventCount = 0; + for (final BackupStream stream : streams) { + recordCount += stream.getRecords().size(); + eventCount += stream.getEventGroups().size(); + } + System.out.println( + String.format( + CliMessages + .LOG_LOGICAL_BACKUP_VERIFIED_ARG_STREAMS_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_5210360B, + streams.size(), + recordCount, + eventCount)); + return 0; + } + + private static int export(final CommandLine line) throws IOException { + final Path source = Path.of(input(line)).toAbsolutePath().normalize(); + final Path target = + Path.of(optionValue(line, OPTION_OUTPUT, OPTION_TARGET)).toAbsolutePath().normalize(); + final List streams = read(line); + final Path root; + if (Files.isDirectory(source)) { + root = source; + } else if (Files.isRegularFile(source) + && "manifest.json".equals(source.getFileName().toString())) { + root = source.getParent(); + } else { + throw new IOException( + String.format( + CliMessages + .EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0, + source)); + } + if (target.startsWith(root)) { + throw new IOException( + CliMessages.EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994); + } + if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { + throw new java.nio.file.FileAlreadyExistsException(target.toString()); + } + final Path parent = target.toAbsolutePath().normalize().getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + final Path temporary = Files.createTempFile(parent, "pipe-logical-backup-", ".tmp"); + boolean published = false; + try { + try (final ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(temporary))) { + for (final Path file : exportFiles(root, streams)) { + final String entryName = root.relativize(file).toString().replace('\\', '/'); + zip.putNextEntry(new ZipEntry(entryName)); + Files.copy(file, zip); + zip.closeEntry(); + } + } + try (final java.nio.channels.FileChannel archiveChannel = + java.nio.channels.FileChannel.open(temporary, StandardOpenOption.WRITE)) { + archiveChannel.force(true); + } + try { + Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE); + } catch (final java.nio.file.AtomicMoveNotSupportedException e) { + Files.move(temporary, target); + } + published = true; + } finally { + if (!published) { + Files.deleteIfExists(temporary); + } + } + System.out.println( + String.format( + CliMessages.LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280, source, target)); + return 0; + } + + private static List exportFiles(final Path root, final List streams) + throws IOException { + final Set files = new LinkedHashSet<>(); + for (final BackupStream stream : streams) { + final Path manifestPath = stream.getManifestPath().toAbsolutePath().normalize(); + if (!manifestPath.startsWith(root)) { + throw new IOException( + String.format( + CliMessages + .EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0, + root)); + } + files.add(manifestPath); + final Path streamDirectory = manifestPath.getParent(); + for (final LogicalBackupManifest.Segment segment : stream.getManifest().segments) { + files.add(streamDirectory.resolve(segment.file).normalize()); + } + } + final List sortedFiles = new ArrayList<>(files); + sortedFiles.sort(Comparator.comparing(Path::toString)); + return sortedFiles; + } + + private static int importBackup(final CommandLine line) throws Exception { + final List streams = read(line); + final boolean dryRun = line.hasOption(OPTION_DRY_RUN); + if (dryRun) { + System.out.println(CliMessages.LOG_DRY_RUN_COMPLETED_NO_DATA_WAS_WRITTEN_38AE244B); + for (final BackupStream stream : streams) { + System.out.println( + String.format( + CliMessages.LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0, + stream.getManifest().streamId, + stream.getRecords().size(), + stream.getEventGroups().size())); + } + return 0; + } + final String host = required(line, OPTION_HOST); + final int port = Integer.parseInt(required(line, OPTION_PORT)); + final String user = line.getOptionValue(OPTION_USER, "root"); + final String password = password(line); + final Path checkpoint = checkpointPath(line); + final ImportCheckpoint checkpointState = readCheckpoint(checkpoint, streams, host, port, user); + try (final IoTDBSyncClient client = + new IoTDBSyncClient( + new ThriftClientProperty.Builder().setConnectionTimeoutMs(20_000).build(), + host, + port, + false, + "", + "")) { + handshake(client, streams.get(0).getManifest().timestampPrecision, user, password); + long importedGroups = 0; + for (final BackupStream stream : streams) { + final String streamId = stream.getManifest().streamId; + final long lastApplied = checkpointState.appliedSequences.getOrDefault(streamId, -1L); + for (final EventGroup group : stream.getEventGroups()) { + if (group.getLastSequence() <= lastApplied) { + continue; + } + for (final LogicalBackupRecord record : group.getRequests()) { + transfer(client, record.toTPipeTransferReq()); + } + checkpointState.appliedSequences.put(streamId, group.getLastSequence()); + writeCheckpoint(checkpoint, checkpointState); + importedGroups++; + } + } + writeCheckpoint(checkpoint, checkpointState); + System.out.println( + String.format( + CliMessages + .LOG_LOGICAL_BACKUP_IMPORT_COMPLETED_ARG_EVENT_GROUPS_CHECKPOINT_ARG_16F6A72D, + importedGroups, + checkpoint)); + } + return 0; + } + + private static List read(final CommandLine line) throws IOException { + final String source = input(line); + try (final BackupInput input = BackupInput.open(Path.of(source))) { + return new LogicalBackupArchiveReader() + .read(input.getPath(), line.hasOption(OPTION_ALLOW_INCOMPLETE)); + } + } + + private static String input(final CommandLine line) throws IOException { + final String value = optionValue(line, OPTION_INPUT, OPTION_SOURCE); + if (value == null || value.isBlank()) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, OPTION_INPUT)); + } + return value; + } + + private static String optionValue( + final CommandLine line, final String preferredOption, final String compatibilityOption) { + return line.hasOption(preferredOption) + ? line.getOptionValue(preferredOption) + : line.getOptionValue(compatibilityOption); + } + + private static String required(final CommandLine line, final String option) throws IOException { + final String value = line.getOptionValue(option); + if (value == null || value.isBlank()) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, option)); + } + return value; + } + + private static String password(final CommandLine line) throws IOException { + final boolean fromStandardInput = line.hasOption(OPTION_PASSWORD_STDIN); + final boolean fromEnvironment = line.hasOption(OPTION_PASSWORD_ENV); + if (fromStandardInput == fromEnvironment) { + throw new IOException( + CliMessages + .EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9); + } + if (fromStandardInput) { + final BufferedReader reader = + new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8)); + final String value = reader.readLine(); + if (value == null) { + throw new IOException( + CliMessages.EXCEPTION_NO_PASSWORD_WAS_READ_FROM_STANDARD_INPUT_6294AB8E); + } + return value; + } + if (fromEnvironment) { + final String value = System.getenv(line.getOptionValue(OPTION_PASSWORD_ENV)); + if (value == null) { + throw new IOException( + CliMessages.EXCEPTION_LOGICAL_BACKUP_PASSWORD_ENVIRONMENT_VARIABLE_IS_NOT_SET_616738A2); + } + return value; + } + throw new IOException( + CliMessages + .EXCEPTION_SPECIFY_EXACTLY_ONE_OF_PASSWORD_STDIN_AND_PASSWORD_ENV_FOR_LOGICAL_BACKUP_IMPORT_A96813D9); + } + + private static void handshake( + final IoTDBSyncClient client, + final String timestampPrecision, + final String user, + final String password) + throws IOException, TException { + final Map params = new HashMap<>(); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLUSTER_ID, getClusterId()); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_TIME_PRECISION, timestampPrecision); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USER_ID, "-1"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USERNAME, user); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_PASSWORD, password); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLI_HOSTNAME, "pipe-logical-backup"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_LOAD_TSFILE_STRATEGY, "sync"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_VALIDATE_TSFILE, "false"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST, "true"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_SKIP_IF, "false"); + final TPipeTransferResp response = + client.pipeTransfer(PipeTransferDataNodeHandshakeV2Req.toTPipeTransferReq(params)); + if (response == null + || response.getStatus() == null + || response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_HANDSHAKE_FAILED_ARG_7CDD4697, + response == null ? "null" : response.getStatus())); + } + } + + private static void transfer(final IoTDBSyncClient client, final TPipeTransferReq request) + throws TException, IOException { + final TPipeTransferResp response = client.pipeTransfer(request); + if (response == null + || response.getStatus() == null + || (response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() + && response.getStatus().getCode() + != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode())) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11, + request.getType(), + response == null ? "null" : response.getStatus())); + } + } + + private static Path checkpointPath(final CommandLine line) throws IOException { + final String configuredCheckpoint = optionValue(line, OPTION_RESUME, OPTION_CHECKPOINT); + if (configuredCheckpoint != null) { + return Path.of(configuredCheckpoint); + } + final Path source = Path.of(input(line)).toAbsolutePath().normalize(); + return source.resolveSibling(source.getFileName() + ".import.checkpoint.json"); + } + + private static ImportCheckpoint readCheckpoint( + final Path checkpoint, + final List streams, + final String host, + final int port, + final String user) + throws IOException { + final Map sourceStreams = sourceStreamIdentities(streams); + if (!Files.exists(checkpoint)) { + final ImportCheckpoint state = new ImportCheckpoint(); + state.formatName = CHECKPOINT_FORMAT_NAME; + state.formatVersion = CHECKPOINT_FORMAT_VERSION; + state.sourceStreams = sourceStreams; + state.targetHost = host; + state.targetPort = port; + state.targetUser = user; + return state; + } + final ImportCheckpoint state; + try { + state = + GSON.fromJson( + Files.readString(checkpoint, StandardCharsets.UTF_8), ImportCheckpoint.class); + } catch (final JsonParseException | NullPointerException e) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C, checkpoint), + e); + } + if (state == null + || !CHECKPOINT_FORMAT_NAME.equals(state.formatName) + || !CHECKPOINT_FORMAT_VERSION.equals(state.formatVersion) + || state.sourceStreams == null + || state.appliedSequences == null) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C, checkpoint)); + } + if (!sourceStreams.equals(state.sourceStreams) + || !host.equals(state.targetHost) + || port != state.targetPort + || !user.equals(state.targetUser) + || !checkpointSequencesAreValid(streams, state.appliedSequences)) { + throw new IOException( + String.format( + CliMessages + .EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_DOES_NOT_MATCH_THE_SOURCE_OR_TARGET_ARG_B978184D, + checkpoint)); + } + return state; + } + + static Map sourceStreamIdentities(final List streams) { + final Map identities = new LinkedHashMap<>(); + for (final BackupStream stream : streams) { + final LogicalBackupManifest manifest = stream.getManifest(); + final MessageDigest digest = LogicalBackupFormat.newSha256(); + updateDigest(digest, manifest.backupId); + updateDigest(digest, manifest.streamId); + updateDigest(digest, manifest.pipeName); + updateDigest(digest, manifest.pipeCreationTime); + updateDigest(digest, manifest.sourceClusterId); + updateDigest(digest, manifest.regionId); + updateDigest(digest, manifest.streamType); + updateDigest(digest, manifest.segments.size()); + for (final LogicalBackupManifest.Segment segment : manifest.segments) { + updateDigest(digest, segment.file); + updateDigest(digest, segment.segmentId); + updateDigest(digest, segment.firstSequence); + updateDigest(digest, segment.lastSequence); + updateDigest(digest, segment.recordCount); + updateDigest(digest, segment.sizeBytes); + updateDigest(digest, segment.sha256); + updateDigest(digest, segment.status); + } + updateDigest(digest, stream.getRecords().size()); + for (final LogicalBackupRecord record : stream.getRecords()) { + digest.update(record.getRecordType().getCode()); + digest.update( + ByteBuffer.allocate(Long.BYTES * 4 + Integer.BYTES + Byte.BYTES + Short.BYTES) + .putLong(record.getSequence()) + .putLong(record.getEventGroupId().getMostSignificantBits()) + .putLong(record.getEventGroupId().getLeastSignificantBits()) + .putInt(record.getOperationIndex()) + .putLong(record.getEventTime()) + .put(record.getRequestVersion()) + .putShort(record.getRequestType()) + .array()); + updateDigest(digest, record.getMetadata()); + updateDigest(digest, record.getPayload()); + } + identities.put(manifest.streamId, LogicalBackupFormat.toHex(digest.digest())); + } + return identities; + } + + private static void updateDigest(final MessageDigest digest, final String value) { + if (value == null) { + digest.update((byte) 0); + return; + } + digest.update((byte) 1); + updateDigest(digest, value.getBytes(StandardCharsets.UTF_8)); + } + + private static void updateDigest(final MessageDigest digest, final byte[] value) { + updateDigest(digest, value.length); + digest.update(value); + } + + private static void updateDigest(final MessageDigest digest, final int value) { + digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(value).array()); + } + + private static void updateDigest(final MessageDigest digest, final long value) { + digest.update(ByteBuffer.allocate(Long.BYTES).putLong(value).array()); + } + + private static boolean checkpointSequencesAreValid( + final List streams, final Map appliedSequences) { + if (appliedSequences.size() > streams.size()) { + return false; + } + for (final BackupStream stream : streams) { + if (!appliedSequences.containsKey(stream.getManifest().streamId)) { + continue; + } + final Long applied = appliedSequences.get(stream.getManifest().streamId); + if (applied == null) { + return false; + } + if (applied == -1) { + continue; + } + if (stream.getEventGroups().stream().noneMatch(group -> group.getLastSequence() == applied)) { + return false; + } + } + return appliedSequences.keySet().stream() + .allMatch( + streamId -> + streams.stream() + .anyMatch(stream -> stream.getManifest().streamId.equals(streamId))); + } + + private static void writeCheckpoint(final Path checkpoint, final ImportCheckpoint checkpointState) + throws IOException { + final Path parent = checkpoint.toAbsolutePath().normalize().getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + final Path temporary = checkpoint.resolveSibling(checkpoint.getFileName() + ".tmp"); + Files.deleteIfExists(temporary); + final ByteBuffer checkpointBytes = StandardCharsets.UTF_8.encode(GSON.toJson(checkpointState)); + try (final java.nio.channels.FileChannel checkpointChannel = + java.nio.channels.FileChannel.open( + temporary, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { + while (checkpointBytes.hasRemaining()) { + checkpointChannel.write(checkpointBytes); + } + checkpointChannel.force(true); + } + try { + Files.move( + temporary, + checkpoint, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (final java.nio.file.AtomicMoveNotSupportedException e) { + Files.move(temporary, checkpoint, StandardCopyOption.REPLACE_EXISTING); + } + } + + private static String getClusterId() { + final byte[] bytes = new byte[16]; + new SecureRandom().nextBytes(bytes); + return "PIPE-LOGICAL-BACKUP-" + UUID.nameUUIDFromBytes(bytes); + } + + private static void printUsage() { + final HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp( + new PrintWriter(System.out), + 120, + CliMessages.LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_RESTORE_STATS_BFF9FDC2, + CliMessages + .LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E, + optionsForHelp(), + 2, + 2, + "", + true); + } + + private static Options optionsForHelp() { + final Options options = new Options(); + addOption(options, OPTION_INPUT, true, false); + addOption(options, OPTION_OUTPUT, true, false); + addOption(options, OPTION_RESUME, true, false); + addOption(options, OPTION_FORMAT, true, false); + addOption(options, OPTION_DEEP, false, false); + addOption(options, OPTION_HOST, true, false); + addOption(options, OPTION_PORT, true, false); + addOption(options, OPTION_USER, true, false); + addOption(options, OPTION_PASSWORD_STDIN, false, false); + addOption(options, OPTION_PASSWORD_ENV, true, false); + addOption(options, OPTION_CHECKPOINT, true, false); + addOption(options, OPTION_DRY_RUN, false, false); + addOption(options, OPTION_ALLOW_INCOMPLETE, false, false); + addOption(options, OPTION_JSON, false, false); + return options; + } + + private static class ImportCheckpoint { + private String formatName; + private String formatVersion; + private Map sourceStreams = new LinkedHashMap<>(); + private String targetHost; + private int targetPort; + private String targetUser; + private Map appliedSequences = new LinkedHashMap<>(); + } + + private static class BackupInput implements AutoCloseable { + private final Path path; + private final Path temporaryDirectory; + + private BackupInput(final Path path, final Path temporaryDirectory) { + this.path = path; + this.temporaryDirectory = temporaryDirectory; + } + + private static BackupInput open(final Path source) throws IOException { + final Path normalizedSource = source.toAbsolutePath().normalize(); + if (!Files.isRegularFile(normalizedSource) + || "manifest.json".equals(normalizedSource.getFileName().toString())) { + return new BackupInput(normalizedSource, null); + } + final Path temporaryDirectory = Files.createTempDirectory("iotdb-logical-backup-"); + try { + extract(normalizedSource, temporaryDirectory); + return new BackupInput(temporaryDirectory, temporaryDirectory); + } catch (final IOException | RuntimeException e) { + deleteRecursively(temporaryDirectory); + throw e; + } + } + + private static void extract(final Path archive, final Path target) throws IOException { + int entryCount = 0; + long totalBytes = 0; + final byte[] buffer = new byte[64 * 1024]; + try (final ZipInputStream zip = new ZipInputStream(Files.newInputStream(archive))) { + ZipEntry entry; + while ((entry = zip.getNextEntry()) != null) { + if (++entryCount > MAX_ARCHIVE_ENTRIES) { + throw new IOException( + CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432); + } + final Path output = target.resolve(entry.getName()).normalize(); + if (!output.startsWith(target)) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152, + entry.getName())); + } + if (entry.isDirectory()) { + Files.createDirectories(output); + } else { + final Path parent = output.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + try (final OutputStream stream = + Files.newOutputStream( + output, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { + int length; + while ((length = zip.read(buffer)) >= 0) { + if (length == 0) { + continue; + } + totalBytes += length; + if (totalBytes > MAX_ARCHIVE_BYTES) { + throw new IOException( + CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432); + } + stream.write(buffer, 0, length); + } + } + } + zip.closeEntry(); + } + } + } + + private Path getPath() { + return path; + } + + @Override + public void close() throws IOException { + if (temporaryDirectory != null) { + deleteRecursively(temporaryDirectory); + } + } + + private static void deleteRecursively(final Path directory) throws IOException { + if (!Files.exists(directory)) { + return; + } + try (final java.util.stream.Stream paths = Files.walk(directory)) { + for (final Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) { + Files.deleteIfExists(path); + } + } + } + } +} diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java new file mode 100644 index 0000000000000..0fe45b22d8970 --- /dev/null +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.pipe; + +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupFormat; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.commons.cli.ParseException; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; + +public class PipeLogicalBackupToolTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testDirectoryExportZipVerifyAndDryRun() throws Exception { + final Path root = temporaryFolder.getRoot().toPath(); + final Path source = root.resolve("source"); + writeBackup(source); + + Assert.assertEquals( + 0, + PipeLogicalBackupTool.run(new String[] {"verify", "--input", source.toString(), "--deep"})); + + final Path archive = root.resolve("backup.zip"); + Assert.assertEquals( + 0, + PipeLogicalBackupTool.run( + new String[] { + "export", + "--input", + source.toString(), + "--output", + archive.toString(), + "--format=binary" + })); + Assert.assertTrue(Files.isRegularFile(archive)); + try (final ZipFile zip = new ZipFile(archive.toFile())) { + Assert.assertNotNull(zip.getEntry(LogicalBackupFormat.MANIFEST_FILE_NAME)); + Assert.assertNotNull(zip.getEntry("segment-00000000000000000000.pwal")); + Assert.assertNull(zip.getEntry(".backup.lock")); + Assert.assertEquals(2, zip.size()); + } + + Assert.assertEquals( + 0, PipeLogicalBackupTool.run(new String[] {"verify", "--input", archive.toString()})); + Assert.assertEquals( + 0, + PipeLogicalBackupTool.run( + new String[] {"import", "--input", archive.toString(), "--dry-run"})); + + final byte[] originalArchive = Files.readAllBytes(archive); + try { + PipeLogicalBackupTool.run( + new String[] {"export", "--input", source.toString(), "--output", archive.toString()}); + Assert.fail(); + } catch (final java.nio.file.FileAlreadyExistsException expected) { + Assert.assertArrayEquals(originalArchive, Files.readAllBytes(archive)); + } + } + + @Test + public void testCheckpointSourceIdentityIncludesContent() throws Exception { + final Path source = temporaryFolder.getRoot().toPath().resolve("source"); + writeBackup(source); + final List streams = new LogicalBackupArchiveReader().read(source, false); + final Map before = PipeLogicalBackupTool.sourceStreamIdentities(streams); + + streams.get(0).getManifest().segments.get(0).sha256 = "different-content"; + final Map after = PipeLogicalBackupTool.sourceStreamIdentities(streams); + + Assert.assertNotEquals(before, after); + } + + @Test + public void testZipSlipIsRejected() throws Exception { + final Path archive = temporaryFolder.getRoot().toPath().resolve("unsafe.zip"); + try (final ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(archive))) { + zip.putNextEntry(new ZipEntry("../outside")); + zip.write(1); + zip.closeEntry(); + } + + try { + PipeLogicalBackupTool.run(new String[] {"verify", "--input", archive.toString()}); + Assert.fail(); + } catch (final IOException expected) { + // Expected. + } + } + + @Test + public void testPlaintextPasswordOptionIsRejected() throws Exception { + final Path source = temporaryFolder.getRoot().toPath().resolve("source"); + writeBackup(source); + + try { + PipeLogicalBackupTool.run( + new String[] { + "import", "--source", source.toString(), "--dry-run", "--password", "secret" + }); + Assert.fail(); + } catch (final ParseException expected) { + // Expected. + } + } + + private static void writeBackup(final Path directory) throws Exception { + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = "backup"; + manifest.pipeName = "pipe"; + manifest.pipeCreationTime = 1; + manifest.streamId = "data-1"; + manifest.streamType = "data"; + manifest.timestampPrecision = "ms"; + try (final LogicalBackupWriter writer = + new LogicalBackupWriter( + directory, manifest, 4096, 1024, LogicalBackupWriter.FsyncPolicy.ALWAYS, 1, 1, false)) { + writer.writeEvent( + UUID.randomUUID(), + 1, + Collections.singletonList( + new TPipeTransferReq() + .setVersion((byte) 1) + .setType((short) 10) + .setBody(ByteBuffer.wrap(new byte[] {3}))), + "metadata"); + } + } +} diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index b818a52e83877..d530fc7cf1bd3 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2643,4 +2643,15 @@ private DataNodePipeMessages() {} "Interrupted while reading OPC UA server operation limits, use defaults: maxNodesPerWrite={}, maxNodesPerNodeManagement={}"; public static final String LOG_FAILED_TO_READ_OPC_UA_SERVER_OPERATION_LIMITS_USE_DEFAULTS_MAXNODESPERWRITE_ARG_MAXNODESPERNODEMANAGEMENT_ARG_65460871 = "Failed to read OPC UA server operation limits, use defaults: maxNodesPerWrite={}, maxNodesPerNodeManagement={}"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_POLICY_ARG_BF42E405 = + "Unsupported logical backup event policy: %s"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 = + "Unsupported logical backup resume policy: %s"; + public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 = + "Unsupported Pipe event for logical backup: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22 = + "Logical backup ID must resolve below the configured backup directory: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_MUST_NOT_BE_EMPTY_99DB3EDB = + "Logical backup directory must not be empty"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index fe768a1f0181c..49575e46abb91 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2467,4 +2467,15 @@ private DataNodePipeMessages() {} "读取 OPC UA 服务器操作限制时被中断，使用默认值：maxNodesPerWrite={}，maxNodesPerNodeManagement={}"; public static final String LOG_FAILED_TO_READ_OPC_UA_SERVER_OPERATION_LIMITS_USE_DEFAULTS_MAXNODESPERWRITE_ARG_MAXNODESPERNODEMANAGEMENT_ARG_65460871 = "读取 OPC UA 服务器操作限制失败，使用默认值：maxNodesPerWrite={}，maxNodesPerNodeManagement={}"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_POLICY_ARG_BF42E405 = + "不支持的逻辑备份事件策略：%s"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 = + "不支持的逻辑备份恢复策略：%s"; + public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 = + "逻辑备份不支持 Pipe 事件：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22 = + "Pipe 逻辑备份 ID 必须解析到配置的备份目录下：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_MUST_NOT_BE_EMPTY_99DB3EDB = + "Pipe 逻辑备份目录不能为空"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java index c8e87890afd69..08061c954afcb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.pipe.sink.protocol.airgap.IoTDBDataRegionAirGapSink; import org.apache.iotdb.db.pipe.sink.protocol.iotconsensusv2.IoTConsensusV2AsyncSink; import org.apache.iotdb.db.pipe.sink.protocol.legacy.IoTDBLegacyPipeSink; +import org.apache.iotdb.db.pipe.sink.protocol.logicalbackup.LogicalBackupSink; import org.apache.iotdb.db.pipe.sink.protocol.opcda.OpcDaSink; import org.apache.iotdb.db.pipe.sink.protocol.opcua.OpcUaSink; import org.apache.iotdb.db.pipe.sink.protocol.thrift.async.IoTDBDataRegionAsyncSink; @@ -74,6 +75,9 @@ protected void initConstructors() { BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingSink::new); pluginConstructors.put( BuiltinPipePlugin.WRITE_BACK_CONNECTOR.getPipePluginName(), WriteBackSink::new); + pluginConstructors.put( + BuiltinPipePlugin.LOGICAL_BACKUP_CONNECTOR.getPipePluginName(), + () -> new LogicalBackupSink("data")); pluginConstructors.put( BuiltinPipePlugin.IOTDB_THRIFT_SINK.getPipePluginName(), IoTDBDataRegionAsyncSink::new); @@ -96,6 +100,9 @@ protected void initConstructors() { BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingSink::new); pluginConstructors.put( BuiltinPipePlugin.WRITE_BACK_SINK.getPipePluginName(), WriteBackSink::new); + pluginConstructors.put( + BuiltinPipePlugin.LOGICAL_BACKUP_SINK.getPipePluginName(), + () -> new LogicalBackupSink("data")); pluginConstructors.put( BuiltinPipePlugin.SUBSCRIPTION_SINK.getPipePluginName(), DoNothingSink::new); pluginConstructors.put( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java index 160ecf54c0158..a4eba6400c178 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.donothing.DoNothingSink; import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeSinkConstructor; import org.apache.iotdb.db.pipe.sink.protocol.airgap.IoTDBSchemaRegionAirGapSink; +import org.apache.iotdb.db.pipe.sink.protocol.logicalbackup.LogicalBackupSink; import org.apache.iotdb.db.pipe.sink.protocol.thrift.sync.IoTDBSchemaRegionSink; import org.apache.iotdb.pipe.api.PipeConnector; @@ -46,6 +47,9 @@ protected void initConstructors() { IoTDBSchemaRegionAirGapSink::new); pluginConstructors.put( BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingSink::new); + pluginConstructors.put( + BuiltinPipePlugin.LOGICAL_BACKUP_CONNECTOR.getPipePluginName(), + () -> new LogicalBackupSink("schema")); pluginConstructors.put( BuiltinPipePlugin.IOTDB_THRIFT_SINK.getPipePluginName(), IoTDBSchemaRegionSink::new); @@ -59,6 +63,9 @@ protected void initConstructors() { BuiltinPipePlugin.IOTDB_AIR_GAP_SINK.getPipePluginName(), IoTDBSchemaRegionAirGapSink::new); pluginConstructors.put( BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingSink::new); + pluginConstructors.put( + BuiltinPipePlugin.LOGICAL_BACKUP_SINK.getPipePluginName(), + () -> new LogicalBackupSink("schema")); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java new file mode 100644 index 0000000000000..5050107c05f98 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java @@ -0,0 +1,591 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.sink.protocol.logicalbackup; + +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; +import org.apache.iotdb.commons.pipe.event.EnrichedEvent; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecordType; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.i18n.DataNodePipeMessages; +import org.apache.iotdb.db.pipe.event.common.deletion.PipeDeleteDataNodeEvent; +import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; +import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionSnapshotEvent; +import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; +import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent; +import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferPlanNodeReq; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferSchemaSnapshotPieceReq; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferSchemaSnapshotSealReq; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletInsertNodeReqV2; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletRawReqV2; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; +import org.apache.iotdb.pipe.api.PipeConnector; +import org.apache.iotdb.pipe.api.annotation.TableModel; +import org.apache.iotdb.pipe.api.annotation.TreeModel; +import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.event.Event; +import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; +import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; +import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.write.record.Tablet; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.UUID; + +@TreeModel +@TableModel +public class LogicalBackupSink implements PipeConnector { + + private static final int FILE_PIECE_BYTES = 64 * 1024; + private LogicalBackupWriter writer; + private String pipeName; + private long creationTime; + private int regionId; + private boolean includeHeartbeat; + private String unsupportedEventPolicy; + private final String streamType; + + public LogicalBackupSink() { + this("unknown"); + } + + public LogicalBackupSink(final String streamType) { + this.streamType = streamType; + } + + @Override + public void validate(final PipeParameterValidator validator) throws Exception { + final PipeParameters parameters = validator.getParameters(); + validator.validateSynonymAttributes( + Collections.singletonList(PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_DIR_KEY), + Collections.singletonList(PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY), + true); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_ID_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_ID_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY); + validateSynonymAttributes( + validator, + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY); + validator.validateAttributeValueRange( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY, + true, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW); + validator.validateAttributeValueRange( + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY, + true, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW); + validator.validateAttributeValueRange( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY, + true, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_ALWAYS, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIODIC, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_NONE); + validator.validateAttributeValueRange( + PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY, + true, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_ALWAYS, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIODIC, + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_NONE); + final String policy = + parameters + .getStringOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY), + PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_DEFAULT_VALUE) + .toLowerCase(Locale.ROOT); + if (!PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_FAIL.equals(policy) + && !PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_SKIP.equals(policy)) { + throw new PipeParameterNotValidException( + String.format( + DataNodePipeMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_POLICY_ARG_BF42E405, + policy)); + } + } + + private static void validateSynonymAttributes( + final PipeParameterValidator validator, + final String connectorAttribute, + final String sinkAttribute) { + validator.validateSynonymAttributes( + Collections.singletonList(connectorAttribute), + Collections.singletonList(sinkAttribute), + false); + } + + @Override + public void customize( + final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration) + throws Exception { + pipeName = configuration.getRuntimeEnvironment().getPipeName(); + creationTime = configuration.getRuntimeEnvironment().getCreationTime(); + regionId = configuration.getRuntimeEnvironment().getRegionId(); + final String root = + parameters.getStringByKeys( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_DIR_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY); + if (root == null || root.isBlank()) { + throw new PipeParameterNotValidException( + DataNodePipeMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_MUST_NOT_BE_EMPTY_99DB3EDB); + } + final String backupId = + parameters.getStringOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_ID_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_ID_KEY), + pipeName + "-" + creationTime); + if (backupId == null || backupId.isBlank()) { + throw new PipeParameterNotValidException( + String.format( + DataNodePipeMessages + .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22, + backupId)); + } + final String streamId = pipeName + "-" + creationTime + "-" + streamType + "-" + regionId; + final Path backupRoot = Paths.get(root).toAbsolutePath().normalize(); + final Path backupIdPath; + try { + backupIdPath = Path.of(backupId); + } catch (final RuntimeException e) { + throw new PipeParameterNotValidException( + String.format( + DataNodePipeMessages + .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22, + backupId)); + } + if (backupIdPath.isAbsolute() + || backupIdPath.getNameCount() != 1 + || ".".equals(backupId) + || "..".equals(backupId)) { + throw new PipeParameterNotValidException( + String.format( + DataNodePipeMessages + .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22, + backupId)); + } + Path directory = backupRoot.resolve(backupId).resolve(streamType + "-" + regionId).normalize(); + if (!directory.startsWith(backupRoot)) { + throw new PipeParameterNotValidException( + String.format( + DataNodePipeMessages + .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22, + backupId)); + } + final String resume = + parameters + .getStringOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY), + PipeSinkConstant.LOGICAL_BACKUP_RESUME_DEFAULT_VALUE) + .toLowerCase(Locale.ROOT); + final boolean append = PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND.equals(resume); + if (!append + && !PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW.equals(resume) + && !PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS.equals(resume)) { + throw new PipeParameterNotValidException( + String.format( + DataNodePipeMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9, + resume)); + } + if (PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW.equals(resume)) { + directory = + directory.resolveSibling( + directory.getFileName() + + "-new-" + + System.currentTimeMillis() + + "-" + + UUID.randomUUID().toString().substring(0, 8)); + } + final long segmentSize = + parameters.getLongOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY), + PipeSinkConstant.LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_DEFAULT_VALUE); + final int maxRecord = + parameters.getIntOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY), + PipeSinkConstant.LOGICAL_BACKUP_MAX_RECORD_BYTES_DEFAULT_VALUE); + final String fsync = + parameters + .getStringOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY), + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_POLICY_DEFAULT_VALUE) + .toUpperCase(Locale.ROOT); + final LogicalBackupWriter.FsyncPolicy fsyncPolicy = + LogicalBackupWriter.FsyncPolicy.valueOf(fsync); + final int fsyncBatch = + parameters.getIntOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY), + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_DEFAULT_VALUE); + final long fsyncPeriod = + parameters.getLongOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY), + PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIOD_MS_DEFAULT_VALUE); + includeHeartbeat = + parameters.getBooleanOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY), + PipeSinkConstant.LOGICAL_BACKUP_INCLUDE_HEARTBEAT_DEFAULT_VALUE); + unsupportedEventPolicy = + parameters + .getStringOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY), + PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_DEFAULT_VALUE) + .toLowerCase(Locale.ROOT); + + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = backupId; + manifest.pipeName = pipeName; + manifest.pipeCreationTime = creationTime; + manifest.sourceClusterId = IoTDBDescriptor.getInstance().getConfig().getClusterId(); + manifest.sourceVersion = IoTDBConstant.VERSION; + manifest.timestampPrecision = + CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); + manifest.streamId = streamId; + manifest.regionId = regionId; + manifest.streamType = streamType; + manifest.sinkTaskId = streamId; + writer = + new LogicalBackupWriter( + directory, + manifest, + segmentSize, + maxRecord, + fsyncPolicy, + fsyncBatch, + fsyncPeriod, + append); + } + + @Override + public void handshake() {} + + @Override + public void heartbeat() throws Exception { + if (writer != null) { + writer.heartbeat(); + } + } + + @Override + public void transfer(final TabletInsertionEvent event) throws Exception { + if (event instanceof PipeInsertNodeTabletInsertionEvent) { + final PipeInsertNodeTabletInsertionEvent insert = (PipeInsertNodeTabletInsertionEvent) event; + transferEnriched(insert, () -> requests(insert)); + } else if (event instanceof PipeRawTabletInsertionEvent) { + final PipeRawTabletInsertionEvent raw = (PipeRawTabletInsertionEvent) event; + transferEnriched( + raw, + () -> + Collections.singletonList( + PipeTransferTabletRawReqV2.toTPipeTransferReq( + raw.convertToTablet(), + raw.isAligned(), + raw.isTableModelEvent() + ? raw.getTableModelDatabaseName() + : raw.getTreeModelDatabaseName()))); + } else { + handleUnsupported(event); + } + } + + @Override + public void transfer(final TsFileInsertionEvent event) throws Exception { + if (!(event instanceof PipeTsFileInsertionEvent)) { + handleUnsupported(event); + return; + } + final PipeTsFileInsertionEvent tsFile = (PipeTsFileInsertionEvent) event; + final List requests = new ArrayList<>(); + if (!tsFile.increaseReferenceCount(getClass().getName())) { + return; + } + try { + tsFile.consumeTabletInsertionEventsWithRetry( + tablet -> { + requests.add( + PipeTransferTabletRawReqV2.toTPipeTransferReq( + tablet.convertToTablet(), + tablet.isAligned(), + tablet.isTableModelEvent() + ? tablet.getTableModelDatabaseName() + : tablet.getTreeModelDatabaseName())); + }, + getClass().getName()); + if (!requests.isEmpty()) { + writer.writeEvent(eventId(tsFile), System.currentTimeMillis(), requests, metadata(tsFile)); + } + } finally { + event.close(); + tsFile.decreaseReferenceCount(getClass().getName(), false); + } + } + + @Override + public void transfer(final Event event) throws Exception { + if (event instanceof PipeDeleteDataNodeEvent) { + final PipeDeleteDataNodeEvent deletion = (PipeDeleteDataNodeEvent) event; + transferEnriched( + deletion, + () -> + Collections.singletonList( + PipeTransferPlanNodeReq.toTPipeTransferReq(deletion.getDeleteDataNode()))); + } else if (event instanceof PipeSchemaRegionWritePlanEvent) { + final PipeSchemaRegionWritePlanEvent schema = (PipeSchemaRegionWritePlanEvent) event; + transferEnriched( + schema, + () -> + Collections.singletonList( + PipeTransferPlanNodeReq.toTPipeTransferReq(schema.getPlanNode()))); + } else if (event instanceof PipeSchemaRegionSnapshotEvent) { + transferSnapshot((PipeSchemaRegionSnapshotEvent) event); + } else if (event instanceof PipeHeartbeatEvent) { + if (includeHeartbeat) { + writer.writeControl( + LogicalBackupRecordType.HEARTBEAT, System.currentTimeMillis(), metadata(event)); + } + } else if (event instanceof PipeTerminateEvent) { + writer.writeControl( + LogicalBackupRecordType.STREAM_END, System.currentTimeMillis(), metadata(event)); + } else { + handleUnsupported(event); + } + } + + private void transferSnapshot(final PipeSchemaRegionSnapshotEvent event) throws Exception { + if (!event.increaseReferenceCount(getClass().getName())) { + return; + } + try { + final List requests = new ArrayList<>(); + final File[] files = + new File[] { + event.getMTreeSnapshotFile(), + event.getTagLogSnapshotFile(), + event.getAttributeSnapshotFile() + }; + for (final File file : files) { + if (file == null) { + continue; + } + try (final FileInputStream input = new FileInputStream(file)) { + final byte[] piece = new byte[FILE_PIECE_BYTES]; + long offset = 0; + int length; + while ((length = input.read(piece)) >= 0) { + if (length == 0) { + continue; + } + requests.add( + PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq( + file.getName(), offset, Arrays.copyOf(piece, length))); + offset += length; + } + } + } + requests.add( + PipeTransferSchemaSnapshotSealReq.toTPipeTransferReq( + event.getTreePattern().getPattern(), + event.getTablePattern().getDatabasePattern(), + event.getTablePattern().getTablePattern(), + event.getTreePattern().isTreeModelDataAllowedToBeCaptured(), + event.getTablePattern().isTableModelDataAllowedToBeCaptured(), + files[0] == null ? null : files[0].getName(), + files[0] == null ? 0 : files[0].length(), + files[1] == null ? null : files[1].getName(), + files[1] == null ? 0 : files[1].length(), + files[2] == null ? null : files[2].getName(), + files[2] == null ? 0 : files[2].length(), + event.getDatabaseName(), + event.toSealTypeString())); + writer.writeEvent(eventId(event), System.currentTimeMillis(), requests, metadata(event)); + } finally { + event.decreaseReferenceCount(getClass().getName(), false); + } + } + + private List requests(final PipeInsertNodeTabletInsertionEvent event) + throws IOException { + final String databaseName = + event.isTableModelEvent() + ? event.getTableModelDatabaseName() + : event.getTreeModelDatabaseName(); + if (!(event.getInsertNode() instanceof RelationalInsertTabletNode)) { + return Collections.singletonList( + PipeTransferTabletInsertNodeReqV2.toTPipeTransferReq( + event.getInsertNode(), databaseName)); + } + + final List tablets = event.convertToTablets(); + final List requests = new ArrayList<>(tablets.size()); + for (int index = 0; index < tablets.size(); index++) { + requests.add( + PipeTransferTabletRawReqV2.toTPipeTransferReq( + tablets.get(index), event.isAligned(index), databaseName)); + } + return requests; + } + + private void transferEnriched(final EnrichedEvent event, final RequestSupplier requestSupplier) + throws Exception { + if (!event.increaseReferenceCount(getClass().getName())) { + return; + } + try { + writer.writeEvent( + eventId(event), System.currentTimeMillis(), requestSupplier.get(), metadata(event)); + } finally { + event.decreaseReferenceCount(getClass().getName(), false); + } + } + + @FunctionalInterface + private interface RequestSupplier { + List get() throws Exception; + } + + private void handleUnsupported(final Event event) throws Exception { + if (PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_SKIP.equals(unsupportedEventPolicy)) { + writer.recordSkippedEvent( + System.currentTimeMillis(), event == null ? "null" : event.getClass().getName()); + return; + } + throw new PipeParameterNotValidException( + String.format( + DataNodePipeMessages.EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97, + event == null ? "null" : event.getClass().getName())); + } + + private UUID eventId(final EnrichedEvent event) { + return UUID.nameUUIDFromBytes( + (event.getClass().getName() + + '\0' + + event.getPipeName() + + '\0' + + event.getCreationTime() + + '\0' + + event.getRegionId() + + '\0' + + commitIds(event)) + .getBytes(StandardCharsets.UTF_8)); + } + + private String commitIds(final EnrichedEvent event) { + try { + return String.valueOf(event.getCommitIds()); + } catch (final UnsupportedOperationException ignored) { + return Long.toString(event.getCommitId()); + } + } + + private String metadata(final Event event) { + if (event instanceof EnrichedEvent) { + final EnrichedEvent enriched = (EnrichedEvent) event; + return "pipe=" + + enriched.getPipeName() + + ";creationTime=" + + enriched.getCreationTime() + + ";regionId=" + + enriched.getRegionId() + + ";commitIds=" + + commitIds(enriched) + + ";eventType=" + + event.getClass().getName(); + } + return "eventType=" + (event == null ? "null" : event.getClass().getName()); + } + + @Override + public void close() throws Exception { + if (writer != null) { + writer.close(); + writer = null; + } + } +} diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml index 3bb68a29562c3..a632dbff5ddef 100644 --- a/iotdb-core/node-commons/pom.xml +++ b/iotdb-core/node-commons/pom.xml @@ -139,6 +139,10 @@ com.google.guava guava + + com.google.code.gson + gson + com.google.code.findbugs jsr305 diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java new file mode 100644 index 0000000000000..1629e2dd00028 --- /dev/null +++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.i18n; + +public final class LogicalBackupMessages { + + public static final String EXCEPTION_DIRECTORY_5F8F22B8 = "directory"; + public static final String EXCEPTION_MANIFEST_7F5CB74A = "manifest"; + public static final String EXCEPTION_FSYNC_POLICY_6D493614 = "fsyncPolicy"; + public static final String EXCEPTION_EVENT_GROUP_ID_C6F6268A = "eventGroupId"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D = + "Logical backup directory already exists: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EVENT_MUST_CONTAIN_AT_LEAST_ONE_REQUEST_0C278BAC = + "Logical backup event must contain at least one request"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9 = + "Logical backup request body must not be null"; + public static final String EXCEPTION_LOGICAL_BACKUP_RECORD_EXCEEDS_MAX_RECORD_BYTES_B9A9C996 = + "Logical backup record exceeds max-record-bytes"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_MANIFEST_FORMAT_ARG_5005F99E = + "Unsupported logical backup manifest format: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1 = + "Logical backup manifest does not match %s: expected %s, found %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800 = + "Logical backup directory is locked: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_WRITER_IS_CLOSED_EE463BBB = + "Logical backup writer is closed"; + public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_SEGMENT_HEADER_ARG_1FE8371A = + "Incomplete logical backup segment header: %s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_SIZE_ARG_4711E278 = + "Invalid logical backup segment footer size: %s"; + public static final String + EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_MAGIC_AT_OFFSET_ARG_IN_ARG_C132B9D5 = + "Invalid logical backup record magic at offset %d in %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_RECORD_HEADER_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_7076DD79 = + "Logical backup record header CRC mismatch at offset %d in %s"; + public static final String + EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_LENGTH_AT_OFFSET_ARG_IN_ARG_A7F4048E = + "Invalid logical backup record length at offset %d in %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_RECORD_PAYLOAD_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_F9436552 = + "Logical backup record payload CRC mismatch at offset %d in %s"; + public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_RECORD_TYPE_ARG_IN_ARG_22BB599D = + "Unknown logical backup record type %d in %s"; + public static final String EXCEPTION_NESTED_LOGICAL_BACKUP_EVENT_GROUPS_IN_ARG_896AC47A = + "Nested logical backup event groups in %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_COMMIT_WITHOUT_BEGIN_IN_ARG_B9031541 = + "Logical backup commit without begin in %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F = + "Logical backup operation index mismatch at offset %d in %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_MISMATCH_AT_OFFSET_ARG_IN_ARG_CCEC352B = + "Logical backup sequence mismatch at offset %d in %s"; + public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_RECORD_TAIL_ARG_F722DEE7 = + "Incomplete logical backup record tail: %s"; + public static final String + EXCEPTION_SEALED_LOGICAL_BACKUP_SEGMENT_CONTAINS_AN_OPEN_EVENT_GROUP_ARG_54C274D8 = + "Sealed logical backup segment contains an open event group: %s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_MAGIC_ARG_8340BA73 = + "Invalid logical backup segment magic: %s"; + public static final String + EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_SEGMENT_MAJOR_VERSION_ARG_ARG_CE07F1CB = + "Unsupported logical backup segment major version %d: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_HEADER_CRC_MISMATCH_ARG_07466D55 = + "Logical backup segment header CRC mismatch: %s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_MAGIC_ARG_D973E2B4 = + "Invalid logical backup segment footer magic: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_ID_MISMATCH_ARG_D832F14D = + "Logical backup segment footer ID mismatch: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_DIGEST_MISMATCH_ARG_85346116 = + "Logical backup segment digest mismatch: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_CRC_MISMATCH_ARG_25FB7D91 = + "Logical backup segment footer CRC mismatch: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_METADATA_MISMATCH_ARG_77D1FADB = + "Logical backup segment footer metadata mismatch: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EVENT_ID_ARG_WAS_ALREADY_WRITTEN_WITH_A_DIFFERENT_DIGEST_6A297330 = + "Logical backup event ID %s was already written with a different digest"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_CONFIGURATION_707BCC99 = + "Invalid logical backup writer configuration: segment-size, max-record, fsync-batch and" + + " fsync-period must be positive"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_OUTSIDE_EVENT_GROUP_8351818F = + "Logical backup request outside an event group at offset %d in %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_CONTROL_RECORD_INSIDE_EVENT_GROUP_C45D158B = + "Logical backup control record inside an event group at offset %d in %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A = + "Logical backup segment ID mismatch: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_GAP_IN_ARG_D8698149 = + "Logical backup sequence gap in %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_HAS_NO_SEGMENTS_B48D5F15 = + "Logical backup has no segments"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845 = + "Invalid logical backup segment path: %s"; + public static final String EXCEPTION_NO_LOGICAL_BACKUP_MANIFEST_FOUND_UNDER_ARG_4ACEA70E = + "No logical backup manifest found under %s"; + public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_STREAM_ID_ARG_1BF0F6EE = + "Duplicate logical backup stream ID: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_HAS_NO_SEGMENTS_ARG_AFB5C138 = + "Logical backup manifest has no segments: %s"; + public static final String EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B = + "Unlisted logical backup segment %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3 = + "Logical backup segment metadata mismatch: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_IS_NOT_SEALED_ARG_937793FF = + "Logical backup segment is not sealed: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_MANIFEST_COUNTERS_DO_NOT_MATCH_SEGMENT_CONTENTS_ARG_946818BB = + "Logical backup manifest counters do not match segment contents: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_SEQUENCE_IS_NOT_CONTINUOUS_ACROSS_SEGMENTS_ARG_8547BC6E = + "Logical backup sequence is not continuous across segments: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7 = + "Logical backup manifest is invalid: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599 = + "Logical backup contains skipped events: %s"; + + public static final String + EXCEPTION_LOGICAL_BACKUP_REQUEST_VERSION_ARG_IS_NOT_SUPPORTED_393457D7 = + "Logical backup request version %d is not supported"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_IS_NOT_ALLOWED_7F1CDD38 = + "Logical backup request type %d is not allowed"; + public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_EVENT_GROUP_ID_ARG_15B04C89 = + "Duplicate logical backup event group ID: %s"; + public static final String + EXCEPTION_SYMBOLIC_LINKS_ARE_NOT_ALLOWED_IN_LOGICAL_BACKUP_DIRECTORY_PATHS_ARG_7E428569 = + "Symbolic links are not allowed in logical backup directory paths: %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_UNAVAILABLE_ARG_85F090AD = + "Logical backup directory is unavailable: %s"; + + private LogicalBackupMessages() {} +} diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java new file mode 100644 index 0000000000000..bfc2bed98fed1 --- /dev/null +++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.i18n; + +public final class LogicalBackupMessages { + + public static final String EXCEPTION_DIRECTORY_5F8F22B8 = "目录"; + public static final String EXCEPTION_MANIFEST_7F5CB74A = "清单"; + public static final String EXCEPTION_FSYNC_POLICY_6D493614 = "fsync 策略"; + public static final String EXCEPTION_EVENT_GROUP_ID_C6F6268A = "事件组 ID"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D = + "Pipe 逻辑备份目录已存在：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EVENT_MUST_CONTAIN_AT_LEAST_ONE_REQUEST_0C278BAC = + "Pipe 逻辑备份事件必须至少包含一个请求"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9 = + "Pipe 逻辑备份请求体不能为空"; + public static final String EXCEPTION_LOGICAL_BACKUP_RECORD_EXCEEDS_MAX_RECORD_BYTES_B9A9C996 = + "Pipe 逻辑备份记录超过 max-record-bytes"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_MANIFEST_FORMAT_ARG_5005F99E = + "不支持的 Pipe 逻辑备份清单格式：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1 = + "Pipe 逻辑备份清单的 %s 不匹配：预期 %s，实际 %s"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800 = + "Pipe 逻辑备份目录已被锁定：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_WRITER_IS_CLOSED_EE463BBB = + "Pipe 逻辑备份写入器已关闭"; + public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_SEGMENT_HEADER_ARG_1FE8371A = + "Pipe 逻辑备份 segment 文件头不完整：%s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_SIZE_ARG_4711E278 = + "Pipe 逻辑备份 segment 文件尾大小无效：%s"; + public static final String + EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_MAGIC_AT_OFFSET_ARG_IN_ARG_C132B9D5 = + "偏移 %d 处的 Pipe 逻辑备份记录魔数无效，文件：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_RECORD_HEADER_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_7076DD79 = + "偏移 %d 处的 Pipe 逻辑备份记录头 CRC 不匹配，文件：%s"; + public static final String + EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_LENGTH_AT_OFFSET_ARG_IN_ARG_A7F4048E = + "偏移 %d 处的 Pipe 逻辑备份记录长度无效，文件：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_RECORD_PAYLOAD_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_F9436552 = + "偏移 %d 处的 Pipe 逻辑备份记录载荷 CRC 不匹配，文件：%s"; + public static final String EXCEPTION_UNKNOWN_LOGICAL_BACKUP_RECORD_TYPE_ARG_IN_ARG_22BB599D = + "未知的 Pipe 逻辑备份记录类型 %d，文件：%s"; + public static final String EXCEPTION_NESTED_LOGICAL_BACKUP_EVENT_GROUPS_IN_ARG_896AC47A = + "Pipe 逻辑备份事件组发生嵌套，文件：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_COMMIT_WITHOUT_BEGIN_IN_ARG_B9031541 = + "Pipe 逻辑备份事件组缺少开始记录，文件：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F = + "偏移 %d 处的 Pipe 逻辑备份操作序号不匹配，文件：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_MISMATCH_AT_OFFSET_ARG_IN_ARG_CCEC352B = + "偏移 %d 处的 Pipe 逻辑备份 sequence 不连续，文件：%s"; + public static final String EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_RECORD_TAIL_ARG_F722DEE7 = + "Pipe 逻辑备份记录尾部不完整：%s"; + public static final String + EXCEPTION_SEALED_LOGICAL_BACKUP_SEGMENT_CONTAINS_AN_OPEN_EVENT_GROUP_ARG_54C274D8 = + "已封存的 Pipe 逻辑备份 segment 包含未提交事件组：%s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_MAGIC_ARG_8340BA73 = + "Pipe 逻辑备份 segment 魔数无效：%s"; + public static final String + EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_SEGMENT_MAJOR_VERSION_ARG_ARG_CE07F1CB = + "不支持的 Pipe 逻辑备份 segment 主版本 %d：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_HEADER_CRC_MISMATCH_ARG_07466D55 = + "Pipe 逻辑备份 segment 文件头 CRC 不匹配：%s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_MAGIC_ARG_D973E2B4 = + "Pipe 逻辑备份 segment 文件尾魔数无效：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_ID_MISMATCH_ARG_D832F14D = + "Pipe 逻辑备份 segment 文件尾 ID 不匹配：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_DIGEST_MISMATCH_ARG_85346116 = + "Pipe 逻辑备份 segment 摘要不匹配：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_CRC_MISMATCH_ARG_25FB7D91 = + "Pipe 逻辑备份 segment 文件尾 CRC 不匹配：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_METADATA_MISMATCH_ARG_77D1FADB = + "Pipe 逻辑备份 segment 文件尾元数据不匹配：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EVENT_ID_ARG_WAS_ALREADY_WRITTEN_WITH_A_DIFFERENT_DIGEST_6A297330 = + "Pipe 逻辑备份事件 ID %s 已使用不同摘要写入"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_CONFIGURATION_707BCC99 = + "Pipe 逻辑备份写入器配置无效：segment-size、max-record、fsync-batch 和 fsync-period" + + " 必须为正数"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_OUTSIDE_EVENT_GROUP_8351818F = + "偏移 %d 处的 Pipe 逻辑备份请求不在事件组中，文件：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_CONTROL_RECORD_INSIDE_EVENT_GROUP_C45D158B = + "偏移 %d 处的 Pipe 逻辑备份控制记录位于事件组内，文件：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A = + "Pipe 逻辑备份 segment ID 不匹配：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEQUENCE_GAP_IN_ARG_D8698149 = + "Pipe 逻辑备份 sequence 在 %s 中不连续"; + public static final String EXCEPTION_LOGICAL_BACKUP_HAS_NO_SEGMENTS_B48D5F15 = + "Pipe 逻辑备份不包含 segment"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845 = + "Pipe 逻辑备份 segment 路径无效：%s"; + public static final String EXCEPTION_NO_LOGICAL_BACKUP_MANIFEST_FOUND_UNDER_ARG_4ACEA70E = + "在 %s 下未找到 Pipe 逻辑备份 manifest"; + public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_STREAM_ID_ARG_1BF0F6EE = + "Pipe 逻辑备份 stream ID 重复：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_HAS_NO_SEGMENTS_ARG_AFB5C138 = + "Pipe 逻辑备份 manifest 不包含 segment：%s"; + public static final String EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B = + "未在 manifest 中列出 Pipe 逻辑备份 segment：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3 = + "Pipe 逻辑备份 segment 元数据不匹配：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_SEGMENT_IS_NOT_SEALED_ARG_937793FF = + "Pipe 逻辑备份 segment 尚未封存：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_MANIFEST_COUNTERS_DO_NOT_MATCH_SEGMENT_CONTENTS_ARG_946818BB = + "Pipe 逻辑备份 manifest 计数与 segment 内容不匹配：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_SEQUENCE_IS_NOT_CONTINUOUS_ACROSS_SEGMENTS_ARG_8547BC6E = + "Pipe 逻辑备份 sequence 在 segment 之间不连续：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7 = + "Pipe 逻辑备份 manifest 无效：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599 = + "Pipe 逻辑备份包含已跳过的事件：%s"; + + public static final String + EXCEPTION_LOGICAL_BACKUP_REQUEST_VERSION_ARG_IS_NOT_SUPPORTED_393457D7 = + "不支持逻辑备份请求版本 %d"; + public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_IS_NOT_ALLOWED_7F1CDD38 = + "不允许逻辑备份请求类型 %d"; + public static final String EXCEPTION_DUPLICATE_LOGICAL_BACKUP_EVENT_GROUP_ID_ARG_15B04C89 = + "逻辑备份事件组 ID 重复：%s"; + public static final String + EXCEPTION_SYMBOLIC_LINKS_ARE_NOT_ALLOWED_IN_LOGICAL_BACKUP_DIRECTORY_PATHS_ARG_7E428569 = + "逻辑备份目录路径中不允许使用符号链接：%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_UNAVAILABLE_ARG_85F090AD = + "逻辑备份目录不可用：%s"; + + private LogicalBackupMessages() {} +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java index 76b45d1e79895..2a10db843618c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java @@ -38,6 +38,7 @@ import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSslSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSyncSink; +import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.logicalbackup.LogicalBackupSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcda.OpcDaSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcua.OpcUaSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.websocket.WebSocketSink; @@ -95,6 +96,7 @@ public enum BuiltinPipePlugin { OPC_UA_CONNECTOR("opc-ua-connector", OpcUaSink.class), OPC_DA_CONNECTOR("opc-da-connector", OpcDaSink.class), WRITE_BACK_CONNECTOR("write-back-connector", WriteBackSink.class), + LOGICAL_BACKUP_CONNECTOR("logical-backup-connector", LogicalBackupSink.class), DO_NOTHING_SINK("do-nothing-sink", DoNothingSink.class), IOTDB_THRIFT_SINK("iotdb-thrift-sink", IoTDBThriftSink.class), @@ -107,6 +109,7 @@ public enum BuiltinPipePlugin { OPC_UA_SINK("opc-ua-sink", OpcUaSink.class), OPC_DA_SINK("opc-da-sink", OpcDaSink.class), WRITE_BACK_SINK("write-back-sink", WriteBackSink.class), + LOGICAL_BACKUP_SINK("logical-backup-sink", LogicalBackupSink.class), SUBSCRIPTION_SINK("subscription-sink", DoNothingSink.class), IOT_CONSENSUS_V2_ASYNC_SINK("iot-consensus-v2-async-sink", IoTConsensusV2AsyncSink.class), // Legacy alias for stale PipeMeta written before the PipeConsensus -> IoTConsensusV2 rename. @@ -178,6 +181,7 @@ public String getClassName() { OPC_UA_CONNECTOR.getPipePluginName().toUpperCase(), OPC_DA_CONNECTOR.getPipePluginName().toUpperCase(), WRITE_BACK_CONNECTOR.getPipePluginName().toUpperCase(), + LOGICAL_BACKUP_CONNECTOR.getPipePluginName().toUpperCase(), IOT_CONSENSUS_V2_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(), PIPE_CONSENSUS_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(), // Sinks diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/sink/logicalbackup/LogicalBackupSink.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/sink/logicalbackup/LogicalBackupSink.java new file mode 100644 index 0000000000000..d5fb9b22021ef --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/sink/logicalbackup/LogicalBackupSink.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.logicalbackup; + +import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.PlaceholderSink; + +public class LogicalBackupSink extends PlaceholderSink {} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java index f2d9ea283c823..6d5dac18c3d9c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java @@ -80,6 +80,55 @@ public class PipeSinkConstant { public static final String SINK_REALTIME_FIRST_KEY = "sink.realtime-first"; public static final boolean CONNECTOR_REALTIME_FIRST_DEFAULT_VALUE = true; + public static final String CONNECTOR_LOGICAL_BACKUP_DIR_KEY = "connector.dir"; + public static final String SINK_LOGICAL_BACKUP_DIR_KEY = "sink.dir"; + public static final String CONNECTOR_LOGICAL_BACKUP_ID_KEY = "connector.backup-id"; + public static final String SINK_LOGICAL_BACKUP_ID_KEY = "sink.backup-id"; + public static final String CONNECTOR_LOGICAL_BACKUP_RESUME_KEY = "connector.resume"; + public static final String SINK_LOGICAL_BACKUP_RESUME_KEY = "sink.resume"; + public static final String LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS = "fail-if-exists"; + public static final String LOGICAL_BACKUP_RESUME_APPEND = "append"; + public static final String LOGICAL_BACKUP_RESUME_NEW = "new"; + public static final String LOGICAL_BACKUP_RESUME_DEFAULT_VALUE = + LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS; + public static final String CONNECTOR_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY = + "connector.segment-size-bytes"; + public static final String SINK_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY = "sink.segment-size-bytes"; + public static final long LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_DEFAULT_VALUE = 256L * MB; + public static final String CONNECTOR_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY = + "connector.max-record-bytes"; + public static final String SINK_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY = "sink.max-record-bytes"; + public static final int LOGICAL_BACKUP_MAX_RECORD_BYTES_DEFAULT_VALUE = 64 * 1024 * 1024; + public static final String CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY = "connector.fsync-policy"; + public static final String SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY = "sink.fsync-policy"; + public static final String LOGICAL_BACKUP_FSYNC_ALWAYS = "always"; + public static final String LOGICAL_BACKUP_FSYNC_BATCH = "batch"; + public static final String LOGICAL_BACKUP_FSYNC_PERIODIC = "periodic"; + public static final String LOGICAL_BACKUP_FSYNC_NONE = "none"; + public static final String LOGICAL_BACKUP_FSYNC_POLICY_DEFAULT_VALUE = + LOGICAL_BACKUP_FSYNC_ALWAYS; + public static final String CONNECTOR_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY = + "connector.fsync-batch-operations"; + public static final String SINK_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY = + "sink.fsync-batch-operations"; + public static final int LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_DEFAULT_VALUE = 1000; + public static final String CONNECTOR_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY = + "connector.fsync-period-ms"; + public static final String SINK_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY = "sink.fsync-period-ms"; + public static final long LOGICAL_BACKUP_FSYNC_PERIOD_MS_DEFAULT_VALUE = 1000; + public static final String CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY = + "connector.on-unsupported-event"; + public static final String SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY = + "sink.on-unsupported-event"; + public static final String LOGICAL_BACKUP_UNSUPPORTED_EVENT_FAIL = "fail"; + public static final String LOGICAL_BACKUP_UNSUPPORTED_EVENT_SKIP = "skip"; + public static final String LOGICAL_BACKUP_UNSUPPORTED_EVENT_DEFAULT_VALUE = + LOGICAL_BACKUP_UNSUPPORTED_EVENT_FAIL; + public static final String CONNECTOR_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY = + "connector.include-heartbeat"; + public static final String SINK_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY = "sink.include-heartbeat"; + public static final boolean LOGICAL_BACKUP_INCLUDE_HEARTBEAT_DEFAULT_VALUE = false; + public static final String CONNECTOR_SERIALIZE_BY_REGION_KEY = "connector.serialize-by-region"; public static final String SINK_SERIALIZE_BY_REGION_KEY = "sink.serialize-by-region"; public static final boolean CONNECTOR_SERIALIZE_BY_REGION_DEFAULT_VALUE = false; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java new file mode 100644 index 0000000000000..6ac9ca23f67bd --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java @@ -0,0 +1,481 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.commons.i18n.LogicalBackupMessages; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.IoTDBSinkRequestVersion; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; + +import com.google.gson.Gson; +import com.google.gson.JsonParseException; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class LogicalBackupArchiveReader { + + private static final Gson GSON = new Gson(); + private static final Set REPLAYABLE_REQUEST_TYPES = + Set.of( + PipeRequestType.TRANSFER_TABLET_INSERT_NODE_V2.getType(), + PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), + PipeRequestType.TRANSFER_PLAN_NODE.getType(), + PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_PIECE.getType(), + PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL.getType()); + + public List read(final Path source, final boolean allowIncomplete) + throws IOException { + final Path normalizedSource = source.toAbsolutePath().normalize(); + final List manifests = discoverManifests(normalizedSource); + if (manifests.isEmpty()) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_NO_LOGICAL_BACKUP_MANIFEST_FOUND_UNDER_ARG_4ACEA70E, + source)); + } + final List streams = new ArrayList<>(); + final Set streamIds = new HashSet<>(); + LogicalBackupManifest archiveIdentity = null; + for (final Path manifest : manifests) { + final BackupStream stream = readStream(manifest, allowIncomplete); + if (archiveIdentity == null) { + archiveIdentity = stream.getManifest(); + } else { + validateArchiveIdentity(archiveIdentity, stream.getManifest()); + } + if (!streamIds.add(stream.getManifest().streamId)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_DUPLICATE_LOGICAL_BACKUP_STREAM_ID_ARG_1BF0F6EE, + stream.getManifest().streamId)); + } + streams.add(stream); + } + streams.sort( + Comparator.comparingInt( + (BackupStream stream) -> + "schema".equalsIgnoreCase(stream.getManifest().streamType) ? 0 : 1) + .thenComparing((BackupStream stream) -> stream.getManifest().streamId)); + return Collections.unmodifiableList(streams); + } + + private static void validateArchiveIdentity( + final LogicalBackupManifest expected, final LogicalBackupManifest actual) throws IOException { + validateArchiveIdentityField("backupId", expected.backupId, actual.backupId); + validateArchiveIdentityField("pipeName", expected.pipeName, actual.pipeName); + validateArchiveIdentityField( + "pipeCreationTime", expected.pipeCreationTime, actual.pipeCreationTime); + validateArchiveIdentityField( + "sourceClusterId", expected.sourceClusterId, actual.sourceClusterId); + validateArchiveIdentityField("sourceVersion", expected.sourceVersion, actual.sourceVersion); + validateArchiveIdentityField( + "timestampPrecision", expected.timestampPrecision, actual.timestampPrecision); + } + + private static void validateArchiveIdentityField( + final String field, final Object expected, final Object actual) throws IOException { + if (!java.util.Objects.equals(expected, actual)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1, + field, + expected, + actual)); + } + } + + private BackupStream readStream(final Path manifestPath, final boolean allowIncomplete) + throws IOException { + final LogicalBackupManifest manifest; + try { + manifest = + GSON.fromJson( + Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class); + } catch (final JsonParseException | NullPointerException e) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7, + manifestPath), + e); + } + validateManifest(manifest, manifestPath); + if (!allowIncomplete && manifest.skippedEventCount > 0) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599, + manifestPath)); + } + + final Path streamDirectory = manifestPath.getParent().toAbsolutePath().normalize(); + final LogicalBackupSegmentReader segmentReader = + new LogicalBackupSegmentReader(manifest.maxRecordBytes); + final List records = new ArrayList<>(); + final Map operationCounts = new LinkedHashMap<>(); + final Set listedSegments = new HashSet<>(); + long expectedSequence = -1; + long expectedSegmentId = 0; + + for (int index = 0; index < manifest.segments.size(); index++) { + final LogicalBackupManifest.Segment segment = manifest.segments.get(index); + if (segment == null) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845, + "null")); + } + final Path segmentPath = resolveSegmentPath(streamDirectory, segment.file); + if (segmentPath == null + || Files.isSymbolicLink(segmentPath) + || !Files.isRegularFile(segmentPath, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845, + segment.file)); + } + listedSegments.add(segmentPath); + final boolean lastSegment = index == manifest.segments.size() - 1; + final LogicalBackupSegmentReader.ScanResult scan = + segmentReader.scan(segmentPath, allowIncomplete && lastSegment); + if (scan.getSegmentId() != segment.segmentId || scan.getSegmentId() != expectedSegmentId++) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A, + segmentPath)); + } + if (!scan.isSealed() && (!allowIncomplete || !lastSegment)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_IS_NOT_SEALED_ARG_937793FF, + segmentPath)); + } + if (scan.isSealed()) { + validateSegmentManifest(segment, scan, segmentPath); + } + for (final LogicalBackupRecord record : scan.getRecords()) { + validateReplayableRequest(record); + } + + final List committedRecords = + scan.hasOpenEventGroup() ? removeOpenEventGroup(scan.getRecords()) : scan.getRecords(); + for (final LogicalBackupRecord record : committedRecords) { + if (expectedSequence >= 0 && record.getSequence() != expectedSequence) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEQUENCE_IS_NOT_CONTINUOUS_ACROSS_SEGMENTS_ARG_8547BC6E, + manifestPath)); + } + expectedSequence = record.getSequence() + 1; + records.add(record); + operationCounts.merge(record.getRecordType().name(), 1L, Long::sum); + } + } + + if (!allowIncomplete) { + validateUnlistedSegments(streamDirectory, listedSegments); + final long skippedEventCount = + operationCounts.getOrDefault(LogicalBackupRecordType.SKIPPED_EVENT.name(), 0L); + if (skippedEventCount > 0) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599, + manifestPath)); + } + if (skippedEventCount != manifest.skippedEventCount + || !operationCounts.equals(manifest.operationCounts) + || (records.isEmpty() + ? manifest.firstSequence != -1 || manifest.lastSequence != -1 + : records.get(0).getSequence() != manifest.firstSequence + || records.get(records.size() - 1).getSequence() != manifest.lastSequence)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_MANIFEST_COUNTERS_DO_NOT_MATCH_SEGMENT_CONTENTS_ARG_946818BB, + manifestPath)); + } + } + return new BackupStream(manifestPath, manifest, records, toEventGroups(records)); + } + + private static void validateReplayableRequest(final LogicalBackupRecord record) + throws IOException { + if (record.getRecordType() != LogicalBackupRecordType.PIPE_REQUEST) { + return; + } + if (record.getRequestVersion() != IoTDBSinkRequestVersion.VERSION_1.getVersion()) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_REQUEST_VERSION_ARG_IS_NOT_SUPPORTED_393457D7, + record.getRequestVersion())); + } + if (!REPLAYABLE_REQUEST_TYPES.contains(record.getRequestType())) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_IS_NOT_ALLOWED_7F1CDD38, + record.getRequestType())); + } + } + + private static List discoverManifests(final Path source) throws IOException { + if (!Files.isSymbolicLink(source) + && Files.isRegularFile(source, LinkOption.NOFOLLOW_LINKS) + && LogicalBackupFormat.MANIFEST_FILE_NAME.equals(source.getFileName().toString())) { + return Collections.singletonList(source); + } + if (!Files.isDirectory(source)) { + return Collections.emptyList(); + } + try (final Stream paths = Files.walk(source)) { + return paths + .filter(path -> !Files.isSymbolicLink(path)) + .filter(path -> Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS)) + .filter( + path -> LogicalBackupFormat.MANIFEST_FILE_NAME.equals(path.getFileName().toString())) + .sorted() + .collect(Collectors.toList()); + } + } + + private static Path resolveSegmentPath(final Path streamDirectory, final String segmentFile) { + if (segmentFile == null) { + return null; + } + final Path relativePath; + try { + relativePath = Path.of(segmentFile); + } catch (final RuntimeException e) { + return null; + } + if (relativePath.isAbsolute() || relativePath.getNameCount() != 1) { + return null; + } + final Path segmentPath = streamDirectory.resolve(relativePath).normalize(); + return segmentPath.startsWith(streamDirectory) + && segmentPath.getParent().equals(streamDirectory) + ? segmentPath + : null; + } + + private static void validateManifest( + final LogicalBackupManifest manifest, final Path manifestPath) throws IOException { + if (manifest == null + || !LogicalBackupFormat.FORMAT_NAME.equals(manifest.formatName) + || !LogicalBackupFormat.FORMAT_VERSION.equals(manifest.formatVersion) + || manifest.streamId == null + || manifest.backupId == null + || manifest.timestampPrecision == null + || manifest.maxRecordBytes <= 0 + || manifest.maxRecordBytes > LogicalBackupFormat.MAX_RECORD_BYTES + || manifest.skippedEventCount < 0) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7, + manifestPath)); + } + if (manifest.segments == null || manifest.segments.isEmpty()) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_HAS_NO_SEGMENTS_ARG_AFB5C138, + manifestPath)); + } + if (manifest.operationCounts == null) { + manifest.operationCounts = new LinkedHashMap<>(); + } + } + + private static void validateSegmentManifest( + final LogicalBackupManifest.Segment segment, + final LogicalBackupSegmentReader.ScanResult scan, + final Path segmentPath) + throws IOException { + final LogicalBackupSegmentReader.Footer footer = scan.getFooter(); + if (!"SEALED".equals(segment.status) + || segment.firstSequence != footer.getFirstSequence() + || segment.lastSequence != footer.getLastSequence() + || segment.recordCount != footer.getRecordCount() + || segment.sizeBytes != scan.getValidLength() + || !LogicalBackupFormat.toHex(footer.getDigest()).equals(segment.sha256)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3, + segmentPath)); + } + } + + private static void validateUnlistedSegments( + final Path streamDirectory, final Set listedSegments) throws IOException { + try (final Stream paths = Files.list(streamDirectory)) { + final Path unlisted = + paths + .filter(Files::isRegularFile) + .filter(path -> path.getFileName().toString().endsWith(".pwal")) + .map(path -> path.toAbsolutePath().normalize()) + .filter(path -> !listedSegments.contains(path)) + .findFirst() + .orElse(null); + if (unlisted != null) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B, + unlisted)); + } + } + } + + private static List removeOpenEventGroup( + final List records) { + for (int i = records.size() - 1; i >= 0; i--) { + if (records.get(i).getRecordType() == LogicalBackupRecordType.EVENT_BEGIN) { + return records.subList(0, i); + } + } + return records; + } + + private static List toEventGroups(final List records) + throws IOException { + final List groups = new ArrayList<>(); + final Set eventGroupIds = new HashSet<>(); + UUID eventGroupId = null; + long firstSequence = -1; + String metadata = ""; + final List requests = new ArrayList<>(); + for (final LogicalBackupRecord record : records) { + if (record.getRecordType() == LogicalBackupRecordType.EVENT_BEGIN) { + eventGroupId = record.getEventGroupId(); + if (!eventGroupIds.add(eventGroupId)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_DUPLICATE_LOGICAL_BACKUP_EVENT_GROUP_ID_ARG_15B04C89, + eventGroupId)); + } + firstSequence = record.getSequence(); + metadata = record.getMetadata(); + requests.clear(); + } else if (record.getRecordType() == LogicalBackupRecordType.PIPE_REQUEST) { + requests.add(record); + } else if (record.getRecordType() == LogicalBackupRecordType.EVENT_COMMIT) { + groups.add( + new EventGroup( + eventGroupId, + firstSequence, + record.getSequence(), + metadata, + new ArrayList<>(requests))); + eventGroupId = null; + requests.clear(); + } + } + return Collections.unmodifiableList(groups); + } + + public static class BackupStream { + private final Path manifestPath; + private final LogicalBackupManifest manifest; + private final List records; + private final List eventGroups; + + private BackupStream( + final Path manifestPath, + final LogicalBackupManifest manifest, + final List records, + final List eventGroups) { + this.manifestPath = manifestPath; + this.manifest = manifest; + this.records = Collections.unmodifiableList(records); + this.eventGroups = eventGroups; + } + + public Path getManifestPath() { + return manifestPath; + } + + public LogicalBackupManifest getManifest() { + return manifest; + } + + public List getRecords() { + return records; + } + + public List getEventGroups() { + return eventGroups; + } + } + + public static class EventGroup { + private final UUID eventGroupId; + private final long firstSequence; + private final long lastSequence; + private final String metadata; + private final List requests; + + private EventGroup( + final UUID eventGroupId, + final long firstSequence, + final long lastSequence, + final String metadata, + final List requests) { + this.eventGroupId = eventGroupId; + this.firstSequence = firstSequence; + this.lastSequence = lastSequence; + this.metadata = metadata; + this.requests = Collections.unmodifiableList(requests); + } + + public UUID getEventGroupId() { + return eventGroupId; + } + + public long getFirstSequence() { + return firstSequence; + } + + public long getLastSequence() { + return lastSequence; + } + + public String getMetadata() { + return metadata; + } + + public List getRequests() { + return requests; + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupFormat.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupFormat.java new file mode 100644 index 0000000000000..6309595202b52 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupFormat.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.zip.CRC32C; + +public final class LogicalBackupFormat { + + public static final String FORMAT_NAME = "iotdb-pipe-logical-backup"; + public static final String FORMAT_VERSION = "1.0"; + public static final short MAJOR_VERSION = 1; + public static final short MINOR_VERSION = 0; + public static final long SEGMENT_MAGIC = 0x494F545057414C31L; + public static final int RECORD_MAGIC = 0x50575231; + public static final int FOOTER_MAGIC = 0x50575346; + public static final int SEGMENT_HEADER_SIZE = 32; + public static final int RECORD_HEADER_SIZE = 56; + public static final int SEGMENT_FOOTER_SIZE = 96; + public static final int SHA256_SIZE = 32; + public static final int MAX_RECORD_BYTES = 256 * 1024 * 1024; + public static final String MANIFEST_FILE_NAME = "manifest.json"; + + private LogicalBackupFormat() {} + + public static int crc32c(final byte[] bytes, final int offset, final int length) { + final CRC32C crc32c = new CRC32C(); + crc32c.update(bytes, offset, length); + return (int) crc32c.getValue(); + } + + public static MessageDigest newSha256() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (final NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + public static String toHex(final byte[] bytes) { + return HexFormat.of().formatHex(bytes); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java new file mode 100644 index 0000000000000..3742ca88d8016 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class LogicalBackupManifest { + + public String formatName = LogicalBackupFormat.FORMAT_NAME; + public String formatVersion = LogicalBackupFormat.FORMAT_VERSION; + public String backupId; + public String status; + public String pipeName; + public long pipeCreationTime; + public String sourceClusterId; + public String sourceVersion; + public String timestampPrecision; + public String createdAt; + public String closedAt; + public String streamId; + public int regionId; + public String sinkTaskId; + public String streamType; + public String fsyncPolicy; + public long segmentSizeBytes; + public int maxRecordBytes; + public long firstSequence = -1; + public long lastSequence = -1; + public long lastDurableSequence = -1; + public String lastEventGroupId; + public String lastEventDigest; + public long lastEventFirstSequence = -1; + public long skippedEventCount; + public boolean recovered; + public Map operationCounts = new LinkedHashMap<>(); + public List segments = new ArrayList<>(); + + public static class Segment { + public String file; + public long segmentId; + public long firstSequence = -1; + public long lastSequence = -1; + public long recordCount; + public long sizeBytes; + public String sha256; + public String status; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecord.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecord.java new file mode 100644 index 0000000000000..b779c4fc4aa3e --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecord.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.UUID; + +public class LogicalBackupRecord { + + private final LogicalBackupRecordType recordType; + private final long sequence; + private final UUID eventGroupId; + private final int operationIndex; + private final long eventTime; + private final byte requestVersion; + private final short requestType; + private final String metadata; + private final byte[] payload; + + public LogicalBackupRecord( + final LogicalBackupRecordType recordType, + final long sequence, + final UUID eventGroupId, + final int operationIndex, + final long eventTime, + final byte requestVersion, + final short requestType, + final String metadata, + final byte[] payload) { + this.recordType = recordType; + this.sequence = sequence; + this.eventGroupId = eventGroupId; + this.operationIndex = operationIndex; + this.eventTime = eventTime; + this.requestVersion = requestVersion; + this.requestType = requestType; + this.metadata = metadata; + this.payload = payload; + } + + public LogicalBackupRecordType getRecordType() { + return recordType; + } + + public long getSequence() { + return sequence; + } + + public UUID getEventGroupId() { + return eventGroupId; + } + + public int getOperationIndex() { + return operationIndex; + } + + public long getEventTime() { + return eventTime; + } + + public byte getRequestVersion() { + return requestVersion; + } + + public short getRequestType() { + return requestType; + } + + public String getMetadata() { + return metadata; + } + + public byte[] getPayload() { + return payload; + } + + public TPipeTransferReq toTPipeTransferReq() { + return new TPipeTransferReq() + .setVersion(requestVersion) + .setType(requestType) + .setBody(ByteBuffer.wrap(payload)); + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof LogicalBackupRecord)) { + return false; + } + final LogicalBackupRecord that = (LogicalBackupRecord) obj; + return sequence == that.sequence + && operationIndex == that.operationIndex + && eventTime == that.eventTime + && requestVersion == that.requestVersion + && requestType == that.requestType + && recordType == that.recordType + && eventGroupId.equals(that.eventGroupId) + && metadata.equals(that.metadata) + && Arrays.equals(payload, that.payload); + } + + @Override + public int hashCode() { + int result = eventGroupId.hashCode(); + result = 31 * result + recordType.hashCode(); + result = 31 * result + Long.hashCode(sequence); + result = 31 * result + operationIndex; + result = 31 * result + Long.hashCode(eventTime); + result = 31 * result + requestVersion; + result = 31 * result + requestType; + result = 31 * result + metadata.hashCode(); + return 31 * result + Arrays.hashCode(payload); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecordType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecordType.java new file mode 100644 index 0000000000000..e188669548ca6 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupRecordType.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import java.util.HashMap; +import java.util.Map; + +public enum LogicalBackupRecordType { + EVENT_BEGIN((byte) 1), + PIPE_REQUEST((byte) 2), + EVENT_COMMIT((byte) 3), + HEARTBEAT((byte) 4), + SKIPPED_EVENT((byte) 5), + STREAM_END((byte) 6); + + private static final Map TYPE_MAP = new HashMap<>(); + + static { + for (final LogicalBackupRecordType type : values()) { + TYPE_MAP.put(type.code, type); + } + } + + private final byte code; + + LogicalBackupRecordType(final byte code) { + this.code = code; + } + + public byte getCode() { + return code; + } + + public static LogicalBackupRecordType valueOf(final byte code) { + return TYPE_MAP.get(code); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSegmentReader.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSegmentReader.java new file mode 100644 index 0000000000000..e400d6133c27d --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSegmentReader.java @@ -0,0 +1,541 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.commons.i18n.LogicalBackupMessages; + +import java.io.EOFException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class LogicalBackupSegmentReader { + + private final int maxRecordBytes; + + public LogicalBackupSegmentReader(final int maxRecordBytes) { + this.maxRecordBytes = maxRecordBytes; + } + + public ScanResult scan(final Path segment, final boolean allowIncompleteTail) throws IOException { + try (final RandomAccessFile reader = new RandomAccessFile(segment.toFile(), "r")) { + final long fileLength = reader.length(); + if (fileLength < LogicalBackupFormat.SEGMENT_HEADER_SIZE) { + throw new EOFException( + String.format( + LogicalBackupMessages + .EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_SEGMENT_HEADER_ARG_1FE8371A, + segment)); + } + + final byte[] segmentHeader = readBytes(reader, LogicalBackupFormat.SEGMENT_HEADER_SIZE); + validateSegmentHeader(segmentHeader, segment); + final ByteBuffer segmentHeaderBuffer = wrap(segmentHeader); + segmentHeaderBuffer.position(Long.BYTES + Short.BYTES * 2); + final long segmentId = segmentHeaderBuffer.getLong(); + final long createdAt = segmentHeaderBuffer.getLong(); + + final MessageDigest digest = LogicalBackupFormat.newSha256(); + digest.update(segmentHeader); + final List records = new ArrayList<>(); + long validLength = LogicalBackupFormat.SEGMENT_HEADER_SIZE; + long lastCommittedLength = validLength; + boolean openEventGroup = false; + UUID openEventGroupId = null; + int nextOperationIndex = 0; + long expectedSequence = -1; + boolean incompleteTail = false; + Footer footer = null; + + while (reader.getFilePointer() < fileLength) { + final long frameOffset = reader.getFilePointer(); + if (fileLength - frameOffset < Integer.BYTES) { + incompleteTail = true; + break; + } + + final int magic = reader.readInt(); + reader.seek(frameOffset); + if (magic == LogicalBackupFormat.FOOTER_MAGIC) { + if (fileLength - frameOffset < LogicalBackupFormat.SEGMENT_FOOTER_SIZE) { + incompleteTail = true; + break; + } + if (fileLength - frameOffset > LogicalBackupFormat.SEGMENT_FOOTER_SIZE) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_SIZE_ARG_4711E278, + segment)); + } + final byte[] footerBytes = readBytes(reader, LogicalBackupFormat.SEGMENT_FOOTER_SIZE); + footer = + parseAndValidateFooter( + footerBytes, + segmentId, + createdAt, + records.isEmpty() ? -1 : records.get(0).getSequence(), + records.isEmpty() ? -1 : records.get(records.size() - 1).getSequence(), + frameOffset, + records.size(), + digest.digest(), + segment); + validLength = reader.getFilePointer(); + lastCommittedLength = validLength; + break; + } + if (magic != LogicalBackupFormat.RECORD_MAGIC) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_MAGIC_AT_OFFSET_ARG_IN_ARG_C132B9D5, + frameOffset, + segment)); + } + if (fileLength - frameOffset < LogicalBackupFormat.RECORD_HEADER_SIZE) { + incompleteTail = true; + break; + } + + final byte[] header = readBytes(reader, LogicalBackupFormat.RECORD_HEADER_SIZE); + final ByteBuffer headerBuffer = wrap(header); + headerBuffer.getInt(); + final byte recordTypeCode = headerBuffer.get(); + final long sequence = headerBuffer.getLong(); + final UUID eventGroupId = new UUID(headerBuffer.getLong(), headerBuffer.getLong()); + final int operationIndex = headerBuffer.getInt(); + final long eventTime = headerBuffer.getLong(); + final byte requestVersion = headerBuffer.get(); + final short requestType = headerBuffer.getShort(); + final int metadataLength = headerBuffer.getInt(); + final int payloadLength = headerBuffer.getInt(); + final int headerCrc = headerBuffer.getInt(); + if (headerCrc + != LogicalBackupFormat.crc32c(header, 0, LogicalBackupFormat.RECORD_HEADER_SIZE - 4)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_RECORD_HEADER_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_7076DD79, + frameOffset, + segment)); + } + if (metadataLength < 0 + || payloadLength < 0 + || metadataLength > maxRecordBytes + || payloadLength > maxRecordBytes + || (long) metadataLength + payloadLength > maxRecordBytes) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_INVALID_LOGICAL_BACKUP_RECORD_LENGTH_AT_OFFSET_ARG_IN_ARG_A7F4048E, + frameOffset, + segment)); + } + final long remainingLength = (long) metadataLength + payloadLength + Integer.BYTES; + if (fileLength - reader.getFilePointer() < remainingLength) { + incompleteTail = true; + break; + } + + final byte[] metadataBytes = readBytes(reader, metadataLength); + final byte[] payload = readBytes(reader, payloadLength); + final int frameCrc = reader.readInt(); + final byte[] frameContent = new byte[metadataLength + payloadLength]; + System.arraycopy(metadataBytes, 0, frameContent, 0, metadataLength); + System.arraycopy(payload, 0, frameContent, metadataLength, payloadLength); + if (frameCrc != LogicalBackupFormat.crc32c(frameContent, 0, frameContent.length)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_RECORD_PAYLOAD_CRC_MISMATCH_AT_OFFSET_ARG_IN_ARG_F9436552, + frameOffset, + segment)); + } + + final LogicalBackupRecordType recordType = LogicalBackupRecordType.valueOf(recordTypeCode); + if (recordType == null) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_UNKNOWN_LOGICAL_BACKUP_RECORD_TYPE_ARG_IN_ARG_22BB599D, + recordTypeCode, + segment)); + } + if (expectedSequence >= 0 && sequence != expectedSequence) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEQUENCE_MISMATCH_AT_OFFSET_ARG_IN_ARG_CCEC352B, + frameOffset, + segment)); + } + final LogicalBackupRecord record = + new LogicalBackupRecord( + recordType, + sequence, + eventGroupId, + operationIndex, + eventTime, + requestVersion, + requestType, + new String(metadataBytes, StandardCharsets.UTF_8), + payload); + records.add(record); + digest.update(header); + digest.update(frameContent); + digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(frameCrc).array()); + validLength = reader.getFilePointer(); + if (recordType == LogicalBackupRecordType.EVENT_BEGIN) { + if (openEventGroup || operationIndex != -1) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_NESTED_LOGICAL_BACKUP_EVENT_GROUPS_IN_ARG_896AC47A, + segment)); + } + openEventGroup = true; + openEventGroupId = eventGroupId; + nextOperationIndex = 0; + } else if (recordType == LogicalBackupRecordType.EVENT_COMMIT) { + if (!openEventGroup) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_COMMIT_WITHOUT_BEGIN_IN_ARG_B9031541, + segment)); + } + if (!eventGroupId.equals(openEventGroupId) || operationIndex != nextOperationIndex) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F, + frameOffset, + segment)); + } + openEventGroup = false; + openEventGroupId = null; + lastCommittedLength = validLength; + } else if (recordType == LogicalBackupRecordType.PIPE_REQUEST) { + if (!openEventGroup) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_REQUEST_OUTSIDE_EVENT_GROUP_8351818F, + frameOffset, + segment)); + } + if (!eventGroupId.equals(openEventGroupId) || operationIndex != nextOperationIndex) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_OPERATION_INDEX_MISMATCH_AT_OFFSET_ARG_IN_ARG_30C3005F, + frameOffset, + segment)); + } + nextOperationIndex++; + } else if (openEventGroup) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_CONTROL_RECORD_INSIDE_EVENT_GROUP_C45D158B, + frameOffset, + segment)); + } else if (!openEventGroup) { + lastCommittedLength = validLength; + } + expectedSequence = sequence + 1; + } + + if (incompleteTail && !allowIncompleteTail) { + throw new EOFException( + String.format( + LogicalBackupMessages.EXCEPTION_INCOMPLETE_LOGICAL_BACKUP_RECORD_TAIL_ARG_F722DEE7, + segment)); + } + if (footer != null && openEventGroup) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_SEALED_LOGICAL_BACKUP_SEGMENT_CONTAINS_AN_OPEN_EVENT_GROUP_ARG_54C274D8, + segment)); + } + return new ScanResult( + segmentId, + createdAt, + records, + footer, + validLength, + lastCommittedLength, + incompleteTail, + openEventGroup); + } + } + + private static void validateSegmentHeader(final byte[] header, final Path segment) + throws IOException { + final ByteBuffer buffer = wrap(header); + if (buffer.getLong() != LogicalBackupFormat.SEGMENT_MAGIC) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_MAGIC_ARG_8340BA73, + segment)); + } + final short majorVersion = buffer.getShort(); + buffer.getShort(); + if (majorVersion != LogicalBackupFormat.MAJOR_VERSION) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_SEGMENT_MAJOR_VERSION_ARG_ARG_CE07F1CB, + majorVersion, + segment)); + } + buffer.position(LogicalBackupFormat.SEGMENT_HEADER_SIZE - Integer.BYTES); + final int headerCrc = buffer.getInt(); + if (headerCrc + != LogicalBackupFormat.crc32c(header, 0, LogicalBackupFormat.SEGMENT_HEADER_SIZE - 4)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEGMENT_HEADER_CRC_MISMATCH_ARG_07466D55, + segment)); + } + } + + private static Footer parseAndValidateFooter( + final byte[] bytes, + final long expectedSegmentId, + final long expectedCreatedAt, + final long expectedFirstSequence, + final long expectedLastSequence, + final long expectedValidBytes, + final long expectedRecordCount, + final byte[] expectedDigest, + final Path segment) + throws IOException { + final ByteBuffer buffer = wrap(bytes); + if (buffer.getInt() != LogicalBackupFormat.FOOTER_MAGIC) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_FOOTER_MAGIC_ARG_D973E2B4, + segment)); + } + final long segmentId = buffer.getLong(); + final long firstSequence = buffer.getLong(); + final long lastSequence = buffer.getLong(); + final long recordCount = buffer.getLong(); + final long validBytes = buffer.getLong(); + final long createdAt = buffer.getLong(); + final long sealedAt = buffer.getLong(); + final byte[] digest = new byte[LogicalBackupFormat.SHA256_SIZE]; + buffer.get(digest); + final int footerCrc = buffer.getInt(); + if (segmentId != expectedSegmentId) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_ID_MISMATCH_ARG_D832F14D, + segment)); + } + if (createdAt != expectedCreatedAt + || firstSequence != expectedFirstSequence + || lastSequence != expectedLastSequence + || validBytes != expectedValidBytes + || recordCount != expectedRecordCount) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_METADATA_MISMATCH_ARG_77D1FADB, + segment)); + } + if (!Arrays.equals(digest, expectedDigest)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_DIGEST_MISMATCH_ARG_85346116, + segment)); + } + if (footerCrc + != LogicalBackupFormat.crc32c(bytes, 0, LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEGMENT_FOOTER_CRC_MISMATCH_ARG_25FB7D91, + segment)); + } + return new Footer( + segmentId, + firstSequence, + lastSequence, + recordCount, + validBytes, + createdAt, + sealedAt, + digest); + } + + private static byte[] readBytes(final RandomAccessFile reader, final int length) + throws IOException { + final byte[] bytes = new byte[length]; + reader.readFully(bytes); + return bytes; + } + + private static ByteBuffer wrap(final byte[] bytes) { + return ByteBuffer.wrap(bytes).order(ByteOrder.BIG_ENDIAN); + } + + public static class ScanResult { + private final long segmentId; + private final long createdAt; + private final List records; + private final Footer footer; + private final long validLength; + private final long lastCommittedLength; + private final boolean incompleteTail; + private final boolean openEventGroup; + + private ScanResult( + final long segmentId, + final long createdAt, + final List records, + final Footer footer, + final long validLength, + final long lastCommittedLength, + final boolean incompleteTail, + final boolean openEventGroup) { + this.segmentId = segmentId; + this.createdAt = createdAt; + this.records = Collections.unmodifiableList(records); + this.footer = footer; + this.validLength = validLength; + this.lastCommittedLength = lastCommittedLength; + this.incompleteTail = incompleteTail; + this.openEventGroup = openEventGroup; + } + + public long getSegmentId() { + return segmentId; + } + + public long getCreatedAt() { + return createdAt; + } + + public List getRecords() { + return records; + } + + public Footer getFooter() { + return footer; + } + + public long getValidLength() { + return validLength; + } + + public long getLastCommittedLength() { + return lastCommittedLength; + } + + public boolean hasIncompleteTail() { + return incompleteTail; + } + + public boolean hasOpenEventGroup() { + return openEventGroup; + } + + public boolean isSealed() { + return footer != null; + } + } + + public static class Footer { + private final long segmentId; + private final long firstSequence; + private final long lastSequence; + private final long recordCount; + private final long validBytes; + private final long createdAt; + private final long sealedAt; + private final byte[] digest; + + private Footer( + final long segmentId, + final long firstSequence, + final long lastSequence, + final long recordCount, + final long validBytes, + final long createdAt, + final long sealedAt, + final byte[] digest) { + this.segmentId = segmentId; + this.firstSequence = firstSequence; + this.lastSequence = lastSequence; + this.recordCount = recordCount; + this.validBytes = validBytes; + this.createdAt = createdAt; + this.sealedAt = sealedAt; + this.digest = digest; + } + + public long getSegmentId() { + return segmentId; + } + + public long getFirstSequence() { + return firstSequence; + } + + public long getLastSequence() { + return lastSequence; + } + + public long getRecordCount() { + return recordCount; + } + + public long getValidBytes() { + return validBytes; + } + + public long getCreatedAt() { + return createdAt; + } + + public long getSealedAt() { + return sealedAt; + } + + public byte[] getDigest() { + return digest; + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java new file mode 100644 index 0000000000000..9c8ba6f9a9a63 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java @@ -0,0 +1,1162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.commons.i18n.LogicalBackupMessages; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonParseException; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; + +public class LogicalBackupWriter implements Closeable { + + public enum FsyncPolicy { + ALWAYS, + BATCH, + PERIODIC, + NONE + } + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + + private final Path directory; + private final Path manifestPath; + private final Path lockPath; + private final int maxRecordBytes; + private final long segmentSizeBytes; + private final FsyncPolicy fsyncPolicy; + private final int fsyncBatchOperations; + private final long fsyncPeriodMs; + private final LogicalBackupManifest manifest; + private final LogicalBackupSegmentReader reader; + + private FileChannel channel; + private FileChannel lockChannel; + private FileLock directoryLock; + private long segmentId; + private long segmentCreatedAt; + private long segmentFirstSequence = -1; + private long segmentLastSequence = -1; + private long segmentRecordCount; + private MessageDigest segmentDigest; + private long nextSequence; + private int operationsSinceFsync; + private long lastFsyncAt; + private boolean closed; + private boolean streamEnded; + private boolean recoveredDuringOpen; + + public LogicalBackupWriter( + final Path directory, + final LogicalBackupManifest manifest, + final long segmentSizeBytes, + final int maxRecordBytes, + final FsyncPolicy fsyncPolicy, + final int fsyncBatchOperations, + final long fsyncPeriodMs, + final boolean append) + throws IOException { + this.directory = + Objects.requireNonNull(directory, LogicalBackupMessages.EXCEPTION_DIRECTORY_5F8F22B8) + .toAbsolutePath() + .normalize(); + this.manifest = + Objects.requireNonNull(manifest, LogicalBackupMessages.EXCEPTION_MANIFEST_7F5CB74A); + this.segmentSizeBytes = segmentSizeBytes; + this.maxRecordBytes = maxRecordBytes; + this.fsyncPolicy = + Objects.requireNonNull(fsyncPolicy, LogicalBackupMessages.EXCEPTION_FSYNC_POLICY_6D493614); + if (segmentSizeBytes + <= LogicalBackupFormat.SEGMENT_HEADER_SIZE + LogicalBackupFormat.SEGMENT_FOOTER_SIZE + || maxRecordBytes <= 0 + || maxRecordBytes > LogicalBackupFormat.MAX_RECORD_BYTES + || fsyncBatchOperations <= 0 + || fsyncPeriodMs <= 0) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_CONFIGURATION_707BCC99); + } + this.fsyncBatchOperations = fsyncBatchOperations; + this.fsyncPeriodMs = fsyncPeriodMs; + this.manifestPath = this.directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + this.lockPath = this.directory.resolve(".backup.lock"); + this.reader = new LogicalBackupSegmentReader(maxRecordBytes); + try { + prepareDirectory(append); + acquireLock(); + if (Files.isSymbolicLink(manifestPath)) { + throw symbolicLinkException(manifestPath); + } + if (Files.exists(manifestPath)) { + if (!append) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D, + directory)); + } + loadExistingManifest(); + } else { + manifest.status = "WRITING"; + manifest.createdAt = Instant.now().toString(); + manifest.fsyncPolicy = fsyncPolicy.name().toLowerCase(Locale.ROOT); + manifest.segmentSizeBytes = segmentSizeBytes; + manifest.maxRecordBytes = maxRecordBytes; + writeManifest(); + } + manifest.status = manifest.recovered ? "RECOVERED" : "WRITING"; + manifest.closedAt = null; + nextSequence = Math.max(0, manifest.lastSequence + 1); + openOrCreateSegment(); + rebuildManifestFromSegments(); + if (recoveredDuringOpen) { + manifest.lastDurableSequence = manifest.lastSequence; + } else { + manifest.lastDurableSequence = + Math.max(-1, Math.min(manifest.lastDurableSequence, manifest.lastSequence)); + } + updateActiveSegmentManifest(); + manifest.status = manifest.recovered ? "RECOVERED" : "WRITING"; + manifest.closedAt = null; + writeManifest(); + lastFsyncAt = System.currentTimeMillis(); + } catch (final IOException | RuntimeException e) { + if (channel != null) { + try { + channel.close(); + } catch (final IOException closeException) { + e.addSuppressed(closeException); + } + channel = null; + } + releaseLock(); + throw e; + } + } + + private void prepareDirectory(final boolean append) throws IOException { + rejectSymbolicLinksInDirectoryPath(); + if (append) { + Files.createDirectories(directory); + rejectSymbolicLinksInDirectoryPath(); + return; + } + final Path parent = directory.getParent(); + if (parent != null) { + Files.createDirectories(parent); + rejectSymbolicLinksInDirectoryPath(); + } + try { + Files.createDirectory(directory); + } catch (final FileAlreadyExistsException e) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_ALREADY_EXISTS_ARG_1C521D1D, + directory), + e); + } + rejectSymbolicLinksInDirectoryPath(); + } + + private void rejectSymbolicLinksInDirectoryPath() throws IOException { + Path current = directory.getRoot(); + for (final Path part : directory) { + current = current == null ? part : current.resolve(part); + if (Files.isSymbolicLink(current)) { + throw symbolicLinkException(current); + } + } + } + + private static IOException symbolicLinkException(final Path path) { + return new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_SYMBOLIC_LINKS_ARE_NOT_ALLOWED_IN_LOGICAL_BACKUP_DIRECTORY_PATHS_ARG_7E428569, + path)); + } + + public synchronized long writeEvent( + final UUID eventGroupId, + final long eventTime, + final List requests, + final String metadata) + throws IOException { + ensureOpen(); + if (requests == null || requests.isEmpty()) { + throw new IOException( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_EVENT_MUST_CONTAIN_AT_LEAST_ONE_REQUEST_0C278BAC); + } + Objects.requireNonNull(eventGroupId, LogicalBackupMessages.EXCEPTION_EVENT_GROUP_ID_C6F6268A); + final String safeMetadata = metadata == null ? "" : metadata; + final String eventDigest = computeEventDigest(requests); + if (eventGroupId.toString().equals(manifest.lastEventGroupId)) { + if (eventDigest.equals(manifest.lastEventDigest)) { + updateActiveSegmentManifest(); + writeManifest(); + return manifest.lastEventFirstSequence; + } + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_EVENT_ID_ARG_WAS_ALREADY_WRITTEN_WITH_A_DIFFERENT_DIGEST_6A297330, + eventGroupId)); + } + rollBeforeEventIfNecessary(requests, safeMetadata); + final long eventStartPosition = channel.position(); + final long firstSequence = nextSequence; + try { + appendRecord( + LogicalBackupRecordType.EVENT_BEGIN, eventGroupId, -1, eventTime, null, safeMetadata); + int operationIndex = 0; + for (final TPipeTransferReq request : requests) { + if (request == null || request.getBody() == null) { + throw new IOException( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9); + } + appendRecord( + LogicalBackupRecordType.PIPE_REQUEST, + eventGroupId, + operationIndex++, + eventTime, + request, + safeMetadata); + } + appendRecord( + LogicalBackupRecordType.EVENT_COMMIT, + eventGroupId, + operationIndex, + eventTime, + null, + safeMetadata); + } catch (final IOException | RuntimeException e) { + try { + rollbackTo(eventStartPosition); + } catch (final IOException rollbackException) { + e.addSuppressed(rollbackException); + } + throw e; + } + if (forceIfNeeded()) { + manifest.lastDurableSequence = manifest.lastSequence; + } + updateActiveSegmentManifest(); + manifest.lastEventGroupId = eventGroupId.toString(); + manifest.lastEventDigest = eventDigest; + manifest.lastEventFirstSequence = firstSequence; + writeManifest(); + return firstSequence; + } + + public synchronized void writeControl( + final LogicalBackupRecordType recordType, final long eventTime, final String metadata) + throws IOException { + ensureOpen(); + final String safeMetadata = metadata == null ? "" : metadata; + rollBeforeControlIfNecessary(safeMetadata); + final long recordStartPosition = channel.position(); + try { + appendRecord(recordType, new UUID(0, 0), -1, eventTime, null, safeMetadata); + } catch (final IOException | RuntimeException e) { + try { + rollbackTo(recordStartPosition); + } catch (final IOException rollbackException) { + e.addSuppressed(rollbackException); + } + throw e; + } + if (forceIfNeeded()) { + manifest.lastDurableSequence = manifest.lastSequence; + } + updateActiveSegmentManifest(); + writeManifest(); + } + + public synchronized void recordSkippedEvent(final long eventTime, final String metadata) + throws IOException { + writeControl(LogicalBackupRecordType.SKIPPED_EVENT, eventTime, metadata); + manifest.skippedEventCount++; + writeManifest(); + } + + public synchronized LogicalBackupManifest getManifest() { + return manifest; + } + + public synchronized void heartbeat() throws IOException { + ensureOpen(); + if (!Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS) + || directoryLock == null + || !directoryLock.isValid()) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_UNAVAILABLE_ARG_85F090AD, + directory)); + } + if (forceIfNeeded()) { + manifest.lastDurableSequence = manifest.lastSequence; + updateActiveSegmentManifest(); + writeManifest(); + } + } + + private void appendRecord( + final LogicalBackupRecordType recordType, + final UUID eventGroupId, + final int operationIndex, + final long eventTime, + final TPipeTransferReq request, + final String metadata) + throws IOException { + final byte[] metadataBytes = metadata.getBytes(StandardCharsets.UTF_8); + final byte[] payload = + request == null + ? new byte[0] + : java.util.Arrays.copyOf(request.getBody(), request.getBody().length); + if ((long) metadataBytes.length + payload.length > maxRecordBytes) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_RECORD_EXCEEDS_MAX_RECORD_BYTES_B9A9C996); + } + final long requiredBytes = + LogicalBackupFormat.RECORD_HEADER_SIZE + + metadataBytes.length + + payload.length + + Integer.BYTES; + final long sequence = nextSequence++; + final ByteBuffer header = + ByteBuffer.allocate(LogicalBackupFormat.RECORD_HEADER_SIZE).order(ByteOrder.BIG_ENDIAN); + header.putInt(LogicalBackupFormat.RECORD_MAGIC); + header.put(recordType.getCode()); + header.putLong(sequence); + header.putLong(eventGroupId.getMostSignificantBits()); + header.putLong(eventGroupId.getLeastSignificantBits()); + header.putInt(operationIndex); + header.putLong(eventTime); + header.put(request == null ? (byte) 0 : request.getVersion()); + header.putShort(request == null ? (short) 0 : request.getType()); + header.putInt(metadataBytes.length); + header.putInt(payload.length); + header.putInt(0); + final byte[] headerBytes = header.array(); + final int headerCrc = + LogicalBackupFormat.crc32c(headerBytes, 0, LogicalBackupFormat.RECORD_HEADER_SIZE - 4); + ByteBuffer.wrap(headerBytes) + .order(ByteOrder.BIG_ENDIAN) + .putInt(LogicalBackupFormat.RECORD_HEADER_SIZE - 4, headerCrc); + final byte[] frameContent = new byte[metadataBytes.length + payload.length]; + System.arraycopy(metadataBytes, 0, frameContent, 0, metadataBytes.length); + System.arraycopy(payload, 0, frameContent, metadataBytes.length, payload.length); + final int frameCrc = LogicalBackupFormat.crc32c(frameContent, 0, frameContent.length); + + writeFully(ByteBuffer.wrap(headerBytes)); + writeFully(ByteBuffer.wrap(frameContent)); + writeFully( + ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.BIG_ENDIAN).putInt(frameCrc).flip()); + segmentDigest.update(headerBytes); + segmentDigest.update(frameContent); + segmentDigest.update( + ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.BIG_ENDIAN).putInt(frameCrc).array()); + if (segmentFirstSequence < 0) { + segmentFirstSequence = sequence; + } + segmentLastSequence = sequence; + segmentRecordCount++; + manifest.firstSequence = manifest.firstSequence < 0 ? sequence : manifest.firstSequence; + manifest.lastSequence = sequence; + final String countKey = recordType.name(); + manifest.operationCounts.put(countKey, manifest.operationCounts.getOrDefault(countKey, 0L) + 1); + operationsSinceFsync++; + streamEnded = recordType == LogicalBackupRecordType.STREAM_END; + } + + private void openOrCreateSegment() throws IOException { + if (manifest.segments.isEmpty()) { + segmentId = 0; + validateSegmentFiles(segmentFileName(segmentId)); + openNewSegment(); + return; + } + final LogicalBackupManifest.Segment last = manifest.segments.get(manifest.segments.size() - 1); + final Path segmentPath = resolveExistingSegmentPath(last.file); + LogicalBackupSegmentReader.ScanResult scan = scanLastSegmentForAppend(segmentPath, last); + if (scan.isSealed()) { + if (!"SEALED".equals(last.status)) { + manifest.recovered = true; + recoveredDuringOpen = true; + } + segmentId = scan.getSegmentId() + 1; + // Validate every existing sealed segment before creating a new file in the backup. + rebuildManifestFromSegments(); + validateSegmentFiles(segmentFileName(segmentId)); + openNewSegment(); + return; + } + validateSegmentFiles(null); + if (scan.hasIncompleteTail() || scan.hasOpenEventGroup()) { + try (final FileChannel truncateChannel = + FileChannel.open(segmentPath, StandardOpenOption.WRITE)) { + truncateChannel.truncate(scan.getLastCommittedLength()); + truncateChannel.force(true); + } + manifest.recovered = true; + manifest.status = "RECOVERED"; + recoveredDuringOpen = true; + scan = reader.scan(segmentPath, false); + } + if (!scan.getRecords().isEmpty()) { + nextSequence = scan.getRecords().get(scan.getRecords().size() - 1).getSequence() + 1; + manifest.lastSequence = scan.getRecords().get(scan.getRecords().size() - 1).getSequence(); + } + segmentId = scan.getSegmentId(); + openExistingSegment(segmentPath, scan); + } + + private LogicalBackupSegmentReader.ScanResult scanLastSegmentForAppend( + final Path segmentPath, final LogicalBackupManifest.Segment segment) throws IOException { + try { + return reader.scan(segmentPath, true); + } catch (final IOException scanException) { + final long recoverableLength = + "SEALED".equals(segment.status) + ? -1 + : findRecoverableTailStart(segmentPath, segment.sizeBytes); + if (recoverableLength < LogicalBackupFormat.SEGMENT_HEADER_SIZE) { + throw scanException; + } + try (final FileChannel truncateChannel = + FileChannel.open(segmentPath, StandardOpenOption.WRITE)) { + truncateChannel.truncate(recoverableLength); + truncateChannel.force(true); + } + manifest.recovered = true; + manifest.status = "RECOVERED"; + recoveredDuringOpen = true; + return reader.scan(segmentPath, false); + } + } + + private static long findRecoverableTailStart( + final Path segmentPath, final long manifestSegmentSize) throws IOException { + final long fileSize = Files.size(segmentPath); + if (fileSize + >= LogicalBackupFormat.SEGMENT_HEADER_SIZE + LogicalBackupFormat.SEGMENT_FOOTER_SIZE) { + final ByteBuffer magic = ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.BIG_ENDIAN); + try (final FileChannel readChannel = FileChannel.open(segmentPath, StandardOpenOption.READ)) { + readChannel.position(fileSize - LogicalBackupFormat.SEGMENT_FOOTER_SIZE); + while (magic.hasRemaining()) { + if (readChannel.read(magic) < 0) { + break; + } + } + } + if (!magic.hasRemaining() && magic.flip().getInt() == LogicalBackupFormat.FOOTER_MAGIC) { + return fileSize - LogicalBackupFormat.SEGMENT_FOOTER_SIZE; + } + } + if (manifestSegmentSize >= LogicalBackupFormat.SEGMENT_HEADER_SIZE + && manifestSegmentSize < fileSize + && fileSize - manifestSegmentSize <= LogicalBackupFormat.SEGMENT_FOOTER_SIZE) { + return manifestSegmentSize; + } + return -1; + } + + private void openExistingSegment( + final Path segmentPath, final LogicalBackupSegmentReader.ScanResult scan) throws IOException { + channel = FileChannel.open(segmentPath, StandardOpenOption.WRITE, StandardOpenOption.READ); + channel.position(channel.size()); + segmentCreatedAt = scan.getCreatedAt(); + segmentDigest = LogicalBackupFormat.newSha256(); + try (final FileChannel digestChannel = FileChannel.open(segmentPath, StandardOpenOption.READ)) { + final ByteBuffer digestBuffer = ByteBuffer.allocate(64 * 1024); + long remaining = scan.getValidLength(); + while (remaining > 0) { + digestBuffer.clear(); + digestBuffer.limit((int) Math.min(digestBuffer.capacity(), remaining)); + final int read = digestChannel.read(digestBuffer); + if (read < 0) { + break; + } + remaining -= read; + segmentDigest.update(digestBuffer.array(), 0, read); + } + } + segmentFirstSequence = -1; + segmentLastSequence = -1; + segmentRecordCount = 0; + for (final LogicalBackupRecord record : scan.getRecords()) { + segmentFirstSequence = segmentFirstSequence < 0 ? record.getSequence() : segmentFirstSequence; + segmentLastSequence = record.getSequence(); + segmentRecordCount++; + } + streamEnded = + !scan.getRecords().isEmpty() + && scan.getRecords().get(scan.getRecords().size() - 1).getRecordType() + == LogicalBackupRecordType.STREAM_END; + } + + private void openNewSegment() throws IOException { + final String fileName = segmentFileName(segmentId); + final Path segmentPath = resolveSegmentPath(fileName); + final Path temporarySegmentPath = resolveSegmentPath(fileName + ".tmp"); + if (Files.exists(segmentPath, LinkOption.NOFOLLOW_LINKS)) { + adoptOrphanSegment(resolveExistingSegmentPath(fileName), fileName); + return; + } + + // A temporary segment is never exposed through the manifest and contains only a header, so it + // is safe to discard after an interrupted creation attempt. + Files.deleteIfExists(temporarySegmentPath); + segmentCreatedAt = System.currentTimeMillis(); + final ByteBuffer header = + ByteBuffer.allocate(LogicalBackupFormat.SEGMENT_HEADER_SIZE).order(ByteOrder.BIG_ENDIAN); + header.putLong(LogicalBackupFormat.SEGMENT_MAGIC); + header.putShort(LogicalBackupFormat.MAJOR_VERSION); + header.putShort(LogicalBackupFormat.MINOR_VERSION); + header.putLong(segmentId); + header.putLong(segmentCreatedAt); + header.putInt(0); + final byte[] bytes = header.array(); + ByteBuffer.wrap(bytes) + .order(ByteOrder.BIG_ENDIAN) + .putInt( + LogicalBackupFormat.SEGMENT_HEADER_SIZE - 4, + LogicalBackupFormat.crc32c(bytes, 0, LogicalBackupFormat.SEGMENT_HEADER_SIZE - 4)); + try (final FileChannel temporaryChannel = + FileChannel.open( + temporarySegmentPath, + StandardOpenOption.CREATE_NEW, + StandardOpenOption.WRITE, + StandardOpenOption.READ)) { + final ByteBuffer headerBuffer = ByteBuffer.wrap(bytes); + while (headerBuffer.hasRemaining()) { + temporaryChannel.write(headerBuffer); + } + temporaryChannel.force(true); + } + try { + Files.move(temporarySegmentPath, segmentPath, StandardCopyOption.ATOMIC_MOVE); + } catch (final AtomicMoveNotSupportedException e) { + Files.move(temporarySegmentPath, segmentPath); + } + + final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false); + openExistingSegment(segmentPath, scan); + addActiveSegmentToManifest(fileName); + } + + private void adoptOrphanSegment(final Path segmentPath, final String fileName) + throws IOException { + final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false); + if (scan.getSegmentId() != segmentId + || scan.isSealed() + || !scan.getRecords().isEmpty() + || scan.getValidLength() != LogicalBackupFormat.SEGMENT_HEADER_SIZE) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B, + segmentPath)); + } + openExistingSegment(segmentPath, scan); + addActiveSegmentToManifest(fileName); + } + + private void addActiveSegmentToManifest(final String fileName) throws IOException { + final LogicalBackupManifest.Segment segment = new LogicalBackupManifest.Segment(); + segment.file = fileName; + segment.segmentId = segmentId; + segment.status = "ACTIVE"; + manifest.segments.add(segment); + writeManifest(); + } + + private void validateSegmentFiles(final String allowedOrphanFile) throws IOException { + final Set listedFiles = new HashSet<>(); + for (final LogicalBackupManifest.Segment segment : manifest.segments) { + listedFiles.add(segment.file); + } + try (final DirectoryStream segmentFiles = Files.newDirectoryStream(directory, "*.pwal")) { + for (final Path segmentFile : segmentFiles) { + final String fileName = segmentFile.getFileName().toString(); + if (!listedFiles.contains(fileName) && !fileName.equals(allowedOrphanFile)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_UNLISTED_LOGICAL_BACKUP_SEGMENT_ARG_FF548C0B, + segmentFile)); + } + } + } + } + + private static String segmentFileName(final long id) { + return String.format(Locale.ROOT, "segment-%020d.pwal", id); + } + + private void sealSegment() throws IOException { + if (channel == null || segmentRecordCount == 0) { + return; + } + final byte[] digest = segmentDigest.digest(); + final ByteBuffer footer = + ByteBuffer.allocate(LogicalBackupFormat.SEGMENT_FOOTER_SIZE).order(ByteOrder.BIG_ENDIAN); + footer.putInt(LogicalBackupFormat.FOOTER_MAGIC); + footer.putLong(segmentId); + footer.putLong(segmentFirstSequence); + footer.putLong(segmentLastSequence); + footer.putLong(segmentRecordCount); + footer.putLong(channel.position()); + footer.putLong(segmentCreatedAt); + footer.putLong(System.currentTimeMillis()); + footer.put(digest); + footer.putInt(0); + final byte[] footerBytes = footer.array(); + ByteBuffer.wrap(footerBytes) + .order(ByteOrder.BIG_ENDIAN) + .putInt( + LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4, + LogicalBackupFormat.crc32c( + footerBytes, 0, LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4)); + writeFully(ByteBuffer.wrap(footerBytes)); + channel.force(true); + final LogicalBackupManifest.Segment segment = + manifest.segments.get(manifest.segments.size() - 1); + segment.firstSequence = segmentFirstSequence; + segment.lastSequence = segmentLastSequence; + segment.recordCount = segmentRecordCount; + segment.sizeBytes = channel.position(); + segment.sha256 = LogicalBackupFormat.toHex(digest); + segment.status = "SEALED"; + manifest.lastDurableSequence = segmentLastSequence; + channel.close(); + channel = null; + writeManifest(); + } + + private boolean forceIfNeeded() throws IOException { + if (channel == null) { + return false; + } + final long now = System.currentTimeMillis(); + final boolean shouldForce = + fsyncPolicy == FsyncPolicy.ALWAYS + || (fsyncPolicy == FsyncPolicy.BATCH && operationsSinceFsync >= fsyncBatchOperations) + || (fsyncPolicy == FsyncPolicy.PERIODIC && now - lastFsyncAt >= fsyncPeriodMs); + if (shouldForce) { + channel.force(true); + operationsSinceFsync = 0; + lastFsyncAt = now; + } + return shouldForce; + } + + private void writeManifest() throws IOException { + final Path temporary = + manifestPath.resolveSibling(LogicalBackupFormat.MANIFEST_FILE_NAME + ".tmp"); + final byte[] manifestBytes = GSON.toJson(manifest).getBytes(StandardCharsets.UTF_8); + Files.deleteIfExists(temporary); + try (final FileChannel manifestChannel = + FileChannel.open(temporary, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { + final ByteBuffer manifestBuffer = ByteBuffer.wrap(manifestBytes); + while (manifestBuffer.hasRemaining()) { + manifestChannel.write(manifestBuffer); + } + manifestChannel.force(true); + } + try { + Files.move( + temporary, + manifestPath, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (final AtomicMoveNotSupportedException e) { + Files.move(temporary, manifestPath, StandardCopyOption.REPLACE_EXISTING); + } + } + + private void loadExistingManifest() throws IOException { + final LogicalBackupManifest existing; + try { + existing = + GSON.fromJson( + Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class); + } catch (final JsonParseException | NullPointerException e) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7, + manifestPath), + e); + } + if (existing == null) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_MANIFEST_IS_INVALID_ARG_CB809CC7, + manifestPath)); + } + if (!LogicalBackupFormat.FORMAT_NAME.equals(existing.formatName) + || !LogicalBackupFormat.FORMAT_VERSION.equals(existing.formatVersion)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_MANIFEST_FORMAT_ARG_5005F99E, + manifestPath)); + } + validateManifestIdentity("backupId", manifest.backupId, existing.backupId); + validateManifestIdentity("streamId", manifest.streamId, existing.streamId); + validateManifestIdentity("pipeName", manifest.pipeName, existing.pipeName); + validateManifestIdentity("streamType", manifest.streamType, existing.streamType); + validateManifestIdentity("sourceClusterId", manifest.sourceClusterId, existing.sourceClusterId); + validateManifestIdentity("sourceVersion", manifest.sourceVersion, existing.sourceVersion); + validateManifestIdentity( + "timestampPrecision", manifest.timestampPrecision, existing.timestampPrecision); + validateManifestIdentity("sinkTaskId", manifest.sinkTaskId, existing.sinkTaskId); + validateManifestIdentity( + "fsyncPolicy", fsyncPolicy.name().toLowerCase(Locale.ROOT), existing.fsyncPolicy); + validateManifestConfiguration("segmentSizeBytes", segmentSizeBytes, existing.segmentSizeBytes); + validateManifestConfiguration("maxRecordBytes", maxRecordBytes, existing.maxRecordBytes); + if (manifest.pipeCreationTime != 0 && manifest.pipeCreationTime != existing.pipeCreationTime) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1, + "pipeCreationTime", + manifest.pipeCreationTime, + existing.pipeCreationTime)); + } + if (manifest.regionId != existing.regionId) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1, + "regionId", + manifest.regionId, + existing.regionId)); + } + manifest.status = existing.status; + manifest.backupId = existing.backupId; + manifest.pipeName = existing.pipeName; + manifest.pipeCreationTime = existing.pipeCreationTime; + manifest.sourceClusterId = existing.sourceClusterId; + manifest.sourceVersion = existing.sourceVersion; + manifest.timestampPrecision = existing.timestampPrecision; + manifest.streamId = existing.streamId; + manifest.regionId = existing.regionId; + manifest.sinkTaskId = existing.sinkTaskId; + manifest.streamType = existing.streamType; + manifest.firstSequence = existing.firstSequence; + manifest.lastSequence = existing.lastSequence; + manifest.lastDurableSequence = existing.lastDurableSequence; + manifest.recovered = existing.recovered; + manifest.segments = existing.segments == null ? new ArrayList<>() : existing.segments; + manifest.operationCounts = + existing.operationCounts == null + ? new java.util.LinkedHashMap<>() + : existing.operationCounts; + manifest.createdAt = existing.createdAt; + manifest.closedAt = existing.closedAt; + manifest.fsyncPolicy = existing.fsyncPolicy; + manifest.segmentSizeBytes = existing.segmentSizeBytes; + manifest.maxRecordBytes = existing.maxRecordBytes; + manifest.lastEventGroupId = existing.lastEventGroupId; + manifest.lastEventDigest = existing.lastEventDigest; + manifest.lastEventFirstSequence = existing.lastEventFirstSequence; + manifest.skippedEventCount = existing.skippedEventCount; + } + + private void validateManifestIdentity( + final String field, final String expected, final String actual) throws IOException { + if (expected != null && !Objects.equals(expected, actual)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1, + field, + expected, + actual)); + } + } + + private void validateManifestConfiguration( + final String field, final long expected, final long actual) throws IOException { + if (expected != actual) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_MANIFEST_DOES_NOT_MATCH_ARG_EXPECTED_ARG_FOUND_ARG_D7BC8AD1, + field, + expected, + actual)); + } + } + + private void acquireLock() throws IOException { + try { + if (Files.isSymbolicLink(lockPath)) { + throw symbolicLinkException(lockPath); + } + lockChannel = FileChannel.open(lockPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + directoryLock = lockChannel.tryLock(); + if (directoryLock == null) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800, + directory)); + } + lockChannel.truncate(0); + lockChannel.write( + StandardCharsets.UTF_8.encode(Long.toString(ProcessHandle.current().pid()))); + lockChannel.force(true); + } catch (final OverlappingFileLockException e) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_IS_LOCKED_ARG_A4366800, + directory), + e); + } + } + + private void releaseLock() { + try { + if (directoryLock != null) { + directoryLock.release(); + directoryLock = null; + } + } catch (final IOException ignored) { + // Best effort during failure/close. + } + try { + if (lockChannel != null) { + lockChannel.close(); + lockChannel = null; + } + } catch (final IOException ignored) { + // Best effort during failure/close. + } + } + + private void writeFully(final ByteBuffer buffer) throws IOException { + while (buffer.hasRemaining()) { + channel.write(buffer); + } + } + + private void ensureOpen() throws IOException { + if (closed || channel == null) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_WRITER_IS_CLOSED_EE463BBB); + } + } + + @Override + public synchronized void close() throws IOException { + if (closed) { + return; + } + try { + if (channel != null) { + if (!streamEnded) { + writeControl(LogicalBackupRecordType.STREAM_END, System.currentTimeMillis(), ""); + } + sealSegment(); + } + manifest.status = manifest.recovered ? "RECOVERED" : "SEALED"; + manifest.closedAt = Instant.now().toString(); + manifest.lastDurableSequence = manifest.lastSequence; + writeManifest(); + } finally { + if (channel != null) { + channel.close(); + channel = null; + } + closed = true; + releaseLock(); + } + } + + private void rollBeforeEventIfNecessary( + final List requests, final String metadata) throws IOException { + final long metadataBytes = metadata.getBytes(StandardCharsets.UTF_8).length; + long requiredBytes = frameSize(metadataBytes, 0) * 2; + for (final TPipeTransferReq request : requests) { + if (request == null || request.getBody() == null) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9); + } + requiredBytes += frameSize(metadataBytes, request.getBody().length); + } + if (channel.position() > LogicalBackupFormat.SEGMENT_HEADER_SIZE + && channel.position() + requiredBytes + LogicalBackupFormat.SEGMENT_FOOTER_SIZE + > segmentSizeBytes) { + sealSegment(); + segmentId++; + openNewSegment(); + } + } + + private void rollBeforeControlIfNecessary(final String metadata) throws IOException { + final long requiredBytes = frameSize(metadata.getBytes(StandardCharsets.UTF_8).length, 0); + if (channel.position() > LogicalBackupFormat.SEGMENT_HEADER_SIZE + && channel.position() + requiredBytes + LogicalBackupFormat.SEGMENT_FOOTER_SIZE + > segmentSizeBytes) { + sealSegment(); + segmentId++; + openNewSegment(); + } + } + + private static long frameSize(final long metadataBytes, final long payloadBytes) { + return LogicalBackupFormat.RECORD_HEADER_SIZE + metadataBytes + payloadBytes + Integer.BYTES; + } + + private static String computeEventDigest(final List requests) + throws IOException { + final MessageDigest digest = LogicalBackupFormat.newSha256(); + for (final TPipeTransferReq request : requests) { + if (request == null || request.getBody() == null) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_BODY_MUST_NOT_BE_NULL_EFFD92D9); + } + updateEventDigest(digest, request.getVersion(), request.getType(), request.getBody()); + } + return LogicalBackupFormat.toHex(digest.digest()); + } + + private static void updateEventDigest( + final MessageDigest digest, + final byte requestVersion, + final short requestType, + final byte[] body) { + digest.update(requestVersion); + digest.update( + ByteBuffer.allocate(Short.BYTES).order(ByteOrder.BIG_ENDIAN).putShort(requestType).array()); + digest.update(body); + } + + private void rollbackTo(final long position) throws IOException { + final Path segmentPath = currentSegmentPath(); + channel.truncate(position); + channel.force(true); + channel.close(); + channel = null; + final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false); + openExistingSegment(segmentPath, scan); + rebuildManifestFromSegments(); + manifest.recovered = true; + manifest.status = "RECOVERED"; + manifest.lastDurableSequence = manifest.lastSequence; + operationsSinceFsync = 0; + lastFsyncAt = System.currentTimeMillis(); + updateActiveSegmentManifest(); + writeManifest(); + } + + private void rebuildManifestFromSegments() throws IOException { + manifest.firstSequence = -1; + manifest.lastSequence = -1; + manifest.lastEventGroupId = null; + manifest.lastEventDigest = null; + manifest.lastEventFirstSequence = -1; + manifest.skippedEventCount = 0; + manifest.operationCounts = new LinkedHashMap<>(); + + long expectedSequence = -1; + long expectedSegmentId = -1; + for (int segmentIndex = 0; segmentIndex < manifest.segments.size(); segmentIndex++) { + final LogicalBackupManifest.Segment segment = manifest.segments.get(segmentIndex); + final Path segmentPath = resolveExistingSegmentPath(segment.file); + final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false); + if (scan.getSegmentId() != segment.segmentId) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A, + segmentPath)); + } + if ((expectedSegmentId >= 0 && scan.getSegmentId() != expectedSegmentId) + || (!scan.isSealed() && segmentIndex != manifest.segments.size() - 1)) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A, + segmentPath)); + } + expectedSegmentId = scan.getSegmentId() + 1; + if (scan.isSealed() && "SEALED".equals(segment.status)) { + final LogicalBackupSegmentReader.Footer footer = scan.getFooter(); + if (segment.firstSequence != footer.getFirstSequence() + || segment.lastSequence != footer.getLastSequence() + || segment.recordCount != footer.getRecordCount() + || segment.sizeBytes != scan.getValidLength() + || !LogicalBackupFormat.toHex(footer.getDigest()).equals(segment.sha256)) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_LOGICAL_BACKUP_SEGMENT_METADATA_MISMATCH_ARG_376FD0B3, + segmentPath)); + } + } + + segment.firstSequence = -1; + segment.lastSequence = -1; + segment.recordCount = scan.getRecords().size(); + segment.sizeBytes = scan.getValidLength(); + segment.status = scan.isSealed() ? "SEALED" : "ACTIVE"; + segment.sha256 = + scan.isSealed() ? LogicalBackupFormat.toHex(scan.getFooter().getDigest()) : null; + + MessageDigest eventDigest = null; + UUID eventGroupId = null; + long eventFirstSequence = -1; + for (final LogicalBackupRecord record : scan.getRecords()) { + if (expectedSequence >= 0 && record.getSequence() != expectedSequence) { + throw new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEQUENCE_GAP_IN_ARG_D8698149, + segmentPath)); + } + expectedSequence = record.getSequence() + 1; + segment.firstSequence = + segment.firstSequence < 0 ? record.getSequence() : segment.firstSequence; + segment.lastSequence = record.getSequence(); + manifest.firstSequence = + manifest.firstSequence < 0 ? record.getSequence() : manifest.firstSequence; + manifest.lastSequence = record.getSequence(); + manifest.operationCounts.merge(record.getRecordType().name(), 1L, Long::sum); + if (record.getRecordType() == LogicalBackupRecordType.SKIPPED_EVENT) { + manifest.skippedEventCount++; + } else if (record.getRecordType() == LogicalBackupRecordType.EVENT_BEGIN) { + eventDigest = LogicalBackupFormat.newSha256(); + eventGroupId = record.getEventGroupId(); + eventFirstSequence = record.getSequence(); + } else if (record.getRecordType() == LogicalBackupRecordType.PIPE_REQUEST) { + updateEventDigest( + eventDigest, + record.getRequestVersion(), + record.getRequestType(), + record.getPayload()); + } else if (record.getRecordType() == LogicalBackupRecordType.EVENT_COMMIT) { + manifest.lastEventGroupId = eventGroupId.toString(); + manifest.lastEventDigest = LogicalBackupFormat.toHex(eventDigest.digest()); + manifest.lastEventFirstSequence = eventFirstSequence; + eventDigest = null; + eventGroupId = null; + eventFirstSequence = -1; + } + } + } + nextSequence = Math.max(0, manifest.lastSequence + 1); + streamEnded = + manifest.lastSequence >= 0 + && !manifest.segments.isEmpty() + && isLastRecordStreamEnd(reader.scan(currentSegmentPath(), false).getRecords()); + } + + private static boolean isLastRecordStreamEnd(final List records) { + return !records.isEmpty() + && records.get(records.size() - 1).getRecordType() == LogicalBackupRecordType.STREAM_END; + } + + private Path currentSegmentPath() throws IOException { + if (manifest.segments.isEmpty()) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_HAS_NO_SEGMENTS_B48D5F15); + } + return resolveSegmentPath(manifest.segments.get(manifest.segments.size() - 1).file); + } + + private Path resolveSegmentPath(final String file) throws IOException { + final Path relativePath; + try { + relativePath = file == null ? null : Path.of(file); + } catch (final RuntimeException e) { + throw invalidSegmentPath(file, e); + } + final Path normalizedDirectory = directory.toAbsolutePath().normalize(); + if (relativePath == null || relativePath.isAbsolute() || relativePath.getNameCount() != 1) { + throw invalidSegmentPath(file, null); + } + final Path resolved = normalizedDirectory.resolve(relativePath).normalize(); + if (!resolved.startsWith(normalizedDirectory) + || !normalizedDirectory.equals(resolved.getParent())) { + throw invalidSegmentPath(file, null); + } + return resolved; + } + + private Path resolveExistingSegmentPath(final String file) throws IOException { + final Path resolved = resolveSegmentPath(file); + if (Files.isSymbolicLink(resolved) + || !Files.isRegularFile(resolved, LinkOption.NOFOLLOW_LINKS)) { + throw invalidSegmentPath(file, null); + } + return resolved; + } + + private static IOException invalidSegmentPath(final String file, final Throwable cause) { + final IOException exception = + new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845, + file)); + if (cause != null) { + exception.initCause(cause); + } + return exception; + } + + private void updateActiveSegmentManifest() throws IOException { + if (channel == null || manifest.segments.isEmpty()) { + return; + } + final LogicalBackupManifest.Segment segment = + manifest.segments.get(manifest.segments.size() - 1); + segment.firstSequence = segmentFirstSequence; + segment.lastSequence = segmentLastSequence; + segment.recordCount = segmentRecordCount; + segment.sizeBytes = channel.position(); + segment.status = "ACTIVE"; + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java new file mode 100644 index 0000000000000..05f6298e95fac --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class LogicalBackupArchiveReaderTest { + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + private static final int MAX_RECORD_BYTES = 1024; + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testReadCompleteArchiveAndOrderSchemaFirst() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + writeStream(root.resolve("data"), "data-stream", "data", 4096); + writeStream(root.resolve("schema"), "schema-stream", "schema", 4096); + + final List streams = new LogicalBackupArchiveReader().read(root, false); + + Assert.assertEquals(2, streams.size()); + Assert.assertEquals("schema-stream", streams.get(0).getManifest().streamId); + Assert.assertEquals("data-stream", streams.get(1).getManifest().streamId); + Assert.assertEquals(1, streams.get(0).getEventGroups().size()); + Assert.assertEquals(4, streams.get(0).getRecords().size()); + Assert.assertEquals( + request(1), + streams.get(0).getEventGroups().get(0).getRequests().get(0).toTPipeTransferReq()); + } + + @Test + public void testManifestCounterMismatchIsRejected() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + writeStream(directory, "stream", "data", 4096); + final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + final LogicalBackupManifest manifest = + GSON.fromJson( + Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class); + manifest.operationCounts.clear(); + Files.writeString(manifestPath, GSON.toJson(manifest), StandardCharsets.UTF_8); + + assertReadFails(directory, false); + } + + @Test + public void testUnlistedSegmentIsRejected() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + writeStream(directory, "stream", "data", 4096); + Files.copy(onlySegment(directory), directory.resolve("unlisted.pwal")); + + assertReadFails(directory, false); + } + + @Test + public void testIncompleteLastSegmentRequiresExplicitOptIn() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + writeStream(directory, "stream", "data", 4096); + final Path segment = onlySegment(directory); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE); + } + + assertReadFails(directory, false); + final List streams = new LogicalBackupArchiveReader().read(directory, true); + Assert.assertEquals(1, streams.size()); + Assert.assertEquals(1, streams.get(0).getEventGroups().size()); + } + + @Test + public void testUnsealedNonLastSegmentIsRejected() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + writeStream(directory, "stream", "data", 500); + final List segments = segments(directory); + Assert.assertTrue(segments.size() > 1); + try (final RandomAccessFile file = new RandomAccessFile(segments.get(0).toFile(), "rw")) { + file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE); + } + + assertReadFails(directory, true); + } + + @Test + public void testParentSegmentPathIsRejected() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + writeStream(directory, "stream", "data", 4096); + final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + final LogicalBackupManifest manifest = + GSON.fromJson( + Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class); + manifest.segments.get(0).file = "../outside.pwal"; + Files.writeString(manifestPath, GSON.toJson(manifest), StandardCharsets.UTF_8); + + assertReadFails(directory, false); + } + + @Test + public void testSkippedEventRequiresExplicitIncompleteOptIn() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final LogicalBackupManifest manifest = manifest("stream", "data"); + try (final LogicalBackupWriter writer = writer(directory, manifest, 4096)) { + writer.recordSkippedEvent(1, "unsupported-event"); + } + + assertReadFails(directory, false); + Assert.assertEquals(1, new LogicalBackupArchiveReader().read(directory, true).size()); + + final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + final LogicalBackupManifest persistedManifest = + GSON.fromJson( + Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class); + persistedManifest.skippedEventCount = 0; + Files.writeString(manifestPath, GSON.toJson(persistedManifest), StandardCharsets.UTF_8); + assertReadFails(directory, false); + } + + @Test + public void testMixedBackupSessionsAreRejected() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + writeStream(root.resolve("data"), "data-stream", "data", 4096); + writeStream(root.resolve("schema"), "schema-stream", "schema", 4096); + final Path manifestPath = + root.resolve("schema").resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + final LogicalBackupManifest manifest = + GSON.fromJson( + Files.readString(manifestPath, StandardCharsets.UTF_8), LogicalBackupManifest.class); + manifest.backupId = "another-backup"; + Files.writeString(manifestPath, GSON.toJson(manifest), StandardCharsets.UTF_8); + + assertReadFails(root, false); + } + + @Test + public void testNonReplayableRequestTypeIsRejected() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, manifest("stream", "data"), 4096)) { + writer.writeEvent( + UUID.randomUUID(), + 1, + Collections.singletonList( + new TPipeTransferReq() + .setVersion((byte) 1) + .setType((short) 200) + .setBody(ByteBuffer.wrap(new byte[] {1}))), + "metadata"); + } + + assertReadFails(directory, false); + } + + @Test + public void testDuplicateEventGroupIdIsRejected() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final UUID eventGroupId = UUID.randomUUID(); + try (final LogicalBackupWriter writer = writer(directory, manifest("stream", "data"), 4096)) { + writer.writeEvent(eventGroupId, 1, Collections.singletonList(request(1)), "metadata"); + writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), "metadata"); + writer.writeEvent(eventGroupId, 3, Collections.singletonList(request(1)), "metadata"); + } + + assertReadFails(directory, false); + } + + private static void writeStream( + final Path directory, final String streamId, final String streamType, final long segmentSize) + throws Exception { + try (final LogicalBackupWriter writer = + writer(directory, manifest(streamId, streamType), segmentSize)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "metadata"); + if (segmentSize < 1000) { + writer.writeEvent( + UUID.randomUUID(), 2, Collections.singletonList(request(new byte[20])), "metadata"); + } + } + } + + private static LogicalBackupManifest manifest(final String streamId, final String streamType) { + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = "backup"; + manifest.pipeName = "pipe"; + manifest.pipeCreationTime = 1; + manifest.streamId = streamId; + manifest.streamType = streamType; + manifest.timestampPrecision = "ms"; + return manifest; + } + + private static LogicalBackupWriter writer( + final Path directory, final LogicalBackupManifest manifest, final long segmentSize) + throws IOException { + return new LogicalBackupWriter( + directory, + manifest, + segmentSize, + MAX_RECORD_BYTES, + LogicalBackupWriter.FsyncPolicy.ALWAYS, + 1, + 1, + false); + } + + private static TPipeTransferReq request(final int value) { + return request(new byte[] {(byte) value}); + } + + private static TPipeTransferReq request(final byte[] body) { + return new TPipeTransferReq() + .setVersion((byte) 1) + .setType((short) 10) + .setBody(ByteBuffer.wrap(body)); + } + + private static Path onlySegment(final Path directory) throws IOException { + final List segments = segments(directory); + Assert.assertEquals(1, segments.size()); + return segments.get(0); + } + + private static List segments(final Path directory) throws IOException { + try (final java.util.stream.Stream files = Files.list(directory)) { + return files + .filter(path -> path.getFileName().toString().endsWith(".pwal")) + .sorted() + .collect(java.util.stream.Collectors.toList()); + } + } + + private static void assertReadFails(final Path source, final boolean allowIncomplete) + throws IOException { + try { + new LogicalBackupArchiveReader().read(source, allowIncomplete); + Assert.fail(); + } catch (final IOException expected) { + // Expected. + } + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java new file mode 100644 index 0000000000000..b6806f3407be0 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java @@ -0,0 +1,588 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.AbstractList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class LogicalBackupWriterTest { + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + private static final int MAX_RECORD_BYTES = 1024; + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testRoundTripAndDuplicateEvent() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final UUID eventId = UUID.randomUUID(); + final TPipeTransferReq request = request((byte) 1, (short) 2, new byte[] {3, 4, 5}); + + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + Assert.assertEquals( + 0, writer.writeEvent(eventId, 123, Collections.singletonList(request), "m")); + Assert.assertEquals( + 0, writer.writeEvent(eventId, 123, Collections.singletonList(request), "m")); + } + + final LogicalBackupSegmentReader.ScanResult result = scanOnlySegment(directory); + Assert.assertTrue(result.isSealed()); + Assert.assertEquals(4, result.getRecords().size()); + Assert.assertEquals( + LogicalBackupRecordType.EVENT_BEGIN, result.getRecords().get(0).getRecordType()); + Assert.assertEquals( + LogicalBackupRecordType.PIPE_REQUEST, result.getRecords().get(1).getRecordType()); + Assert.assertEquals(request, result.getRecords().get(1).toTPipeTransferReq()); + Assert.assertEquals( + LogicalBackupRecordType.EVENT_COMMIT, result.getRecords().get(2).getRecordType()); + Assert.assertEquals( + LogicalBackupRecordType.STREAM_END, result.getRecords().get(3).getRecordType()); + Assert.assertEquals(0, result.getFooter().getFirstSequence()); + Assert.assertEquals(3, result.getFooter().getLastSequence()); + } + + @Test + public void testDuplicateEventWithDifferentDigestFails() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final UUID eventId = UUID.randomUUID(); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(eventId, 1, Collections.singletonList(request(1)), ""); + try { + writer.writeEvent(eventId, 1, Collections.singletonList(request(2)), ""); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains(eventId.toString())); + } + } + } + + @Test + public void testOnlineRollbackAfterPartialEvent() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + final List requests = + new AbstractList() { + private int accesses; + + @Override + public TPipeTransferReq get(final int index) { + if (++accesses == 6) { + throw new IllegalStateException(); + } + return request(index + 1); + } + + @Override + public int size() { + return 2; + } + }; + try { + writer.writeEvent(UUID.randomUUID(), 1, requests, ""); + Assert.fail(); + } catch (final IllegalStateException expected) { + // Expected. + } + Assert.assertEquals( + 0, + writer.writeEvent( + UUID.randomUUID(), 2, Collections.singletonList(request(3)), "after-rollback")); + } + + final List records = scanOnlySegment(directory).getRecords(); + Assert.assertEquals(4, records.size()); + Assert.assertEquals(0, records.get(0).getSequence()); + Assert.assertEquals("after-rollback", records.get(0).getMetadata()); + } + + @Test + public void testRecoverIncompleteEventAndRebuildCounts() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), ""); + } + final Path segment = onlySegment(directory); + final long beginAndRequestLength = + LogicalBackupFormat.SEGMENT_HEADER_SIZE + frameSize(0, 0) + frameSize(0, 1); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + file.setLength(beginAndRequestLength); + } + + try (final LogicalBackupWriter writer = writer(directory, 4096, true)) { + Assert.assertEquals( + 0, writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), "")); + Assert.assertTrue(writer.getManifest().recovered); + Assert.assertEquals(Long.valueOf(1), writer.getManifest().operationCounts.get("EVENT_BEGIN")); + Assert.assertEquals( + Long.valueOf(1), writer.getManifest().operationCounts.get("PIPE_REQUEST")); + Assert.assertEquals( + Long.valueOf(1), writer.getManifest().operationCounts.get("EVENT_COMMIT")); + } + } + + @Test + public void testRecoverIncompleteFrame() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), ""); + } + final Path segment = onlySegment(directory); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 2); + } + + try (final LogicalBackupWriter writer = writer(directory, 4096, true)) { + Assert.assertTrue(writer.getManifest().recovered); + Assert.assertEquals(2, writer.getManifest().lastSequence); + } + } + + @Test + public void testRecoverPartialFooter() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), ""); + } + final Path segment = onlySegment(directory); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + file.setLength(file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE + 8); + } + + try (final LogicalBackupWriter writer = writer(directory, 4096, true)) { + Assert.assertTrue(writer.getManifest().recovered); + Assert.assertEquals( + 4, writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), "")); + } + Assert.assertTrue( + new LogicalBackupSegmentReader(MAX_RECORD_BYTES) + .scan(segments(directory).get(segments(directory).size() - 1), false) + .isSealed()); + } + + @Test + public void testRecoverCorruptFooterWhenManifestStillMarksSegmentActive() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), ""); + } + final Path segment = onlySegment(directory); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + final long lastByteOffset = file.length() - 1; + file.seek(lastByteOffset); + final int lastByte = file.readUnsignedByte(); + file.seek(lastByteOffset); + file.writeByte(lastByte ^ 1); + } + final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + final LogicalBackupManifest manifest = + GSON.fromJson(Files.readString(manifestPath), LogicalBackupManifest.class); + manifest.status = "WRITING"; + manifest.closedAt = null; + manifest.segments.get(0).status = "ACTIVE"; + manifest.segments.get(0).sizeBytes -= LogicalBackupFormat.SEGMENT_FOOTER_SIZE; + manifest.segments.get(0).sha256 = null; + Files.writeString(manifestPath, GSON.toJson(manifest)); + + try (final LogicalBackupWriter writer = writer(directory, 4096, true)) { + Assert.assertTrue(writer.getManifest().recovered); + } + Assert.assertTrue(scanOnlySegment(directory).isSealed()); + } + + @Test + public void testChecksumDamageFails() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), "metadata"); + } + final Path segment = onlySegment(directory); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + final long payloadOffset = + LogicalBackupFormat.SEGMENT_HEADER_SIZE + + frameSize("metadata".length(), 0) + + LogicalBackupFormat.RECORD_HEADER_SIZE + + "metadata".length(); + file.seek(payloadOffset); + file.writeByte(file.readByte() ^ 1); + } + try { + new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segment, false); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains("CRC")); + } + } + + @Test + public void testFooterSequenceDamageFails() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), ""); + } + final Path segment = onlySegment(directory); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + final long footerOffset = file.length() - LogicalBackupFormat.SEGMENT_FOOTER_SIZE; + final byte[] footer = new byte[LogicalBackupFormat.SEGMENT_FOOTER_SIZE]; + file.seek(footerOffset); + file.readFully(footer); + ByteBuffer.wrap(footer).order(ByteOrder.BIG_ENDIAN).putLong(Integer.BYTES + Long.BYTES, 99); + ByteBuffer.wrap(footer) + .order(ByteOrder.BIG_ENDIAN) + .putInt( + LogicalBackupFormat.SEGMENT_FOOTER_SIZE - Integer.BYTES, + LogicalBackupFormat.crc32c( + footer, 0, LogicalBackupFormat.SEGMENT_FOOTER_SIZE - Integer.BYTES)); + file.seek(footerOffset); + file.write(footer); + } + try { + new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segment, false); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains("metadata")); + } + } + + @Test + public void testSegmentRollover() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 500, false)) { + writer.writeEvent( + UUID.randomUUID(), 1, Collections.singletonList(request(new byte[20])), "m"); + writer.writeEvent( + UUID.randomUUID(), 2, Collections.singletonList(request(new byte[20])), "m"); + } + final List segments = segments(directory); + Assert.assertEquals(2, segments.size()); + Assert.assertTrue( + new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segments.get(0), false).isSealed()); + Assert.assertTrue( + new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segments.get(1), false).isSealed()); + } + + @Test + public void testSealedSegmentAdvancesDurableSequenceWithFsyncNone() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final LogicalBackupManifest manifest = manifest(); + try (final LogicalBackupWriter writer = + new LogicalBackupWriter( + directory, + manifest, + 500, + MAX_RECORD_BYTES, + LogicalBackupWriter.FsyncPolicy.NONE, + 1, + 1, + false)) { + writer.writeEvent( + UUID.randomUUID(), 1, Collections.singletonList(request(new byte[20])), "m"); + writer.writeEvent( + UUID.randomUUID(), 2, Collections.singletonList(request(new byte[20])), "m"); + Assert.assertEquals(2, writer.getManifest().lastDurableSequence); + } + } + + @Test + public void testControlRecordRollover() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 258, false)) { + writer.writeControl(LogicalBackupRecordType.HEARTBEAT, 1, "first"); + writer.writeControl(LogicalBackupRecordType.HEARTBEAT, 2, "second"); + Assert.assertEquals(2, writer.getManifest().segments.size()); + Assert.assertEquals(1, writer.getManifest().lastSequence); + } + } + + @Test + public void testDirectoryLock() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) { + try { + writer(directory, 4096, true); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains("locked")); + } + } + } + + @Test + public void testFailIfDirectoryExists() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath(); + try { + writer(directory, 4096, false); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains(directory.toString())); + } + } + + @Test + public void testAppendRejectsConfigurationMismatch() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) { + // Create the backup. + } + try { + writer(directory, 8192, true); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains("segmentSizeBytes")); + } + } + + @Test + public void testAppendRejectsParentSegmentPath() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + final Path directory = root.resolve("backup"); + try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) { + // Create the backup. + } + final Path manifestPath = directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + final LogicalBackupManifest manifest = + GSON.fromJson(Files.readString(manifestPath), LogicalBackupManifest.class); + final String originalSegment = manifest.segments.get(0).file; + Files.copy(directory.resolve(originalSegment), root.resolve("outside.pwal")); + manifest.segments.get(0).file = "../outside.pwal"; + Files.writeString(manifestPath, GSON.toJson(manifest)); + + try { + writer(directory, 4096, true); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains("segment path")); + } + + manifest.segments.get(0).file = originalSegment; + Files.writeString(manifestPath, GSON.toJson(manifest)); + try (final LogicalBackupWriter ignored = writer(directory, 4096, true)) { + // A failed append must release the directory lock. + } + } + + @Test + public void testAppendAdoptsOrphanHeaderOnlySegmentAndClearsClosedState() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) { + // Create and seal the initial segment. + } + + final Path orphan = directory.resolve(segmentFileName(1)); + Files.write(orphan, segmentHeader(1, 123)); + try (final LogicalBackupWriter writer = writer(directory, 4096, true)) { + Assert.assertEquals("WRITING", writer.getManifest().status); + Assert.assertNull(writer.getManifest().closedAt); + Assert.assertEquals(2, writer.getManifest().segments.size()); + Assert.assertEquals( + 1, writer.writeEvent(UUID.randomUUID(), 2, Collections.singletonList(request(2)), "")); + } + + final LogicalBackupSegmentReader.ScanResult scan = + new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(orphan, false); + Assert.assertTrue(scan.isSealed()); + Assert.assertEquals(123, scan.getCreatedAt()); + } + + @Test + public void testAppendRejectsNonEmptyOrphanSegment() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) { + // Create and seal the initial segment. + } + + final Path orphan = directory.resolve(segmentFileName(1)); + final byte[] invalidOrphan = + Arrays.copyOf(segmentHeader(1, 123), LogicalBackupFormat.SEGMENT_HEADER_SIZE + 1); + Files.write(orphan, invalidOrphan); + try { + writer(directory, 4096, true); + Assert.fail(); + } catch (final IOException expected) { + // The extra byte is an incomplete record tail and must never be adopted. + } + } + + @Test + public void testAppendRejectsUnexpectedUnlistedSegment() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter ignored = writer(directory, 4096, false)) { + // Create and seal the initial segment. + } + + final Path unexpected = directory.resolve(segmentFileName(99)); + Files.write(unexpected, segmentHeader(99, 123)); + try { + writer(directory, 4096, true); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(expected.getMessage().contains("Unlisted")); + } + } + + @Test + public void testPeriodicFsyncAdvancesOnHeartbeat() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = "backup"; + manifest.pipeName = "pipe"; + manifest.pipeCreationTime = 1; + manifest.streamId = "stream"; + manifest.streamType = "data"; + try (final LogicalBackupWriter writer = + new LogicalBackupWriter( + directory, + manifest, + 4096, + MAX_RECORD_BYTES, + LogicalBackupWriter.FsyncPolicy.PERIODIC, + 1000, + 100, + false)) { + writer.writeEvent(UUID.randomUUID(), 1, Collections.singletonList(request(1)), ""); + Assert.assertEquals(-1, writer.getManifest().lastDurableSequence); + Thread.sleep(150); + writer.heartbeat(); + Assert.assertEquals( + writer.getManifest().lastSequence, writer.getManifest().lastDurableSequence); + } + } + + @Test + public void testMaxRecordBytesHasHardLimit() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + try { + new LogicalBackupWriter( + directory, + manifest, + 4096, + LogicalBackupFormat.MAX_RECORD_BYTES + 1, + LogicalBackupWriter.FsyncPolicy.ALWAYS, + 1, + 1, + false); + Assert.fail(); + } catch (final IOException expected) { + // Expected. + } + } + + private static LogicalBackupWriter writer( + final Path directory, final long segmentSize, final boolean append) throws IOException { + return new LogicalBackupWriter( + directory, + manifest(), + segmentSize, + MAX_RECORD_BYTES, + LogicalBackupWriter.FsyncPolicy.ALWAYS, + 1, + 1, + append); + } + + private static LogicalBackupManifest manifest() { + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = "backup"; + manifest.pipeName = "pipe"; + manifest.pipeCreationTime = 1; + manifest.streamId = "stream"; + manifest.streamType = "data"; + return manifest; + } + + private static TPipeTransferReq request(final int value) { + return request(new byte[] {(byte) value}); + } + + private static TPipeTransferReq request(final byte[] body) { + return request((byte) 1, (short) 2, body); + } + + private static TPipeTransferReq request(final byte version, final short type, final byte[] body) { + return new TPipeTransferReq().setVersion(version).setType(type).setBody(ByteBuffer.wrap(body)); + } + + private static LogicalBackupSegmentReader.ScanResult scanOnlySegment(final Path directory) + throws IOException { + return new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(onlySegment(directory), false); + } + + private static Path onlySegment(final Path directory) throws IOException { + final List segments = segments(directory); + Assert.assertEquals(1, segments.size()); + return segments.get(0); + } + + private static List segments(final Path directory) throws IOException { + try (final java.util.stream.Stream files = Files.list(directory)) { + final Path[] segments = + files + .filter(path -> path.getFileName().toString().endsWith(".pwal")) + .sorted() + .toArray(Path[]::new); + return Arrays.asList(segments); + } + } + + private static long frameSize(final int metadataBytes, final int payloadBytes) { + return LogicalBackupFormat.RECORD_HEADER_SIZE + metadataBytes + payloadBytes + Integer.BYTES; + } + + private static String segmentFileName(final long segmentId) { + return String.format(java.util.Locale.ROOT, "segment-%020d.pwal", segmentId); + } + + private static byte[] segmentHeader(final long segmentId, final long createdAt) { + final ByteBuffer header = + ByteBuffer.allocate(LogicalBackupFormat.SEGMENT_HEADER_SIZE).order(ByteOrder.BIG_ENDIAN); + header.putLong(LogicalBackupFormat.SEGMENT_MAGIC); + header.putShort(LogicalBackupFormat.MAJOR_VERSION); + header.putShort(LogicalBackupFormat.MINOR_VERSION); + header.putLong(segmentId); + header.putLong(createdAt); + header.putInt(0); + final byte[] bytes = header.array(); + ByteBuffer.wrap(bytes) + .order(ByteOrder.BIG_ENDIAN) + .putInt( + LogicalBackupFormat.SEGMENT_HEADER_SIZE - Integer.BYTES, + LogicalBackupFormat.crc32c( + bytes, 0, LogicalBackupFormat.SEGMENT_HEADER_SIZE - Integer.BYTES)); + return bytes; + } +} diff --git a/scripts/tools/export-pipe-logical-backup.sh b/scripts/tools/export-pipe-logical-backup.sh new file mode 100644 index 0000000000000..8a0a4f8a1834b --- /dev/null +++ b/scripts/tools/export-pipe-logical-backup.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +exec "$(dirname "$0")/pipe-logical-backup.sh" export "$@" diff --git a/scripts/tools/import-pipe-logical-backup.sh b/scripts/tools/import-pipe-logical-backup.sh new file mode 100644 index 0000000000000..6e4e069210ded --- /dev/null +++ b/scripts/tools/import-pipe-logical-backup.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +exec "$(dirname "$0")/pipe-logical-backup.sh" import "$@" diff --git a/scripts/tools/pipe-logical-backup.sh b/scripts/tools/pipe-logical-backup.sh new file mode 100644 index 0000000000000..a0f6a3529f746 --- /dev/null +++ b/scripts/tools/pipe-logical-backup.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +if [ -n "${IOTDB_INCLUDE}" ] && [ -r "${IOTDB_INCLUDE}" ]; then + . "${IOTDB_INCLUDE}" +fi + +if [ -z "${IOTDB_HOME}" ]; then + IOTDB_HOME="$(cd "$(dirname "$0")/.."; pwd)" + export IOTDB_HOME +fi + +if [ -n "${JAVA_HOME}" ] && [ -x "${JAVA_HOME}/bin/java" ]; then + JAVA="${JAVA_HOME}/bin/java" +else + JAVA=java +fi + +CLASSPATH="" +for jar in "${IOTDB_HOME}"/lib/*.jar; do + CLASSPATH="${CLASSPATH}:${jar}" +done + +exec "${JAVA}" -Dsun.jnu.encoding=UTF-8 -Dfile.encoding=UTF-8 \ + -DIOTDB_HOME="${IOTDB_HOME}" -cp "${CLASSPATH}" \ + org.apache.iotdb.tool.pipe.PipeLogicalBackupTool "$@" diff --git a/scripts/tools/windows/export-pipe-logical-backup.bat b/scripts/tools/windows/export-pipe-logical-backup.bat new file mode 100644 index 0000000000000..42e1df8afb274 --- /dev/null +++ b/scripts/tools/windows/export-pipe-logical-backup.bat @@ -0,0 +1,21 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM + +@call "%~dp0pipe-logical-backup.bat" export %* +@exit /B %ERRORLEVEL% diff --git a/scripts/tools/windows/import-pipe-logical-backup.bat b/scripts/tools/windows/import-pipe-logical-backup.bat new file mode 100644 index 0000000000000..0fba332d5a2e5 --- /dev/null +++ b/scripts/tools/windows/import-pipe-logical-backup.bat @@ -0,0 +1,21 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM + +@call "%~dp0pipe-logical-backup.bat" import %* +@exit /B %ERRORLEVEL% diff --git a/scripts/tools/windows/pipe-logical-backup.bat b/scripts/tools/windows/pipe-logical-backup.bat new file mode 100644 index 0000000000000..14e3180e46994 --- /dev/null +++ b/scripts/tools/windows/pipe-logical-backup.bat @@ -0,0 +1,40 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM + +@echo off +if "%OS%" == "Windows_NT" setlocal + +pushd %~dp0..\.. +if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD% +popd + +if NOT DEFINED JAVA_HOME goto :no_java +set CLASSPATH=%CLASSPATH%;"%IOTDB_HOME%\lib\*" +"%JAVA_HOME%\bin\java" -Dsun.jnu.encoding=UTF-8 -Dfile.encoding=UTF-8 ^ + -DIOTDB_HOME="%IOTDB_HOME%" -cp %CLASSPATH% ^ + org.apache.iotdb.tool.pipe.PipeLogicalBackupTool %* +set ret_code=%ERRORLEVEL% +goto :finally + +:no_java +echo JAVA_HOME environment variable must be set! +set ret_code=1 + +:finally +endlocal & exit /B %ret_code% From ce4b30d0ea26d433ca780e03fb691100ab15c268 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Sep 2026 12:03:37 +0800 Subject: [PATCH 02/17] Fix logical backup tablet serialization error handling --- .../logicalbackup/LogicalBackupSink.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java index 5050107c05f98..594d49922f7e0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java @@ -51,6 +51,7 @@ import org.apache.iotdb.pipe.api.event.Event; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; +import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; @@ -389,13 +390,17 @@ public void transfer(final TsFileInsertionEvent event) throws Exception { try { tsFile.consumeTabletInsertionEventsWithRetry( tablet -> { - requests.add( - PipeTransferTabletRawReqV2.toTPipeTransferReq( - tablet.convertToTablet(), - tablet.isAligned(), - tablet.isTableModelEvent() - ? tablet.getTableModelDatabaseName() - : tablet.getTreeModelDatabaseName())); + try { + requests.add( + PipeTransferTabletRawReqV2.toTPipeTransferReq( + tablet.convertToTablet(), + tablet.isAligned(), + tablet.isTableModelEvent() + ? tablet.getTableModelDatabaseName() + : tablet.getTreeModelDatabaseName())); + } catch (final IOException e) { + throw new PipeException(e.getMessage(), e); + } }, getClass().getName()); if (!requests.isEmpty()) { From 218dcb43001c4f99f5905d0cba032ee7e62478c1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Sep 2026 12:06:00 +0800 Subject: [PATCH 03/17] Harden logical backup failover and import recovery --- .../apache/iotdb/cli/i18n/CliMessages.java | 5 +- .../apache/iotdb/cli/i18n/CliMessages.java | 5 +- .../tool/pipe/PipeLogicalBackupTool.java | 191 +++++++++++++----- .../tool/pipe/PipeLogicalBackupToolTest.java | 111 +++++++++- .../iotdb/db/i18n/DataNodePipeMessages.java | 3 + .../iotdb/db/i18n/DataNodePipeMessages.java | 3 + .../logicalbackup/LogicalBackupSink.java | 25 +++ .../logicalbackup/LogicalBackupSinkTest.java | 79 ++++++++ .../LogicalBackupWriterTest.java | 30 +++ 9 files changed, 390 insertions(+), 62 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java diff --git a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java index 77888e65768a0..8aa2feccc907a 100644 --- a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java @@ -130,9 +130,8 @@ private CliMessages() {} public static final String LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E = "Use --input to specify the input. Export also requires --output; import requires --host and --port. Use --password-stdin or --password-env to avoid command-line passwords."; - public static final String - LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_RESTORE_STATS_BFF9FDC2 = - "pipe-logical-backup "; + public static final String LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE = + "pipe-logical-backup "; public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 = "Logical backup archive entry is unsafe: %s"; public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 = diff --git a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java index 530b504b29ce0..35346133f19e5 100644 --- a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java @@ -126,9 +126,8 @@ private CliMessages() {} public static final String LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E = "使用 --input 指定输入。导出还需要 --output；导入需要 --host 和 --port。请使用 --password-stdin 或 --password-env，避免密码出现在命令行中。"; - public static final String - LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_RESTORE_STATS_BFF9FDC2 = - "pipe-logical-backup "; + public static final String LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE = + "pipe-logical-backup "; public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 = "逻辑备份归档项路径不安全：%s"; public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 = diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java index 5d80243f3adf5..1d2315f17c04a 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java @@ -80,8 +80,6 @@ public final class PipeLogicalBackupTool { private static final String COMMAND_VERIFY = "verify"; private static final String COMMAND_EXPORT = "export"; private static final String COMMAND_IMPORT = "import"; - private static final String COMMAND_RESTORE = "restore"; - private static final String COMMAND_STATS = "stats"; private static final String OPTION_INPUT = "input"; private static final String OPTION_OUTPUT = "output"; private static final String OPTION_RESUME = "resume"; @@ -137,10 +135,7 @@ static int run(final String[] args) throws Exception { case COMMAND_EXPORT: return export(line); case COMMAND_IMPORT: - case COMMAND_RESTORE: return importBackup(line); - case COMMAND_STATS: - return inspect(line); default: throw new ParseException( String.format( @@ -352,19 +347,13 @@ private static int importBackup(final CommandLine line) throws Exception { handshake(client, streams.get(0).getManifest().timestampPrecision, user, password); long importedGroups = 0; for (final BackupStream stream : streams) { - final String streamId = stream.getManifest().streamId; - final long lastApplied = checkpointState.appliedSequences.getOrDefault(streamId, -1L); - for (final EventGroup group : stream.getEventGroups()) { - if (group.getLastSequence() <= lastApplied) { - continue; - } - for (final LogicalBackupRecord record : group.getRequests()) { - transfer(client, record.toTPipeTransferReq()); - } - checkpointState.appliedSequences.put(streamId, group.getLastSequence()); - writeCheckpoint(checkpoint, checkpointState); - importedGroups++; - } + importedGroups += + importStream( + checkpoint, + checkpointState, + stream, + request -> transfer(client, request), + state -> writeCheckpoint(checkpoint, state)); } writeCheckpoint(checkpoint, checkpointState); System.out.println( @@ -377,6 +366,49 @@ private static int importBackup(final CommandLine line) throws Exception { return 0; } + static long importStream( + final Path checkpoint, + final ImportCheckpoint checkpointState, + final BackupStream stream, + final RequestTransfer requestTransfer, + final CheckpointPersister checkpointPersister) + throws Exception { + final String streamId = stream.getManifest().streamId; + final long lastApplied = checkpointState.appliedSequences.getOrDefault(streamId, -1L); + long importedGroups = 0; + for (final EventGroup group : stream.getEventGroups()) { + if (group.getLastSequence() <= lastApplied) { + continue; + } + + int nextRequestIndex = 0; + if (checkpointState.inProgress == null) { + checkpointState.inProgress = ImportProgress.start(streamId, group); + checkpointPersister.persist(checkpointState); + } else if (checkpointState.inProgress.matches(streamId, group)) { + nextRequestIndex = checkpointState.inProgress.nextRequestIndex; + } else { + throw invalidCheckpoint(checkpoint); + } + + final List requests = group.getRequests(); + if (nextRequestIndex < 0 || nextRequestIndex > requests.size()) { + throw invalidCheckpoint(checkpoint); + } + for (int index = nextRequestIndex; index < requests.size(); index++) { + requestTransfer.transfer(requests.get(index).toTPipeTransferReq()); + checkpointState.inProgress.nextRequestIndex = index + 1; + checkpointPersister.persist(checkpointState); + } + + checkpointState.appliedSequences.put(streamId, group.getLastSequence()); + checkpointState.inProgress = null; + checkpointPersister.persist(checkpointState); + importedGroups++; + } + return importedGroups; + } + private static List read(final CommandLine line) throws IOException { final String source = input(line); try (final BackupInput input = BackupInput.open(Path.of(source))) { @@ -472,14 +504,15 @@ private static void handshake( } } - private static void transfer(final IoTDBSyncClient client, final TPipeTransferReq request) + static void transfer(final IoTDBSyncClient client, final TPipeTransferReq request) throws TException, IOException { final TPipeTransferResp response = client.pipeTransfer(request); if (response == null || response.getStatus() == null || (response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() + && response.getStatus().getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode() && response.getStatus().getCode() - != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode())) { + != TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode())) { throw new IOException( String.format( CliMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11, @@ -497,7 +530,7 @@ private static Path checkpointPath(final CommandLine line) throws IOException { return source.resolveSibling(source.getFileName() + ".import.checkpoint.json"); } - private static ImportCheckpoint readCheckpoint( + static ImportCheckpoint readCheckpoint( final Path checkpoint, final List streams, final String host, @@ -539,7 +572,7 @@ private static ImportCheckpoint readCheckpoint( || !host.equals(state.targetHost) || port != state.targetPort || !user.equals(state.targetUser) - || !checkpointSequencesAreValid(streams, state.appliedSequences)) { + || !checkpointStateIsValid(streams, state)) { throw new IOException( String.format( CliMessages @@ -615,34 +648,66 @@ private static void updateDigest(final MessageDigest digest, final long value) { digest.update(ByteBuffer.allocate(Long.BYTES).putLong(value).array()); } - private static boolean checkpointSequencesAreValid( - final List streams, final Map appliedSequences) { + private static boolean checkpointStateIsValid( + final List streams, final ImportCheckpoint state) { + final Map appliedSequences = state.appliedSequences; if (appliedSequences.size() > streams.size()) { return false; } + boolean foundIncompleteStream = false; + boolean matchedInProgress = false; for (final BackupStream stream : streams) { - if (!appliedSequences.containsKey(stream.getManifest().streamId)) { - continue; - } - final Long applied = appliedSequences.get(stream.getManifest().streamId); - if (applied == null) { + final String streamId = stream.getManifest().streamId; + final Long applied = appliedSequences.getOrDefault(streamId, -1L); + if (applied == null || applied < -1) { return false; } - if (applied == -1) { - continue; + + int appliedGroupIndex = -1; + if (applied != -1) { + for (int index = 0; index < stream.getEventGroups().size(); index++) { + if (stream.getEventGroups().get(index).getLastSequence() == applied) { + appliedGroupIndex = index; + break; + } + } + if (appliedGroupIndex < 0) { + return false; + } } - if (stream.getEventGroups().stream().noneMatch(group -> group.getLastSequence() == applied)) { + + final boolean streamComplete = appliedGroupIndex == stream.getEventGroups().size() - 1; + if (foundIncompleteStream && appliedGroupIndex >= 0) { return false; } + if (!streamComplete && !foundIncompleteStream) { + foundIncompleteStream = true; + if (state.inProgress != null) { + final EventGroup nextGroup = stream.getEventGroups().get(appliedGroupIndex + 1); + if (!state.inProgress.matches(streamId, nextGroup) + || state.inProgress.nextRequestIndex < 0 + || state.inProgress.nextRequestIndex > nextGroup.getRequests().size()) { + return false; + } + matchedInProgress = true; + } + } } - return appliedSequences.keySet().stream() - .allMatch( - streamId -> - streams.stream() - .anyMatch(stream -> stream.getManifest().streamId.equals(streamId))); + return (state.inProgress == null || matchedInProgress) + && appliedSequences.keySet().stream() + .allMatch( + streamId -> + streams.stream() + .anyMatch(stream -> stream.getManifest().streamId.equals(streamId))); + } + + private static IOException invalidCheckpoint(final Path checkpoint) { + return new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C, checkpoint)); } - private static void writeCheckpoint(final Path checkpoint, final ImportCheckpoint checkpointState) + static void writeCheckpoint(final Path checkpoint, final ImportCheckpoint checkpointState) throws IOException { final Path parent = checkpoint.toAbsolutePath().normalize().getParent(); if (parent != null) { @@ -681,7 +746,7 @@ private static void printUsage() { formatter.printHelp( new PrintWriter(System.out), 120, - CliMessages.LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_RESTORE_STATS_BFF9FDC2, + CliMessages.LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE, CliMessages .LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E, optionsForHelp(), @@ -710,14 +775,46 @@ private static Options optionsForHelp() { return options; } - private static class ImportCheckpoint { - private String formatName; - private String formatVersion; - private Map sourceStreams = new LinkedHashMap<>(); - private String targetHost; - private int targetPort; - private String targetUser; - private Map appliedSequences = new LinkedHashMap<>(); + static class ImportCheckpoint { + String formatName; + String formatVersion; + Map sourceStreams = new LinkedHashMap<>(); + String targetHost; + int targetPort; + String targetUser; + Map appliedSequences = new LinkedHashMap<>(); + ImportProgress inProgress; + } + + static class ImportProgress { + String streamId; + String eventGroupId; + long lastSequence; + int nextRequestIndex; + + static ImportProgress start(final String streamId, final EventGroup group) { + final ImportProgress progress = new ImportProgress(); + progress.streamId = streamId; + progress.eventGroupId = group.getEventGroupId().toString(); + progress.lastSequence = group.getLastSequence(); + return progress; + } + + boolean matches(final String candidateStreamId, final EventGroup group) { + return candidateStreamId.equals(streamId) + && group.getEventGroupId().toString().equals(eventGroupId) + && group.getLastSequence() == lastSequence; + } + } + + @FunctionalInterface + interface RequestTransfer { + void transfer(TPipeTransferReq request) throws Exception; + } + + @FunctionalInterface + interface CheckpointPersister { + void persist(ImportCheckpoint state) throws IOException; } private static class BackupInput implements AutoCloseable { diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java index 0fe45b22d8970..6e7af8726ab35 100644 --- a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java @@ -19,27 +19,35 @@ package org.apache.iotdb.tool.pipe; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.pipe.sink.client.IoTDBSyncClient; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupFormat; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; import org.apache.commons.cli.ParseException; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import java.util.zip.ZipOutputStream; @@ -141,7 +149,93 @@ public void testPlaintextPasswordOptionIsRejected() throws Exception { } } + @Test + public void testRemovedCommandAliasesAreRejected() throws Exception { + final Path source = temporaryFolder.getRoot().toPath().resolve("source"); + writeBackup(source); + + for (final String command : Arrays.asList("restore", "stats")) { + try { + PipeLogicalBackupTool.run(new String[] {command, "--input", source.toString()}); + Assert.fail(); + } catch (final ParseException expected) { + // Expected. + } + } + } + + @Test + public void testTransferAcceptsIdempotentConflict() throws Exception { + final IoTDBSyncClient client = Mockito.mock(IoTDBSyncClient.class); + final TPipeTransferReq request = request(1); + Mockito.when(client.pipeTransfer(request)) + .thenReturn( + new TPipeTransferResp( + new TSStatus( + TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()))); + + PipeLogicalBackupTool.transfer(client, request); + } + + @Test + public void testImportCheckpointResumesAtRequestGranularity() throws Exception { + final Path root = temporaryFolder.getRoot().toPath(); + final Path source = root.resolve("source"); + writeBackup(source, Arrays.asList(request(1), request(2))); + final List streams = new LogicalBackupArchiveReader().read(source, false); + final BackupStream stream = streams.get(0); + final Path checkpoint = root.resolve("import.checkpoint.json"); + final PipeLogicalBackupTool.ImportCheckpoint state = + PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); + final List transferredBodies = new ArrayList<>(); + final AtomicBoolean failCheckpointOnce = new AtomicBoolean(true); + + try { + PipeLogicalBackupTool.importStream( + checkpoint, + state, + stream, + request -> transferredBodies.add((int) request.getBody()[0]), + checkpointState -> { + if (checkpointState.inProgress != null + && checkpointState.inProgress.nextRequestIndex == 1 + && failCheckpointOnce.getAndSet(false)) { + throw new IOException("simulated checkpoint failure"); + } + PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState); + }); + Assert.fail(); + } catch (final IOException expected) { + // The request succeeded, but advancing its checkpoint was interrupted. + } + + Assert.assertEquals(Collections.singletonList(1), transferredBodies); + final PipeLogicalBackupTool.ImportCheckpoint durableState = + PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); + Assert.assertNotNull(durableState.inProgress); + Assert.assertEquals(0, durableState.inProgress.nextRequestIndex); + + Assert.assertEquals( + 1, + PipeLogicalBackupTool.importStream( + checkpoint, + durableState, + stream, + request -> transferredBodies.add((int) request.getBody()[0]), + checkpointState -> PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState))); + Assert.assertEquals(Arrays.asList(1, 1, 2), transferredBodies); + Assert.assertNull(durableState.inProgress); + Assert.assertEquals( + Long.valueOf(stream.getEventGroups().get(0).getLastSequence()), + durableState.appliedSequences.get(stream.getManifest().streamId)); + } + private static void writeBackup(final Path directory) throws Exception { + writeBackup(directory, Collections.singletonList(request(3))); + } + + private static void writeBackup(final Path directory, final List requests) + throws Exception { final LogicalBackupManifest manifest = new LogicalBackupManifest(); manifest.backupId = "backup"; manifest.pipeName = "pipe"; @@ -152,15 +246,14 @@ private static void writeBackup(final Path directory) throws Exception { try (final LogicalBackupWriter writer = new LogicalBackupWriter( directory, manifest, 4096, 1024, LogicalBackupWriter.FsyncPolicy.ALWAYS, 1, 1, false)) { - writer.writeEvent( - UUID.randomUUID(), - 1, - Collections.singletonList( - new TPipeTransferReq() - .setVersion((byte) 1) - .setType((short) 10) - .setBody(ByteBuffer.wrap(new byte[] {3}))), - "metadata"); + writer.writeEvent(UUID.randomUUID(), 1, requests, "metadata"); } } + + private static TPipeTransferReq request(final int value) { + return new TPipeTransferReq() + .setVersion((byte) 1) + .setType((short) 10) + .setBody(ByteBuffer.wrap(new byte[] {(byte) value})); + } } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index d530fc7cf1bd3..3a914bc80c763 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2647,6 +2647,9 @@ private DataNodePipeMessages() {} "Unsupported logical backup event policy: %s"; public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 = "Unsupported logical backup resume policy: %s"; + public static final String + EXCEPTION_DATAREGION_LOGICAL_BACKUP_REQUIRES_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_9F7ADEA8 = + "DataRegion logical backup requires explicit sink.resume=append (or connector.resume=append) and a shared backup directory"; public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 = "Unsupported Pipe event for logical backup: %s"; public static final String diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index 49575e46abb91..08803638ea60a 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2471,6 +2471,9 @@ private DataNodePipeMessages() {} "不支持的逻辑备份事件策略：%s"; public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 = "不支持的逻辑备份恢复策略：%s"; + public static final String + EXCEPTION_DATAREGION_LOGICAL_BACKUP_REQUIRES_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_9F7ADEA8 = + "DataRegion 逻辑备份要求显式配置 sink.resume=append（或 connector.resume=append），并使用共享备份目录"; public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 = "逻辑备份不支持 Pipe 事件：%s"; public static final String diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java index 594d49922f7e0..ef152b58b7c34 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java @@ -160,6 +160,7 @@ public void validate(final PipeParameterValidator validator) throws Exception { PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH, PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIODIC, PipeSinkConstant.LOGICAL_BACKUP_FSYNC_NONE); + validateDataRegionResumePolicy(parameters); final String policy = parameters .getStringOrDefault( @@ -187,6 +188,29 @@ private static void validateSynonymAttributes( false); } + private void validateDataRegionResumePolicy(final PipeParameters parameters) + throws PipeParameterNotValidException { + if (!"data".equals(streamType)) { + return; + } + final boolean explicitlyConfigured = + parameters.hasAnyAttributes( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY); + final String resume = + parameters.getStringOrDefault( + Arrays.asList( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY), + PipeSinkConstant.LOGICAL_BACKUP_RESUME_DEFAULT_VALUE); + if (!explicitlyConfigured + || !PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND.equalsIgnoreCase(resume)) { + throw new PipeParameterNotValidException( + DataNodePipeMessages + .EXCEPTION_DATAREGION_LOGICAL_BACKUP_REQUIRES_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_9F7ADEA8); + } + } + @Override public void customize( final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration) @@ -253,6 +277,7 @@ public void customize( PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY), PipeSinkConstant.LOGICAL_BACKUP_RESUME_DEFAULT_VALUE) .toLowerCase(Locale.ROOT); + validateDataRegionResumePolicy(parameters); final boolean append = PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND.equals(resume); if (!append && !PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW.equals(resume) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java new file mode 100644 index 0000000000000..37c3d622b4f55 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.sink.protocol.logicalbackup; + +import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class LogicalBackupSinkTest { + + @Test + public void testDataRegionRequiresExplicitAppendResumePolicy() throws Exception { + final Map attributes = new HashMap<>(); + attributes.put(PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY, "backup"); + + assertInvalid(new LogicalBackupSink("data"), attributes); + + attributes.put( + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS); + assertInvalid(new LogicalBackupSink("data"), attributes); + + attributes.put( + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND); + new LogicalBackupSink("data") + .validate(new PipeParameterValidator(new PipeParameters(attributes))); + + attributes.remove(PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY); + attributes.put( + PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND); + new LogicalBackupSink("data") + .validate(new PipeParameterValidator(new PipeParameters(attributes))); + } + + @Test + public void testSchemaRegionKeepsDefaultResumePolicy() throws Exception { + final Map attributes = new HashMap<>(); + attributes.put(PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY, "backup"); + + new LogicalBackupSink("schema") + .validate(new PipeParameterValidator(new PipeParameters(attributes))); + } + + private static void assertInvalid( + final LogicalBackupSink sink, final Map attributes) throws Exception { + try { + sink.validate(new PipeParameterValidator(new PipeParameters(new HashMap<>(attributes)))); + Assert.fail(); + } catch (final PipeParameterNotValidException expected) { + Assert.assertTrue(expected.getMessage().contains("append")); + } + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java index b6806f3407be0..ec7d5d63ed9fc 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java @@ -91,6 +91,31 @@ public void testDuplicateEventWithDifferentDigestFails() throws Exception { } } + @Test + public void testAppendContinuesAndDeduplicatesLastEventAfterRestart() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final UUID firstEventId = UUID.randomUUID(); + final TPipeTransferReq firstRequest = replayableRequest(1); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + Assert.assertEquals( + 0, writer.writeEvent(firstEventId, 1, Collections.singletonList(firstRequest), "first")); + } + + try (final LogicalBackupWriter writer = writer(directory, 4096, true)) { + Assert.assertEquals( + 0, writer.writeEvent(firstEventId, 1, Collections.singletonList(firstRequest), "first")); + Assert.assertEquals( + 4, + writer.writeEvent( + UUID.randomUUID(), 2, Collections.singletonList(replayableRequest(2)), "second")); + } + + final List streams = + new LogicalBackupArchiveReader().read(directory, false); + Assert.assertEquals(1, streams.size()); + Assert.assertEquals(2, streams.get(0).getEventGroups().size()); + } + @Test public void testOnlineRollbackAfterPartialEvent() throws Exception { final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); @@ -522,6 +547,7 @@ private static LogicalBackupManifest manifest() { manifest.pipeCreationTime = 1; manifest.streamId = "stream"; manifest.streamType = "data"; + manifest.timestampPrecision = "ms"; return manifest; } @@ -533,6 +559,10 @@ private static TPipeTransferReq request(final byte[] body) { return request((byte) 1, (short) 2, body); } + private static TPipeTransferReq replayableRequest(final int value) { + return request((byte) 1, (short) 10, new byte[] {(byte) value}); + } + private static TPipeTransferReq request(final byte version, final short type, final byte[] body) { return new TPipeTransferReq().setVersion(version).setType(type).setBody(ByteBuffer.wrap(body)); } From 4451babdf117666b14245a04ea1561b1fdafbe0c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 21 Sep 2026 16:43:51 +0800 Subject: [PATCH 04/17] Add managed logical backup export workflow --- .../apache/iotdb/cli/i18n/CliMessages.java | 21 +- .../apache/iotdb/cli/i18n/CliMessages.java | 21 +- .../tool/pipe/PipeLogicalBackupTool.java | 228 ++++++++++++++++-- 3 files changed, 250 insertions(+), 20 deletions(-) diff --git a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java index 8aa2feccc907a..064dd16db8b05 100644 --- a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java @@ -128,8 +128,8 @@ private CliMessages() {} public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11 = "Logical backup request type %d failed: %s"; public static final String - LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E = - "Use --input to specify the input. Export also requires --output; import requires --host and --port. Use --password-stdin or --password-env to avoid command-line passwords."; + LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_EXPORT_CAN_USE_BACKUP_DIR_TO_CREATE_MONITOR_AND_CLEAN_UP_A_SNAPSHOT_PIPE_AUTOMATICALLY_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_F5B223CB = + "Use --input to specify the input. Export also requires --output; export can use --backup-dir to create, monitor, and clean up a snapshot Pipe automatically. Import requires --host and --port. Use --password-stdin or --password-env to avoid command-line passwords."; public static final String LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE = "pipe-logical-backup "; public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 = @@ -151,5 +151,22 @@ private CliMessages() {} "Logical backup checkpoint is invalid: %s"; public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1 = "Unsupported logical backup export format: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_TIMEOUT_AND_POLL_INTERVAL_MUST_BE_POSITIVE_38622769 = + "Logical backup export timeout and poll interval must be positive"; + public static final String + LOG_LOGICAL_BACKUP_EXPORT_ARG_STATE_ARG_REMAINING_ARG_ESTIMATED_SECONDS_ARG_6256B85B = + "Logical backup export %s: state=%s, remaining=%s, estimated_seconds=%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_EXPORT_PIPE_ARG_FAILED_ARG_F52EB190 = + "Logical backup export pipe %s failed: %s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_PIPE_ARG_TIMED_OUT_AFTER_ARG_SECONDS_76CF4B4B = + "Logical backup export pipe %s timed out after %d seconds"; + public static final String + EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_MUST_BE_AN_INTEGER_C9B397BE = + "Logical backup option --%s must be an integer"; + public static final String + EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_HAS_INVALID_IDENTIFIER_ARG_BB015204 = + "Logical backup option --%s has invalid identifier: %s"; } diff --git a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java index 35346133f19e5..d08e2053fa8cb 100644 --- a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java @@ -124,8 +124,8 @@ private CliMessages() {} public static final String EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11 = "逻辑备份请求类型 %d 执行失败：%s"; public static final String - LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E = - "使用 --input 指定输入。导出还需要 --output；导入需要 --host 和 --port。请使用 --password-stdin 或 --password-env，避免密码出现在命令行中。"; + LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_EXPORT_CAN_USE_BACKUP_DIR_TO_CREATE_MONITOR_AND_CLEAN_UP_A_SNAPSHOT_PIPE_AUTOMATICALLY_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_F5B223CB = + "使用 --input 指定输入。导出还需要 --output；导出也可使用 --backup-dir 自动创建、监控并清理快照 Pipe。导入需要 --host 和 --port。请使用 --password-stdin 或 --password-env，避免密码出现在命令行中。"; public static final String LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE = "pipe-logical-backup "; public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 = @@ -148,4 +148,21 @@ private CliMessages() {} public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1 = "不支持的逻辑备份导出格式：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_TIMEOUT_AND_POLL_INTERVAL_MUST_BE_POSITIVE_38622769 = + "逻辑备份导出 timeout 和轮询间隔必须为正数"; + public static final String + LOG_LOGICAL_BACKUP_EXPORT_ARG_STATE_ARG_REMAINING_ARG_ESTIMATED_SECONDS_ARG_6256B85B = + "逻辑备份导出 %s：状态=%s，剩余=%s，预计秒数=%s"; + public static final String EXCEPTION_LOGICAL_BACKUP_EXPORT_PIPE_ARG_FAILED_ARG_F52EB190 = + "逻辑备份导出 pipe %s 失败：%s"; + public static final String + EXCEPTION_LOGICAL_BACKUP_EXPORT_PIPE_ARG_TIMED_OUT_AFTER_ARG_SECONDS_76CF4B4B = + "逻辑备份导出 pipe %s 在 %d 秒后超时"; + public static final String + EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_MUST_BE_AN_INTEGER_C9B397BE = + "逻辑备份选项 --%s 必须是整数"; + public static final String + EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_HAS_INVALID_IDENTIFIER_ARG_BB015204 = + "逻辑备份选项 --%s 的标识符无效：%s"; } diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java index 1d2315f17c04a..218f367093178 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java @@ -30,9 +30,11 @@ import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecord; import org.apache.iotdb.commons.pipe.sink.payload.thrift.common.PipeTransferHandshakeConstant; import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferDataNodeHandshakeV2Req; +import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; +import org.apache.iotdb.session.Session; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -97,6 +99,11 @@ public final class PipeLogicalBackupTool { private static final String OPTION_DRY_RUN = "dry-run"; private static final String OPTION_ALLOW_INCOMPLETE = "allow-incomplete"; private static final String OPTION_JSON = "json"; + private static final String OPTION_BACKUP_DIR = "backup-dir"; + private static final String OPTION_PIPE_NAME = "pipe-name"; + private static final String OPTION_BACKUP_ID = "backup-id"; + private static final String OPTION_TIMEOUT_SECONDS = "timeout-seconds"; + private static final String OPTION_POLL_INTERVAL_SECONDS = "poll-interval-seconds"; private static final int MAX_ARCHIVE_ENTRIES = 1_000_000; private static final long MAX_ARCHIVE_BYTES = 4L * 1024 * 1024 * 1024 * 1024; private static final String CHECKPOINT_FORMAT_NAME = @@ -162,9 +169,15 @@ private static CommandLine parse(final String command, final String[] args) addOption(options, OPTION_DRY_RUN, false, false); addOption(options, OPTION_ALLOW_INCOMPLETE, false, false); addOption(options, OPTION_JSON, false, false); + addOption(options, OPTION_BACKUP_DIR, true, false); + addOption(options, OPTION_PIPE_NAME, true, false); + addOption(options, OPTION_BACKUP_ID, true, false); + addOption(options, OPTION_TIMEOUT_SECONDS, true, false); + addOption(options, OPTION_POLL_INTERVAL_SECONDS, true, false); final CommandLine line = new DefaultParser().parse(options, args); final String input = optionValue(line, OPTION_INPUT, OPTION_SOURCE); - if (input == null || input.isBlank()) { + if ((input == null || input.isBlank()) + && (!COMMAND_EXPORT.equals(command) || !line.hasOption(OPTION_BACKUP_DIR))) { throw new ParseException( String.format( CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, OPTION_INPUT)); @@ -231,32 +244,108 @@ private static int verify(final CommandLine line) throws IOException { return 0; } - private static int export(final CommandLine line) throws IOException { + private static int export(final CommandLine line) throws Exception { + if (!line.hasOption(OPTION_INPUT) && !line.hasOption(OPTION_SOURCE)) { + return managedExport(line); + } final Path source = Path.of(input(line)).toAbsolutePath().normalize(); final Path target = Path.of(optionValue(line, OPTION_OUTPUT, OPTION_TARGET)).toAbsolutePath().normalize(); final List streams = read(line); + return exportArchive(source, target, streams); + } + + private static int managedExport(final CommandLine line) throws Exception { + final String host = required(line, OPTION_HOST); + final int port = Integer.parseInt(required(line, OPTION_PORT)); + final String user = line.getOptionValue(OPTION_USER, "root"); + final String password = password(line); + final Path backupRoot = Path.of(required(line, OPTION_BACKUP_DIR)).toAbsolutePath().normalize(); + final Path target = + Path.of(optionValue(line, OPTION_OUTPUT, OPTION_TARGET)).toAbsolutePath().normalize(); + final String pipeName = + line.getOptionValue( + OPTION_PIPE_NAME, + "logical_backup_export_" + UUID.randomUUID().toString().replace('-', '_')); + validateIdentifier(pipeName, OPTION_PIPE_NAME); + final String backupId = line.getOptionValue(OPTION_BACKUP_ID, pipeName); + validateIdentifier(backupId, OPTION_BACKUP_ID); + final long timeoutSeconds = longOption(line, OPTION_TIMEOUT_SECONDS, 3600L); + final long pollIntervalSeconds = longOption(line, OPTION_POLL_INTERVAL_SECONDS, 2L); + if (timeoutSeconds <= 0 || pollIntervalSeconds <= 0) { + throw new IOException( + CliMessages + .EXCEPTION_LOGICAL_BACKUP_EXPORT_TIMEOUT_AND_POLL_INTERVAL_MUST_BE_POSITIVE_38622769); + } + + final String createSql = + "CREATE PIPE " + + pipeName + + " WITH SOURCE ('source'='iotdb-source','source.mode'='snapshot','source.inclusion'='all'," + + "'source.capture.tree'='true','source.capture.table'='true'," + + "'source.realtime.enable'='false') WITH SINK ('sink'='logical-backup-sink','sink.dir'='" + + sqlString(backupRoot.toString()) + + "','sink.backup-id'='" + + sqlString(backupId) + + "','sink.resume'='append')"; + boolean created = false; + try (final Session session = + new Session.Builder().host(host).port(port).username(user).password(password).build()) { + session.open(); + session.executeNonQueryStatement(createSql); + created = true; + session.executeNonQueryStatement("START PIPE " + pipeName); + waitForSnapshotPipe(session, pipeName, timeoutSeconds, pollIntervalSeconds); + final Path source = backupRoot.resolve(backupId).normalize(); + final List streams = new LogicalBackupArchiveReader().read(source, false); + return exportArchive(source, target, streams); + } finally { + if (created) { + try (final Session cleanup = + new Session.Builder().host(host).port(port).username(user).password(password).build()) { + cleanup.open(); + try { + cleanup.executeNonQueryStatement("STOP PIPE " + pipeName); + } catch (final Exception ignored) { + // A completed snapshot pipe may already have been auto-dropped. + } + try { + cleanup.executeNonQueryStatement("DROP PIPE " + pipeName); + } catch (final Exception ignored) { + // Keep the original export failure when cleanup races with auto-drop. + } + } catch (final Exception ignored) { + // The archive is complete once the generated pipe disappears; cleanup is best effort. + } + } + } + } + + private static int exportArchive( + final Path source, final Path target, final List streams) throws IOException { + final Path normalizedSource = source.toAbsolutePath().normalize(); + final Path normalizedTarget = target.toAbsolutePath().normalize(); final Path root; - if (Files.isDirectory(source)) { - root = source; - } else if (Files.isRegularFile(source) - && "manifest.json".equals(source.getFileName().toString())) { - root = source.getParent(); + if (Files.isDirectory(normalizedSource)) { + root = normalizedSource; + } else if (Files.isRegularFile(normalizedSource) + && "manifest.json".equals(normalizedSource.getFileName().toString())) { + root = normalizedSource.getParent(); } else { throw new IOException( String.format( CliMessages .EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0, - source)); + normalizedSource)); } - if (target.startsWith(root)) { + if (normalizedTarget.startsWith(root)) { throw new IOException( CliMessages.EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994); } - if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) { - throw new java.nio.file.FileAlreadyExistsException(target.toString()); + if (Files.exists(normalizedTarget, LinkOption.NOFOLLOW_LINKS)) { + throw new java.nio.file.FileAlreadyExistsException(normalizedTarget.toString()); } - final Path parent = target.toAbsolutePath().normalize().getParent(); + final Path parent = normalizedTarget.getParent(); if (parent != null) { Files.createDirectories(parent); } @@ -276,9 +365,9 @@ private static int export(final CommandLine line) throws IOException { archiveChannel.force(true); } try { - Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE); + Files.move(temporary, normalizedTarget, StandardCopyOption.ATOMIC_MOVE); } catch (final java.nio.file.AtomicMoveNotSupportedException e) { - Files.move(temporary, target); + Files.move(temporary, normalizedTarget); } published = true; } finally { @@ -288,10 +377,112 @@ private static int export(final CommandLine line) throws IOException { } System.out.println( String.format( - CliMessages.LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280, source, target)); + CliMessages.LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280, + normalizedSource, + normalizedTarget)); return 0; } + private static void waitForSnapshotPipe( + final Session session, + final String pipeName, + final long timeoutSeconds, + final long pollSeconds) + throws Exception { + final long deadline = System.nanoTime() + timeoutSeconds * 1_000_000_000L; + while (System.nanoTime() < deadline) { + final PipeStatus status = readPipeStatus(session, pipeName); + if (status == null) { + return; + } else { + System.out.println( + String.format( + CliMessages + .LOG_LOGICAL_BACKUP_EXPORT_ARG_STATE_ARG_REMAINING_ARG_ESTIMATED_SECONDS_ARG_6256B85B, + pipeName, + status.state, + status.remainingEvents, + status.estimatedSeconds)); + if (status.state != null + && (status.state.equalsIgnoreCase("ERROR") + || status.state.equalsIgnoreCase("STOPPED"))) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_EXPORT_PIPE_ARG_FAILED_ARG_F52EB190, + pipeName, + status.exceptionMessage)); + } + } + Thread.sleep(pollSeconds * 1000L); + } + throw new IOException( + String.format( + CliMessages + .EXCEPTION_LOGICAL_BACKUP_EXPORT_PIPE_ARG_TIMED_OUT_AFTER_ARG_SECONDS_76CF4B4B, + pipeName, + timeoutSeconds)); + } + + private static PipeStatus readPipeStatus(final Session session, final String pipeName) + throws Exception { + try (final SessionDataSet dataSet = session.executeQueryStatement("SHOW PIPE " + pipeName)) { + final SessionDataSet.DataIterator iterator = dataSet.iterator(); + if (!iterator.next()) { + return null; + } + final PipeStatus status = new PipeStatus(); + status.state = value(iterator, dataSet, "State"); + status.remainingEvents = value(iterator, dataSet, "RemainingEventCount"); + status.estimatedSeconds = value(iterator, dataSet, "EstimatedRemainingSeconds"); + status.exceptionMessage = value(iterator, dataSet, "ExceptionMessage"); + return status; + } + } + + private static String value( + final SessionDataSet.DataIterator iterator, final SessionDataSet dataSet, final String name) + throws Exception { + for (final String column : dataSet.getColumnNames()) { + if (column.equalsIgnoreCase(name)) { + return iterator.getString(column); + } + } + return "Unknown"; + } + + private static long longOption(final CommandLine line, final String name, final long fallback) + throws ParseException { + try { + return Long.parseLong(line.getOptionValue(name, Long.toString(fallback))); + } catch (final NumberFormatException e) { + throw new ParseException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_MUST_BE_AN_INTEGER_C9B397BE, name)); + } + } + + private static void validateIdentifier(final String value, final String option) + throws IOException { + if (!value.matches("[A-Za-z_][A-Za-z0-9_]{0,127}")) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_HAS_INVALID_IDENTIFIER_ARG_BB015204, + option, + value)); + } + } + + private static String sqlString(final String value) { + return value.replace("'", "''"); + } + + private static final class PipeStatus { + private String state; + private String remainingEvents; + private String estimatedSeconds; + private String exceptionMessage; + } + private static List exportFiles(final Path root, final List streams) throws IOException { final Set files = new LinkedHashSet<>(); @@ -748,7 +939,7 @@ private static void printUsage() { 120, CliMessages.LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE, CliMessages - .LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_4677380E, + .LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_EXPORT_CAN_USE_BACKUP_DIR_TO_CREATE_MONITOR_AND_CLEAN_UP_A_SNAPSHOT_PIPE_AUTOMATICALLY_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_F5B223CB, optionsForHelp(), 2, 2, @@ -772,6 +963,11 @@ private static Options optionsForHelp() { addOption(options, OPTION_DRY_RUN, false, false); addOption(options, OPTION_ALLOW_INCOMPLETE, false, false); addOption(options, OPTION_JSON, false, false); + addOption(options, OPTION_BACKUP_DIR, true, false); + addOption(options, OPTION_PIPE_NAME, true, false); + addOption(options, OPTION_BACKUP_ID, true, false); + addOption(options, OPTION_TIMEOUT_SECONDS, true, false); + addOption(options, OPTION_POLL_INTERVAL_SECONDS, true, false); return options; } From d38a79627b7c859eb3fcbae1f394aeb88443776b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 22 Sep 2026 16:01:26 +0800 Subject: [PATCH 05/17] Rename logical backup user-facing scripts --- .../src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java | 2 +- .../src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java | 2 +- .../{export-pipe-logical-backup.sh => export-logical-backup.sh} | 2 +- .../{import-pipe-logical-backup.sh => import-logical-backup.sh} | 2 +- scripts/tools/{pipe-logical-backup.sh => logical-backup.sh} | 0 ...export-pipe-logical-backup.bat => export-logical-backup.bat} | 2 +- ...import-pipe-logical-backup.bat => import-logical-backup.bat} | 2 +- .../windows/{pipe-logical-backup.bat => logical-backup.bat} | 0 8 files changed, 6 insertions(+), 6 deletions(-) rename scripts/tools/{export-pipe-logical-backup.sh => export-logical-backup.sh} (93%) rename scripts/tools/{import-pipe-logical-backup.sh => import-logical-backup.sh} (93%) rename scripts/tools/{pipe-logical-backup.sh => logical-backup.sh} (100%) rename scripts/tools/windows/{export-pipe-logical-backup.bat => export-logical-backup.bat} (94%) rename scripts/tools/windows/{import-pipe-logical-backup.bat => import-logical-backup.bat} (94%) rename scripts/tools/windows/{pipe-logical-backup.bat => logical-backup.bat} (100%) diff --git a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java index 064dd16db8b05..1316415682f59 100644 --- a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java @@ -131,7 +131,7 @@ private CliMessages() {} LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_EXPORT_CAN_USE_BACKUP_DIR_TO_CREATE_MONITOR_AND_CLEAN_UP_A_SNAPSHOT_PIPE_AUTOMATICALLY_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_F5B223CB = "Use --input to specify the input. Export also requires --output; export can use --backup-dir to create, monitor, and clean up a snapshot Pipe automatically. Import requires --host and --port. Use --password-stdin or --password-env to avoid command-line passwords."; public static final String LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE = - "pipe-logical-backup "; + "logical-backup "; public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 = "Logical backup archive entry is unsafe: %s"; public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 = diff --git a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java index d08e2053fa8cb..c26071d0bf5f0 100644 --- a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java @@ -127,7 +127,7 @@ private CliMessages() {} LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_EXPORT_CAN_USE_BACKUP_DIR_TO_CREATE_MONITOR_AND_CLEAN_UP_A_SNAPSHOT_PIPE_AUTOMATICALLY_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_F5B223CB = "使用 --input 指定输入。导出还需要 --output；导出也可使用 --backup-dir 自动创建、监控并清理快照 Pipe。导入需要 --host 和 --port。请使用 --password-stdin 或 --password-env，避免密码出现在命令行中。"; public static final String LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE = - "pipe-logical-backup "; + "logical-backup "; public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152 = "逻辑备份归档项路径不安全：%s"; public static final String EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432 = diff --git a/scripts/tools/export-pipe-logical-backup.sh b/scripts/tools/export-logical-backup.sh similarity index 93% rename from scripts/tools/export-pipe-logical-backup.sh rename to scripts/tools/export-logical-backup.sh index 8a0a4f8a1834b..fcc072bb5c1ea 100644 --- a/scripts/tools/export-pipe-logical-backup.sh +++ b/scripts/tools/export-logical-backup.sh @@ -18,4 +18,4 @@ # under the License. # -exec "$(dirname "$0")/pipe-logical-backup.sh" export "$@" +exec "$(dirname "$0")/logical-backup.sh" export "$@" diff --git a/scripts/tools/import-pipe-logical-backup.sh b/scripts/tools/import-logical-backup.sh similarity index 93% rename from scripts/tools/import-pipe-logical-backup.sh rename to scripts/tools/import-logical-backup.sh index 6e4e069210ded..ebd38227a2896 100644 --- a/scripts/tools/import-pipe-logical-backup.sh +++ b/scripts/tools/import-logical-backup.sh @@ -18,4 +18,4 @@ # under the License. # -exec "$(dirname "$0")/pipe-logical-backup.sh" import "$@" +exec "$(dirname "$0")/logical-backup.sh" import "$@" diff --git a/scripts/tools/pipe-logical-backup.sh b/scripts/tools/logical-backup.sh similarity index 100% rename from scripts/tools/pipe-logical-backup.sh rename to scripts/tools/logical-backup.sh diff --git a/scripts/tools/windows/export-pipe-logical-backup.bat b/scripts/tools/windows/export-logical-backup.bat similarity index 94% rename from scripts/tools/windows/export-pipe-logical-backup.bat rename to scripts/tools/windows/export-logical-backup.bat index 42e1df8afb274..d79ce5a43d986 100644 --- a/scripts/tools/windows/export-pipe-logical-backup.bat +++ b/scripts/tools/windows/export-logical-backup.bat @@ -17,5 +17,5 @@ @REM under the License. @REM -@call "%~dp0pipe-logical-backup.bat" export %* +@call "%~dp0logical-backup.bat" export %* @exit /B %ERRORLEVEL% diff --git a/scripts/tools/windows/import-pipe-logical-backup.bat b/scripts/tools/windows/import-logical-backup.bat similarity index 94% rename from scripts/tools/windows/import-pipe-logical-backup.bat rename to scripts/tools/windows/import-logical-backup.bat index 0fba332d5a2e5..bfe85ba3943fe 100644 --- a/scripts/tools/windows/import-pipe-logical-backup.bat +++ b/scripts/tools/windows/import-logical-backup.bat @@ -17,5 +17,5 @@ @REM under the License. @REM -@call "%~dp0pipe-logical-backup.bat" import %* +@call "%~dp0logical-backup.bat" import %* @exit /B %ERRORLEVEL% diff --git a/scripts/tools/windows/pipe-logical-backup.bat b/scripts/tools/windows/logical-backup.bat similarity index 100% rename from scripts/tools/windows/pipe-logical-backup.bat rename to scripts/tools/windows/logical-backup.bat From bc0a6e2d6249ec4ae9afb0232b4cb7fd5865ab4f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:19:59 +0800 Subject: [PATCH 06/17] Add ConfigNode logical backup support --- .../tool/pipe/PipeLogicalBackupTool.java | 42 ++++ .../tool/pipe/PipeLogicalBackupToolTest.java | 39 +++- .../PipeConfigRegionSinkConstructor.java | 3 + .../logicalbackup/LogicalBackupSink.java | 183 ++++++++++++++++++ .../PipeConfigRegionSinkConstructorTest.java | 49 +++++ .../sink/PipeConfigNodeThriftRequestTest.java | 7 + .../iotdb/db/i18n/DataNodePipeMessages.java | 4 +- .../iotdb/db/i18n/DataNodePipeMessages.java | 4 +- .../logicalbackup/LogicalBackupSink.java | 32 +-- .../logicalbackup/LogicalBackupSinkTest.java | 14 ++ .../plugin/builtin/BuiltinPipePlugin.java | 2 +- .../LogicalBackupArchiveReader.java | 18 +- .../request/PipeTransferHandshakeV2Req.java | 29 ++- .../LogicalBackupArchiveReaderTest.java | 43 +++- 14 files changed, 430 insertions(+), 39 deletions(-) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructorTest.java diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java index 218f367093178..631353027a614 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java @@ -29,6 +29,8 @@ import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecord; import org.apache.iotdb.commons.pipe.sink.payload.thrift.common.PipeTransferHandshakeConstant; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferHandshakeV2Req; import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferDataNodeHandshakeV2Req; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.rpc.TSStatusCode; @@ -536,6 +538,10 @@ private static int importBackup(final CommandLine line) throws Exception { "", "")) { handshake(client, streams.get(0).getManifest().timestampPrecision, user, password); + if (containsConfigStream(streams)) { + configNodeHandshake( + client, streams.get(0).getManifest().timestampPrecision, user, password); + } long importedGroups = 0; for (final BackupStream stream : streams) { importedGroups += @@ -557,6 +563,11 @@ private static int importBackup(final CommandLine line) throws Exception { return 0; } + static boolean containsConfigStream(final List streams) { + return streams.stream() + .anyMatch(stream -> "config".equalsIgnoreCase(stream.getManifest().streamType)); + } + static long importStream( final Path checkpoint, final ImportCheckpoint checkpointState, @@ -695,6 +706,37 @@ private static void handshake( } } + private static void configNodeHandshake( + final IoTDBSyncClient client, + final String timestampPrecision, + final String user, + final String password) + throws IOException, TException { + final Map params = new HashMap<>(); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLUSTER_ID, getClusterId()); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_TIME_PRECISION, timestampPrecision); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USER_ID, "-1"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_USERNAME, user); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_PASSWORD, password); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_CLI_HOSTNAME, "pipe-logical-backup"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_LOAD_TSFILE_STRATEGY, "sync"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_VALIDATE_TSFILE, "false"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST, "true"); + params.put(PipeTransferHandshakeConstant.HANDSHAKE_KEY_SKIP_IF, "false"); + final TPipeTransferResp response = + client.pipeTransfer( + PipeTransferHandshakeV2Req.toTPipeTransferReq( + PipeRequestType.HANDSHAKE_CONFIGNODE_V2, params)); + if (response == null + || response.getStatus() == null + || response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_HANDSHAKE_FAILED_ARG_7CDD4697, + response == null ? "null" : response.getStatus())); + } + } + static void transfer(final IoTDBSyncClient client, final TPipeTransferReq request) throws TException, IOException { final TPipeTransferResp response = client.pipeTransfer(request); diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java index 6e7af8726ab35..f39b87f0754ea 100644 --- a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java @@ -116,6 +116,22 @@ public void testCheckpointSourceIdentityIncludesContent() throws Exception { Assert.assertNotEquals(before, after); } + @Test + public void testConfigStreamDetection() throws Exception { + final Path root = temporaryFolder.getRoot().toPath(); + final Path data = root.resolve("data"); + final Path config = root.resolve("config"); + writeBackup(data); + writeBackup(config, "config-0", "config", Collections.singletonList(request((short) 200))); + + Assert.assertFalse( + PipeLogicalBackupTool.containsConfigStream( + new LogicalBackupArchiveReader().read(data, false))); + Assert.assertTrue( + PipeLogicalBackupTool.containsConfigStream( + new LogicalBackupArchiveReader().read(config, false))); + } + @Test public void testZipSlipIsRejected() throws Exception { final Path archive = temporaryFolder.getRoot().toPath().resolve("unsafe.zip"); @@ -236,12 +252,21 @@ private static void writeBackup(final Path directory) throws Exception { private static void writeBackup(final Path directory, final List requests) throws Exception { + writeBackup(directory, "data-1", "data", requests); + } + + private static void writeBackup( + final Path directory, + final String streamId, + final String streamType, + final List requests) + throws Exception { final LogicalBackupManifest manifest = new LogicalBackupManifest(); manifest.backupId = "backup"; manifest.pipeName = "pipe"; manifest.pipeCreationTime = 1; - manifest.streamId = "data-1"; - manifest.streamType = "data"; + manifest.streamId = streamId; + manifest.streamType = streamType; manifest.timestampPrecision = "ms"; try (final LogicalBackupWriter writer = new LogicalBackupWriter( @@ -251,9 +276,17 @@ private static void writeBackup(final Path directory, final List requests = new ArrayList<>(); + appendFile(requests, e.getSnapshotFile()); + appendFile(requests, e.getTemplateFile()); + final File snapshot = e.getSnapshotFile(); + final File template = e.getTemplateFile(); + requests.add( + PipeTransferConfigSnapshotSealReq.toTPipeTransferReq( + e.getTreePattern().getPattern(), + e.getTablePattern().getDatabasePattern(), + e.getTablePattern().getTablePattern(), + e.getTreePattern().isTreeModelDataAllowedToBeCaptured(), + e.getTablePattern().isTableModelDataAllowedToBeCaptured(), + snapshot.getName(), + snapshot.length(), + template == null ? null : template.getName(), + template == null ? 0 : template.length(), + e.getFileType(), + e.toSealTypeString(), + e.getAuthUserName())); + writer.writeEvent( + UUID.randomUUID(), System.currentTimeMillis(), requests, "config-snapshot"); + } finally { + e.decreaseReferenceCount(getClass().getName(), false); + } + } + } + } + + private static void appendFile(final List requests, final File file) + throws Exception { + if (file == null) return; + try (FileInputStream input = new FileInputStream(file)) { + final byte[] buffer = new byte[FILE_PIECE_BYTES]; + long offset = 0; + int length; + while ((length = input.read(buffer)) > 0) { + requests.add( + PipeTransferConfigSnapshotPieceReq.toTPipeTransferReq( + file.getName(), offset, Arrays.copyOf(buffer, length))); + offset += length; + } + } + } + + @Override + public void handshake() {} + + @Override + public void heartbeat() throws Exception { + if (writer != null) writer.heartbeat(); + } + + @Override + public void close() throws Exception { + if (writer != null) { + writer.close(); + writer = null; + } + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructorTest.java new file mode 100644 index 0000000000000..3031e2c6b216d --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructorTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.pipe.agent.plugin; + +import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; +import org.apache.iotdb.confignode.manager.pipe.sink.protocol.logicalbackup.LogicalBackupSink; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; + +public class PipeConfigRegionSinkConstructorTest { + + @Test + public void testLogicalBackupAliasesUseConfigRegionSink() { + final PipeConfigRegionSinkConstructor constructor = new PipeConfigRegionSinkConstructor(); + + Assert.assertTrue( + constructor.reflectPlugin( + new PipeParameters( + Collections.singletonMap( + PipeSinkConstant.CONNECTOR_KEY, "logical-backup-connector"))) + instanceof LogicalBackupSink); + Assert.assertTrue( + constructor.reflectPlugin( + new PipeParameters( + Collections.singletonMap(PipeSinkConstant.SINK_KEY, "logical-backup-sink"))) + instanceof LogicalBackupSink); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/sink/PipeConfigNodeThriftRequestTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/sink/PipeConfigNodeThriftRequestTest.java index bd77d72d1bef1..e8a17e5d524d6 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/sink/PipeConfigNodeThriftRequestTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/sink/PipeConfigNodeThriftRequestTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.IoTDBSinkRequestVersion; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferFileSealReqV2; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferHandshakeV2Req; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.confignode.consensus.request.write.cq.ActiveCQPlan; import org.apache.iotdb.confignode.manager.pipe.sink.payload.PipeTransferConfigNodeHandshakeV1Req; @@ -93,12 +94,18 @@ public void testPipeTransferConfigHandshakeV2Req() throws IOException { final PipeTransferConfigNodeHandshakeV2Req req = PipeTransferConfigNodeHandshakeV2Req.toTPipeTransferReq(params); + final TPipeTransferReq commonReq = + PipeTransferHandshakeV2Req.toTPipeTransferReq( + PipeRequestType.HANDSHAKE_CONFIGNODE_V2, params); final int originalBodyPosition = req.body.position(); final PipeTransferConfigNodeHandshakeV2Req deserializeReq = PipeTransferConfigNodeHandshakeV2Req.fromTPipeTransferReq(req); Assert.assertEquals(req.getVersion(), deserializeReq.getVersion()); Assert.assertEquals(req.getType(), deserializeReq.getType()); + Assert.assertEquals(req.getVersion(), commonReq.getVersion()); + Assert.assertEquals(req.getType(), commonReq.getType()); + Assert.assertArrayEquals(req.getBody(), commonReq.getBody()); Assert.assertEquals(originalBodyPosition, req.body.position()); Assert.assertEquals(params, deserializeReq.getParams()); } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index 8c6200718838b..5744a1bd34cff 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2654,8 +2654,8 @@ private DataNodePipeMessages() {} public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 = "Unsupported logical backup resume policy: %s"; public static final String - EXCEPTION_DATAREGION_LOGICAL_BACKUP_REQUIRES_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_9F7ADEA8 = - "DataRegion logical backup requires explicit sink.resume=append (or connector.resume=append) and a shared backup directory"; + EXCEPTION_CONFIGREGION_AND_DATAREGION_LOGICAL_BACKUP_REQUIRE_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_B041371B = + "ConfigRegion and DataRegion logical backup require explicit sink.resume=append (or connector.resume=append) and a shared backup directory"; public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 = "Unsupported Pipe event for logical backup: %s"; public static final String diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index 44f7c02691993..d5200ee93dfa7 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2478,8 +2478,8 @@ private DataNodePipeMessages() {} public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9 = "不支持的逻辑备份恢复策略：%s"; public static final String - EXCEPTION_DATAREGION_LOGICAL_BACKUP_REQUIRES_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_9F7ADEA8 = - "DataRegion 逻辑备份要求显式配置 sink.resume=append（或 connector.resume=append），并使用共享备份目录"; + EXCEPTION_CONFIGREGION_AND_DATAREGION_LOGICAL_BACKUP_REQUIRE_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_B041371B = + "ConfigRegion 和 DataRegion 逻辑备份要求显式配置 sink.resume=append（或 connector.resume=append），并使用共享备份目录"; public static final String EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97 = "逻辑备份不支持 Pipe 事件：%s"; public static final String diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java index ef152b58b7c34..c48238a5ce225 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java @@ -75,12 +75,12 @@ public class LogicalBackupSink implements PipeConnector { private static final int FILE_PIECE_BYTES = 64 * 1024; - private LogicalBackupWriter writer; + protected LogicalBackupWriter writer; private String pipeName; private long creationTime; private int regionId; - private boolean includeHeartbeat; - private String unsupportedEventPolicy; + protected boolean includeHeartbeat; + protected String unsupportedEventPolicy; private final String streamType; public LogicalBackupSink() { @@ -160,7 +160,7 @@ public void validate(final PipeParameterValidator validator) throws Exception { PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH, PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIODIC, PipeSinkConstant.LOGICAL_BACKUP_FSYNC_NONE); - validateDataRegionResumePolicy(parameters); + validateSharedDirectoryResumePolicy(parameters); final String policy = parameters .getStringOrDefault( @@ -188,9 +188,9 @@ private static void validateSynonymAttributes( false); } - private void validateDataRegionResumePolicy(final PipeParameters parameters) + private void validateSharedDirectoryResumePolicy(final PipeParameters parameters) throws PipeParameterNotValidException { - if (!"data".equals(streamType)) { + if (!"data".equals(streamType) && !"config".equals(streamType)) { return; } final boolean explicitlyConfigured = @@ -207,7 +207,7 @@ private void validateDataRegionResumePolicy(final PipeParameters parameters) || !PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND.equalsIgnoreCase(resume)) { throw new PipeParameterNotValidException( DataNodePipeMessages - .EXCEPTION_DATAREGION_LOGICAL_BACKUP_REQUIRES_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_9F7ADEA8); + .EXCEPTION_CONFIGREGION_AND_DATAREGION_LOGICAL_BACKUP_REQUIRE_EXPLICIT_SINK_RESUME_APPEND_OR_CONNECTOR_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_B041371B); } } @@ -277,7 +277,7 @@ public void customize( PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY), PipeSinkConstant.LOGICAL_BACKUP_RESUME_DEFAULT_VALUE) .toLowerCase(Locale.ROOT); - validateDataRegionResumePolicy(parameters); + validateSharedDirectoryResumePolicy(parameters); final boolean append = PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND.equals(resume); if (!append && !PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW.equals(resume) @@ -349,7 +349,7 @@ public void customize( manifest.backupId = backupId; manifest.pipeName = pipeName; manifest.pipeCreationTime = creationTime; - manifest.sourceClusterId = IoTDBDescriptor.getInstance().getConfig().getClusterId(); + manifest.sourceClusterId = getSourceClusterId(); manifest.sourceVersion = IoTDBConstant.VERSION; manifest.timestampPrecision = CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); @@ -369,6 +369,10 @@ public void customize( append); } + protected String getSourceClusterId() { + return IoTDBDescriptor.getInstance().getConfig().getClusterId(); + } + @Override public void handshake() {} @@ -542,7 +546,7 @@ private List requests(final PipeInsertNodeTabletInsertionEvent return requests; } - private void transferEnriched(final EnrichedEvent event, final RequestSupplier requestSupplier) + protected void transferEnriched(final EnrichedEvent event, final RequestSupplier requestSupplier) throws Exception { if (!event.increaseReferenceCount(getClass().getName())) { return; @@ -556,11 +560,11 @@ private void transferEnriched(final EnrichedEvent event, final RequestSupplier r } @FunctionalInterface - private interface RequestSupplier { + protected interface RequestSupplier { List get() throws Exception; } - private void handleUnsupported(final Event event) throws Exception { + protected void handleUnsupported(final Event event) throws Exception { if (PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_SKIP.equals(unsupportedEventPolicy)) { writer.recordSkippedEvent( System.currentTimeMillis(), event == null ? "null" : event.getClass().getName()); @@ -572,7 +576,7 @@ private void handleUnsupported(final Event event) throws Exception { event == null ? "null" : event.getClass().getName())); } - private UUID eventId(final EnrichedEvent event) { + protected UUID eventId(final EnrichedEvent event) { return UUID.nameUUIDFromBytes( (event.getClass().getName() + '\0' @@ -594,7 +598,7 @@ private String commitIds(final EnrichedEvent event) { } } - private String metadata(final Event event) { + protected String metadata(final Event event) { if (event instanceof EnrichedEvent) { final EnrichedEvent enriched = (EnrichedEvent) event; return "pipe=" diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java index 37c3d622b4f55..2354e8309ab93 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java @@ -67,6 +67,20 @@ public void testSchemaRegionKeepsDefaultResumePolicy() throws Exception { .validate(new PipeParameterValidator(new PipeParameters(attributes))); } + @Test + public void testConfigRegionRequiresExplicitAppendResumePolicy() throws Exception { + final Map attributes = new HashMap<>(); + attributes.put(PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY, "backup"); + + assertInvalid(new LogicalBackupSink("config"), attributes); + + attributes.put( + PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY, + PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND); + new LogicalBackupSink("config") + .validate(new PipeParameterValidator(new PipeParameters(attributes))); + } + private static void assertInvalid( final LogicalBackupSink sink, final Map attributes) throws Exception { try { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java index 11859bb6fe614..d9ff67f160e81 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java @@ -38,7 +38,6 @@ import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSslSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSyncSink; -import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.logicalbackup.LogicalBackupSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcda.OpcDaSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcua.OpcUaSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.websocket.WebSocketSink; @@ -224,6 +223,7 @@ public String getClassName() { WEBSOCKET_SINK.getPipePluginName().toUpperCase(), OPC_UA_SINK.getPipePluginName().toUpperCase(), OPC_DA_SINK.getPipePluginName().toUpperCase(), + LOGICAL_BACKUP_SINK.getPipePluginName().toUpperCase(), SUBSCRIPTION_SINK.getPipePluginName().toUpperCase(), IOT_CONSENSUS_V2_ASYNC_SINK.getPipePluginName().toUpperCase(), PIPE_CONSENSUS_ASYNC_SINK.getPipePluginName().toUpperCase()))); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java index 6ac9ca23f67bd..b378a16136c10 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java @@ -52,7 +52,10 @@ public class LogicalBackupArchiveReader { PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), PipeRequestType.TRANSFER_PLAN_NODE.getType(), PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_PIECE.getType(), - PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL.getType()); + PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL.getType(), + PipeRequestType.TRANSFER_CONFIG_PLAN.getType(), + PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_PIECE.getType(), + PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_SEAL.getType()); public List read(final Path source, final boolean allowIncomplete) throws IOException { @@ -84,12 +87,21 @@ public List read(final Path source, final boolean allowIncomplete) } streams.sort( Comparator.comparingInt( - (BackupStream stream) -> - "schema".equalsIgnoreCase(stream.getManifest().streamType) ? 0 : 1) + (BackupStream stream) -> streamOrder(stream.getManifest().streamType)) .thenComparing((BackupStream stream) -> stream.getManifest().streamId)); return Collections.unmodifiableList(streams); } + private static int streamOrder(final String streamType) { + if ("config".equalsIgnoreCase(streamType)) { + return 0; + } + if ("schema".equalsIgnoreCase(streamType)) { + return 1; + } + return 2; + } + private static void validateArchiveIdentity( final LogicalBackupManifest expected, final LogicalBackupManifest actual) throws IOException { validateArchiveIdentityField("backupId", expected.backupId, actual.backupId); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java index 472679494729b..d7e45cad5ae75 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferHandshakeV2Req.java @@ -41,6 +41,20 @@ public Map getParams() { protected abstract PipeRequestType getPlanType(); + public static TPipeTransferReq toTPipeTransferReq( + final PipeRequestType requestType, final Map params) throws IOException { + final TPipeTransferReq request = new TPipeTransferReq(); + request.version = IoTDBSinkRequestVersion.VERSION_1.getVersion(); + request.type = requestType.getType(); + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + writeParams(params, outputStream); + request.body = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + return request; + } + /////////////////////////////// Thrift /////////////////////////////// protected final PipeTransferHandshakeV2Req convertToTPipeTransferReq(Map params) @@ -49,11 +63,7 @@ protected final PipeTransferHandshakeV2Req convertToTPipeTransferReq(Map entry : params.entrySet()) { - ReadWriteIOUtils.write(entry.getKey(), outputStream); - ReadWriteIOUtils.write(entry.getValue(), outputStream); - } + writeParams(params, outputStream); this.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); } @@ -61,6 +71,15 @@ protected final PipeTransferHandshakeV2Req convertToTPipeTransferReq(Map params, final DataOutputStream outputStream) throws IOException { + ReadWriteIOUtils.write(params.size(), outputStream); + for (final Map.Entry entry : params.entrySet()) { + ReadWriteIOUtils.write(entry.getKey(), outputStream); + ReadWriteIOUtils.write(entry.getValue(), outputStream); + } + } + protected final PipeTransferHandshakeV2Req translateFromTPipeTransferReq( TPipeTransferReq transferReq) { Map params = new HashMap<>(); diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java index 05f6298e95fac..10db588f2c3c9 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java @@ -35,6 +35,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.UUID; @@ -47,16 +48,18 @@ public class LogicalBackupArchiveReaderTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @Test - public void testReadCompleteArchiveAndOrderSchemaFirst() throws Exception { + public void testReadCompleteArchiveAndOrderConfigThenSchemaThenData() throws Exception { final Path root = temporaryFolder.newFolder().toPath(); writeStream(root.resolve("data"), "data-stream", "data", 4096); writeStream(root.resolve("schema"), "schema-stream", "schema", 4096); + writeStream(root.resolve("config"), "config-stream", "config", 4096); final List streams = new LogicalBackupArchiveReader().read(root, false); - Assert.assertEquals(2, streams.size()); - Assert.assertEquals("schema-stream", streams.get(0).getManifest().streamId); - Assert.assertEquals("data-stream", streams.get(1).getManifest().streamId); + Assert.assertEquals(3, streams.size()); + Assert.assertEquals("config-stream", streams.get(0).getManifest().streamId); + Assert.assertEquals("schema-stream", streams.get(1).getManifest().streamId); + Assert.assertEquals("data-stream", streams.get(2).getManifest().streamId); Assert.assertEquals(1, streams.get(0).getEventGroups().size()); Assert.assertEquals(4, streams.get(0).getRecords().size()); Assert.assertEquals( @@ -175,7 +178,7 @@ public void testNonReplayableRequestTypeIsRejected() throws Exception { Collections.singletonList( new TPipeTransferReq() .setVersion((byte) 1) - .setType((short) 200) + .setType(Short.MAX_VALUE) .setBody(ByteBuffer.wrap(new byte[] {1}))), "metadata"); } @@ -183,6 +186,23 @@ public void testNonReplayableRequestTypeIsRejected() throws Exception { assertReadFails(directory, false); } + @Test + public void testConfigRequestTypesAreReplayable() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = + writer(directory, manifest("config-0", "config"), 4096)) { + writer.writeEvent( + UUID.randomUUID(), + 1, + Arrays.asList(request((short) 200), request((short) 201), request((short) 202)), + "metadata"); + } + + final List streams = new LogicalBackupArchiveReader().read(directory, false); + Assert.assertEquals(1, streams.size()); + Assert.assertEquals(3, streams.get(0).getEventGroups().get(0).getRequests().size()); + } + @Test public void testDuplicateEventGroupIdIsRejected() throws Exception { final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); @@ -239,10 +259,15 @@ private static TPipeTransferReq request(final int value) { } private static TPipeTransferReq request(final byte[] body) { - return new TPipeTransferReq() - .setVersion((byte) 1) - .setType((short) 10) - .setBody(ByteBuffer.wrap(body)); + return request((short) 10, body); + } + + private static TPipeTransferReq request(final short type) { + return request(type, new byte[] {1}); + } + + private static TPipeTransferReq request(final short type, final byte[] body) { + return new TPipeTransferReq().setVersion((byte) 1).setType(type).setBody(ByteBuffer.wrap(body)); } private static Path onlySegment(final Path directory) throws IOException { From 7311c1454f2cd6d46a0b92c35c7566d7a130258d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:10:42 +0800 Subject: [PATCH 07/17] Fix logical backup builtin plugin import --- .../commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java | 1 + 1 file changed, 1 insertion(+) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java index d9ff67f160e81..c06b8cf458692 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java @@ -38,6 +38,7 @@ import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSslSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSyncSink; +import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.logicalbackup.LogicalBackupSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcda.OpcDaSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcua.OpcUaSink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.websocket.WebSocketSink; From 6355496a6f82ae4bbd25b34f29eabc5996f2ca30 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:40:57 +0800 Subject: [PATCH 08/17] feat: add logical backup dump and restore workflow --- .../manual/IoTDBPipeLogicalBackupIT.java | 298 +++++ iotdb-client/cli/LOGICAL_BACKUP.md | 138 +++ .../apache/iotdb/cli/i18n/CliMessages.java | 40 + .../apache/iotdb/cli/i18n/CliMessages.java | 40 + .../tool/pipe/LogicalBackupDumpOptions.java | 275 +++++ .../tool/pipe/PipeLogicalBackupTool.java | 1059 +++++++++++++---- .../tool/pipe/PipeLogicalBackupToolTest.java | 258 +++- .../heartbeat/PipeHeartbeatParser.java | 67 +- .../logicalbackup/LogicalBackupSink.java | 97 +- .../heartbeat/PipeHeartbeatParserTest.java | 32 + .../subtask/sink/PipeSinkSubtaskManager.java | 32 +- .../logicalbackup/LogicalBackupSink.java | 227 ++-- .../dataregion/IoTDBDataRegionSource.java | 11 + .../PipeRealtimeDataRegionSource.java | 25 + .../sink/PipeSinkSubtaskManagerTest.java | 21 + .../logicalbackup/LogicalBackupSinkTest.java | 105 ++ .../PipeRealtimeDataRegionSourceTest.java | 20 + .../commons/i18n/LogicalBackupMessages.java | 10 + .../commons/i18n/LogicalBackupMessages.java | 10 + .../LogicalBackupArchiveReader.java | 147 ++- .../logicalbackup/LogicalBackupCatalog.java | 277 +++++ .../logicalbackup/LogicalBackupManifest.java | 4 + .../logicalbackup/LogicalBackupSettings.java | 199 ++++ .../logicalbackup/LogicalBackupWriter.java | 21 +- .../pipe/source/IoTDBNonDataRegionSource.java | 51 +- .../LogicalBackupArchiveReaderTest.java | 105 +- .../LogicalBackupSettingsTest.java | 99 ++ .../LogicalBackupWriterTest.java | 11 +- .../source/IoTDBNonDataRegionSourceTest.java | 60 +- 29 files changed, 3260 insertions(+), 479 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeLogicalBackupIT.java create mode 100644 iotdb-client/cli/LOGICAL_BACKUP.md create mode 100644 iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/LogicalBackupDumpOptions.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupCatalog.java create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettings.java create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettingsTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeLogicalBackupIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeLogicalBackupIT.java new file mode 100644 index 0000000000000..4629bcb887506 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeLogicalBackupIT.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.manual; + +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupCatalog; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.tool.pipe.PipeLogicalBackupTool; + +import org.awaitility.Awaitility; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.TimeUnit; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeLogicalBackupIT extends AbstractPipeDualTreeModelManualIT { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Override + protected void setupConfig() { + super.setupConfig(); + for (final BaseEnv env : Arrays.asList(senderEnv, receiverEnv)) { + env.getConfig().getConfigNodeJVMConfig().setInitHeapSize(128).setMaxHeapSize(256); + env.getConfig().getDataNodeJVMConfig().setInitHeapSize(256).setMaxHeapSize(768); + } + } + + @Test + public void testBoundedContinuousTableCapture() throws Exception { + try (final Connection connection = senderEnv.getTableConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("create database backup_db"); + statement.execute("use backup_db"); + statement.execute("create table measurements(device STRING TAG, reading INT32 FIELD)"); + statement.execute("create table other_table(device STRING TAG, reading INT32 FIELD)"); + statement.execute("insert into measurements(time,device,reading) values(1,'d1',10)"); + statement.execute("insert into other_table(time,device,reading) values(1,'d1',100)"); + statement.execute("flush"); + } + final String pipeName = "logical_backup_continuous_table"; + final CompletableFuture writes = + CompletableFuture.runAsync( + () -> { + try { + Awaitility.await() + .atMost(30, TimeUnit.SECONDS) + .until( + () -> { + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement(); + final ResultSet result = + statement.executeQuery("show pipe " + pipeName)) { + return result.next(); + } + }); + try (final Connection connection = senderEnv.getTableConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("use backup_db"); + statement.execute( + "insert into measurements(time,device,reading) values(2,'d1',20)"); + statement.execute( + "insert into measurements(time,device,reading) values(4,'d1',40)"); + statement.execute( + "insert into other_table(time,device,reading) values(2,'d1',200)"); + statement.execute("delete from measurements where time <= 1"); + } + } catch (final Exception e) { + throw new CompletionException(e); + } + }); + final Path parent = temporaryFolder.getRoot().toPath().toAbsolutePath(); + runConnected( + senderEnv, + "dump", + "--backup-dir", + parent.toString(), + "--backup-id", + "bounded", + "--pipe-name", + pipeName, + "--scope", + "schema,data", + "--model", + "table", + "--include", + "backup_db/measurements", + "--start-time", + "0", + "--end-time", + "3", + "--mode", + "continuous", + "--until", + Instant.now().plusSeconds(20).toString(), + "--timeout-seconds", + "180", + "--poll-interval-seconds", + "1"); + writes.get(30, TimeUnit.SECONDS); + final Path backup = parent.resolve("bounded"); + final LogicalBackupCatalog catalog = LogicalBackupCatalog.load(backup.resolve("manifest.json")); + Assert.assertEquals("SEALED", catalog.status); + Assert.assertEquals("continuous", catalog.mode); + assertLogicalRequests(backup); + runConnected(receiverEnv, "restore", "--input", backup.toString()); + try (final Connection connection = receiverEnv.getTableConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("use backup_db"); + try (final ResultSet result = statement.executeQuery("select * from measurements")) { + Assert.assertTrue(result.next()); + Assert.assertEquals(2L, result.getLong("time")); + Assert.assertEquals("d1", result.getString("device")); + Assert.assertEquals(20, result.getInt("reading")); + Assert.assertFalse(result.next()); + } + try (final ResultSet result = statement.executeQuery("show tables")) { + while (result.next()) { + Assert.assertNotEquals("other_table", result.getString(1)); + } + } + } + } + + @Test + public void testDumpPackPreflightRestoreWithScopeAndTimeFilters() throws Exception { + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "create database root.backup", + "create aligned timeseries root.backup.d(s1 INT32, s2 BOOLEAN)", + "insert into root.backup.d(time,s1,s2) aligned values(1,10,true)", + "insert into root.backup.d(time,s2) aligned values(2,true)", + "insert into root.backup.d(time,s1,s2) aligned values(3,30,false)", + "insert into root.backup.d(time,s1,s2) aligned values(4,40,true)", + "create database root.unselected", + "create timeseries root.unselected.d.s with datatype=INT32,encoding=PLAIN", + "insert into root.unselected.d(time,s) values(2,100)", + "create user logical_backup_excluded_user 'test_password'", + "flush"), + null); + final Path parent = temporaryFolder.getRoot().toPath().toAbsolutePath(); + runConnected( + senderEnv, + "dump", + "--backup-dir", + parent.toString(), + "--backup-id", + "snapshot", + "--scope", + "config,schema,data", + "--model", + "tree", + "--include", + "root.backup.**", + "--start-time", + "2", + "--end-time", + "3", + "--timeout-seconds", + "180", + "--poll-interval-seconds", + "1"); + final Path backup = parent.resolve("snapshot"); + final LogicalBackupCatalog catalog = LogicalBackupCatalog.load(backup.resolve("manifest.json")); + Assert.assertEquals("SEALED", catalog.status); + Assert.assertFalse(catalog.includeAuth); + Assert.assertFalse(catalog.streams.isEmpty()); + assertLogicalRequests(backup); + final Path zip = parent.resolve("snapshot.zip"); + Assert.assertEquals( + 0, + PipeLogicalBackupTool.run(new String[] {"verify", "--input", backup.toString(), "--json"})); + Assert.assertEquals( + 0, + PipeLogicalBackupTool.run( + new String[] {"pack", "--input", backup.toString(), "--output", zip.toString()})); + Assert.assertEquals( + 0, + PipeLogicalBackupTool.run( + new String[] {"restore", "--input", zip.toString(), "--dry-run"})); + runConnected(receiverEnv, "restore", "--input", zip.toString(), "--dry-run"); + try (final Connection connection = receiverEnv.getConnection(); + final Statement statement = connection.createStatement(); + final ResultSet result = statement.executeQuery("show timeseries root.backup.**")) { + Assert.assertFalse(result.next()); + } + final Path checkpoint = parent.resolve("restore-checkpoint.json"); + runConnected( + receiverEnv, "restore", "--input", zip.toString(), "--checkpoint", checkpoint.toString()); + runConnected( + receiverEnv, + "restore", + "--input", + zip.toString(), + "--checkpoint", + checkpoint.toString(), + "--resume"); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.backup.**", + "Time,root.backup.d.s1,root.backup.d.s2,", + new HashSet<>(Arrays.asList("2,null,true,", "3,30,false,"))); + try (final Connection connection = receiverEnv.getConnection(); + final Statement statement = connection.createStatement()) { + try (final ResultSet result = statement.executeQuery("show timeseries root.unselected.**")) { + Assert.assertFalse(result.next()); + } + try (final ResultSet result = statement.executeQuery("list user")) { + while (result.next()) { + for (int column = 1; column <= result.getMetaData().getColumnCount(); column++) { + Assert.assertNotEquals("logical_backup_excluded_user", result.getString(column)); + } + } + } + } + } + + private static void assertLogicalRequests(final Path backup) throws Exception { + final Set logicalRequests = + Set.of( + PipeRequestType.TRANSFER_CONFIG_PLAN.getType(), + PipeRequestType.TRANSFER_PLAN_NODE.getType(), + PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), + PipeRequestType.TRANSFER_TABLET_INSERT_NODE_V2.getType()); + new LogicalBackupArchiveReader() + .read(backup, false).stream() + .flatMap(stream -> stream.getEventGroups().stream()) + .flatMap(group -> group.getRequests().stream()) + .forEach( + record -> Assert.assertTrue(logicalRequests.contains(record.getRequestType()))); + } + + private static void runConnected(final BaseEnv env, final String command, final String... options) + throws Exception { + final List args = + new ArrayList<>( + Arrays.asList( + command, + "--host", + env.getIP(), + "--port", + env.getPort(), + "--user", + "root", + "--password-stdin")); + args.addAll(Arrays.asList(options)); + final InputStream previous = System.in; + try { + System.setIn(new ByteArrayInputStream("root\n".getBytes(StandardCharsets.UTF_8))); + Assert.assertEquals(0, PipeLogicalBackupTool.run(args.toArray(new String[0]))); + } finally { + System.setIn(previous); + } + } +} diff --git a/iotdb-client/cli/LOGICAL_BACKUP.md b/iotdb-client/cli/LOGICAL_BACKUP.md new file mode 100644 index 0000000000000..2fa605314e425 --- /dev/null +++ b/iotdb-client/cli/LOGICAL_BACKUP.md @@ -0,0 +1,138 @@ + + +# Logical backup + +Use `tools/logical-backup.sh` on Linux/macOS or +`tools/windows/logical-backup.bat` on Windows. `--help` lists the supported options. +Commands exit with 0 on success and 1 on failure. Errors go to stderr; `--json` +provides a machine-readable result on stdout. + +## Capture, check, transport, restore + +```sh +tools/logical-backup.sh dump --host source-host --port 6667 --user root \ + --password-stdin --backup-dir /mnt/shared-backups --backup-id migration_001 \ + --scope config,schema,data --model tree,table --mode snapshot + +tools/logical-backup.sh inspect --input /mnt/shared-backups/migration_001 --json +tools/logical-backup.sh verify --input /mnt/shared-backups/migration_001 --json +tools/logical-backup.sh pack --input /mnt/shared-backups/migration_001 \ + --output /archives/migration_001.zip + +# Local verification; no target connection is opened. +tools/logical-backup.sh restore --input /archives/migration_001.zip --dry-run + +# Target version, precision, authentication and protocol capability checks. +tools/logical-backup.sh restore --input /archives/migration_001.zip \ + --host target-host --port 6667 --user root --password-stdin --dry-run + +tools/logical-backup.sh restore --input /archives/migration_001.zip \ + --host target-host --port 6667 --user root --password-stdin \ + --checkpoint /archives/migration_001.restore.json + +# After an interrupted restore, retain all connection and input options and add --resume. +tools/logical-backup.sh restore --input /archives/migration_001.zip \ + --host target-host --port 6667 --user root --password-stdin \ + --checkpoint /archives/migration_001.restore.json --resume +``` + +Passwords are read from stdin or `--password-env NAME`. Plaintext password +arguments are rejected. Manifests and checkpoints contain no connection secrets. + +`dump` generates a directory, and `pack` only packages an existing directory. +`pack` rejects connection options. Output and checkpoint files must be outside +the input backup, and existing output archives are never overwritten. + +All source ConfigNodes, DataNodes, and the CLI must access the same backup +directory at the specified path. This is a shared filesystem workflow, not a +remote download protocol. Do not modify a directory while it is being checked, +packed or restored. + +## Scope and boundaries + +`--scope` is required and selects `config`, `schema`, `data`, or a comma-separated +combination. Schema capture includes its ConfigRegion definitions, such as +databases and tables. `--model` defaults to `tree,table`. + +Authorization objects are excluded by default. Capturing them requires +`--scope config` (possibly combined with other scopes) and `--include-auth true`. +Restoring them also requires `--include-auth true`; inspect/dry-run reports this +selection. Legacy config archives are conservatively treated as containing +authorization objects. + +Object filters require a single model. Tree data accepts source path inclusion +and exclusion lists; when capturing config/schema, the source supports a single +prefix or full path without exclusions. Table filters use +`database-regex/table-regex`; exclusions must match either whole databases or +whole table-name patterns. Unsupported combinations fail before task creation. +Data `--start-time` and `--end-time` are inclusive and use source timestamp +precision. These filters do not remove the schema needed to describe selected +objects. + +`snapshot` captures finite historical streams. `continuous` requires an ISO-8601 +`--until` boundary. Each source stops accepting realtime work at its capture +boundary, drains accepted work, and acknowledges completion before task removal. +The boundary uses source clocks; streams do not represent a global transaction. +`--stop-sequence` is reserved and rejected because this server does not expose a +stable global source sequence. `sourceHighWatermark` records each stream's durable +archive sequence; it is not a cluster-wide source log position. + +The root `manifest.json` is the portable entry point. It declares scope, +models, filters, capture settings, streams, relative file paths, full-file SHA-256 +digests, counts, status and capture/seal times. Stream manifests describe the +versioned `.pwal` frames, CRCs, segment digests and committed event groups. Legacy +stream-only directories remain readable; packing them adds a root manifest +without changing the input. + +Config and Schema snapshots are decoded into logical plans before capture; +data files are decoded into logical tablet requests. Each Region has its own +ordered stream and writer. Pipe parameter validation creates no backup files. + +## Interrupted work and conflicts + +`dump --resume fail-if-exists` is the default. `new` chooses a new backup identity +if the requested directory exists. `append` requires the same identity, scope +and writing settings. A timed-out command reports the backup, directory and +Pipe identity; query `SHOW PIPE` or rerun the same dump with `--resume append`. +An already sealed matching backup is returned after verification. + +Writer sizes, fsync policy and batch/period combinations are validated before +writing. Unknown events fail by default; `--on-unsupported skip` records a gap +and seals the result as `INCOMPLETE`. Inspect, verify, pack and restore reject +incomplete backups unless `--allow-incomplete` is explicit. That flag never +bypasses checksum errors, truncated segments, missing files or invalid groups. + +Restore validates locally before connecting, then replays config, schema and +data, sorting equal-type streams by stream ID. Dry-run sends no replay requests. +Target preflight checks version, precision, authentication and protocol +handshakes; object conflicts and object-level permissions are checked during +replay. Only `--conflict fail` is supported. Target errors stop replay and retain +the checkpoint; no schema is silently deleted, overwritten or skipped. + +Checkpoints bind the archive/content digest, backup format and identity, target +host/port/user, completed groups and the next request within a group. A request +that may have succeeded immediately before an interrupted checkpoint update can +be retried; only that uncertain request accepts an idempotent-conflict response. +There is no transaction across requests or streams. Completed checkpoints remain +available for audit and repeat invocation. + +`export` and `import` are deprecated compatibility aliases. Offline `export` is +`pack`; connected `export` performs `dump` then `pack`; `import` is `restore`. +Both aliases print a migration notice on stderr. diff --git a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java index 1316415682f59..9f49fc238e6e7 100644 --- a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java @@ -169,4 +169,44 @@ private CliMessages() {} EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_HAS_INVALID_IDENTIFIER_ARG_BB015204 = "Logical backup option --%s has invalid identifier: %s"; + public static final String LOG_EXPORT_IS_DEPRECATED_USE_DUMP_TO_GENERATE_A_BACKUP_AND_PACK_TO_CREATE_A_ZIP_47709D2D = + "export is deprecated; use dump to generate a backup and pack to create a ZIP."; + public static final String LOG_IMPORT_IS_DEPRECATED_USE_RESTORE_2F1AE2CE = + "import is deprecated; use restore."; + public static final String LOG_LOGICAL_BACKUP_ARG_1E5C23CE = + "Logical backup: %s"; + public static final String LOG_LOGICAL_BACKUP_PACKED_FROM_ARG_TO_ARG_CB9D5304 = + "Logical backup packed from %s to %s"; +public static final String EXCEPTION_STOP_SEQUENCE_REQUIRES_A_STABLE_SOURCE_SEQUENCE_CAPABILITY_USE_UNTIL_WITH_THIS_SERVER_VERSION_8323B09E = + "--stop-sequence requires a stable source sequence capability; use --until with this server version."; + public static final String EXCEPTION_OBJECT_FILTERS_REQUIRE_MODEL_TREE_OR_MODEL_TABLE_SPLIT_MIXED_MODEL_FILTERED_DUMPS_INTO_SEPARATE_BACKUPS_0488699E = + "Object filters require --model tree or --model table; split mixed-model filtered dumps into separate backups."; + public static final String EXCEPTION_TABLE_FILTERS_USE_DATABASE_REGEX_TABLE_REGEX_EXCLUSIONS_MUST_USE_FOR_EITHER_DATABASE_OR_TABLE_42F0FA23 = + "Table filters use database-regex/table-regex; exclusions must use .* for either database or table."; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_DUMP_OPTION_ARG_894E11E0 = + "Invalid logical backup dump option: --%s"; + public static final String EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81 = + "Invalid or conflicting logical backup arguments."; + public static final String EXCEPTION_ONLY_CONFLICT_FAIL_IS_SUPPORTED_RESTORE_NEVER_IMPLICITLY_OVERWRITES_OR_DELETES_TARGET_OBJECTS_70C9523E = + "Only --conflict fail is supported; restore never implicitly overwrites or deletes target objects."; + public static final String EXCEPTION_THE_EXISTING_BACKUP_TASK_DOES_NOT_MATCH_THIS_BACKUP_IDENTITY_OR_CAPTURE_OPTIONS_50E6388F = + "The existing backup task does not match this backup identity or capture options."; + public static final String EXCEPTION_ANOTHER_DUMP_PROCESS_IS_MANAGING_THIS_BACKUP_FE8EE828 = + "Another dump process is managing this backup."; + public static final String EXCEPTION_BACKUP_ARG_TASK_ARG_DIRECTORY_ARG_ARG_THE_TASK_IS_RETAINED_QUERY_SHOW_PIPE_OR_REPEAT_DUMP_WITH_THE_SAME_OPTIONS_AND_RESUME_APPEND_6F8C1E87 = + "Backup %s, task %s, directory %s: %s. The task is retained; query SHOW PIPE or repeat dump with the same options and --resume append."; + public static final String EXCEPTION_THE_BACKUP_TASK_HAS_NOT_PRODUCED_SEALED_STREAMS_YET_RESUME_THE_SAME_TASK_TO_FINISH_SEALING_87C9B599 = + "The backup task has not produced sealed streams yet; resume the same task to finish sealing."; + public static final String EXCEPTION_AN_EXISTING_CHECKPOINT_REQUIRES_RESUME_RESUME_REQUIRES_AN_EXISTING_CHECKPOINT_FB6D0B7D = + "An existing checkpoint requires --resume; --resume requires an existing checkpoint."; + public static final String EXCEPTION_THE_TARGET_TIMESTAMP_PRECISION_DOES_NOT_MATCH_THE_BACKUP_70B225CB = + "The target timestamp precision does not match the backup."; + public static final String EXCEPTION_THE_TARGET_VERSION_OR_DATA_MODEL_IS_INCOMPATIBLE_WITH_THE_BACKUP_220EB40C = + "The target version or data model is incompatible with the backup."; + public static final String EXCEPTION_THE_CHECKPOINT_MUST_BE_OUTSIDE_THE_INPUT_BACKUP_15B70F59 = + "The checkpoint must be outside the input backup."; + public static final String EXCEPTION_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_INSPECT_THE_BACKUP_OR_USE_DRY_RUN_FIRST_83DDCD79 = + "Restoring authorization objects requires --include-auth true; inspect the backup or use --dry-run first."; + public static final String LOG_LOGICAL_BACKUP_DUMP_VERIFY_INSPECT_PACK_RESTORE_OPTIONS_DUMP_CONNECT_TO_A_SOURCE_AND_SEAL_A_BACKUP_DIRECTORY_DOES_NOT_CREATE_A_ZIP_INSPECT_SHOW_BACKUP_IDENTITY_SCOPE_STREAMS_SEGMENTS_COUNTS_AND_STATUS_VERIFY_CHECK_STRUCTURE_CHECKSUMS_AND_SEMANTICS_WITHOUT_CHANGING_THE_INPUT_PACK_VERIFY_AND_PACKAGE_AN_EXISTING_DIRECTORY_AS_ZIP_ACCEPTS_NO_CONNECTION_OPTIONS_RESTORE_PREFLIGHT_AND_REPLAY_A_DIRECTORY_ROOT_MANIFEST_JSON_OR_ZIP_INSPECT_VERIFY_INPUT_PATH_JSON_ALLOW_INCOMPLETE_PACK_INPUT_DIR_OUTPUT_ZIP_JSON_ALLOW_INCOMPLETE_DUMP_HOST_HOST_PORT_PORT_USER_ROOT_PASSWORD_STDIN_PASSWORD_ENV_NAME_BACKUP_DIR_DIR_BACKUP_ID_ID_SCOPE_CONFIG_SCHEMA_DATA_MODEL_TREE_TABLE_INCLUDE_PATTERN_EXCLUDE_PATTERN_START_TIME_TIME_END_TIME_TIME_INCLUDE_AUTH_TRUE_FALSE_MODE_SNAPSHOT_CONTINUOUS_UNTIL_ISO_8601_RESUME_FAIL_IF_EXISTS_APPEND_NEW_PIPE_NAME_NAME_SEGMENT_SIZE_BYTES_268435456_MAX_RECORD_BYTES_67108864_FSYNC_POLICY_ALWAYS_BATCH_PERIODIC_NONE_FSYNC_BATCH_OPERATIONS_N_FSYNC_PERIOD_MS_N_ON_UNSUPPORTED_FAIL_SKIP_INCLUDE_HEARTBEAT_TRUE_FALSE_TIMEOUT_SECONDS_3600_POLL_INTERVAL_SECONDS_2_JSON_ALLOW_INCOMPLETE_CONTINUOUS_REQUIRES_UNTIL_STOP_SEQUENCE_IS_RESERVED_AND_CURRENTLY_REJECTED_OBJECT_FILTERS_REQUIRE_ONE_MODEL_TREE_USES_PATH_LISTS_TABLE_USES_DATABASE_REGEX_TABLE_REGEX_TREE_CONFIG_SCHEMA_CAPTURE_ACCEPTS_A_SINGLE_PREFIX_FULL_PATH_WITHOUT_EXCLUSIONS_TABLE_EXCLUSIONS_MUST_SELECT_WHOLE_DATABASES_OR_WHOLE_TABLE_NAME_PATTERNS_RESTORE_INPUT_PATH_HOST_HOST_PORT_PORT_USER_USER_PASSWORD_STDIN_PASSWORD_ENV_NAME_DRY_RUN_CHECKPOINT_FILE_RESUME_CONFLICT_FAIL_INCLUDE_AUTH_TRUE_FALSE_JSON_ALLOW_INCOMPLETE_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_LEGACY_CONFIG_IS_CONSERVATIVE_A_TARGET_CONNECTION_IS_REQUIRED_FOR_REPLAY_LOCAL_DRY_RUN_NEEDS_NO_CONNECTION_TARGET_PREFLIGHT_CHECKS_VERSION_PRECISION_AUTHENTICATION_AND_HANDSHAKE_SUPPORT_OBJECT_CONFLICTS_AND_PER_OBJECT_PRIVILEGES_ARE_CHECKED_DURING_REPLAY_DEPRECATED_ALIASES_EXPORT_INPUT_DIR_OUTPUT_ZIP_PACK_EXPORT_WITH_SOURCE_CONNECTION_OPTIONS_DUMP_THEN_PACK_IMPORT_RESTORE_EXIT_CODE_0_SUCCESS_1_FAILURE_ERRORS_AND_DEPRECATION_NOTICES_GO_TO_STDERR_B1983BE8 = + "logical-backup [options]\n dump Connect to a source and seal a backup directory; does not create a ZIP.\n inspect Show backup identity, scope, streams, segments, counts and status.\n verify Check structure, checksums and semantics without changing the input.\n pack Verify and package an existing directory as ZIP; accepts no connection options.\n restore Preflight and replay a directory, root manifest.json or ZIP.\n\ninspect/verify: --input PATH [--json] [--allow-incomplete]\npack: --input DIR --output ZIP [--json] [--allow-incomplete]\ndump: --host HOST --port PORT [--user root] (--password-stdin|--password-env NAME)\n --backup-dir DIR --backup-id ID --scope config,schema,data [--model tree,table]\n [--include PATTERN] [--exclude PATTERN] [--start-time TIME] [--end-time TIME]\n [--include-auth true|false] [--mode snapshot|continuous] [--until ISO-8601]\n [--resume fail-if-exists|append|new] [--pipe-name NAME]\n [--segment-size-bytes 268435456] [--max-record-bytes 67108864]\n [--fsync-policy always|batch|periodic|none] [--fsync-batch-operations N|--fsync-period-ms N]\n [--on-unsupported fail|skip] [--include-heartbeat true|false]\n [--timeout-seconds 3600] [--poll-interval-seconds 2] [--json] [--allow-incomplete]\n continuous requires --until; --stop-sequence is reserved and currently rejected.\n Object filters require one model: Tree uses path lists; Table uses database-regex/table-regex.\n Tree config/schema capture accepts a single prefix/full path without exclusions.\n Table exclusions must select whole databases or whole table-name patterns.\nrestore: --input PATH [--host HOST --port PORT --user USER\n (--password-stdin|--password-env NAME)] [--dry-run] [--checkpoint FILE] [--resume]\n [--conflict fail] [--include-auth true|false] [--json] [--allow-incomplete]\n Restoring authorization objects requires --include-auth true (legacy config is conservative).\n A target connection is required for replay; local dry-run needs no connection.\n Target preflight checks version, precision, authentication and handshake support;\n object conflicts and per-object privileges are checked during replay.\n\nDeprecated aliases: export --input DIR --output ZIP = pack;\n export with source connection options = dump then pack; import = restore.\nExit code: 0 success, 1 failure. Errors and deprecation notices go to stderr."; } diff --git a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java index c26071d0bf5f0..b8733d990479c 100644 --- a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java @@ -165,4 +165,44 @@ private CliMessages() {} public static final String EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_HAS_INVALID_IDENTIFIER_ARG_BB015204 = "逻辑备份选项 --%s 的标识符无效：%s"; + public static final String LOG_EXPORT_IS_DEPRECATED_USE_DUMP_TO_GENERATE_A_BACKUP_AND_PACK_TO_CREATE_A_ZIP_47709D2D = + "export 已弃用；请使用 dump 生成备份，使用 pack 创建 ZIP。"; + public static final String LOG_IMPORT_IS_DEPRECATED_USE_RESTORE_2F1AE2CE = + "import 已弃用；请使用 restore。"; + public static final String LOG_LOGICAL_BACKUP_ARG_1E5C23CE = + "逻辑备份：%s"; + public static final String LOG_LOGICAL_BACKUP_PACKED_FROM_ARG_TO_ARG_CB9D5304 = + "已将逻辑备份从 %s 打包至 %s"; +public static final String EXCEPTION_STOP_SEQUENCE_REQUIRES_A_STABLE_SOURCE_SEQUENCE_CAPABILITY_USE_UNTIL_WITH_THIS_SERVER_VERSION_8323B09E = + "--stop-sequence 需要源端提供稳定序号能力；当前服务端版本请使用 --until。"; + public static final String EXCEPTION_OBJECT_FILTERS_REQUIRE_MODEL_TREE_OR_MODEL_TABLE_SPLIT_MIXED_MODEL_FILTERED_DUMPS_INTO_SEPARATE_BACKUPS_0488699E = + "对象过滤需要 --model tree 或 --model table；请将混合模型过滤拆为独立备份。"; + public static final String EXCEPTION_TABLE_FILTERS_USE_DATABASE_REGEX_TABLE_REGEX_EXCLUSIONS_MUST_USE_FOR_EITHER_DATABASE_OR_TABLE_42F0FA23 = + "Table 过滤使用数据库正则/表名正则；排除过滤中数据库或表名必须有一项为 .*。"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_DUMP_OPTION_ARG_894E11E0 = + "逻辑备份 dump 选项无效：--%s"; + public static final String EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81 = + "逻辑备份参数无效或相互冲突。"; + public static final String EXCEPTION_ONLY_CONFLICT_FAIL_IS_SUPPORTED_RESTORE_NEVER_IMPLICITLY_OVERWRITES_OR_DELETES_TARGET_OBJECTS_70C9523E = + "当前仅支持 --conflict fail；restore 不会隐式覆盖或删除目标对象。"; + public static final String EXCEPTION_THE_EXISTING_BACKUP_TASK_DOES_NOT_MATCH_THIS_BACKUP_IDENTITY_OR_CAPTURE_OPTIONS_50E6388F = + "已有备份任务与本次备份身份或捕获选项不一致。"; + public static final String EXCEPTION_ANOTHER_DUMP_PROCESS_IS_MANAGING_THIS_BACKUP_FE8EE828 = + "其他 dump 进程正在管理此备份。"; + public static final String EXCEPTION_BACKUP_ARG_TASK_ARG_DIRECTORY_ARG_ARG_THE_TASK_IS_RETAINED_QUERY_SHOW_PIPE_OR_REPEAT_DUMP_WITH_THE_SAME_OPTIONS_AND_RESUME_APPEND_6F8C1E87 = + "备份 %s，任务 %s，目录 %s：%s。任务已保留；可使用 SHOW PIPE 查询，或使用相同选项和 --resume append 继续 dump。"; + public static final String EXCEPTION_THE_BACKUP_TASK_HAS_NOT_PRODUCED_SEALED_STREAMS_YET_RESUME_THE_SAME_TASK_TO_FINISH_SEALING_87C9B599 = + "备份任务尚未产出封存的流；请继续原任务以完成封存。"; + public static final String EXCEPTION_AN_EXISTING_CHECKPOINT_REQUIRES_RESUME_RESUME_REQUIRES_AN_EXISTING_CHECKPOINT_FB6D0B7D = + "使用已有 checkpoint 必须指定 --resume；--resume 要求 checkpoint 已存在。"; + public static final String EXCEPTION_THE_TARGET_TIMESTAMP_PRECISION_DOES_NOT_MATCH_THE_BACKUP_70B225CB = + "目标时间精度与备份不一致。"; + public static final String EXCEPTION_THE_TARGET_VERSION_OR_DATA_MODEL_IS_INCOMPATIBLE_WITH_THE_BACKUP_220EB40C = + "目标版本或数据模型与备份不兼容。"; + public static final String EXCEPTION_THE_CHECKPOINT_MUST_BE_OUTSIDE_THE_INPUT_BACKUP_15B70F59 = + "checkpoint 必须位于输入备份之外。"; + public static final String EXCEPTION_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_INSPECT_THE_BACKUP_OR_USE_DRY_RUN_FIRST_83DDCD79 = + "恢复权限对象必须显式指定 --include-auth true；请先 inspect 备份或执行 --dry-run。"; + public static final String LOG_LOGICAL_BACKUP_DUMP_VERIFY_INSPECT_PACK_RESTORE_OPTIONS_DUMP_CONNECT_TO_A_SOURCE_AND_SEAL_A_BACKUP_DIRECTORY_DOES_NOT_CREATE_A_ZIP_INSPECT_SHOW_BACKUP_IDENTITY_SCOPE_STREAMS_SEGMENTS_COUNTS_AND_STATUS_VERIFY_CHECK_STRUCTURE_CHECKSUMS_AND_SEMANTICS_WITHOUT_CHANGING_THE_INPUT_PACK_VERIFY_AND_PACKAGE_AN_EXISTING_DIRECTORY_AS_ZIP_ACCEPTS_NO_CONNECTION_OPTIONS_RESTORE_PREFLIGHT_AND_REPLAY_A_DIRECTORY_ROOT_MANIFEST_JSON_OR_ZIP_INSPECT_VERIFY_INPUT_PATH_JSON_ALLOW_INCOMPLETE_PACK_INPUT_DIR_OUTPUT_ZIP_JSON_ALLOW_INCOMPLETE_DUMP_HOST_HOST_PORT_PORT_USER_ROOT_PASSWORD_STDIN_PASSWORD_ENV_NAME_BACKUP_DIR_DIR_BACKUP_ID_ID_SCOPE_CONFIG_SCHEMA_DATA_MODEL_TREE_TABLE_INCLUDE_PATTERN_EXCLUDE_PATTERN_START_TIME_TIME_END_TIME_TIME_INCLUDE_AUTH_TRUE_FALSE_MODE_SNAPSHOT_CONTINUOUS_UNTIL_ISO_8601_RESUME_FAIL_IF_EXISTS_APPEND_NEW_PIPE_NAME_NAME_SEGMENT_SIZE_BYTES_268435456_MAX_RECORD_BYTES_67108864_FSYNC_POLICY_ALWAYS_BATCH_PERIODIC_NONE_FSYNC_BATCH_OPERATIONS_N_FSYNC_PERIOD_MS_N_ON_UNSUPPORTED_FAIL_SKIP_INCLUDE_HEARTBEAT_TRUE_FALSE_TIMEOUT_SECONDS_3600_POLL_INTERVAL_SECONDS_2_JSON_ALLOW_INCOMPLETE_CONTINUOUS_REQUIRES_UNTIL_STOP_SEQUENCE_IS_RESERVED_AND_CURRENTLY_REJECTED_OBJECT_FILTERS_REQUIRE_ONE_MODEL_TREE_USES_PATH_LISTS_TABLE_USES_DATABASE_REGEX_TABLE_REGEX_TREE_CONFIG_SCHEMA_CAPTURE_ACCEPTS_A_SINGLE_PREFIX_FULL_PATH_WITHOUT_EXCLUSIONS_TABLE_EXCLUSIONS_MUST_SELECT_WHOLE_DATABASES_OR_WHOLE_TABLE_NAME_PATTERNS_RESTORE_INPUT_PATH_HOST_HOST_PORT_PORT_USER_USER_PASSWORD_STDIN_PASSWORD_ENV_NAME_DRY_RUN_CHECKPOINT_FILE_RESUME_CONFLICT_FAIL_INCLUDE_AUTH_TRUE_FALSE_JSON_ALLOW_INCOMPLETE_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_LEGACY_CONFIG_IS_CONSERVATIVE_A_TARGET_CONNECTION_IS_REQUIRED_FOR_REPLAY_LOCAL_DRY_RUN_NEEDS_NO_CONNECTION_TARGET_PREFLIGHT_CHECKS_VERSION_PRECISION_AUTHENTICATION_AND_HANDSHAKE_SUPPORT_OBJECT_CONFLICTS_AND_PER_OBJECT_PRIVILEGES_ARE_CHECKED_DURING_REPLAY_DEPRECATED_ALIASES_EXPORT_INPUT_DIR_OUTPUT_ZIP_PACK_EXPORT_WITH_SOURCE_CONNECTION_OPTIONS_DUMP_THEN_PACK_IMPORT_RESTORE_EXIT_CODE_0_SUCCESS_1_FAILURE_ERRORS_AND_DEPRECATION_NOTICES_GO_TO_STDERR_B1983BE8 = + "logical-backup [选项]\n dump 连接源端并封存备份目录，不生成 ZIP。\n inspect 查看备份身份、范围、流、segment、计数和状态。\n verify 检查结构、校验和与语义，不修改输入。\n pack 校验已有目录并打包为 ZIP，不接受数据库连接参数。\n restore 预检并回放备份目录、根 manifest.json 或 ZIP。\n\ninspect/verify: --input PATH [--json] [--allow-incomplete]\npack: --input DIR --output ZIP [--json] [--allow-incomplete]\ndump: --host HOST --port PORT [--user root] (--password-stdin|--password-env NAME)\n --backup-dir DIR --backup-id ID --scope config,schema,data [--model tree,table]\n [--include PATTERN] [--exclude PATTERN] [--start-time TIME] [--end-time TIME]\n [--include-auth true|false] [--mode snapshot|continuous] [--until ISO-8601]\n [--resume fail-if-exists|append|new] [--pipe-name NAME]\n [--segment-size-bytes 268435456] [--max-record-bytes 67108864]\n [--fsync-policy always|batch|periodic|none] [--fsync-batch-operations N|--fsync-period-ms N]\n [--on-unsupported fail|skip] [--include-heartbeat true|false]\n [--timeout-seconds 3600] [--poll-interval-seconds 2] [--json] [--allow-incomplete]\n continuous 必须提供 --until；--stop-sequence 是预留参数，当前会被拒绝。\n 对象过滤只允许选择一个模型：Tree 使用路径列表，Table 使用数据库正则/表名正则。\n Tree config/schema 捕获只接受单个前缀或完整路径，不支持排除过滤。\n Table 排除过滤必须覆盖整个数据库或整个表名模式。\nrestore: --input PATH [--host HOST --port PORT --user USER\n (--password-stdin|--password-env NAME)] [--dry-run] [--checkpoint FILE] [--resume]\n [--conflict fail] [--include-auth true|false] [--json] [--allow-incomplete]\n 恢复权限对象必须指定 --include-auth true；旧 config 备份按包含权限对象处理。\n 实际恢复需要目标连接；本地 dry-run 无需连接。\n 目标预检检查版本、时间精度、认证和握手能力；\n 对象冲突及对象级权限在回放时检查。\n\n弃用别名：export --input DIR --output ZIP = pack；\n 带源端连接参数的 export = dump 后接 pack；import = restore。\n退出码：0 成功，1 失败。错误及弃用提示输出到标准错误。"; } diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/LogicalBackupDumpOptions.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/LogicalBackupDumpOptions.java new file mode 100644 index 0000000000000..e28e288df0b42 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/LogicalBackupDumpOptions.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.pipe; + +import org.apache.iotdb.cli.i18n.CliMessages; +import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePatternOperations; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupCatalog; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupSettings; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; + +import com.google.gson.Gson; +import org.apache.commons.cli.CommandLine; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +/** Converts the public dump contract to source and sink parameters before any remote mutation. */ +final class LogicalBackupDumpOptions { + + final LogicalBackupCatalog catalog = new LogicalBackupCatalog(); + final Map source = new LinkedHashMap<>(); + final Map sink = new LinkedHashMap<>(); + final String resume; + final long until; + + LogicalBackupDumpOptions(final CommandLine line, final boolean legacy) throws IOException { + catalog.scope = + choices( + line.getOptionValue("scope", legacy ? "config,schema,data" : ""), + List.of("config", "schema", "data"), + "scope"); + catalog.model = + choices(line.getOptionValue("model", "tree,table"), List.of("tree", "table"), "model"); + catalog.mode = line.getOptionValue("mode", "snapshot"); + if (!List.of("snapshot", "continuous").contains(catalog.mode)) { + throw invalid("mode"); + } + final String includeAuth = line.getOptionValue("include-auth", "false"); + if (!List.of("true", "false").contains(includeAuth)) { + throw invalid("include-auth"); + } + catalog.includeAuth = Boolean.parseBoolean(includeAuth); + if (catalog.includeAuth && !catalog.scope.contains("config")) { + throw invalid("include-auth"); + } + if (line.hasOption("stop-sequence")) { + throw new IOException( + CliMessages + .EXCEPTION_STOP_SEQUENCE_REQUIRES_A_STABLE_SOURCE_SEQUENCE_CAPABILITY_USE_UNTIL_WITH_THIS_SERVER_VERSION_8323B09E); + } + if ("continuous".equals(catalog.mode)) { + if (!line.hasOption("until")) { + throw invalid("until"); + } + try { + until = Instant.parse(line.getOptionValue("until")).toEpochMilli(); + } catch (final java.time.DateTimeException e) { + throw invalid("until"); + } + if (until <= System.currentTimeMillis() && !"append".equals(line.getOptionValue("resume"))) { + throw invalid("until"); + } + } else { + if (line.hasOption("until")) { + throw invalid("until"); + } + until = -1; + } + resume = line.getOptionValue("resume", "fail-if-exists"); + if (!List.of("fail-if-exists", "append", "new").contains(resume)) { + throw invalid("resume"); + } + catalog.captureOptions.put("scope", String.join(",", catalog.scope)); + catalog.captureOptions.put("model", String.join(",", catalog.model)); + catalog.captureOptions.put("mode", catalog.mode); + catalog.captureOptions.put("include-auth", includeAuth); + for (final String key : List.of("include", "exclude", "start-time", "end-time", "until")) { + if (line.hasOption(key)) { + catalog.filters.put(key, line.getOptionValue(key)); + catalog.captureOptions.put(key, line.getOptionValue(key)); + } + } + catalog.filters.put("time-boundary", "closed-closed"); + final List inclusion = new ArrayList<>(); + if (catalog.scope.contains("config")) { + inclusion.add("schema.database"); + inclusion.add("schema.ttl"); + inclusion.add("schema.timeseries.template.create"); + inclusion.add("schema.timeseries.template.set"); + inclusion.add("schema.timeseries.template.unset"); + inclusion.add("schema.timeseries.template.alter"); + inclusion.add("schema.timeseries.template.drop"); + } + if (catalog.scope.contains("schema")) { + inclusion.add("schema"); + } + if (catalog.scope.contains("data")) { + inclusion.add("data"); + } + if (catalog.includeAuth) { + inclusion.add("auth"); + } + source.put("source", "iotdb-source"); + source.put("source.inclusion", String.join(",", inclusion)); + source.put("source.inclusion.exclusion", catalog.includeAuth ? "" : "auth"); + source.put("source.skipif", ""); + source.put("source.capture.tree", Boolean.toString(catalog.model.contains("tree"))); + source.put("source.capture.table", Boolean.toString(catalog.model.contains("table"))); + source.put("source.mode", "snapshot".equals(catalog.mode) ? "snapshot" : "live"); + source.put("source.realtime.enable", Boolean.toString("continuous".equals(catalog.mode))); + source.put("source.logical-backup.snapshot", Boolean.toString("snapshot".equals(catalog.mode))); + if (until > 0) { + source.put("source.logical-backup.until", Long.toString(until)); + source.put("source.realtime.mode", "forced-log"); + } + // Decode TsFiles so the source applies exact path/time filters to logical rows. + source.put("source.mods.enable", "true"); + for (final String key : List.of("start-time", "end-time")) { + if (line.hasOption(key)) { + source.put("source." + key, line.getOptionValue(key)); + } + } + applyObjectFilters(line); + sink.put("sink", "logical-backup-sink"); + sink.put("sink.resume", "append"); + final Map defaults = new LinkedHashMap<>(); + defaults.put("segment-size-bytes", "268435456"); + defaults.put("max-record-bytes", "67108864"); + defaults.put("fsync-policy", "always"); + defaults.put("on-unsupported", "fail"); + defaults.put("include-heartbeat", "false"); + for (final Map.Entry entry : defaults.entrySet()) { + catalog.captureOptions.put( + entry.getKey(), line.getOptionValue(entry.getKey(), entry.getValue())); + sink.put( + "sink." + + ("on-unsupported".equals(entry.getKey()) ? "on-unsupported-event" : entry.getKey()), + catalog.captureOptions.get(entry.getKey())); + } + for (final String key : List.of("fsync-batch-operations", "fsync-period-ms")) { + if (line.hasOption(key)) { + sink.put("sink." + key, line.getOptionValue(key)); + catalog.captureOptions.put(key, line.getOptionValue(key)); + } + } + sink.put("sink.capture-options", new Gson().toJson(catalog.captureOptions)); + // Reuse the same validation as all three server-side writers. + final Map validation = new LinkedHashMap<>(sink); + validation.put("sink.dir", line.getOptionValue("backup-dir", ".")); + new LogicalBackupSettings(new PipeParameters(validation), "backup"); + } + + private void applyObjectFilters(final CommandLine line) throws IOException { + if (!line.hasOption("include") && !line.hasOption("exclude")) { + return; + } + if (catalog.model.size() != 1) { + throw new IOException( + CliMessages + .EXCEPTION_OBJECT_FILTERS_REQUIRE_MODEL_TREE_OR_MODEL_TABLE_SPLIT_MIXED_MODEL_FILTERED_DUMPS_INTO_SEPARATE_BACKUPS_0488699E); + } + if (catalog.model.contains("tree")) { + if (line.hasOption("include")) { + source.put("source.path.inclusion", line.getOptionValue("include")); + } + if (line.hasOption("exclude")) { + source.put("source.path.exclusion", line.getOptionValue("exclude")); + } + final TreePattern pattern = + TreePattern.parsePipePatternFromSourceParameters(new PipeParameters(source)); + if (!pattern.isLegal() + || ((catalog.scope.contains("config") || catalog.scope.contains("schema")) + && !(pattern instanceof IoTDBTreePatternOperations + && ((IoTDBTreePatternOperations) pattern).isPrefixOrFullPath()))) { + throw invalid("include/exclude"); + } + return; + } + // TablePattern accepts independent database/table regular expressions. Reject correlated + // exclusions rather than silently backing up their Cartesian product. + final String[] include = tableFilter(line.getOptionValue("include", ".*/.*")); + final String[] exclude = + line.hasOption("exclude") ? tableFilter(line.getOptionValue("exclude")) : null; + if (exclude != null && !".*".equals(exclude[0]) && !".*".equals(exclude[1])) { + throw new IOException( + CliMessages + .EXCEPTION_TABLE_FILTERS_USE_DATABASE_REGEX_TABLE_REGEX_EXCLUSIONS_MUST_USE_FOR_EITHER_DATABASE_OR_TABLE_42F0FA23); + } + for (int index = 0; index < 2; index++) { + String regex = include[index]; + if (exclude != null && !".*".equals(exclude[index])) { + regex = "(?!(?:" + exclude[index] + ")$)(?:" + regex + ")"; + } + source.put(index == 0 ? "source.database-name" : "source.table-name", regex); + } + } + + private static String[] tableFilter(final String filter) throws IOException { + final String[] parts = filter.split("/", -1); + if (parts.length != 2 || parts[0].isBlank() || parts[1].isBlank()) { + throw new IOException( + CliMessages + .EXCEPTION_TABLE_FILTERS_USE_DATABASE_REGEX_TABLE_REGEX_EXCLUSIONS_MUST_USE_FOR_EITHER_DATABASE_OR_TABLE_42F0FA23); + } + try { + Pattern.compile(parts[0]); + Pattern.compile(parts[1]); + } catch (final java.util.regex.PatternSyntaxException e) { + throw invalid("include/exclude"); + } + return parts; + } + + private static List choices( + final String value, final List allowed, final String key) throws IOException { + final List values = Arrays.asList(value.split(",", -1)); + if (values.isEmpty() + || !allowed.containsAll(values) + || values.stream().distinct().count() != values.size()) { + throw invalid(key); + } + // Canonical ordering makes equivalent requests compare equal when resuming. + final List canonical = new ArrayList<>(allowed); + canonical.retainAll(values); + return canonical; + } + + String createSql(final String pipeName, final String root, final String backupId) { + final Map sinkOptions = new LinkedHashMap<>(sink); + sinkOptions.put("sink.dir", root); + sinkOptions.put("sink.backup-id", backupId); + return "CREATE PIPE " + + pipeName + + " WITH SOURCE (" + + sqlOptions(source) + + ") WITH SINK (" + + sqlOptions(sinkOptions) + + ")"; + } + + private static String sqlOptions(final Map options) { + final List values = new ArrayList<>(); + options.forEach((key, value) -> values.add("'" + key + "'='" + value.replace("'", "''") + "'")); + return String.join(",", values); + } + + private static IOException invalid(final String key) { + return new IOException( + String.format(CliMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_DUMP_OPTION_ARG_894E11E0, key)); + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java index 631353027a614..e73067ed69ea9 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java @@ -25,9 +25,11 @@ import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.EventGroup; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupCatalog; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupFormat; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecord; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupSettings; import org.apache.iotdb.commons.pipe.sink.payload.thrift.common.PipeTransferHandshakeConstant; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferHandshakeV2Req; @@ -43,7 +45,6 @@ import com.google.gson.JsonParseException; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; @@ -53,7 +54,6 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; -import java.io.PrintWriter; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -73,13 +73,16 @@ import java.util.Set; import java.util.UUID; import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; +import java.util.zip.ZipFile; import java.util.zip.ZipOutputStream; /** Command line inspector, verifier, archiver and importer for Pipe logical backups. */ public final class PipeLogicalBackupTool { private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + private static final String COMMAND_DUMP = "dump"; + private static final String COMMAND_PACK = "pack"; + private static final String COMMAND_RESTORE = "restore"; private static final String COMMAND_INSPECT = "inspect"; private static final String COMMAND_VERIFY = "verify"; private static final String COMMAND_EXPORT = "export"; @@ -129,71 +132,199 @@ public static void main(final String[] args) { } } - static int run(final String[] args) throws Exception { + public static int run(final String[] args) throws Exception { if (args.length == 0 || "--help".equals(args[0]) || "-h".equals(args[0])) { printUsage(); return 0; } final String command = args[0].toLowerCase(java.util.Locale.ROOT); + if (!List.of( + COMMAND_DUMP, + COMMAND_PACK, + COMMAND_RESTORE, + COMMAND_INSPECT, + COMMAND_VERIFY, + COMMAND_EXPORT, + COMMAND_IMPORT) + .contains(command)) { + throw new ParseException( + String.format( + CliMessages.EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619, command)); + } + if (java.util.Arrays.asList(args).contains("--help") + || java.util.Arrays.asList(args).contains("-h")) { + printUsage(); + return 0; + } final CommandLine line = parse(command, java.util.Arrays.copyOfRange(args, 1, args.length)); switch (command) { case COMMAND_INSPECT: return inspect(line); case COMMAND_VERIFY: return verify(line); + case COMMAND_DUMP: + final Path dumped = dump(line, false); + printDumpResult(line, dumped); + return 0; + case COMMAND_PACK: + return pack(line); + case COMMAND_RESTORE: + return importBackup(line); case COMMAND_EXPORT: - return export(line); + System.err.println( + CliMessages + .LOG_EXPORT_IS_DEPRECATED_USE_DUMP_TO_GENERATE_A_BACKUP_AND_PACK_TO_CREATE_A_ZIP_47709D2D); + if (!line.hasOption(OPTION_INPUT) && !line.hasOption(OPTION_SOURCE)) { + final Path source = dump(line, true); + final List streams = + new LogicalBackupArchiveReader() + .read(source, line.hasOption(OPTION_ALLOW_INCOMPLETE)); + return exportArchive( + source, + Path.of(optionValue(line, OPTION_OUTPUT, OPTION_TARGET)), + streams, + line.hasOption(OPTION_JSON)); + } + return pack(line); case COMMAND_IMPORT: + System.err.println(CliMessages.LOG_IMPORT_IS_DEPRECATED_USE_RESTORE_2F1AE2CE); return importBackup(line); default: - throw new ParseException( - String.format( - CliMessages.EXCEPTION_UNKNOWN_LOGICAL_BACKUP_COMMAND_ARG_79275619, command)); + throw new AssertionError(command); } } - private static CommandLine parse(final String command, final String[] args) - throws ParseException { + static CommandLine parse(final String command, final String[] args) throws ParseException { final Options options = new Options(); - addOption(options, OPTION_INPUT, true, false); - addOption(options, OPTION_OUTPUT, true, false); - addOption(options, OPTION_RESUME, true, false); - addOption(options, OPTION_FORMAT, true, false); - addOption(options, OPTION_DEEP, false, false); - addOption(options, OPTION_SOURCE, true, false); - addOption(options, OPTION_TARGET, true, false); - addOption(options, OPTION_HOST, true, false); - addOption(options, OPTION_PORT, true, false); - addOption(options, OPTION_USER, true, false); - addOption(options, OPTION_PASSWORD_STDIN, false, false); - addOption(options, OPTION_PASSWORD_ENV, true, false); - addOption(options, OPTION_CHECKPOINT, true, false); - addOption(options, OPTION_DRY_RUN, false, false); - addOption(options, OPTION_ALLOW_INCOMPLETE, false, false); - addOption(options, OPTION_JSON, false, false); - addOption(options, OPTION_BACKUP_DIR, true, false); - addOption(options, OPTION_PIPE_NAME, true, false); - addOption(options, OPTION_BACKUP_ID, true, false); - addOption(options, OPTION_TIMEOUT_SECONDS, true, false); - addOption(options, OPTION_POLL_INTERVAL_SECONDS, true, false); + final Set names = new LinkedHashSet<>(List.of(OPTION_JSON, OPTION_ALLOW_INCOMPLETE)); + final boolean legacyExport = COMMAND_EXPORT.equals(command); + final boolean legacyImport = COMMAND_IMPORT.equals(command); + final boolean dump = COMMAND_DUMP.equals(command) || legacyExport; + final boolean restore = COMMAND_RESTORE.equals(command) || legacyImport; + if (!COMMAND_DUMP.equals(command)) { + names.add(OPTION_INPUT); + } + if (COMMAND_PACK.equals(command) || legacyExport) { + names.add(OPTION_OUTPUT); + } + if (dump || restore) { + names.addAll( + List.of( + OPTION_HOST, + OPTION_PORT, + OPTION_USER, + OPTION_PASSWORD_STDIN, + OPTION_PASSWORD_ENV, + OPTION_RESUME)); + } + if (restore) { + names.addAll(List.of(OPTION_CHECKPOINT, OPTION_DRY_RUN, "conflict", "include-auth")); + } + if (dump) { + names.addAll( + List.of( + OPTION_BACKUP_DIR, + OPTION_BACKUP_ID, + OPTION_PIPE_NAME, + OPTION_TIMEOUT_SECONDS, + OPTION_POLL_INTERVAL_SECONDS, + "scope", + "model", + "include", + "exclude", + "start-time", + "end-time", + "include-auth", + "mode", + "until", + "stop-sequence", + "segment-size-bytes", + "max-record-bytes", + "fsync-policy", + "fsync-batch-operations", + "fsync-period-ms", + "on-unsupported", + "include-heartbeat")); + } + if (legacyExport || legacyImport) { + names.add(OPTION_SOURCE); + if (legacyExport) { + names.addAll(List.of(OPTION_TARGET, OPTION_FORMAT)); + } + } + if (COMMAND_VERIFY.equals(command)) { + names.add(OPTION_DEEP); // Accepted for compatibility; verification is always deep. + } + final Set flags = + Set.of( + OPTION_JSON, + OPTION_ALLOW_INCOMPLETE, + OPTION_DRY_RUN, + OPTION_PASSWORD_STDIN, + OPTION_DEEP); + for (final String name : names) { + if (OPTION_RESUME.equals(name) && restore) { + options.addOption( + Option.builder().longOpt(name).hasArg(legacyImport).optionalArg(legacyImport).build()); + } else { + addOption(options, name, !flags.contains(name), false); + } + } final CommandLine line = new DefaultParser().parse(options, args); - final String input = optionValue(line, OPTION_INPUT, OPTION_SOURCE); - if ((input == null || input.isBlank()) - && (!COMMAND_EXPORT.equals(command) || !line.hasOption(OPTION_BACKUP_DIR))) { + if (!line.getArgList().isEmpty() + || (line.hasOption(OPTION_INPUT) && line.hasOption(OPTION_SOURCE)) + || (line.hasOption(OPTION_OUTPUT) && line.hasOption(OPTION_TARGET))) { + throw new ParseException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + if (!COMMAND_DUMP.equals(command) + && !(legacyExport && line.hasOption(OPTION_BACKUP_DIR)) + && optionValue(line, OPTION_INPUT, OPTION_SOURCE) == null) { throw new ParseException( String.format( CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, OPTION_INPUT)); } - final String output = optionValue(line, OPTION_OUTPUT, OPTION_TARGET); - if (COMMAND_EXPORT.equals(command) && (output == null || output.isBlank())) { + if ((COMMAND_PACK.equals(command) || legacyExport) + && optionValue(line, OPTION_OUTPUT, OPTION_TARGET) == null) { throw new ParseException( - CliMessages.EXCEPTION_OUTPUT_IS_REQUIRED_FOR_LOGICAL_BACKUP_EXPORT_603340B1); + String.format( + CliMessages.EXCEPTION_LOGICAL_BACKUP_OPTION_ARG_IS_REQUIRED_1E7449AA, OPTION_OUTPUT)); } - final String format = line.getOptionValue(OPTION_FORMAT, "binary"); - if (COMMAND_EXPORT.equals(command) && !"binary".equalsIgnoreCase(format)) { + if (legacyExport && !"binary".equals(line.getOptionValue(OPTION_FORMAT, "binary"))) { throw new ParseException( String.format( - CliMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1, format)); + CliMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EXPORT_FORMAT_ARG_A6D7DEB1, + line.getOptionValue(OPTION_FORMAT))); + } + if (line.hasOption(OPTION_PORT)) { + final long port = longOption(line, OPTION_PORT, 6667); + if (port <= 0 || port > 65535) { + throw new ParseException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + } + if (restore && !"fail".equals(line.getOptionValue("conflict", "fail"))) { + throw new ParseException( + CliMessages + .EXCEPTION_ONLY_CONFLICT_FAIL_IS_SUPPORTED_RESTORE_NEVER_IMPLICITLY_OVERWRITES_OR_DELETES_TARGET_OBJECTS_70C9523E); + } + // Offline export is exactly pack, including its refusal of source connection options. + if (legacyExport && optionValue(line, OPTION_INPUT, OPTION_SOURCE) != null) { + final Set packOptions = + Set.of( + OPTION_INPUT, + OPTION_SOURCE, + OPTION_OUTPUT, + OPTION_TARGET, + OPTION_FORMAT, + OPTION_JSON, + OPTION_ALLOW_INCOMPLETE); + for (final Option option : line.getOptions()) { + if (!packOptions.contains(option.getLongOpt())) { + throw new ParseException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + } } return line; } @@ -203,128 +334,228 @@ private static void addOption( options.addOption(Option.builder().longOpt(name).hasArg(hasArg).required(required).build()); } - private static int inspect(final CommandLine line) throws IOException { - final List streams = read(line); - if (line.hasOption(OPTION_JSON)) { - final List> output = new ArrayList<>(); - for (final BackupStream stream : streams) { - final Map item = new HashMap<>(); - item.put("manifest", stream.getManifest()); - item.put("recordCount", stream.getRecords().size()); - item.put("eventGroupCount", stream.getEventGroups().size()); - output.add(item); - } - System.out.println(GSON.toJson(output)); - return 0; + private static Map summary(final List streams) { + final Map result = new LinkedHashMap<>(); + if (!streams.isEmpty() && streams.get(0).getCatalog() != null) { + result.put("manifest", streams.get(0).getCatalog()); } + final List> details = new ArrayList<>(); for (final BackupStream stream : streams) { - System.out.println( - String.format( - CliMessages.LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0, - stream.getManifest().streamId, - stream.getRecords().size(), - stream.getEventGroups().size())); + final Map item = new LinkedHashMap<>(); + item.put("manifest", stream.getManifest()); + item.put("recordCount", stream.getRecords().size()); + item.put("eventGroupCount", stream.getEventGroups().size()); + details.add(item); } + result.put("streams", details); + return result; + } + + private static int inspect(final CommandLine line) throws IOException { + final List streams = read(line); + printSummary(line, summary(streams)); return 0; } private static int verify(final CommandLine line) throws IOException { final List streams = read(line); - long recordCount = 0; - long eventCount = 0; - for (final BackupStream stream : streams) { - recordCount += stream.getRecords().size(); - eventCount += stream.getEventGroups().size(); - } - System.out.println( - String.format( - CliMessages - .LOG_LOGICAL_BACKUP_VERIFIED_ARG_STREAMS_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_5210360B, - streams.size(), - recordCount, - eventCount)); + final Map result = summary(streams); + result.put("structural", "passed"); + result.put("checksum", "passed"); + result.put("semantic", "passed"); + printSummary(line, result); return 0; } - private static int export(final CommandLine line) throws Exception { - if (!line.hasOption(OPTION_INPUT) && !line.hasOption(OPTION_SOURCE)) { - return managedExport(line); + private static void printSummary(final CommandLine line, final Map result) { + if (line.hasOption(OPTION_JSON)) { + System.out.println(GSON.toJson(result)); + } else { + System.out.println( + String.format(CliMessages.LOG_LOGICAL_BACKUP_ARG_1E5C23CE, GSON.toJson(result))); } + } + + private static int pack(final CommandLine line) throws IOException { final Path source = Path.of(input(line)).toAbsolutePath().normalize(); final Path target = Path.of(optionValue(line, OPTION_OUTPUT, OPTION_TARGET)).toAbsolutePath().normalize(); final List streams = read(line); - return exportArchive(source, target, streams); + return exportArchive(source, target, streams, line.hasOption(OPTION_JSON)); } - private static int managedExport(final CommandLine line) throws Exception { + private static Path dump(final CommandLine line, final boolean legacy) throws Exception { + final LogicalBackupDumpOptions options = new LogicalBackupDumpOptions(line, legacy); final String host = required(line, OPTION_HOST); final int port = Integer.parseInt(required(line, OPTION_PORT)); final String user = line.getOptionValue(OPTION_USER, "root"); - final String password = password(line); final Path backupRoot = Path.of(required(line, OPTION_BACKUP_DIR)).toAbsolutePath().normalize(); - final Path target = - Path.of(optionValue(line, OPTION_OUTPUT, OPTION_TARGET)).toAbsolutePath().normalize(); - final String pipeName = - line.getOptionValue( - OPTION_PIPE_NAME, - "logical_backup_export_" + UUID.randomUUID().toString().replace('-', '_')); - validateIdentifier(pipeName, OPTION_PIPE_NAME); - final String backupId = line.getOptionValue(OPTION_BACKUP_ID, pipeName); - validateIdentifier(backupId, OPTION_BACKUP_ID); + String backupId = + legacy + ? line.getOptionValue( + OPTION_BACKUP_ID, "backup_" + UUID.randomUUID().toString().replace('-', '_')) + : required(line, OPTION_BACKUP_ID); + LogicalBackupSettings.validateBackupId(backupId); + if ("new".equals(options.resume) + && Files.exists(backupRoot.resolve(backupId), LinkOption.NOFOLLOW_LINKS)) { + backupId += "-new-" + UUID.randomUUID().toString().substring(0, 8); + LogicalBackupSettings.validateBackupId(backupId); + } + final Path directory = backupRoot.resolve(backupId); + LogicalBackupCatalog.rejectSymbolicLinks(directory); + final boolean append = "append".equals(options.resume); + final boolean existed = Files.exists(directory, LinkOption.NOFOLLOW_LINKS); + if (existed && !append) { + throw new java.nio.file.FileAlreadyExistsException(directory.toString()); + } + final String password = password(line); final long timeoutSeconds = longOption(line, OPTION_TIMEOUT_SECONDS, 3600L); final long pollIntervalSeconds = longOption(line, OPTION_POLL_INTERVAL_SECONDS, 2L); - if (timeoutSeconds <= 0 || pollIntervalSeconds <= 0) { + if (timeoutSeconds <= 0 + || timeoutSeconds > 31_536_000L + || pollIntervalSeconds <= 0 + || pollIntervalSeconds > 3600L) { throw new IOException( CliMessages .EXCEPTION_LOGICAL_BACKUP_EXPORT_TIMEOUT_AND_POLL_INTERVAL_MUST_BE_POSITIVE_38622769); } - - final String createSql = - "CREATE PIPE " - + pipeName - + " WITH SOURCE ('source'='iotdb-source','source.mode'='snapshot','source.inclusion'='all'," - + "'source.capture.tree'='true','source.capture.table'='true'," - + "'source.realtime.enable'='false') WITH SINK ('sink'='logical-backup-sink','sink.dir'='" - + sqlString(backupRoot.toString()) - + "','sink.backup-id'='" - + sqlString(backupId) - + "','sink.resume'='append')"; - boolean created = false; - try (final Session session = - new Session.Builder().host(host).port(port).username(user).password(password).build()) { - session.open(); - session.executeNonQueryStatement(createSql); - created = true; - session.executeNonQueryStatement("START PIPE " + pipeName); - waitForSnapshotPipe(session, pipeName, timeoutSeconds, pollIntervalSeconds); - final Path source = backupRoot.resolve(backupId).normalize(); - final List streams = new LogicalBackupArchiveReader().read(source, false); - return exportArchive(source, target, streams); - } finally { - if (created) { - try (final Session cleanup = - new Session.Builder().host(host).port(port).username(user).password(password).build()) { - cleanup.open(); - try { - cleanup.executeNonQueryStatement("STOP PIPE " + pipeName); - } catch (final Exception ignored) { - // A completed snapshot pipe may already have been auto-dropped. - } - try { - cleanup.executeNonQueryStatement("DROP PIPE " + pipeName); - } catch (final Exception ignored) { - // Keep the original export failure when cleanup races with auto-drop. - } - } catch (final Exception ignored) { - // The archive is complete once the generated pipe disappears; cleanup is best effort. + LogicalBackupCatalog catalog = options.catalog; + catalog.backupId = backupId; + catalog.pipeName = + line.getOptionValue( + OPTION_PIPE_NAME, "logical_backup_" + UUID.randomUUID().toString().replace('-', '_')); + validateIdentifier(catalog.pipeName, OPTION_PIPE_NAME); + if (existed) { + final LogicalBackupCatalog previous = + LogicalBackupCatalog.load(directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME)); + if (previous == null + || !catalog.backupId.equals(previous.backupId) + || !catalog.captureOptions.equals(previous.captureOptions) + || (line.hasOption(OPTION_PIPE_NAME) && !catalog.pipeName.equals(previous.pipeName))) { + throw new IOException( + CliMessages + .EXCEPTION_THE_EXISTING_BACKUP_TASK_DOES_NOT_MATCH_THIS_BACKUP_IDENTITY_OR_CAPTURE_OPTIONS_50E6388F); + } + catalog = previous; + if ("SEALED".equals(catalog.status) || "INCOMPLETE".equals(catalog.status)) { + new LogicalBackupArchiveReader().read(directory, line.hasOption(OPTION_ALLOW_INCOMPLETE)); + return directory; + } + } else { + Files.createDirectories(backupRoot); + Files.createDirectory(directory); + } + // Keep the lock inode: deleting it after unlock can let two later callers lock different files. + final Path lock = directory.resolve(".backup.lock"); + try (final java.nio.channels.FileChannel channel = + java.nio.channels.FileChannel.open( + lock, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + final java.nio.channels.FileLock ignored = channel.tryLock()) { + if (ignored == null) { + throw new IOException( + CliMessages.EXCEPTION_ANOTHER_DUMP_PROCESS_IS_MANAGING_THIS_BACKUP_FE8EE828); + } + catalog.write(directory); + final String pipeName = catalog.pipeName; + try (final Session session = + new Session.Builder().host(host).port(port).username(user).password(password).build()) { + session.open(); + final PipeStatus existing = readPipeStatus(session, pipeName); + if (existing != null && !append) { + throw new IOException( + CliMessages + .EXCEPTION_THE_EXISTING_BACKUP_TASK_DOES_NOT_MATCH_THIS_BACKUP_IDENTITY_OR_CAPTURE_OPTIONS_50E6388F); } + if (existing == null && (!existed || streamManifests(directory).isEmpty())) { + session.executeNonQueryStatement( + options.createSql(pipeName, backupRoot.toString(), backupId)); + session.executeNonQueryStatement("START PIPE " + pipeName); + } + waitForPipe( + session, + pipeName, + timeoutSeconds, + pollIntervalSeconds, + options.until, + line.hasOption(OPTION_JSON)); + final List streams = + awaitSealedStreams(directory, Math.min(timeoutSeconds, 120L), pollIntervalSeconds); + catalog.seal(directory, streams); + } catch (final Exception e) { + throw new IOException( + String.format( + CliMessages + .EXCEPTION_BACKUP_ARG_TASK_ARG_DIRECTORY_ARG_ARG_THE_TASK_IS_RETAINED_QUERY_SHOW_PIPE_OR_REPEAT_DUMP_WITH_THE_SAME_OPTIONS_AND_RESUME_APPEND_6F8C1E87, + backupId, + pipeName, + directory, + e.getMessage()), + e); } } + new LogicalBackupArchiveReader().read(directory, line.hasOption(OPTION_ALLOW_INCOMPLETE)); + return directory; + } + + private static List streamManifests(final Path root) throws IOException { + try (final java.util.stream.Stream paths = Files.walk(root)) { + return paths + .filter(path -> !path.equals(root.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME))) + .filter( + path -> LogicalBackupFormat.MANIFEST_FILE_NAME.equals(path.getFileName().toString())) + .sorted() + .toList(); + } + } + + private static List readGeneratedStreams(final Path root) throws Exception { + final List streams = new ArrayList<>(); + for (final Path manifest : streamManifests(root)) { + streams.addAll(new LogicalBackupArchiveReader().read(manifest, true)); + } + if (streams.isEmpty() + || streams.stream() + .anyMatch( + stream -> + stream.getManifest().segments.stream() + .anyMatch(segment -> !"SEALED".equals(segment.status)))) { + throw new IOException( + CliMessages + .EXCEPTION_THE_BACKUP_TASK_HAS_NOT_PRODUCED_SEALED_STREAMS_YET_RESUME_THE_SAME_TASK_TO_FINISH_SEALING_87C9B599); + } + return streams; + } + + private static List awaitSealedStreams( + final Path root, final long timeoutSeconds, final long pollSeconds) throws Exception { + final long deadline = System.nanoTime() + timeoutSeconds * 1_000_000_000L; + Exception lastFailure; + do { + try { + return readGeneratedStreams(root); + } catch (final IOException e) { + lastFailure = e; + } + Thread.sleep(Math.min(pollSeconds * 1000L, 1000L)); + } while (System.nanoTime() < deadline); + throw lastFailure; + } + + private static void printDumpResult(final CommandLine line, final Path directory) + throws IOException { + final LogicalBackupCatalog catalog = + LogicalBackupCatalog.load(directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME)); + final Map result = new LinkedHashMap<>(); + result.put("backupId", catalog.backupId); + result.put("manifest", directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME).toString()); + result.put("status", catalog.status); + result.put("sourceHighWatermark", catalog.sourceHighWatermark); + printSummary(line, result); } private static int exportArchive( - final Path source, final Path target, final List streams) throws IOException { + final Path source, final Path target, final List streams, final boolean json) + throws IOException { final Path normalizedSource = source.toAbsolutePath().normalize(); final Path normalizedTarget = target.toAbsolutePath().normalize(); final Path root; @@ -340,7 +571,9 @@ private static int exportArchive( .EXCEPTION_LOGICAL_BACKUP_EXPORT_SOURCE_MUST_BE_A_DIRECTORY_OR_MANIFEST_ARG_0BBDE9F0, normalizedSource)); } - if (normalizedTarget.startsWith(root)) { + LogicalBackupCatalog.rejectSymbolicLinks(root); + LogicalBackupCatalog.rejectSymbolicLinks(normalizedTarget); + if (normalizedTarget.startsWith(root.toRealPath())) { throw new IOException( CliMessages.EXCEPTION_LOGICAL_BACKUP_EXPORT_TARGET_MUST_NOT_BE_INSIDE_SOURCE_A499F994); } @@ -352,16 +585,58 @@ private static int exportArchive( Files.createDirectories(parent); } final Path temporary = Files.createTempFile(parent, "pipe-logical-backup-", ".tmp"); + final boolean incomplete = + streams.stream() + .anyMatch( + stream -> + stream.getManifest().skippedEventCount > 0 + || "INCOMPLETE".equals(stream.getManifest().status) + || (stream.getCatalog() != null + && "INCOMPLETE".equals(stream.getCatalog().status))); boolean published = false; try { + final boolean legacy = + LogicalBackupCatalog.load(root.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME)) == null; + final boolean relocateLegacyStream = + legacy + && streams.stream() + .anyMatch(stream -> stream.getManifestPath().getParent().equals(root)); try (final ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(temporary))) { + if (legacy) { + final LogicalBackupManifest first = streams.get(0).getManifest(); + final LogicalBackupCatalog catalog = new LogicalBackupCatalog(); + catalog.backupId = first.backupId; + catalog.pipeName = first.pipeName; + catalog.scope = + streams.stream().map(stream -> stream.getManifest().streamType).distinct().toList(); + catalog.model = List.of("tree", "table"); + catalog.includeAuth = containsConfigStream(streams); + catalog.captureOptions.putAll(first.captureOptions); + catalog.collect(root, streams); + if (relocateLegacyStream) { + for (final LogicalBackupCatalog.StreamEntry entry : catalog.streams) { + entry.manifest.file = "legacy-stream/" + entry.manifest.file; + for (final LogicalBackupCatalog.FileEntry segment : entry.segments) { + segment.file = "legacy-stream/" + segment.file; + } + } + } + zip.putNextEntry(new ZipEntry(LogicalBackupFormat.MANIFEST_FILE_NAME)); + zip.write(GSON.toJson(catalog).getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } for (final Path file : exportFiles(root, streams)) { - final String entryName = root.relativize(file).toString().replace('\\', '/'); + final String entryName = + (relocateLegacyStream ? "legacy-stream/" : "") + + root.relativize(file).toString().replace('\\', '/'); zip.putNextEntry(new ZipEntry(entryName)); Files.copy(file, zip); zip.closeEntry(); } } + try (final BackupInput packed = BackupInput.open(temporary)) { + new LogicalBackupArchiveReader().read(packed.getPath(), incomplete); + } try (final java.nio.channels.FileChannel archiveChannel = java.nio.channels.FileChannel.open(temporary, StandardOpenOption.WRITE)) { archiveChannel.force(true); @@ -377,19 +652,33 @@ private static int exportArchive( Files.deleteIfExists(temporary); } } - System.out.println( - String.format( - CliMessages.LOG_LOGICAL_BACKUP_EXPORTED_FROM_ARG_TO_ARG_B3E8D280, - normalizedSource, - normalizedTarget)); + if (json) { + System.out.println( + GSON.toJson( + Map.of( + "input", + normalizedSource.toString(), + "output", + normalizedTarget.toString(), + "status", + incomplete ? "INCOMPLETE" : "SEALED"))); + } else { + System.out.println( + String.format( + CliMessages.LOG_LOGICAL_BACKUP_PACKED_FROM_ARG_TO_ARG_CB9D5304, + normalizedSource, + normalizedTarget)); + } return 0; } - private static void waitForSnapshotPipe( + private static void waitForPipe( final Session session, final String pipeName, final long timeoutSeconds, - final long pollSeconds) + final long pollSeconds, + final long until, + final boolean json) throws Exception { final long deadline = System.nanoTime() + timeoutSeconds * 1_000_000_000L; while (System.nanoTime() < deadline) { @@ -397,14 +686,15 @@ private static void waitForSnapshotPipe( if (status == null) { return; } else { - System.out.println( - String.format( - CliMessages - .LOG_LOGICAL_BACKUP_EXPORT_ARG_STATE_ARG_REMAINING_ARG_ESTIMATED_SECONDS_ARG_6256B85B, - pipeName, - status.state, - status.remainingEvents, - status.estimatedSeconds)); + (json ? System.err : System.out) + .println( + String.format( + CliMessages + .LOG_LOGICAL_BACKUP_EXPORT_ARG_STATE_ARG_REMAINING_ARG_ESTIMATED_SECONDS_ARG_6256B85B, + pipeName, + status.state, + status.remainingEvents, + status.estimatedSeconds)); if (status.state != null && (status.state.equalsIgnoreCase("ERROR") || status.state.equalsIgnoreCase("STOPPED"))) { @@ -488,6 +778,10 @@ private static final class PipeStatus { private static List exportFiles(final Path root, final List streams) throws IOException { final Set files = new LinkedHashSet<>(); + final Path rootManifest = root.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + if (LogicalBackupCatalog.load(rootManifest) != null) { + files.add(rootManifest); + } for (final BackupStream stream : streams) { final Path manifestPath = stream.getManifestPath().toAbsolutePath().normalize(); if (!manifestPath.startsWith(root)) { @@ -510,25 +804,102 @@ private static List exportFiles(final Path root, final List private static int importBackup(final CommandLine line) throws Exception { final List streams = read(line); - final boolean dryRun = line.hasOption(OPTION_DRY_RUN); - if (dryRun) { - System.out.println(CliMessages.LOG_DRY_RUN_COMPLETED_NO_DATA_WAS_WRITTEN_38AE244B); - for (final BackupStream stream : streams) { - System.out.println( - String.format( - CliMessages.LOG_STREAM_ARG_ARG_RECORDS_ARG_COMMITTED_EVENT_GROUPS_DE6BBAD0, - stream.getManifest().streamId, - stream.getRecords().size(), - stream.getEventGroups().size())); - } + final Map report = summary(streams); + final boolean authorizationObjects = + streams.get(0).getCatalog() == null + ? containsConfigStream(streams) + : streams.get(0).getCatalog().includeAuth; + final String includeAuth = line.getOptionValue("include-auth", "false"); + if (!List.of("true", "false").contains(includeAuth)) { + throw new IOException( + String.format( + CliMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_DUMP_OPTION_ARG_894E11E0, + "include-auth")); + } + report.put("authorizationObjects", authorizationObjects); + report.put("authorizationRestoreSelected", Boolean.parseBoolean(includeAuth)); + if (authorizationObjects) { + report.put( + "authorizationSensitiveFields", + List.of("password-hashes", "role-memberships", "privileges")); + } + report.put("conflict", "fail"); + report.put( + "requestCount", + streams.stream() + .flatMap(stream -> stream.getEventGroups().stream()) + .mapToLong(group -> group.getRequests().size()) + .sum()); + report.put("dryRun", line.hasOption(OPTION_DRY_RUN)); + final boolean targetProvided = + line.hasOption(OPTION_HOST) + || line.hasOption(OPTION_PORT) + || line.hasOption(OPTION_USER) + || line.hasOption(OPTION_PASSWORD_STDIN) + || line.hasOption(OPTION_PASSWORD_ENV); + if (!targetProvided && line.hasOption(OPTION_DRY_RUN)) { + report.put("targetPreflight", "not-requested"); + printSummary(line, report); return 0; } + if (authorizationObjects + && !Boolean.parseBoolean(includeAuth) + && !line.hasOption(OPTION_DRY_RUN)) { + throw new IOException( + CliMessages + .EXCEPTION_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_INSPECT_THE_BACKUP_OR_USE_DRY_RUN_FIRST_83DDCD79); + } final String host = required(line, OPTION_HOST); final int port = Integer.parseInt(required(line, OPTION_PORT)); final String user = line.getOptionValue(OPTION_USER, "root"); final String password = password(line); + report.put("target", Map.of("host", host, "port", port, "user", user)); final Path checkpoint = checkpointPath(line); - final ImportCheckpoint checkpointState = readCheckpoint(checkpoint, streams, host, port, user); + final boolean resume = line.hasOption(OPTION_RESUME); + if (!line.hasOption(OPTION_DRY_RUN) + && Files.exists(checkpoint, LinkOption.NOFOLLOW_LINKS) != resume) { + throw new IOException( + CliMessages + .EXCEPTION_AN_EXISTING_CHECKPOINT_REQUIRES_RESUME_RESUME_REQUIRES_AN_EXISTING_CHECKPOINT_FB6D0B7D); + } + final ImportCheckpoint state = + line.hasOption(OPTION_DRY_RUN) + ? null + : readCheckpoint(checkpoint, streams, host, port, user); + if (state != null) { + final String sourceDigest = sourceDigest(Path.of(input(line)), streams); + if (state.sourceSha256 != null && !state.sourceSha256.equals(sourceDigest)) { + throw invalidCheckpoint(checkpoint); + } + state.sourceSha256 = sourceDigest; + } + try (final Session session = + new Session.Builder().host(host).port(port).username(user).password(password).build()) { + session.open(); + final String precision = session.getTimestampPrecision(); + if (!streams.get(0).getManifest().timestampPrecision.equals(precision)) { + throw new IOException( + CliMessages + .EXCEPTION_THE_TARGET_TIMESTAMP_PRECISION_DOES_NOT_MATCH_THE_BACKUP_70B225CB); + } + String version = "unknown"; + try (final SessionDataSet dataSet = session.executeQueryStatement("SHOW VERSION")) { + final SessionDataSet.DataIterator iterator = dataSet.iterator(); + if (iterator.next()) { + version = iterator.getString(dataSet.getColumnNames().get(0)); + } + } + final LogicalBackupCatalog catalog = streams.get(0).getCatalog(); + final String sourceVersion = streams.get(0).getManifest().sourceVersion; + if ((sourceVersion != null && !version.split("\\.")[0].equals(sourceVersion.split("\\.")[0])) + || (catalog != null && catalog.model.contains("table") && version.startsWith("1."))) { + throw new IOException( + CliMessages + .EXCEPTION_THE_TARGET_VERSION_OR_DATA_MODEL_IS_INCOMPATIBLE_WITH_THE_BACKUP_220EB40C); + } + report.put("targetVersion", version); + report.put("timestampPrecision", precision); + } try (final IoTDBSyncClient client = new IoTDBSyncClient( new ThriftClientProperty.Builder().setConnectionTimeoutMs(20_000).build(), @@ -537,32 +908,87 @@ private static int importBackup(final CommandLine line) throws Exception { false, "", "")) { - handshake(client, streams.get(0).getManifest().timestampPrecision, user, password); - if (containsConfigStream(streams)) { - configNodeHandshake( - client, streams.get(0).getManifest().timestampPrecision, user, password); + preflight(client, streams, user, password); + report.put("targetPreflight", "passed"); + report.put("conflictCheck", "on-replay"); + if (line.hasOption(OPTION_DRY_RUN)) { + printSummary(line, report); + return 0; } - long importedGroups = 0; - for (final BackupStream stream : streams) { - importedGroups += - importStream( - checkpoint, - checkpointState, - stream, - request -> transfer(client, request), - state -> writeCheckpoint(checkpoint, state)); + (line.hasOption(OPTION_JSON) ? System.err : System.out).println(GSON.toJson(report)); + final java.util.concurrent.atomic.AtomicBoolean uncertainRequest = + new java.util.concurrent.atomic.AtomicBoolean( + state.inProgress != null + && state.inProgress.uncertain + && streams.stream() + .flatMap(stream -> stream.getEventGroups().stream()) + .anyMatch( + group -> + group + .getEventGroupId() + .toString() + .equals(state.inProgress.eventGroupId) + && state.inProgress.nextRequestIndex + < group.getRequests().size())); + try { + state.status = "RUNNING"; + state.lastError = null; + long importedGroups = 0; + for (final BackupStream stream : streams) { + importedGroups += + importStream( + checkpoint, + state, + stream, + request -> transfer(client, request, uncertainRequest.getAndSet(false)), + checkpointState -> writeCheckpoint(checkpoint, checkpointState)); + } + state.status = "COMPLETED"; + writeCheckpoint(checkpoint, state); + report.put("status", state.status); + report.put("importedEventGroups", importedGroups); + report.put("checkpoint", checkpoint.toString()); + printSummary(line, report); + } catch (final Exception e) { + state.status = "FAILED"; + state.lastError = e.getClass().getSimpleName(); + try { + writeCheckpoint(checkpoint, state); + } catch (final IOException checkpointFailure) { + e.addSuppressed(checkpointFailure); + } + throw e; } - writeCheckpoint(checkpoint, checkpointState); - System.out.println( - String.format( - CliMessages - .LOG_LOGICAL_BACKUP_IMPORT_COMPLETED_ARG_EVENT_GROUPS_CHECKPOINT_ARG_16F6A72D, - importedGroups, - checkpoint)); } return 0; } + static void preflight( + final IoTDBSyncClient client, + final List streams, + final String user, + final String password) + throws IOException, TException { + handshake(client, streams.get(0).getManifest().timestampPrecision, user, password); + if (containsConfigStream(streams)) { + configNodeHandshake(client, streams.get(0).getManifest().timestampPrecision, user, password); + } + } + + private static String sourceDigest(final Path input, final List streams) + throws IOException { + if (Files.isRegularFile(input) + && !LogicalBackupFormat.MANIFEST_FILE_NAME.equals(input.getFileName().toString())) { + return LogicalBackupCatalog.sha256(input); + } + final MessageDigest digest = LogicalBackupFormat.newSha256(); + updateDigest(digest, GSON.toJson(sourceStreamIdentities(streams))); + if (!streams.isEmpty() && streams.get(0).getCatalog() != null) { + updateDigest(digest, GSON.toJson(streams.get(0).getCatalog())); + } + return LogicalBackupFormat.toHex(digest.digest()); + } + static boolean containsConfigStream(final List streams) { return streams.stream() .anyMatch(stream -> "config".equalsIgnoreCase(stream.getManifest().streamType)); @@ -598,8 +1024,16 @@ static long importStream( throw invalidCheckpoint(checkpoint); } for (int index = nextRequestIndex; index < requests.size(); index++) { - requestTransfer.transfer(requests.get(index).toTPipeTransferReq()); + checkpointState.inProgress.uncertain = true; + checkpointPersister.persist(checkpointState); + try { + requestTransfer.transfer(requests.get(index).toTPipeTransferReq()); + } catch (final RestoreRequestRejectedException e) { + checkpointState.inProgress.uncertain = false; + throw e; + } checkpointState.inProgress.nextRequestIndex = index + 1; + checkpointState.inProgress.uncertain = false; checkpointPersister.persist(checkpointState); } @@ -737,16 +1171,20 @@ private static void configNodeHandshake( } } - static void transfer(final IoTDBSyncClient client, final TPipeTransferReq request) + static void transfer( + final IoTDBSyncClient client, + final TPipeTransferReq request, + final boolean allowIdempotentReplay) throws TException, IOException { final TPipeTransferResp response = client.pipeTransfer(request); if (response == null || response.getStatus() == null || (response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() && response.getStatus().getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode() - && response.getStatus().getCode() - != TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode())) { - throw new IOException( + && (!allowIdempotentReplay + || response.getStatus().getCode() + != TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()))) { + throw new RestoreRequestRejectedException( String.format( CliMessages.EXCEPTION_LOGICAL_BACKUP_REQUEST_TYPE_ARG_FAILED_ARG_75EE2D11, request.getType(), @@ -755,12 +1193,29 @@ static void transfer(final IoTDBSyncClient client, final TPipeTransferReq reques } private static Path checkpointPath(final CommandLine line) throws IOException { - final String configuredCheckpoint = optionValue(line, OPTION_RESUME, OPTION_CHECKPOINT); - if (configuredCheckpoint != null) { - return Path.of(configuredCheckpoint); - } + final String configuredCheckpoint = + line.hasOption(OPTION_CHECKPOINT) + ? line.getOptionValue(OPTION_CHECKPOINT) + : line.getOptionValue(OPTION_RESUME); final Path source = Path.of(input(line)).toAbsolutePath().normalize(); - return source.resolveSibling(source.getFileName() + ".import.checkpoint.json"); + final Path checkpoint = + (configuredCheckpoint == null + ? source.resolveSibling(source.getFileName() + ".restore.checkpoint.json") + : Path.of(configuredCheckpoint)) + .toAbsolutePath() + .normalize(); + LogicalBackupCatalog.rejectSymbolicLinks(checkpoint); + final Path sourceRoot = + Files.isDirectory(source) + ? source + : LogicalBackupFormat.MANIFEST_FILE_NAME.equals(source.getFileName().toString()) + ? source.getParent() + : source; + if (checkpoint.startsWith(sourceRoot)) { + throw new IOException( + CliMessages.EXCEPTION_THE_CHECKPOINT_MUST_BE_OUTSIDE_THE_INPUT_BACKUP_15B70F59); + } + return checkpoint; } static ImportCheckpoint readCheckpoint( @@ -776,6 +1231,8 @@ static ImportCheckpoint readCheckpoint( state.formatName = CHECKPOINT_FORMAT_NAME; state.formatVersion = CHECKPOINT_FORMAT_VERSION; state.sourceStreams = sourceStreams; + state.backupId = streams.get(0).getManifest().backupId; + state.backupFormatVersion = streams.get(0).getManifest().formatVersion; state.targetHost = host; state.targetPort = port; state.targetUser = user; @@ -801,7 +1258,9 @@ static ImportCheckpoint readCheckpoint( String.format( CliMessages.EXCEPTION_LOGICAL_BACKUP_CHECKPOINT_IS_INVALID_ARG_71E82F4C, checkpoint)); } - if (!sourceStreams.equals(state.sourceStreams) + if (!streams.get(0).getManifest().backupId.equals(state.backupId) + || !streams.get(0).getManifest().formatVersion.equals(state.backupFormatVersion) + || !sourceStreams.equals(state.sourceStreams) || !host.equals(state.targetHost) || port != state.targetPort || !user.equals(state.targetUser) @@ -820,6 +1279,11 @@ static Map sourceStreamIdentities(final List strea for (final BackupStream stream : streams) { final LogicalBackupManifest manifest = stream.getManifest(); final MessageDigest digest = LogicalBackupFormat.newSha256(); + updateDigest(digest, manifest.formatName); + updateDigest(digest, manifest.formatVersion); + updateDigest(digest, manifest.timestampPrecision); + updateDigest(digest, manifest.sourceVersion); + updateDigest(digest, GSON.toJson(manifest.captureOptions)); updateDigest(digest, manifest.backupId); updateDigest(digest, manifest.streamId); updateDigest(digest, manifest.pipeName); @@ -946,12 +1410,13 @@ static void writeCheckpoint(final Path checkpoint, final ImportCheckpoint checkp if (parent != null) { Files.createDirectories(parent); } - final Path temporary = checkpoint.resolveSibling(checkpoint.getFileName() + ".tmp"); - Files.deleteIfExists(temporary); + LogicalBackupCatalog.rejectSymbolicLinks(checkpoint); + checkpointState.updatedAt = java.time.Instant.now().toString(); + final Path temporary = Files.createTempFile(parent, ".checkpoint-", ".tmp"); final ByteBuffer checkpointBytes = StandardCharsets.UTF_8.encode(GSON.toJson(checkpointState)); try (final java.nio.channels.FileChannel checkpointChannel = java.nio.channels.FileChannel.open( - temporary, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { + temporary, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE)) { while (checkpointBytes.hasRemaining()) { checkpointChannel.write(checkpointBytes); } @@ -975,47 +1440,20 @@ private static String getClusterId() { } private static void printUsage() { - final HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp( - new PrintWriter(System.out), - 120, - CliMessages.LOG_PIPE_LOGICAL_BACKUP_INSPECT_VERIFY_EXPORT_IMPORT_6D62F9CE, + System.out.println( CliMessages - .LOG_USE_INPUT_TO_SPECIFY_THE_INPUT_EXPORT_ALSO_REQUIRES_OUTPUT_EXPORT_CAN_USE_BACKUP_DIR_TO_CREATE_MONITOR_AND_CLEAN_UP_A_SNAPSHOT_PIPE_AUTOMATICALLY_IMPORT_REQUIRES_HOST_AND_PORT_USE_PASSWORD_STDIN_OR_PASSWORD_ENV_TO_AVOID_COMMAND_LINE_PASSWORDS_F5B223CB, - optionsForHelp(), - 2, - 2, - "", - true); - } - - private static Options optionsForHelp() { - final Options options = new Options(); - addOption(options, OPTION_INPUT, true, false); - addOption(options, OPTION_OUTPUT, true, false); - addOption(options, OPTION_RESUME, true, false); - addOption(options, OPTION_FORMAT, true, false); - addOption(options, OPTION_DEEP, false, false); - addOption(options, OPTION_HOST, true, false); - addOption(options, OPTION_PORT, true, false); - addOption(options, OPTION_USER, true, false); - addOption(options, OPTION_PASSWORD_STDIN, false, false); - addOption(options, OPTION_PASSWORD_ENV, true, false); - addOption(options, OPTION_CHECKPOINT, true, false); - addOption(options, OPTION_DRY_RUN, false, false); - addOption(options, OPTION_ALLOW_INCOMPLETE, false, false); - addOption(options, OPTION_JSON, false, false); - addOption(options, OPTION_BACKUP_DIR, true, false); - addOption(options, OPTION_PIPE_NAME, true, false); - addOption(options, OPTION_BACKUP_ID, true, false); - addOption(options, OPTION_TIMEOUT_SECONDS, true, false); - addOption(options, OPTION_POLL_INTERVAL_SECONDS, true, false); - return options; + .LOG_LOGICAL_BACKUP_DUMP_VERIFY_INSPECT_PACK_RESTORE_OPTIONS_DUMP_CONNECT_TO_A_SOURCE_AND_SEAL_A_BACKUP_DIRECTORY_DOES_NOT_CREATE_A_ZIP_INSPECT_SHOW_BACKUP_IDENTITY_SCOPE_STREAMS_SEGMENTS_COUNTS_AND_STATUS_VERIFY_CHECK_STRUCTURE_CHECKSUMS_AND_SEMANTICS_WITHOUT_CHANGING_THE_INPUT_PACK_VERIFY_AND_PACKAGE_AN_EXISTING_DIRECTORY_AS_ZIP_ACCEPTS_NO_CONNECTION_OPTIONS_RESTORE_PREFLIGHT_AND_REPLAY_A_DIRECTORY_ROOT_MANIFEST_JSON_OR_ZIP_INSPECT_VERIFY_INPUT_PATH_JSON_ALLOW_INCOMPLETE_PACK_INPUT_DIR_OUTPUT_ZIP_JSON_ALLOW_INCOMPLETE_DUMP_HOST_HOST_PORT_PORT_USER_ROOT_PASSWORD_STDIN_PASSWORD_ENV_NAME_BACKUP_DIR_DIR_BACKUP_ID_ID_SCOPE_CONFIG_SCHEMA_DATA_MODEL_TREE_TABLE_INCLUDE_PATTERN_EXCLUDE_PATTERN_START_TIME_TIME_END_TIME_TIME_INCLUDE_AUTH_TRUE_FALSE_MODE_SNAPSHOT_CONTINUOUS_UNTIL_ISO_8601_RESUME_FAIL_IF_EXISTS_APPEND_NEW_PIPE_NAME_NAME_SEGMENT_SIZE_BYTES_268435456_MAX_RECORD_BYTES_67108864_FSYNC_POLICY_ALWAYS_BATCH_PERIODIC_NONE_FSYNC_BATCH_OPERATIONS_N_FSYNC_PERIOD_MS_N_ON_UNSUPPORTED_FAIL_SKIP_INCLUDE_HEARTBEAT_TRUE_FALSE_TIMEOUT_SECONDS_3600_POLL_INTERVAL_SECONDS_2_JSON_ALLOW_INCOMPLETE_CONTINUOUS_REQUIRES_UNTIL_STOP_SEQUENCE_IS_RESERVED_AND_CURRENTLY_REJECTED_OBJECT_FILTERS_REQUIRE_ONE_MODEL_TREE_USES_PATH_LISTS_TABLE_USES_DATABASE_REGEX_TABLE_REGEX_TREE_CONFIG_SCHEMA_CAPTURE_ACCEPTS_A_SINGLE_PREFIX_FULL_PATH_WITHOUT_EXCLUSIONS_TABLE_EXCLUSIONS_MUST_SELECT_WHOLE_DATABASES_OR_WHOLE_TABLE_NAME_PATTERNS_RESTORE_INPUT_PATH_HOST_HOST_PORT_PORT_USER_USER_PASSWORD_STDIN_PASSWORD_ENV_NAME_DRY_RUN_CHECKPOINT_FILE_RESUME_CONFLICT_FAIL_INCLUDE_AUTH_TRUE_FALSE_JSON_ALLOW_INCOMPLETE_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_LEGACY_CONFIG_IS_CONSERVATIVE_A_TARGET_CONNECTION_IS_REQUIRED_FOR_REPLAY_LOCAL_DRY_RUN_NEEDS_NO_CONNECTION_TARGET_PREFLIGHT_CHECKS_VERSION_PRECISION_AUTHENTICATION_AND_HANDSHAKE_SUPPORT_OBJECT_CONFLICTS_AND_PER_OBJECT_PRIVILEGES_ARE_CHECKED_DURING_REPLAY_DEPRECATED_ALIASES_EXPORT_INPUT_DIR_OUTPUT_ZIP_PACK_EXPORT_WITH_SOURCE_CONNECTION_OPTIONS_DUMP_THEN_PACK_IMPORT_RESTORE_EXIT_CODE_0_SUCCESS_1_FAILURE_ERRORS_AND_DEPRECATION_NOTICES_GO_TO_STDERR_B1983BE8); } static class ImportCheckpoint { String formatName; String formatVersion; + String sourceSha256; + String backupId; + String backupFormatVersion; + String status = "RUNNING"; + String updatedAt; + String lastError; Map sourceStreams = new LinkedHashMap<>(); String targetHost; int targetPort; @@ -1029,6 +1467,7 @@ static class ImportProgress { String eventGroupId; long lastSequence; int nextRequestIndex; + boolean uncertain; static ImportProgress start(final String streamId, final EventGroup group) { final ImportProgress progress = new ImportProgress(); @@ -1045,6 +1484,12 @@ boolean matches(final String candidateStreamId, final EventGroup group) { } } + private static class RestoreRequestRejectedException extends IOException { + private RestoreRequestRejectedException(final String message) { + super(message); + } + } + @FunctionalInterface interface RequestTransfer { void transfer(TPipeTransferReq request) throws Exception; @@ -1076,7 +1521,11 @@ private static BackupInput open(final Path source) throws IOException { return new BackupInput(temporaryDirectory, temporaryDirectory); } catch (final IOException | RuntimeException e) { deleteRecursively(temporaryDirectory); - throw e; + if (e instanceof IOException) { + throw (IOException) e; + } + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81, e); } } @@ -1084,46 +1533,150 @@ private static void extract(final Path archive, final Path target) throws IOExce int entryCount = 0; long totalBytes = 0; final byte[] buffer = new byte[64 * 1024]; - try (final ZipInputStream zip = new ZipInputStream(Files.newInputStream(archive))) { - ZipEntry entry; - while ((entry = zip.getNextEntry()) != null) { + try (final ZipFile zip = new ZipFile(archive.toFile())) { + final Map entries = new LinkedHashMap<>(); + final Set allowed = new LinkedHashSet<>(); + final java.util.Enumeration enumeration = zip.entries(); + while (enumeration.hasMoreElements()) { + final ZipEntry entry = enumeration.nextElement(); if (++entryCount > MAX_ARCHIVE_ENTRIES) { throw new IOException( CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432); } - final Path output = target.resolve(entry.getName()).normalize(); - if (!output.startsWith(target)) { + LogicalBackupCatalog.resolve( + target, + entry.isDirectory() + ? entry.getName().substring(0, entry.getName().length() - 1) + : entry.getName()); + if (entries.putIfAbsent(entry.getName(), entry) != null + || entry.getSize() < 0 + || entry.getSize() > MAX_ARCHIVE_BYTES + || (entry.getSize() > 1024 * 1024 + && entry.getSize() / Math.max(1, entry.getCompressedSize()) > 1000)) { throw new IOException( - String.format( - CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_ENTRY_IS_UNSAFE_ARG_3E548152, - entry.getName())); + CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432); } - if (entry.isDirectory()) { - Files.createDirectories(output); - } else { - final Path parent = output.getParent(); - if (parent != null) { - Files.createDirectories(parent); + } + final ZipEntry rootManifest = entries.get(LogicalBackupFormat.MANIFEST_FILE_NAME); + if (rootManifest != null) { + final com.google.gson.JsonObject rootJson = manifestJson(zip, rootManifest); + if (rootJson.has("streams")) { + final LogicalBackupCatalog catalog = + GSON.fromJson(rootJson, LogicalBackupCatalog.class); + allowed.add(LogicalBackupFormat.MANIFEST_FILE_NAME); + if (catalog.streams == null) { + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); } - try (final OutputStream stream = - Files.newOutputStream( - output, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { - int length; - while ((length = zip.read(buffer)) >= 0) { - if (length == 0) { - continue; + for (final LogicalBackupCatalog.StreamEntry stream : catalog.streams) { + if (stream == null + || stream.manifest == null + || stream.segments == null + || !allowed.add(stream.manifest.file)) { + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + for (final LogicalBackupCatalog.FileEntry segment : stream.segments) { + if (segment == null || !allowed.add(segment.file)) { + throw new IOException( + CliMessages + .EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); } - totalBytes += length; - if (totalBytes > MAX_ARCHIVE_BYTES) { + } + } + } + } + if (allowed.isEmpty()) { + // Read archives produced by the initial stream-only format with the same allowlist. + for (final ZipEntry entry : entries.values()) { + if (!entry.isDirectory() && entry.getName().endsWith("manifest.json")) { + final LogicalBackupManifest manifest = + GSON.fromJson(manifestJson(zip, entry), LogicalBackupManifest.class); + if (manifest.segments == null || !allowed.add(entry.getName())) { + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + final Path parent = Path.of(entry.getName()).getParent(); + for (final LogicalBackupManifest.Segment segment : manifest.segments) { + if (segment == null + || segment.file == null + || !allowed.add( + parent == null + ? segment.file + : parent.resolve(segment.file).toString().replace('\\', '/'))) { throw new IOException( - CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432); + CliMessages + .EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); } - stream.write(buffer, 0, length); } } } - zip.closeEntry(); } + if (allowed.isEmpty()) { + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + for (final String file : allowed) { + LogicalBackupCatalog.resolve(target, file); + if (!entries.containsKey(file) || entries.get(file).isDirectory()) { + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + } + for (final ZipEntry entry : entries.values()) { + if (entry.isDirectory()) { + if (allowed.stream().noneMatch(file -> file.startsWith(entry.getName()))) { + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + continue; + } + if (!allowed.contains(entry.getName())) { + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + final Path output = LogicalBackupCatalog.resolve(target, entry.getName()); + Files.createDirectories(output.getParent()); + try (final java.io.InputStream input = zip.getInputStream(entry); + final OutputStream stream = + Files.newOutputStream( + output, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { + int length; + while ((length = input.read(buffer)) >= 0) { + if (length == 0) { + continue; + } + totalBytes += length; + if (totalBytes > MAX_ARCHIVE_BYTES) { + throw new IOException( + CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432); + } + stream.write(buffer, 0, length); + } + } + } + } + } + + private static com.google.gson.JsonObject manifestJson(final ZipFile zip, final ZipEntry entry) + throws IOException { + if (entry.isDirectory() || entry.getSize() > 16 * 1024 * 1024) { + throw new IOException( + CliMessages.EXCEPTION_LOGICAL_BACKUP_ARCHIVE_EXCEEDS_SAFETY_LIMIT_FFC54432); + } + try (final java.io.InputStream input = zip.getInputStream(entry)) { + final byte[] bytes = input.readNBytes(16 * 1024 * 1024 + 1); + final com.google.gson.JsonObject json = + GSON.fromJson( + new String(bytes, StandardCharsets.UTF_8), com.google.gson.JsonObject.class); + if (bytes.length > 16 * 1024 * 1024 || json == null) { + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81); + } + return json; + } catch (final JsonParseException | IllegalStateException e) { + throw new IOException( + CliMessages.EXCEPTION_INVALID_OR_CONFLICTING_LOGICAL_BACKUP_ARGUMENTS_76233F81, e); } } diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java index f39b87f0754ea..4fee78c4b15b0 100644 --- a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java @@ -23,9 +23,11 @@ import org.apache.iotdb.commons.pipe.sink.client.IoTDBSyncClient; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupCatalog; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupFormat; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; @@ -37,8 +39,11 @@ import org.junit.rules.TemporaryFolder; import org.mockito.Mockito; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.PrintStream; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -57,7 +62,7 @@ public class PipeLogicalBackupToolTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @Test - public void testDirectoryExportZipVerifyAndDryRun() throws Exception { + public void testDirectoryPackZipVerifyAndRestoreDryRun() throws Exception { final Path root = temporaryFolder.getRoot().toPath(); final Path source = root.resolve("source"); writeBackup(source); @@ -70,20 +75,14 @@ public void testDirectoryExportZipVerifyAndDryRun() throws Exception { Assert.assertEquals( 0, PipeLogicalBackupTool.run( - new String[] { - "export", - "--input", - source.toString(), - "--output", - archive.toString(), - "--format=binary" - })); + new String[] {"pack", "--input", source.toString(), "--output", archive.toString()})); Assert.assertTrue(Files.isRegularFile(archive)); try (final ZipFile zip = new ZipFile(archive.toFile())) { Assert.assertNotNull(zip.getEntry(LogicalBackupFormat.MANIFEST_FILE_NAME)); - Assert.assertNotNull(zip.getEntry("segment-00000000000000000000.pwal")); + Assert.assertNotNull(zip.getEntry("legacy-stream/manifest.json")); + Assert.assertNotNull(zip.getEntry("legacy-stream/segment-00000000000000000000.pwal")); Assert.assertNull(zip.getEntry(".backup.lock")); - Assert.assertEquals(2, zip.size()); + Assert.assertEquals(3, zip.size()); } Assert.assertEquals( @@ -91,7 +90,7 @@ public void testDirectoryExportZipVerifyAndDryRun() throws Exception { Assert.assertEquals( 0, PipeLogicalBackupTool.run( - new String[] {"import", "--input", archive.toString(), "--dry-run"})); + new String[] {"restore", "--input", archive.toString(), "--dry-run"})); final byte[] originalArchive = Files.readAllBytes(archive); try { @@ -103,6 +102,61 @@ public void testDirectoryExportZipVerifyAndDryRun() throws Exception { } } + @Test + public void testPackPreservesIncompleteStatus() throws Exception { + final Path root = temporaryFolder.getRoot().toPath(); + final Path source = root.resolve("incomplete"); + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = "backup"; + manifest.pipeName = "pipe"; + manifest.pipeCreationTime = 1; + manifest.streamId = "data-1"; + manifest.streamType = "data"; + manifest.timestampPrecision = "ms"; + try (final LogicalBackupWriter writer = + new LogicalBackupWriter( + source, manifest, 4096, 1024, LogicalBackupWriter.FsyncPolicy.ALWAYS, 1, 1, false)) { + writer.recordSkippedEvent(1, "unsupported event"); + } + final Path archive = root.resolve("incomplete.zip"); + try { + PipeLogicalBackupTool.run( + new String[] {"pack", "--input", source.toString(), "--output", archive.toString()}); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertFalse(Files.exists(archive)); + } + final PrintStream previous = System.out; + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try { + System.setOut(new PrintStream(bytes, true, StandardCharsets.UTF_8)); + PipeLogicalBackupTool.run( + new String[] { + "pack", + "--input", + source.toString(), + "--output", + archive.toString(), + "--allow-incomplete", + "--json" + }); + final com.google.gson.JsonObject json = + new com.google.gson.Gson() + .fromJson(bytes.toString(StandardCharsets.UTF_8), com.google.gson.JsonObject.class); + Assert.assertEquals("INCOMPLETE", json.get("status").getAsString()); + bytes.reset(); + PipeLogicalBackupTool.run( + new String[] {"verify", "--input", archive.toString(), "--allow-incomplete", "--json"}); + final com.google.gson.JsonObject verified = + new com.google.gson.Gson() + .fromJson(bytes.toString(StandardCharsets.UTF_8), com.google.gson.JsonObject.class); + Assert.assertEquals( + "INCOMPLETE", verified.getAsJsonObject("manifest").get("status").getAsString()); + } finally { + System.setOut(previous); + } + } + @Test public void testCheckpointSourceIdentityIncludesContent() throws Exception { final Path source = temporaryFolder.getRoot().toPath().resolve("source"); @@ -166,11 +220,11 @@ public void testPlaintextPasswordOptionIsRejected() throws Exception { } @Test - public void testRemovedCommandAliasesAreRejected() throws Exception { + public void testUnknownCommandsAreRejected() throws Exception { final Path source = temporaryFolder.getRoot().toPath().resolve("source"); writeBackup(source); - for (final String command : Arrays.asList("restore", "stats")) { + for (final String command : Arrays.asList("stats", "generate")) { try { PipeLogicalBackupTool.run(new String[] {command, "--input", source.toString()}); Assert.fail(); @@ -180,6 +234,180 @@ public void testRemovedCommandAliasesAreRejected() throws Exception { } } + @Test + public void testRootManifestPackAndJsonVerification() throws Exception { + final Path root = temporaryFolder.getRoot().toPath(); + final Path source = root.resolve("backup"); + writeBackup(source.resolve("data-1")); + final LogicalBackupCatalog catalog = new LogicalBackupCatalog(); + catalog.backupId = "backup"; + catalog.pipeName = "pipe"; + catalog.scope = Collections.singletonList("data"); + catalog.model = Collections.singletonList("tree"); + catalog.seal(source, new LogicalBackupArchiveReader().read(source, false)); + final byte[] originalManifest = Files.readAllBytes(source.resolve("manifest.json")); + final Path archive = root.resolve("rooted.zip"); + PipeLogicalBackupTool.run( + new String[] {"pack", "--input", source.toString(), "--output", archive.toString()}); + Assert.assertArrayEquals(originalManifest, Files.readAllBytes(source.resolve("manifest.json"))); + try (final ZipFile zip = new ZipFile(archive.toFile())) { + Assert.assertNotNull(zip.getEntry("manifest.json")); + Assert.assertNotNull(zip.getEntry("data-1/manifest.json")); + Assert.assertEquals(3, zip.size()); + } + final PrintStream previous = System.out; + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try { + System.setOut(new PrintStream(bytes, true, StandardCharsets.UTF_8)); + PipeLogicalBackupTool.run(new String[] {"verify", "--input", archive.toString(), "--json"}); + final com.google.gson.JsonObject json = + new com.google.gson.Gson() + .fromJson(bytes.toString(StandardCharsets.UTF_8), com.google.gson.JsonObject.class); + Assert.assertEquals("passed", json.get("checksum").getAsString()); + Assert.assertEquals("backup", json.getAsJsonObject("manifest").get("backupId").getAsString()); + } finally { + System.setOut(previous); + } + } + + @Test + public void testDeprecatedAliasesReportMigration() throws Exception { + final Path root = temporaryFolder.getRoot().toPath(); + final Path source = root.resolve("source"); + writeBackup(source); + final PrintStream previous = System.err; + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try { + System.setErr(new PrintStream(bytes, true, StandardCharsets.UTF_8)); + PipeLogicalBackupTool.run( + new String[] { + "export", + "--input", + source.toString(), + "--output", + root.resolve("legacy.zip").toString() + }); + PipeLogicalBackupTool.run(new String[] {"import", "--input", source.toString(), "--dry-run"}); + } finally { + System.setErr(previous); + } + final String notices = bytes.toString(StandardCharsets.UTF_8); + Assert.assertTrue(notices.contains("dump")); + Assert.assertTrue(notices.contains("pack")); + Assert.assertTrue(notices.contains("restore")); + } + + @Test + public void testCommandsRejectOptionsWithOtherResponsibilities() throws Exception { + final List invalid = + Arrays.asList( + new String[] { + "pack", "--input", "backup", "--output", "backup.zip", "--host", "localhost" + }, + new String[] {"dump", "--input", "backup"}, + new String[] {"dump", "--output", "backup.zip"}, + new String[] {"inspect", "--input", "backup", "--password-stdin"}, + new String[] {"restore", "--input", "backup", "--resume", "checkpoint.json"}, + new String[] {"restore", "--input", "backup", "--conflict", "overwrite"}); + for (final String[] args : invalid) { + try { + PipeLogicalBackupTool.run(args); + Assert.fail(Arrays.toString(args)); + } catch (final ParseException expected) { + // Command-specific validation must fail before opening files or connections. + } + } + } + + @Test + public void testDumpScopeAuthAndWriterOptionsAreExplicit() throws Exception { + final LogicalBackupDumpOptions options = + new LogicalBackupDumpOptions( + PipeLogicalBackupTool.parse( + "dump", + new String[] { + "--scope", + "schema,data", + "--model", + "tree", + "--include", + "root.a.**", + "--fsync-policy", + "periodic", + "--fsync-period-ms", + "42" + }), + false); + Assert.assertEquals("schema,data", options.source.get("source.inclusion")); + Assert.assertEquals("auth", options.source.get("source.inclusion.exclusion")); + Assert.assertEquals("false", options.source.get("source.capture.table")); + Assert.assertEquals("root.a.**", options.source.get("source.path.inclusion")); + Assert.assertEquals("42", options.sink.get("sink.fsync-period-ms")); + Assert.assertFalse(options.catalog.includeAuth); + final LogicalBackupDumpOptions auth = + new LogicalBackupDumpOptions( + PipeLogicalBackupTool.parse( + "dump", new String[] {"--scope", "config", "--include-auth", "true"}), + false); + Assert.assertTrue(auth.source.get("source.inclusion").contains("auth")); + } + + @Test + public void testDumpRejectsAmbiguousOrUnboundedCaptureBeforeCreatingTask() throws Exception { + for (final String[] args : + Arrays.asList( + new String[] {}, + new String[] {"--scope", "data", "--mode", "continuous"}, + new String[] {"--scope", "data", "--include-auth", "true"}, + new String[] {"--scope", "data", "--model", "tree,table", "--include", "root.a.**"}, + new String[] {"--scope", "data", "--segment-size-bytes", "1024"}, + new String[] {"--scope", "data", "--fsync-period-ms", "1000"}, + new String[] {"--scope", "data", "--mode", "continuous", "--stop-sequence", "100"})) { + try { + new LogicalBackupDumpOptions(PipeLogicalBackupTool.parse("dump", args), false); + Assert.fail(Arrays.toString(args)); + } catch (final IOException expected) { + // No unsupported option may broaden capture or leave an unbounded task. + } + } + } + + @Test + public void testPreflightSendsOnlyHandshakeRequests() throws Exception { + final Path source = temporaryFolder.getRoot().toPath().resolve("source"); + writeBackup(source, "config-0", "config", Collections.singletonList(request((short) 200))); + final List streams = new LogicalBackupArchiveReader().read(source, false); + final IoTDBSyncClient client = Mockito.mock(IoTDBSyncClient.class); + Mockito.when(client.pipeTransfer(Mockito.any())) + .thenReturn( + new TPipeTransferResp(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()))); + PipeLogicalBackupTool.preflight(client, streams, "root", "secret"); + final org.mockito.ArgumentCaptor requests = + org.mockito.ArgumentCaptor.forClass(TPipeTransferReq.class); + Mockito.verify(client, Mockito.times(2)).pipeTransfer(requests.capture()); + Assert.assertEquals( + PipeRequestType.HANDSHAKE_DATANODE_V2.getType(), requests.getAllValues().get(0).getType()); + Assert.assertEquals( + PipeRequestType.HANDSHAKE_CONFIGNODE_V2.getType(), + requests.getAllValues().get(1).getType()); + } + + @Test + public void testFreshRestoreRejectsIdempotentConflict() throws Exception { + final IoTDBSyncClient client = Mockito.mock(IoTDBSyncClient.class); + Mockito.when(client.pipeTransfer(Mockito.any())) + .thenReturn( + new TPipeTransferResp( + new TSStatus( + TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()))); + try { + PipeLogicalBackupTool.transfer(client, request(1), false); + Assert.fail(); + } catch (final IOException expected) { + // Only the uncertain request at a durable resume point may accept an idempotent conflict. + } + } + @Test public void testTransferAcceptsIdempotentConflict() throws Exception { final IoTDBSyncClient client = Mockito.mock(IoTDBSyncClient.class); @@ -190,7 +418,7 @@ public void testTransferAcceptsIdempotentConflict() throws Exception { new TSStatus( TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()))); - PipeLogicalBackupTool.transfer(client, request); + PipeLogicalBackupTool.transfer(client, request, true); } @Test diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java index f837493481158..fe946004e3be8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.commons.consensus.index.ProgressIndex; +import org.apache.iotdb.commons.consensus.index.impl.MetaProgressIndex; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeCriticalException; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeException; import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent; @@ -32,13 +33,18 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTemporaryMetaInCoordinator; import org.apache.iotdb.commons.pipe.config.PipeConfig; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; import org.apache.iotdb.confignode.i18n.ManagerMessages; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.confignode.manager.pipe.resource.PipeConfigNodeResourceManager; +import org.apache.iotdb.confignode.manager.pipe.source.ConfigRegionListeningFilter; import org.apache.iotdb.confignode.persistence.pipe.PipeTaskInfo; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.db.pipe.source.dataregion.DataRegionListeningFilter; +import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningFilter; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -185,12 +191,21 @@ private void parseHeartbeatAndSaveMetaChangeLocally( // pipes must remain alive because they may receive work in the future. final boolean isFiniteInternalPipe = !staticMeta.isSourceExternal() - && PipeTaskAgent.isHistoryOnlyPipe(staticMeta.getSourceParameters()); + && (PipeTaskAgent.isHistoryOnlyPipe(staticMeta.getSourceParameters()) + || (staticMeta + .getSourceParameters() + .getLongOrDefault("source.logical-backup.until", -1L) + > 0 + && System.currentTimeMillis() + >= staticMeta + .getSourceParameters() + .getLongOrDefault("source.logical-backup.until", -1L))); final boolean hasReliableDataRegionReport = pipeHeartbeat.hasCompletedDataRegionReport(staticMeta); if (isFiniteInternalPipe && hasReliableDataRegionReport - && temporaryMeta.getCompletedDataRegionIds().containsAll(requiredDataRegionIds)) { + && temporaryMeta.getCompletedDataRegionIds().containsAll(requiredDataRegionIds) + && metadataSnapshotsCompleted(pipeMetaFromCoordinator)) { PipeLogger.log( LOGGER::info, ManagerMessages.ALL_DATANODES_REPORTED_HISTORICAL_PIPE_COMPLETED, @@ -329,6 +344,54 @@ private Set collectRequiredDataRegionIds(final PipeMeta pipeMetaFromCoo return requiredDataRegionIds; } + private boolean metadataSnapshotsCompleted(final PipeMeta pipeMeta) { + final PipeParameters parameters = pipeMeta.getStaticMeta().getSourceParameters(); + if (!parameters.getBooleanOrDefault("source.logical-backup.snapshot", false) + && parameters.getLongOrDefault("source.logical-backup.until", -1L) <= 0) { + return true; + } + try { + for (final Map.Entry entry : + pipeMeta.getRuntimeMeta().getConsensusGroupId2TaskMetaMap().entrySet()) { + final boolean required; + if (entry.getKey() == Integer.MIN_VALUE) { + required = !ConfigRegionListeningFilter.parseListeningPlanTypeSet(parameters).isEmpty(); + } else { + final TConsensusGroupId region = + new TConsensusGroupId(TConsensusGroupType.SchemaRegion, entry.getKey()); + if (!configManager.getPartitionManager().isRegionGroupExists(region) + || !SchemaRegionListeningFilter.shouldSchemaRegionBeListened(parameters)) { + continue; + } + final String database = configManager.getPartitionManager().getRegionDatabase(region); + final TDatabaseSchema schema = + configManager.getClusterSchemaManager().getDatabaseSchemaByName(database); + required = + schema.isIsTableModel() + ? TablePattern.isTableModelDataAllowToBeCaptured(parameters) + : TreePattern.isTreeModelDataAllowToBeCaptured(parameters); + } + if (required + && entry + .getValue() + .getProgressIndex() + .getProgressIndexByType(MetaProgressIndex.class) + .map( + index -> + index.getIndex() + != org.apache.iotdb.commons.pipe.source.IoTDBNonDataRegionSource + .LOGICAL_BACKUP_COMPLETED_INDEX) + .orElse(true)) { + return false; + } + } + return true; + } catch (final Exception e) { + // A failed metadata lookup must never turn a partial snapshot into a completed backup. + return false; + } + } + private boolean shouldKeepDataRegionAsRequired( final PipeStaticMeta staticMeta, final TConsensusGroupId dataRegionId) { if (staticMeta.isSourceExternal()) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java index 8c7f28f6c7146..789953a05d4da 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java @@ -8,12 +8,24 @@ * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.iotdb.confignode.manager.pipe.sink.protocol.logicalbackup; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.i18n.LogicalBackupMessages; +import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskTemporaryRuntimeEnvironment; +import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecordType; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupSettings; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter; import org.apache.iotdb.confignode.manager.pipe.event.PipeConfigRegionSnapshotEvent; import org.apache.iotdb.confignode.manager.pipe.event.PipeConfigRegionWritePlanEvent; @@ -21,6 +33,8 @@ import org.apache.iotdb.confignode.manager.pipe.sink.payload.PipeTransferConfigSnapshotPieceReq; import org.apache.iotdb.confignode.manager.pipe.sink.payload.PipeTransferConfigSnapshotSealReq; import org.apache.iotdb.confignode.service.ConfigNode; +import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; +import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent; import org.apache.iotdb.pipe.api.PipeConnector; import org.apache.iotdb.pipe.api.annotation.TableModel; import org.apache.iotdb.pipe.api.annotation.TreeModel; @@ -34,8 +48,6 @@ import java.io.File; import java.io.FileInputStream; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -47,11 +59,15 @@ public class LogicalBackupSink implements PipeConnector { private static final int FILE_PIECE_BYTES = 64 * 1024; private LogicalBackupWriter writer; + private LogicalBackupSettings settings; @Override public void validate(final PipeParameterValidator validator) throws Exception { validator.validateSynonymAttributes( Collections.singletonList("connector.dir"), Collections.singletonList("sink.dir"), true); + final LogicalBackupSettings validated = + new LogicalBackupSettings(validator.getParameters(), "backup"); + validated.requireSharedResume(validator.getParameters()); } @Override @@ -61,18 +77,13 @@ public void customize( final String pipeName = configuration.getRuntimeEnvironment().getPipeName(); final long creationTime = configuration.getRuntimeEnvironment().getCreationTime(); final int regionId = configuration.getRuntimeEnvironment().getRegionId(); - final String root = parameters.getStringByKeys("connector.dir", "sink.dir"); - final String backupId = - parameters.getStringOrDefault( - Arrays.asList("connector.backup-id", "sink.backup-id"), pipeName + "-" + creationTime); - final Path directory = - Paths.get(root) - .toAbsolutePath() - .normalize() - .resolve(backupId) - .resolve("config-" + regionId); + settings = new LogicalBackupSettings(parameters, pipeName + "-" + creationTime); + settings.requireSharedResume(parameters); + if (configuration.getRuntimeEnvironment() instanceof PipeTaskTemporaryRuntimeEnvironment) { + return; + } final LogicalBackupManifest manifest = new LogicalBackupManifest(); - manifest.backupId = backupId; + manifest.backupId = settings.backupId; manifest.pipeName = pipeName; manifest.pipeCreationTime = creationTime; manifest.sourceClusterId = @@ -83,23 +94,18 @@ public void customize( manifest.streamId = pipeName + "-" + creationTime + "-config-" + regionId; manifest.regionId = regionId; manifest.streamType = "config"; - writer = - new LogicalBackupWriter( - directory, - manifest, - 64 * 1024 * 1024, - 64 * 1024 * 1024, - LogicalBackupWriter.FsyncPolicy.BATCH, - 100, - 1000, - true); + writer = settings.open("config", regionId, manifest); } @Override - public void transfer(final TabletInsertionEvent event) throws Exception {} + public void transfer(final TabletInsertionEvent event) throws Exception { + unsupported(event); + } @Override - public void transfer(final TsFileInsertionEvent event) throws Exception {} + public void transfer(final TsFileInsertionEvent event) throws Exception { + unsupported(event); + } @Override public void transfer(final Event event) throws Exception { @@ -108,7 +114,7 @@ public void transfer(final Event event) throws Exception { if (e.increaseReferenceCount(getClass().getName())) { try { writer.writeEvent( - UUID.randomUUID(), + eventId(e), System.currentTimeMillis(), Collections.singletonList( PipeTransferConfigPlanReq.toTPipeTransferReq(e.getConfigPhysicalPlan())), @@ -140,13 +146,48 @@ public void transfer(final Event event) throws Exception { e.getFileType(), e.toSealTypeString(), e.getAuthUserName())); - writer.writeEvent( - UUID.randomUUID(), System.currentTimeMillis(), requests, "config-snapshot"); + writer.writeEvent(eventId(e), System.currentTimeMillis(), requests, "config-snapshot"); } finally { e.decreaseReferenceCount(getClass().getName(), false); } } + } else if (event instanceof PipeHeartbeatEvent) { + if (settings.includeHeartbeat) { + writer.writeControl( + LogicalBackupRecordType.HEARTBEAT, System.currentTimeMillis(), "heartbeat"); + } + } else if (event instanceof PipeTerminateEvent) { + // close() writes STREAM_END after every preceding request has been committed. + } else { + unsupported(event); + } + } + + private void unsupported(final Event event) throws Exception { + final String name = event == null ? "null" : event.getClass().getName(); + if ("skip".equals(settings.unsupported)) { + writer.recordSkippedEvent(System.currentTimeMillis(), name); + } else { + throw new java.io.IOException( + String.format( + LogicalBackupMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_ARG_30920E60, name)); + } + } + + private static UUID eventId(final EnrichedEvent event) { + String identity = + event.getClass().getName() + + "\0" + + event.getPipeName() + + "\0" + + event.getCreationTime() + + "\0" + + event.getCommitId(); + if (event instanceof PipeConfigRegionSnapshotEvent) { + final PipeConfigRegionSnapshotEvent snapshot = (PipeConfigRegionSnapshotEvent) event; + identity += "\0" + snapshot.getFileType() + "\0" + snapshot.getSnapshotFile().getName(); } + return UUID.nameUUIDFromBytes(identity.getBytes(java.nio.charset.StandardCharsets.UTF_8)); } private static void appendFile(final List requests, final File file) diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java index b76156bb6192a..e066266396e98 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TPipeCompletedDataRegion; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.consensus.index.impl.MetaProgressIndex; import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeCriticalException; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeSinkCriticalException; @@ -75,6 +76,37 @@ public class PipeHeartbeatParserTest { private boolean originalSeparatedPipeHeartbeatEnabled; + @Test + public void testBackupWaitsForCommittedConfigSnapshotAfterDataCompletes() throws Exception { + CommonDescriptor.getInstance().getConfig().setSeperatedPipeHeartbeatEnabled(false); + final Map source = new HashMap<>(); + source.put("source.realtime.enable", "false"); + source.put("source.logical-backup.snapshot", "true"); + source.put("source.inclusion", "schema.database"); + final PipeMeta meta = createPipeMeta(source, Integer.MIN_VALUE); + final PipeTaskInfo info = new PipeTaskInfo(); + info.createPipe(new CreatePipePlanV2(meta.getStaticMeta(), meta.getRuntimeMeta())); + final ParserTestContext context = createParserTestContext(1, info); + context.parser.parseHeartbeat( + 1, + createPipeHeartbeatWithCompletedRegions( + meta, true, Collections.singletonList(Integer.MIN_VALUE))); + Assert.assertNotNull(info.getPipeMetaByPipeName("test_pipe")); + info.getPipeMetaByPipeName("test_pipe") + .getRuntimeMeta() + .getConsensusGroupId2TaskMetaMap() + .get(Integer.MIN_VALUE) + .updateProgressIndex( + new MetaProgressIndex( + org.apache.iotdb.commons.pipe.source.IoTDBNonDataRegionSource + .LOGICAL_BACKUP_COMPLETED_INDEX)); + context.parser.parseHeartbeat( + 1, + createPipeHeartbeatWithCompletedRegions( + meta, true, Collections.singletonList(Integer.MIN_VALUE))); + Assert.assertNull(info.getPipeMetaByPipeName("test_pipe")); + } + @Before public void setUp() { originalSeparatedPipeHeartbeatEnabled = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java index 7ac1f7ba12a45..ef404d4e5f8d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java @@ -81,12 +81,13 @@ public synchronized String register( connectorKey); final boolean isDataRegionSink = isDataRegionSink(environment.getRegionId()); + final boolean isLogicalBackupSink = isLogicalBackupSink(connectorKey); final int sinkNum = calculateSinkSubtaskNum(pipeSinkParameters, isDataRegionSink, connectorKey); boolean realTimeFirst = false; final String attributeSortedString = generateAttributeSortedString(pipeSinkParameters, environment.getRegionId()); final String attributeDisplayString = generateAttributeDisplayString(pipeSinkParameters); - if (isDataRegionSink) { + if (isDataRegionSink && !isLogicalBackupSink) { realTimeFirst = pipeSinkParameters.getBooleanOrDefault( Arrays.asList( @@ -95,11 +96,16 @@ public synchronized String register( PipeSinkConstant.CONNECTOR_REALTIME_FIRST_DEFAULT_VALUE); } final String attributeDisplayStringWithPrefix = - isDataRegionSink - ? PipeSinkConstant.isSerializeByRegionEnabled(pipeSinkParameters) - ? "data_region_" + environment.getRegionId() + "_" + attributeDisplayString - : "data_" + attributeDisplayString - : "schema_" + attributeDisplayString; + isLogicalBackupSink + ? (isDataRegionSink ? "data_region_" : "schema_region_") + + environment.getRegionId() + + "_" + + attributeDisplayString + : isDataRegionSink + ? PipeSinkConstant.isSerializeByRegionEnabled(pipeSinkParameters) + ? "data_region_" + environment.getRegionId() + "_" + attributeDisplayString + : "data_" + attributeDisplayString + : "schema_" + attributeDisplayString; environment.setAttributeSortedString(attributeDisplayStringWithPrefix); final PipeSinkSubtaskKey pipeSinkSubtaskKey = new PipeSinkSubtaskKey( @@ -386,6 +392,13 @@ public static int calculateSinkSubtaskNum( public static String generateAttributeSortedString( final PipeParameters pipeSinkParameters, final int regionId) { final String attributeSortedString = generateAttributeSortedString(pipeSinkParameters); + if (isLogicalBackupSink(getConnectorKey(pipeSinkParameters))) { + // A backup stream belongs to exactly one region and must have one ordered writer. + return (isDataRegionSink(regionId) ? "data_region_" : "schema_region_") + + regionId + + "_" + + attributeSortedString; + } if (isDataRegionSink(regionId)) { return PipeSinkConstant.isSerializeByRegionEnabled(pipeSinkParameters) ? "data_region_" + regionId + "_" + attributeSortedString @@ -398,6 +411,11 @@ private static String getConnectorKey(final PipeParameters pipeSinkParameters) { return PipeSinkConstant.getConnectorOrSinkNameWithDefault(pipeSinkParameters).toLowerCase(); } + private static boolean isLogicalBackupSink(final String connectorKey) { + return BuiltinPipePlugin.LOGICAL_BACKUP_SINK.getPipePluginName().equals(connectorKey) + || BuiltinPipePlugin.LOGICAL_BACKUP_CONNECTOR.getPipePluginName().equals(connectorKey); + } + private static boolean isDataRegionSink(final int regionId) { return StorageEngine.getInstance().getAllDataRegionIds().contains(new DataRegionId(regionId)) || PipeRuntimeMeta.isSourceExternal(regionId); @@ -407,7 +425,7 @@ private static int calculateSinkSubtaskNum( final PipeParameters pipeSinkParameters, final boolean isDataRegionSink, final String connectorKey) { - if (!isDataRegionSink) { + if (!isDataRegionSink || isLogicalBackupSink(connectorKey)) { // Do not allow parallel tasks for schema region connectors to avoid the potential disorder of // the schema region data transfer. return 1; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java index c48238a5ce225..a1ef953f4a007 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java @@ -22,10 +22,13 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; +import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskTemporaryRuntimeEnvironment; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupRecordType; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupSettings; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter; +import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.event.common.deletion.PipeDeleteDataNodeEvent; @@ -41,7 +44,11 @@ import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferSchemaSnapshotSealReq; import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletInsertNodeReqV2; import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletRawReqV2; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractDeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; +import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.pipe.api.PipeConnector; import org.apache.iotdb.pipe.api.annotation.TableModel; import org.apache.iotdb.pipe.api.annotation.TreeModel; @@ -55,19 +62,19 @@ import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; +import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.write.record.Tablet; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.Optional; import java.util.UUID; @TreeModel @@ -82,6 +89,8 @@ public class LogicalBackupSink implements PipeConnector { protected boolean includeHeartbeat; protected String unsupportedEventPolicy; private final String streamType; + private long backupStartTime = Long.MIN_VALUE; + private long backupEndTime = Long.MAX_VALUE; public LogicalBackupSink() { this("unknown"); @@ -161,6 +170,7 @@ public void validate(final PipeParameterValidator validator) throws Exception { PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIODIC, PipeSinkConstant.LOGICAL_BACKUP_FSYNC_NONE); validateSharedDirectoryResumePolicy(parameters); + new LogicalBackupSettings(parameters, "backup"); final String policy = parameters .getStringOrDefault( @@ -218,135 +228,27 @@ public void customize( pipeName = configuration.getRuntimeEnvironment().getPipeName(); creationTime = configuration.getRuntimeEnvironment().getCreationTime(); regionId = configuration.getRuntimeEnvironment().getRegionId(); - final String root = - parameters.getStringByKeys( - PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_DIR_KEY, - PipeSinkConstant.SINK_LOGICAL_BACKUP_DIR_KEY); - if (root == null || root.isBlank()) { - throw new PipeParameterNotValidException( - DataNodePipeMessages.EXCEPTION_LOGICAL_BACKUP_DIRECTORY_MUST_NOT_BE_EMPTY_99DB3EDB); - } - final String backupId = - parameters.getStringOrDefault( - Arrays.asList( - PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_ID_KEY, - PipeSinkConstant.SINK_LOGICAL_BACKUP_ID_KEY), - pipeName + "-" + creationTime); - if (backupId == null || backupId.isBlank()) { - throw new PipeParameterNotValidException( - String.format( - DataNodePipeMessages - .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22, - backupId)); - } - final String streamId = pipeName + "-" + creationTime + "-" + streamType + "-" + regionId; - final Path backupRoot = Paths.get(root).toAbsolutePath().normalize(); - final Path backupIdPath; - try { - backupIdPath = Path.of(backupId); - } catch (final RuntimeException e) { - throw new PipeParameterNotValidException( - String.format( - DataNodePipeMessages - .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22, - backupId)); - } - if (backupIdPath.isAbsolute() - || backupIdPath.getNameCount() != 1 - || ".".equals(backupId) - || "..".equals(backupId)) { - throw new PipeParameterNotValidException( - String.format( - DataNodePipeMessages - .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22, - backupId)); - } - Path directory = backupRoot.resolve(backupId).resolve(streamType + "-" + regionId).normalize(); - if (!directory.startsWith(backupRoot)) { - throw new PipeParameterNotValidException( - String.format( - DataNodePipeMessages - .EXCEPTION_LOGICAL_BACKUP_ID_MUST_RESOLVE_BELOW_THE_CONFIGURED_BACKUP_DIRECTORY_ARG_25BE4B22, - backupId)); - } - final String resume = - parameters - .getStringOrDefault( - Arrays.asList( - PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_RESUME_KEY, - PipeSinkConstant.SINK_LOGICAL_BACKUP_RESUME_KEY), - PipeSinkConstant.LOGICAL_BACKUP_RESUME_DEFAULT_VALUE) - .toLowerCase(Locale.ROOT); validateSharedDirectoryResumePolicy(parameters); - final boolean append = PipeSinkConstant.LOGICAL_BACKUP_RESUME_APPEND.equals(resume); - if (!append - && !PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW.equals(resume) - && !PipeSinkConstant.LOGICAL_BACKUP_RESUME_FAIL_IF_EXISTS.equals(resume)) { - throw new PipeParameterNotValidException( - String.format( - DataNodePipeMessages.EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_RESUME_POLICY_ARG_583FCCF9, - resume)); + final LogicalBackupSettings settings = + new LogicalBackupSettings(parameters, pipeName + "-" + creationTime); + includeHeartbeat = settings.includeHeartbeat; + unsupportedEventPolicy = settings.unsupported; + if (settings.captureOptions.containsKey("start-time")) { + backupStartTime = + DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone( + settings.captureOptions.get("start-time")); } - if (PipeSinkConstant.LOGICAL_BACKUP_RESUME_NEW.equals(resume)) { - directory = - directory.resolveSibling( - directory.getFileName() - + "-new-" - + System.currentTimeMillis() - + "-" - + UUID.randomUUID().toString().substring(0, 8)); + if (settings.captureOptions.containsKey("end-time")) { + backupEndTime = + DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone( + settings.captureOptions.get("end-time")); } - final long segmentSize = - parameters.getLongOrDefault( - Arrays.asList( - PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY, - PipeSinkConstant.SINK_LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_KEY), - PipeSinkConstant.LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_DEFAULT_VALUE); - final int maxRecord = - parameters.getIntOrDefault( - Arrays.asList( - PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY, - PipeSinkConstant.SINK_LOGICAL_BACKUP_MAX_RECORD_BYTES_KEY), - PipeSinkConstant.LOGICAL_BACKUP_MAX_RECORD_BYTES_DEFAULT_VALUE); - final String fsync = - parameters - .getStringOrDefault( - Arrays.asList( - PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_POLICY_KEY, - PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_POLICY_KEY), - PipeSinkConstant.LOGICAL_BACKUP_FSYNC_POLICY_DEFAULT_VALUE) - .toUpperCase(Locale.ROOT); - final LogicalBackupWriter.FsyncPolicy fsyncPolicy = - LogicalBackupWriter.FsyncPolicy.valueOf(fsync); - final int fsyncBatch = - parameters.getIntOrDefault( - Arrays.asList( - PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY, - PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_KEY), - PipeSinkConstant.LOGICAL_BACKUP_FSYNC_BATCH_OPERATIONS_DEFAULT_VALUE); - final long fsyncPeriod = - parameters.getLongOrDefault( - Arrays.asList( - PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY, - PipeSinkConstant.SINK_LOGICAL_BACKUP_FSYNC_PERIOD_MS_KEY), - PipeSinkConstant.LOGICAL_BACKUP_FSYNC_PERIOD_MS_DEFAULT_VALUE); - includeHeartbeat = - parameters.getBooleanOrDefault( - Arrays.asList( - PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY, - PipeSinkConstant.SINK_LOGICAL_BACKUP_INCLUDE_HEARTBEAT_KEY), - PipeSinkConstant.LOGICAL_BACKUP_INCLUDE_HEARTBEAT_DEFAULT_VALUE); - unsupportedEventPolicy = - parameters - .getStringOrDefault( - Arrays.asList( - PipeSinkConstant.CONNECTOR_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY, - PipeSinkConstant.SINK_LOGICAL_BACKUP_UNSUPPORTED_EVENT_KEY), - PipeSinkConstant.LOGICAL_BACKUP_UNSUPPORTED_EVENT_DEFAULT_VALUE) - .toLowerCase(Locale.ROOT); - + if (configuration.getRuntimeEnvironment() instanceof PipeTaskTemporaryRuntimeEnvironment) { + return; + } + final String streamId = pipeName + "-" + creationTime + "-" + streamType + "-" + regionId; final LogicalBackupManifest manifest = new LogicalBackupManifest(); - manifest.backupId = backupId; + manifest.backupId = settings.backupId; manifest.pipeName = pipeName; manifest.pipeCreationTime = creationTime; manifest.sourceClusterId = getSourceClusterId(); @@ -357,16 +259,7 @@ public void customize( manifest.regionId = regionId; manifest.streamType = streamType; manifest.sinkTaskId = streamId; - writer = - new LogicalBackupWriter( - directory, - manifest, - segmentSize, - maxRecord, - fsyncPolicy, - fsyncBatch, - fsyncPeriod, - append); + writer = settings.open(streamType, regionId, manifest); } protected String getSourceClusterId() { @@ -447,9 +340,13 @@ public void transfer(final Event event) throws Exception { final PipeDeleteDataNodeEvent deletion = (PipeDeleteDataNodeEvent) event; transferEnriched( deletion, - () -> - Collections.singletonList( - PipeTransferPlanNodeReq.toTPipeTransferReq(deletion.getDeleteDataNode()))); + () -> { + final Optional node = + boundedDeletion(deletion.getDeleteDataNode(), backupStartTime, backupEndTime); + return node.isPresent() + ? Collections.singletonList(PipeTransferPlanNodeReq.toTPipeTransferReq(node.get())) + : Collections.emptyList(); + }); } else if (event instanceof PipeSchemaRegionWritePlanEvent) { final PipeSchemaRegionWritePlanEvent schema = (PipeSchemaRegionWritePlanEvent) event; transferEnriched( @@ -465,8 +362,8 @@ public void transfer(final Event event) throws Exception { LogicalBackupRecordType.HEARTBEAT, System.currentTimeMillis(), metadata(event)); } } else if (event instanceof PipeTerminateEvent) { - writer.writeControl( - LogicalBackupRecordType.STREAM_END, System.currentTimeMillis(), metadata(event)); + // The framework commits this ordered completion signal. close() writes the final STREAM_END; + // a continuous source also emits a historical completion signal before its realtime events. } else { handleUnsupported(event); } @@ -524,13 +421,14 @@ private void transferSnapshot(final PipeSchemaRegionSnapshotEvent event) throws } } - private List requests(final PipeInsertNodeTabletInsertionEvent event) + List requests(final PipeInsertNodeTabletInsertionEvent event) throws IOException { final String databaseName = event.isTableModelEvent() ? event.getTableModelDatabaseName() : event.getTreeModelDatabaseName(); - if (!(event.getInsertNode() instanceof RelationalInsertTabletNode)) { + if (!(event.getInsertNode() instanceof RelationalInsertTabletNode) + && !event.shouldParseTimeOrPattern()) { return Collections.singletonList( PipeTransferTabletInsertNodeReqV2.toTPipeTransferReq( event.getInsertNode(), databaseName)); @@ -539,6 +437,9 @@ private List requests(final PipeInsertNodeTabletInsertionEvent final List tablets = event.convertToTablets(); final List requests = new ArrayList<>(tablets.size()); for (int index = 0; index < tablets.size(); index++) { + if (tablets.get(index).getRowSize() == 0) { + continue; + } requests.add( PipeTransferTabletRawReqV2.toTPipeTransferReq( tablets.get(index), event.isAligned(index), databaseName)); @@ -546,14 +447,48 @@ private List requests(final PipeInsertNodeTabletInsertionEvent return requests; } + static Optional boundedDeletion( + final AbstractDeleteDataNode node, final long start, final long end) { + if (node instanceof DeleteDataNode) { + final DeleteDataNode tree = (DeleteDataNode) node; + final long first = Math.max(start, tree.getDeleteStartTime()); + final long last = Math.min(end, tree.getDeleteEndTime()); + return first > last + ? Optional.empty() + : Optional.of(new DeleteDataNode(tree.getPlanNodeId(), tree.getPathList(), first, last)); + } + if (node instanceof RelationalDeleteDataNode) { + final RelationalDeleteDataNode table = (RelationalDeleteDataNode) node; + final List entries = new ArrayList<>(); + for (final TableDeletionEntry entry : table.getModEntries()) { + final long first = Math.max(start, entry.getStartTime()); + final long last = Math.min(end, entry.getEndTime()); + if (first <= last) { + entries.add(new TableDeletionEntry(entry.getPredicate(), new TimeRange(first, last))); + } + } + return entries.isEmpty() + ? Optional.empty() + : Optional.of( + new RelationalDeleteDataNode( + table.getPlanNodeId(), entries, table.getDatabaseName())); + } + throw new IllegalArgumentException( + String.format( + DataNodePipeMessages.EXCEPTION_UNSUPPORTED_PIPE_EVENT_FOR_LOGICAL_BACKUP_ARG_521AEE97, + node.getClass().getName())); + } + protected void transferEnriched(final EnrichedEvent event, final RequestSupplier requestSupplier) throws Exception { if (!event.increaseReferenceCount(getClass().getName())) { return; } try { - writer.writeEvent( - eventId(event), System.currentTimeMillis(), requestSupplier.get(), metadata(event)); + final List requests = requestSupplier.get(); + if (!requests.isEmpty()) { + writer.writeEvent(eventId(event), System.currentTimeMillis(), requests, metadata(event)); + } } finally { event.decreaseReferenceCount(getClass().getName(), false); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java index 5d677e100472f..01d65b54f9c00 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.queryengine.common.SqlDialect; import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; +import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent; import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics; import org.apache.iotdb.db.pipe.metric.overview.PipeTsFileToTabletsMetrics; import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionSourceMetrics; @@ -136,6 +137,7 @@ public class IoTDBDataRegionSource extends IoTDBSource { private PipeHistoricalDataRegionSource historicalSource; private PipeRealtimeDataRegionSource realtimeSource; + private boolean logicalBackupCompletionSent; private DataRegionWatermarkInjector watermarkInjector; @@ -619,6 +621,15 @@ public Event supply() throws Exception { } if (Objects.isNull(event)) { event = realtimeSource.supply(); + if (event == null + && !logicalBackupCompletionSent + && realtimeSource.isLogicalBackupDrained()) { + final PipeTerminateEvent completion = + new PipeTerminateEvent(pipeName, creationTime, pipeTaskMeta, regionId, true); + completion.increaseReferenceCount(IoTDBDataRegionSource.class.getName()); + logicalBackupCompletionSent = true; + event = completion; + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java index 92ad93e09f1b3..9879a3a3ec5d1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java @@ -146,6 +146,8 @@ public abstract class PipeRealtimeDataRegionSource implements PipeExtractor { protected String userName; protected String cliHostname; protected boolean skipIfNoPrivileges = true; + private long logicalBackupUntil = -1; + private final Object logicalBackupFence = new Object(); protected PipeRealtimeDataRegionSource() { // Do nothing @@ -218,6 +220,7 @@ public void validate(final PipeParameterValidator validator) throws Exception { public void customize( final PipeParameters parameters, final PipeExtractorRuntimeConfiguration configuration) throws Exception { + logicalBackupUntil = parameters.getLongOrDefault("source.logical-backup.until", -1L); final PipeRuntimeEnvironment environment = configuration.getRuntimeEnvironment(); final Pair insertionDeletionListeningOptionPair = @@ -362,6 +365,28 @@ private void clearPendingQueue() { * @param event the {@link Event} from the {@link StorageEngine} */ public final void extract(final PipeRealtimeEvent event) { + if (logicalBackupUntil > 0) { + synchronized (logicalBackupFence) { + if (System.currentTimeMillis() >= logicalBackupUntil) { + event.decreaseReferenceCount(PipeRealtimeDataRegionSource.class.getName(), false); + return; + } + extractInternal(event); + } + } else { + extractInternal(event); + } + } + + public boolean isLogicalBackupDrained() { + synchronized (logicalBackupFence) { + return logicalBackupUntil > 0 + && System.currentTimeMillis() >= logicalBackupUntil + && pendingQueue.isEmpty(); + } + } + + private void extractInternal(final PipeRealtimeEvent event) { // The progress report event shall be directly extracted if (event.getEvent() instanceof ProgressReportEvent) { extractProgressReportEvent(event); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java index 8c2fd176587db..3711867604e31 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManagerTest.java @@ -38,6 +38,27 @@ public class PipeSinkSubtaskManagerTest { + @Test + public void testLogicalBackupUsesOneWriterPerRegion() { + for (final BuiltinPipePlugin plugin : + new BuiltinPipePlugin[] { + BuiltinPipePlugin.LOGICAL_BACKUP_SINK, BuiltinPipePlugin.LOGICAL_BACKUP_CONNECTOR + }) { + final Map attributes = new HashMap<>(); + attributes.put(PipeSinkConstant.SINK_KEY, plugin.getPipePluginName()); + attributes.put(PipeSinkConstant.SINK_IOTDB_PARALLEL_TASKS_KEY, "3"); + attributes.put(PipeSinkConstant.SINK_SERIALIZE_BY_REGION_KEY, "false"); + final PipeParameters parameters = new PipeParameters(attributes); + Assert.assertEquals(1, PipeSinkSubtaskManager.calculateSinkSubtaskNum(parameters, -1)); + Assert.assertNotEquals( + PipeSinkSubtaskManager.generateAttributeSortedString(parameters, -1), + PipeSinkSubtaskManager.generateAttributeSortedString(parameters, -2)); + Assert.assertNotEquals( + PipeSinkSubtaskManager.generateAttributeSortedString(parameters, Integer.MAX_VALUE), + PipeSinkSubtaskManager.generateAttributeSortedString(parameters, Integer.MAX_VALUE - 1)); + } + } + @Test public void testSubtasksAreSharedOnlyWithinSamePipe() { // Initialize the task agent used by PipeEventCommitManager before registering subtasks. diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java index 2354e8309ab93..f2f0407d9b9e3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java @@ -19,19 +19,124 @@ package org.apache.iotdb.db.pipe.sink.protocol.logicalbackup; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; +import org.apache.iotdb.commons.pipe.config.plugin.configuraion.PipeTaskRuntimeConfiguration; +import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskTemporaryRuntimeEnvironment; +import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletRawReqV2; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; public class LogicalBackupSinkTest { + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testTemporaryValidationDoesNotCreateBackupStreams() throws Exception { + final Path directory = temporaryFolder.getRoot().toPath().resolve("backup"); + final PipeParameters parameters = + new PipeParameters( + Map.of( + "sink.dir", + directory.toString(), + "sink.backup-id", + "backup", + "sink.resume", + "append")); + try (final LogicalBackupSink sink = new LogicalBackupSink("data")) { + sink.validate(new PipeParameterValidator(parameters)); + sink.customize( + parameters, + new PipeTaskRuntimeConfiguration(new PipeTaskTemporaryRuntimeEnvironment("pipe"))); + sink.handshake(); + } + Assert.assertFalse(Files.exists(directory)); + } + + @Test + public void testDeleteTimeFilterPreservesPathsAndNeverExpandsTheRange() throws Exception { + final DeleteDataNode original = + new DeleteDataNode( + new PlanNodeId("deletion"), + Collections.singletonList(new MeasurementPath("root.backup.d.s1")), + 10L, + 20L); + final DeleteDataNode bounded = + (DeleteDataNode) LogicalBackupSink.boundedDeletion(original, 12L, 15L).get(); + Assert.assertEquals(original.getPathList(), bounded.getPathList()); + Assert.assertEquals(12L, bounded.getDeleteStartTime()); + Assert.assertEquals(15L, bounded.getDeleteEndTime()); + Assert.assertEquals(10L, original.getDeleteStartTime()); + Assert.assertEquals(20L, original.getDeleteEndTime()); + Assert.assertFalse(LogicalBackupSink.boundedDeletion(original, 21L, 30L).isPresent()); + } + + @Test + public void testInsertNodeIsFilteredBeforeItIsArchived() throws Exception { + final InsertTabletNode node = + new InsertTabletNode( + new PlanNodeId("backup-test"), + new PartialPath("root.backup.d"), + true, + new String[] {"s1", "s2"}, + new TSDataType[] {TSDataType.INT32, TSDataType.INT32}, + new MeasurementSchema[] { + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32) + }, + new long[] {1, 2, 3}, + null, + new Object[] {new int[] {10, 20, 30}, new int[] {40, 50, 60}}, + 3); + final PipeInsertNodeTabletInsertionEvent event = + new PipeInsertNodeTabletInsertionEvent( + false, + "root.backup", + node, + null, + 0, + null, + new IoTDBTreePattern("root.backup.d.s1"), + null, + null, + null, + null, + true, + 2L, + 2L); + final List requests = new LogicalBackupSink("data").requests(event); + Assert.assertEquals(1, requests.size()); + final InsertTabletStatement restored = + PipeTransferTabletRawReqV2.fromTPipeTransferReq(requests.get(0)).constructStatement(); + Assert.assertArrayEquals(new String[] {"s1"}, restored.getMeasurements()); + Assert.assertArrayEquals(new long[] {2}, restored.getTimes()); + Assert.assertArrayEquals(new int[] {20}, (int[]) restored.getColumns()[0]); + Assert.assertTrue(restored.isAligned()); + } + @Test public void testDataRegionRequiresExplicitAppendResumePolicy() throws Exception { final Map attributes = new HashMap<>(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSourceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSourceTest.java index 4677f2126b6c6..b3b7f954ebb09 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSourceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSourceTest.java @@ -34,6 +34,26 @@ public class PipeRealtimeDataRegionSourceTest { private static final String TEST_REFERENCE_HOLDER = PipeRealtimeDataRegionSourceTest.class.getName(); + @Test + public void logicalBackupBoundaryRejectsNewEventsAndDrainsAcceptedEvents() throws Exception { + try (final ProgressReportTestSource source = new ProgressReportTestSource()) { + final java.lang.reflect.Field until = + PipeRealtimeDataRegionSource.class.getDeclaredField("logicalBackupUntil"); + until.setAccessible(true); + until.set(source, System.currentTimeMillis() + 60_000L); + final PipeRealtimeEvent accepted = createHeartbeatEvent(); + source.extract(accepted); + until.set(source, System.currentTimeMillis() - 1L); + final PipeRealtimeEvent late = createHeartbeatEvent(); + source.extract(late); + Assert.assertTrue(late.getEvent().isReleased()); + Assert.assertFalse(source.isLogicalBackupDrained()); + Assert.assertSame(accepted, source.supply()); + accepted.decreaseReferenceCount(TEST_REFERENCE_HOLDER, false); + Assert.assertTrue(source.isLogicalBackupDrained()); + } + } + @Test public void progressReportEventReleasesDroppedHeartbeatEvent() throws Exception { try (final ProgressReportTestSource source = new ProgressReportTestSource()) { diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java index 1629e2dd00028..deb5f8629fd7a 100644 --- a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java @@ -148,4 +148,14 @@ public final class LogicalBackupMessages { "Logical backup directory is unavailable: %s"; private LogicalBackupMessages() {} + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_ROOT_MANIFEST_FILE_CHECKSUM_OR_STATE_ARG_3EDC33F3 = + "Invalid logical backup root manifest, file, checksum or state: %s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_SETTINGS_4E94FAAB = + "Invalid logical backup writer settings."; + public static final String EXCEPTION_CONFIGREGION_AND_DATAREGION_BACKUPS_REQUIRE_EXPLICIT_SINK_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_BECBAB90 = + "ConfigRegion and DataRegion backups require explicit sink.resume=append and a shared backup directory."; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SETTING_ARG_6D261EEB = + "Invalid logical backup setting: %s"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_ARG_30920E60 = + "Unsupported logical backup event: %s"; } diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java index bfc2bed98fed1..b4cf5a28670c9 100644 --- a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java +++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/LogicalBackupMessages.java @@ -148,4 +148,14 @@ public final class LogicalBackupMessages { "逻辑备份目录不可用：%s"; private LogicalBackupMessages() {} + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_ROOT_MANIFEST_FILE_CHECKSUM_OR_STATE_ARG_3EDC33F3 = + "逻辑备份根清单、文件、校验和或状态无效：%s"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_SETTINGS_4E94FAAB = + "逻辑备份写入参数无效。"; + public static final String EXCEPTION_CONFIGREGION_AND_DATAREGION_BACKUPS_REQUIRE_EXPLICIT_SINK_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_BECBAB90 = + "ConfigRegion 和 DataRegion 备份必须显式设置 sink.resume=append，并使用共享备份目录。"; + public static final String EXCEPTION_INVALID_LOGICAL_BACKUP_SETTING_ARG_6D261EEB = + "逻辑备份参数无效：%s"; + public static final String EXCEPTION_UNSUPPORTED_LOGICAL_BACKUP_EVENT_ARG_30920E60 = + "不支持的逻辑备份事件：%s"; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java index b378a16136c10..f90522acbf820 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReader.java @@ -60,7 +60,18 @@ public class LogicalBackupArchiveReader { public List read(final Path source, final boolean allowIncomplete) throws IOException { final Path normalizedSource = source.toAbsolutePath().normalize(); - final List manifests = discoverManifests(normalizedSource); + LogicalBackupCatalog.rejectSymbolicLinks(normalizedSource); + final Path root = + Files.isDirectory(normalizedSource) ? normalizedSource : normalizedSource.getParent(); + final LogicalBackupCatalog catalog = + LogicalBackupCatalog.load( + Files.isDirectory(normalizedSource) + ? normalizedSource.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME) + : normalizedSource); + final List manifests = + catalog == null + ? discoverManifests(normalizedSource) + : catalogManifests(root, catalog, allowIncomplete); if (manifests.isEmpty()) { throw new IOException( String.format( @@ -72,6 +83,7 @@ public List read(final Path source, final boolean allowIncomplete) LogicalBackupManifest archiveIdentity = null; for (final Path manifest : manifests) { final BackupStream stream = readStream(manifest, allowIncomplete); + stream.catalog = catalog; if (archiveIdentity == null) { archiveIdentity = stream.getManifest(); } else { @@ -85,6 +97,9 @@ public List read(final Path source, final boolean allowIncomplete) } streams.add(stream); } + if (catalog != null) { + validateCatalog(root, catalog, streams); + } streams.sort( Comparator.comparingInt( (BackupStream stream) -> streamOrder(stream.getManifest().streamType)) @@ -92,6 +107,104 @@ public List read(final Path source, final boolean allowIncomplete) return Collections.unmodifiableList(streams); } + private static List catalogManifests( + final Path root, final LogicalBackupCatalog catalog, final boolean allowIncomplete) + throws IOException { + if ((!"SEALED".equals(catalog.status) && !"INCOMPLETE".equals(catalog.status)) + || (!allowIncomplete + && ("INCOMPLETE".equals(catalog.status) || catalog.skippedEventCount > 0))) { + throw LogicalBackupCatalog.invalid(root); + } + final List manifests = new ArrayList<>(); + final Set files = new HashSet<>(); + files.add(root.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME)); + for (final LogicalBackupCatalog.StreamEntry entry : catalog.streams) { + if (entry == null || entry.manifest == null || entry.segments == null) { + throw LogicalBackupCatalog.invalid(root); + } + final Path manifest = validateCatalogFile(root, entry.manifest, files); + if (!LogicalBackupFormat.MANIFEST_FILE_NAME.equals(manifest.getFileName().toString())) { + throw LogicalBackupCatalog.invalid(manifest); + } + manifests.add(manifest); + for (final LogicalBackupCatalog.FileEntry segment : entry.segments) { + validateCatalogFile(root, segment, files); + } + } + try (final Stream paths = Files.walk(root)) { + for (final Path path : paths.toList()) { + if (Files.isSymbolicLink(path) + || (!Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS) + && !files.contains(path) + && !".backup.lock".equals(path.getFileName().toString()))) { + throw LogicalBackupCatalog.invalid(path); + } + } + } + return manifests; + } + + private static Path validateCatalogFile( + final Path root, final LogicalBackupCatalog.FileEntry entry, final Set files) + throws IOException { + if (entry == null) { + throw LogicalBackupCatalog.invalid(root); + } + final Path path = LogicalBackupCatalog.resolve(root, entry.file); + if (!files.add(path) + || !Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS) + || entry.sizeBytes != Files.size(path) + || !LogicalBackupCatalog.sha256(path).equals(entry.sha256)) { + throw LogicalBackupCatalog.invalid(path); + } + return path; + } + + private static void validateCatalog( + final Path root, final LogicalBackupCatalog catalog, final List streams) + throws IOException { + long skipped = 0; + final Map counts = new LinkedHashMap<>(); + final Map highWatermarks = new LinkedHashMap<>(); + for (int index = 0; index < streams.size(); index++) { + final BackupStream stream = streams.get(index); + final LogicalBackupManifest manifest = stream.manifest; + final LogicalBackupCatalog.StreamEntry entry = catalog.streams.get(index); + if (!manifest.streamId.equals(entry.streamId) + || !manifest.streamType.equals(entry.streamType) + || !manifest.backupId.equals(catalog.backupId) + || !java.util.Objects.equals(manifest.pipeName, catalog.pipeName) + || !java.util.Objects.equals(manifest.sourceClusterId, catalog.sourceClusterId) + || !java.util.Objects.equals(manifest.sourceVersion, catalog.sourceVersion) + || !java.util.Objects.equals(manifest.timestampPrecision, catalog.timestampPrecision) + || !java.util.Objects.equals(manifest.captureOptions, catalog.captureOptions) + || entry.segments.size() != manifest.segments.size()) { + throw LogicalBackupCatalog.invalid(root); + } + if (("data".equals(manifest.streamType) && !catalog.scope.contains("data")) + || ("schema".equals(manifest.streamType) && !catalog.scope.contains("schema")) + || ("config".equals(manifest.streamType) + && !catalog.scope.contains("config") + && !catalog.scope.contains("schema"))) { + throw LogicalBackupCatalog.invalid(root); + } + for (int segment = 0; segment < manifest.segments.size(); segment++) { + if (!LogicalBackupCatalog.resolve(root, entry.segments.get(segment).file) + .equals(stream.manifestPath.getParent().resolve(manifest.segments.get(segment).file))) { + throw LogicalBackupCatalog.invalid(root); + } + } + skipped += manifest.skippedEventCount; + manifest.operationCounts.forEach((key, value) -> counts.merge(key, value, Long::sum)); + highWatermarks.put(manifest.streamId, manifest.lastDurableSequence); + } + if (skipped != catalog.skippedEventCount + || !counts.equals(catalog.operationCounts) + || !highWatermarks.equals(catalog.sourceHighWatermark)) { + throw LogicalBackupCatalog.invalid(root); + } + } + private static int streamOrder(final String streamType) { if ("config".equalsIgnoreCase(streamType)) { return 0; @@ -131,6 +244,10 @@ private static void validateArchiveIdentityField( private BackupStream readStream(final Path manifestPath, final boolean allowIncomplete) throws IOException { final LogicalBackupManifest manifest; + LogicalBackupCatalog.rejectSymbolicLinks(manifestPath); + if (Files.size(manifestPath) > 16 * 1024 * 1024) { + throw LogicalBackupCatalog.invalid(manifestPath); + } try { manifest = GSON.fromJson( @@ -143,7 +260,9 @@ private BackupStream readStream(final Path manifestPath, final boolean allowInco e); } validateManifest(manifest, manifestPath); - if (!allowIncomplete && manifest.skippedEventCount > 0) { + if ((!"SEALED".equals(manifest.status) && !"INCOMPLETE".equals(manifest.status)) + || (!allowIncomplete + && (manifest.skippedEventCount > 0 || "INCOMPLETE".equals(manifest.status)))) { throw new IOException( String.format( LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599, @@ -158,6 +277,7 @@ private BackupStream readStream(final Path manifestPath, final boolean allowInco final Set listedSegments = new HashSet<>(); long expectedSequence = -1; long expectedSegmentId = 0; + boolean allSealed = true; for (int index = 0; index < manifest.segments.size(); index++) { final LogicalBackupManifest.Segment segment = manifest.segments.get(index); @@ -176,22 +296,23 @@ private BackupStream readStream(final Path manifestPath, final boolean allowInco LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SEGMENT_PATH_ARG_6485A845, segment.file)); } - listedSegments.add(segmentPath); - final boolean lastSegment = index == manifest.segments.size() - 1; - final LogicalBackupSegmentReader.ScanResult scan = - segmentReader.scan(segmentPath, allowIncomplete && lastSegment); + if (!listedSegments.add(segmentPath)) { + throw LogicalBackupCatalog.invalid(segmentPath); + } + final LogicalBackupSegmentReader.ScanResult scan = segmentReader.scan(segmentPath, false); if (scan.getSegmentId() != segment.segmentId || scan.getSegmentId() != expectedSegmentId++) { throw new IOException( String.format( LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_ID_MISMATCH_ARG_9FE7E88A, segmentPath)); } - if (!scan.isSealed() && (!allowIncomplete || !lastSegment)) { + if (!scan.isSealed() || scan.hasOpenEventGroup()) { throw new IOException( String.format( LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_SEGMENT_IS_NOT_SEALED_ARG_937793FF, segmentPath)); } + allSealed &= scan.isSealed(); if (scan.isSealed()) { validateSegmentManifest(segment, scan, segmentPath); } @@ -215,11 +336,11 @@ private BackupStream readStream(final Path manifestPath, final boolean allowInco } } - if (!allowIncomplete) { - validateUnlistedSegments(streamDirectory, listedSegments); + validateUnlistedSegments(streamDirectory, listedSegments); + if (allSealed) { final long skippedEventCount = operationCounts.getOrDefault(LogicalBackupRecordType.SKIPPED_EVENT.name(), 0L); - if (skippedEventCount > 0) { + if (!allowIncomplete && skippedEventCount > 0) { throw new IOException( String.format( LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_CONTAINS_SKIPPED_EVENTS_ARG_E69FD599, @@ -308,6 +429,7 @@ private static void validateManifest( || !LogicalBackupFormat.FORMAT_NAME.equals(manifest.formatName) || !LogicalBackupFormat.FORMAT_VERSION.equals(manifest.formatVersion) || manifest.streamId == null + || !List.of("config", "schema", "data").contains(manifest.streamType) || manifest.backupId == null || manifest.timestampPrecision == null || manifest.maxRecordBytes <= 0 @@ -421,6 +543,7 @@ public static class BackupStream { private final LogicalBackupManifest manifest; private final List records; private final List eventGroups; + private LogicalBackupCatalog catalog; private BackupStream( final Path manifestPath, @@ -441,6 +564,10 @@ public LogicalBackupManifest getManifest() { return manifest; } + public LogicalBackupCatalog getCatalog() { + return catalog; + } + public List getRecords() { return records; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupCatalog.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupCatalog.java new file mode 100644 index 0000000000000..3b9d79b1fed12 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupCatalog.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.commons.i18n.LogicalBackupMessages; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonParseException; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** The portable, authoritative root manifest of a backup task. */ +public class LogicalBackupCatalog { + + private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + + public String formatName = LogicalBackupFormat.FORMAT_NAME; + public String formatVersion = LogicalBackupFormat.FORMAT_VERSION; + public String backupId; + public String pipeName; + public String status = "WRITING"; + public String sourceClusterId; + public String sourceVersion; + public String timestampPrecision; + public List scope = new ArrayList<>(); + public List model = new ArrayList<>(); + public Map filters = new LinkedHashMap<>(); + public Map captureOptions = new LinkedHashMap<>(); + public boolean includeAuth; + public String mode = "snapshot"; + public String createdAt = Instant.now().toString(); + public String capturedAt; + public String sealedAt; + public Map sourceHighWatermark = new LinkedHashMap<>(); + public Map operationCounts = new LinkedHashMap<>(); + public long skippedEventCount; + public boolean recovered; + public List streams = new ArrayList<>(); + + public static class StreamEntry { + public String streamId; + public String streamType; + public FileEntry manifest; + public List segments = new ArrayList<>(); + } + + public static class FileEntry { + public String file; + public long sizeBytes; + public String sha256; + } + + public void seal(final Path root, final List backupStreams) throws IOException { + collect(root, backupStreams); + write(root); + } + + public void collect(final Path root, final List backupStreams) throws IOException { + streams.clear(); + operationCounts.clear(); + sourceHighWatermark.clear(); + skippedEventCount = 0; + recovered = false; + for (final BackupStream stream : backupStreams) { + final LogicalBackupManifest manifest = stream.getManifest(); + if (!backupId.equals(manifest.backupId)) { + throw invalid(root); + } + if (streams.isEmpty()) { + sourceClusterId = manifest.sourceClusterId; + sourceVersion = manifest.sourceVersion; + timestampPrecision = manifest.timestampPrecision; + } + if (!java.util.Objects.equals(pipeName, manifest.pipeName) + || !java.util.Objects.equals(sourceClusterId, manifest.sourceClusterId) + || !java.util.Objects.equals(sourceVersion, manifest.sourceVersion) + || !java.util.Objects.equals(timestampPrecision, manifest.timestampPrecision) + || !captureOptions.equals(manifest.captureOptions)) { + throw invalid(root); + } + final StreamEntry entry = new StreamEntry(); + entry.streamId = manifest.streamId; + entry.streamType = manifest.streamType; + entry.manifest = describe(root, stream.getManifestPath()); + for (final LogicalBackupManifest.Segment segment : manifest.segments) { + entry.segments.add( + describe(root, stream.getManifestPath().getParent().resolve(segment.file))); + } + streams.add(entry); + manifest.operationCounts.forEach( + (key, value) -> operationCounts.merge(key, value, Long::sum)); + sourceHighWatermark.put(manifest.streamId, manifest.lastDurableSequence); + skippedEventCount += manifest.skippedEventCount; + recovered |= manifest.recovered; + } + capturedAt = Instant.now().toString(); + sealedAt = capturedAt; + status = skippedEventCount > 0 ? "INCOMPLETE" : "SEALED"; + } + + public void write(final Path root) throws IOException { + rejectSymbolicLinks(root); + Files.createDirectories(root); + final Path target = root.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME); + if (Files.isSymbolicLink(target)) { + throw invalid(target); + } + final Path temporary = Files.createTempFile(root, ".manifest-", ".tmp"); + try { + final ByteBuffer bytes = StandardCharsets.UTF_8.encode(GSON.toJson(this)); + try (final FileChannel channel = FileChannel.open(temporary, StandardOpenOption.WRITE)) { + while (bytes.hasRemaining()) { + channel.write(bytes); + } + channel.force(true); + } + try { + Files.move( + temporary, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (final java.nio.file.AtomicMoveNotSupportedException e) { + Files.move(temporary, target, StandardCopyOption.REPLACE_EXISTING); + } + } finally { + Files.deleteIfExists(temporary); + } + } + + /** Returns null for a legacy stream manifest; malformed root manifests always fail. */ + public static LogicalBackupCatalog load(final Path manifest) throws IOException { + rejectSymbolicLinks(manifest); + if (!Files.isRegularFile(manifest, LinkOption.NOFOLLOW_LINKS)) { + return null; + } + if (Files.size(manifest) > 16 * 1024 * 1024) { + throw invalid(manifest); + } + try { + final com.google.gson.JsonObject json = + GSON.fromJson(Files.readString(manifest), com.google.gson.JsonObject.class); + if (json == null) { + throw invalid(manifest); + } + if (json.has("streamId")) { + return null; + } + final LogicalBackupCatalog catalog = GSON.fromJson(json, LogicalBackupCatalog.class); + if (!LogicalBackupFormat.FORMAT_NAME.equals(catalog.formatName) + || !LogicalBackupFormat.FORMAT_VERSION.equals(catalog.formatVersion) + || catalog.backupId == null + || catalog.scope == null + || catalog.model == null + || catalog.filters == null + || catalog.captureOptions == null + || catalog.streams == null + || catalog.operationCounts == null + || catalog.sourceHighWatermark == null + || catalog.skippedEventCount < 0 + || catalog.scope.isEmpty() + || !List.of("config", "schema", "data").containsAll(catalog.scope) + || catalog.model.isEmpty() + || !List.of("tree", "table").containsAll(catalog.model) + || !List.of("WRITING", "RECOVERED", "SEALED", "INCOMPLETE").contains(catalog.status)) { + throw invalid(manifest); + } + return catalog; + } catch (final JsonParseException | IllegalStateException e) { + throw new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_INVALID_LOGICAL_BACKUP_ROOT_MANIFEST_FILE_CHECKSUM_OR_STATE_ARG_3EDC33F3, + manifest), + e); + } + } + + public static FileEntry describe(final Path root, final Path path) throws IOException { + final Path normalizedRoot = root.toAbsolutePath().normalize(); + final Path normalizedPath = path.toAbsolutePath().normalize(); + if (!normalizedPath.startsWith(normalizedRoot)) { + throw invalid(path); + } + final FileEntry entry = new FileEntry(); + entry.file = normalizedRoot.relativize(normalizedPath).toString().replace('\\', '/'); + resolve(normalizedRoot, entry.file); + entry.sizeBytes = Files.size(normalizedPath); + entry.sha256 = sha256(normalizedPath); + return entry; + } + + public static Path resolve(final Path root, final String relative) throws IOException { + if (relative == null + || relative.isBlank() + || relative.indexOf('\\') >= 0 + || relative.indexOf(':') >= 0 + || relative.startsWith("/") + || relative.chars().anyMatch(Character::isISOControl)) { + throw invalid(root); + } + for (final String part : relative.split("/", -1)) { + if (part.isEmpty() || ".".equals(part) || "..".equals(part)) { + throw invalid(root); + } + } + final Path normalizedRoot = root.toAbsolutePath().normalize(); + final Path path = normalizedRoot.resolve(relative).normalize(); + if (!path.startsWith(normalizedRoot) || path.equals(normalizedRoot)) { + throw invalid(path); + } + rejectSymbolicLinks(path); + return path; + } + + public static void rejectSymbolicLinks(final Path path) throws IOException { + final Path absolute = path.toAbsolutePath().normalize(); + Path current = absolute.getRoot(); + for (final Path part : absolute) { + current = current.resolve(part); + if (Files.isSymbolicLink(current)) { + throw invalid(current); + } + } + } + + public static String sha256(final Path path) throws IOException { + final MessageDigest digest = LogicalBackupFormat.newSha256(); + try (final InputStream input = Files.newInputStream(path)) { + final byte[] buffer = new byte[64 * 1024]; + int length; + while ((length = input.read(buffer)) != -1) { + digest.update(buffer, 0, length); + } + } + return LogicalBackupFormat.toHex(digest.digest()); + } + + static IOException invalid(final Path path) { + return new IOException( + String.format( + LogicalBackupMessages + .EXCEPTION_INVALID_LOGICAL_BACKUP_ROOT_MANIFEST_FILE_CHECKSUM_OR_STATE_ARG_3EDC33F3, + path)); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java index 3742ca88d8016..891ed03ff6e81 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupManifest.java @@ -37,11 +37,15 @@ public class LogicalBackupManifest { public String timestampPrecision; public String createdAt; public String closedAt; + public String sealedAt; + public Map captureOptions = new LinkedHashMap<>(); public String streamId; public int regionId; public String sinkTaskId; public String streamType; public String fsyncPolicy; + public int fsyncBatchOperations; + public long fsyncPeriodMs; public long segmentSizeBytes; public int maxRecordBytes; public long firstSequence = -1; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettings.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettings.java new file mode 100644 index 0000000000000..5a95830d03e8a --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettings.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.commons.i18n.LogicalBackupMessages; +import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.UUID; + +/** Settings shared by ConfigRegion, SchemaRegion and DataRegion backup writers. */ +public final class LogicalBackupSettings { + + public final Path root; + public final String backupId; + public final String resume; + public final long segmentSize; + public final int maxRecord; + public final LogicalBackupWriter.FsyncPolicy fsync; + public final int fsyncBatch; + public final long fsyncPeriod; + public final boolean includeHeartbeat; + public final String unsupported; + public final Map captureOptions; + + public LogicalBackupSettings(final PipeParameters parameters, final String defaultBackupId) + throws IOException { + try { + final String directory = value(parameters, "dir", ""); + if (directory.isBlank()) { + throw invalid("dir"); + } + root = Path.of(directory).toAbsolutePath().normalize(); + LogicalBackupCatalog.rejectSymbolicLinks(root); + backupId = value(parameters, "backup-id", defaultBackupId); + validateBackupId(backupId); + resume = value(parameters, "resume", "fail-if-exists").toLowerCase(Locale.ROOT); + if (!List.of("fail-if-exists", "append", "new").contains(resume)) { + throw invalid("resume"); + } + segmentSize = + Long.parseLong( + value( + parameters, + "segment-size-bytes", + Long.toString(PipeSinkConstant.LOGICAL_BACKUP_SEGMENT_SIZE_BYTES_DEFAULT_VALUE))); + maxRecord = + Integer.parseInt( + value( + parameters, + "max-record-bytes", + Integer.toString( + PipeSinkConstant.LOGICAL_BACKUP_MAX_RECORD_BYTES_DEFAULT_VALUE))); + fsync = + LogicalBackupWriter.FsyncPolicy.valueOf( + value(parameters, "fsync-policy", "always").toUpperCase(Locale.ROOT)); + fsyncBatch = Integer.parseInt(value(parameters, "fsync-batch-operations", "1000")); + fsyncPeriod = Long.parseLong(value(parameters, "fsync-period-ms", "1000")); + includeHeartbeat = booleanValue(parameters, "include-heartbeat"); + unsupported = value(parameters, "on-unsupported-event", "fail"); + if (maxRecord <= 0 + || maxRecord > LogicalBackupFormat.MAX_RECORD_BYTES + || segmentSize < maxRecord + || fsyncBatch <= 0 + || fsyncPeriod <= 0 + || !List.of("fail", "skip").contains(unsupported) + || (has(parameters, "fsync-batch-operations") + && fsync != LogicalBackupWriter.FsyncPolicy.BATCH) + || (has(parameters, "fsync-period-ms") + && fsync != LogicalBackupWriter.FsyncPolicy.PERIODIC)) { + throw invalid("writer"); + } + captureOptions = + new Gson() + .fromJson( + value(parameters, "capture-options", "{}"), + new TypeToken>() {}.getType()); + if (captureOptions == null) { + throw invalid("capture-options"); + } + if (!List.of( + "scope", + "model", + "mode", + "include-auth", + "include", + "exclude", + "start-time", + "end-time", + "until", + "segment-size-bytes", + "max-record-bytes", + "fsync-policy", + "on-unsupported", + "include-heartbeat", + "fsync-batch-operations", + "fsync-period-ms") + .containsAll(captureOptions.keySet()) + || captureOptions.containsValue(null)) { + throw invalid("capture-options"); + } + } catch (final IllegalArgumentException | com.google.gson.JsonParseException e) { + throw new IOException( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_WRITER_SETTINGS_4E94FAAB, e); + } + } + + public void requireSharedResume(final PipeParameters parameters) throws IOException { + if (!has(parameters, "resume") || !"append".equals(resume)) { + throw new IOException( + LogicalBackupMessages + .EXCEPTION_CONFIGREGION_AND_DATAREGION_BACKUPS_REQUIRE_EXPLICIT_SINK_RESUME_APPEND_AND_A_SHARED_BACKUP_DIRECTORY_BECBAB90); + } + } + + public LogicalBackupWriter open( + final String streamType, final int regionId, final LogicalBackupManifest manifest) + throws IOException { + Path directory = root.resolve(backupId).resolve(streamType + "-" + regionId); + if ("new".equals(resume)) { + directory = directory.resolveSibling(directory.getFileName() + "-new-" + UUID.randomUUID()); + } + manifest.captureOptions = new LinkedHashMap<>(captureOptions); + return new LogicalBackupWriter( + directory, + manifest, + segmentSize, + maxRecord, + fsync, + fsyncBatch, + fsyncPeriod, + "append".equals(resume)); + } + + public static void validateBackupId(final String value) throws IOException { + if (value == null + || !value.matches("[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}") + || value.endsWith(".") + || value.matches("(?i)(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(\\..*)?")) { + throw invalid("backup-id"); + } + } + + private static boolean booleanValue(final PipeParameters parameters, final String key) + throws IOException { + final String value = value(parameters, key, "false"); + if (!"true".equalsIgnoreCase(value) && !"false".equalsIgnoreCase(value)) { + throw invalid(key); + } + return Boolean.parseBoolean(value); + } + + private static boolean has(final PipeParameters parameters, final String key) { + return parameters.hasAnyAttributes("connector." + key, "sink." + key); + } + + private static String value( + final PipeParameters parameters, final String key, final String fallback) throws IOException { + if (parameters.getAttribute().containsKey("connector." + key) + && parameters.getAttribute().containsKey("sink." + key)) { + throw invalid(key); + } + return parameters.getStringOrDefault( + Arrays.asList("connector." + key, "sink." + key), fallback); + } + + private static IOException invalid(final String key) { + return new IOException( + String.format( + LogicalBackupMessages.EXCEPTION_INVALID_LOGICAL_BACKUP_SETTING_ARG_6D261EEB, key)); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java index 9c8ba6f9a9a63..86536587efef9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java @@ -115,6 +115,7 @@ public LogicalBackupWriter( <= LogicalBackupFormat.SEGMENT_HEADER_SIZE + LogicalBackupFormat.SEGMENT_FOOTER_SIZE || maxRecordBytes <= 0 || maxRecordBytes > LogicalBackupFormat.MAX_RECORD_BYTES + || segmentSizeBytes < maxRecordBytes || fsyncBatchOperations <= 0 || fsyncPeriodMs <= 0) { throw new IOException( @@ -144,6 +145,8 @@ public LogicalBackupWriter( manifest.status = "WRITING"; manifest.createdAt = Instant.now().toString(); manifest.fsyncPolicy = fsyncPolicy.name().toLowerCase(Locale.ROOT); + manifest.fsyncBatchOperations = fsyncBatchOperations; + manifest.fsyncPeriodMs = fsyncPeriodMs; manifest.segmentSizeBytes = segmentSizeBytes; manifest.maxRecordBytes = maxRecordBytes; writeManifest(); @@ -759,6 +762,17 @@ private void loadExistingManifest() throws IOException { "fsyncPolicy", fsyncPolicy.name().toLowerCase(Locale.ROOT), existing.fsyncPolicy); validateManifestConfiguration("segmentSizeBytes", segmentSizeBytes, existing.segmentSizeBytes); validateManifestConfiguration("maxRecordBytes", maxRecordBytes, existing.maxRecordBytes); + if (existing.fsyncBatchOperations > 0) { + validateManifestConfiguration( + "fsyncBatchOperations", fsyncBatchOperations, existing.fsyncBatchOperations); + } + if (existing.fsyncPeriodMs > 0) { + validateManifestConfiguration("fsyncPeriodMs", fsyncPeriodMs, existing.fsyncPeriodMs); + } + validateManifestIdentity( + "captureOptions", + GSON.toJson(manifest.captureOptions), + GSON.toJson(existing.captureOptions)); if (manifest.pipeCreationTime != 0 && manifest.pipeCreationTime != existing.pipeCreationTime) { throw new IOException( String.format( @@ -799,7 +813,11 @@ private void loadExistingManifest() throws IOException { : existing.operationCounts; manifest.createdAt = existing.createdAt; manifest.closedAt = existing.closedAt; + manifest.sealedAt = existing.sealedAt; + manifest.captureOptions = existing.captureOptions; manifest.fsyncPolicy = existing.fsyncPolicy; + manifest.fsyncBatchOperations = fsyncBatchOperations; + manifest.fsyncPeriodMs = fsyncPeriodMs; manifest.segmentSizeBytes = existing.segmentSizeBytes; manifest.maxRecordBytes = existing.maxRecordBytes; manifest.lastEventGroupId = existing.lastEventGroupId; @@ -904,8 +922,9 @@ public synchronized void close() throws IOException { } sealSegment(); } - manifest.status = manifest.recovered ? "RECOVERED" : "SEALED"; + manifest.status = manifest.skippedEventCount > 0 ? "INCOMPLETE" : "SEALED"; manifest.closedAt = Instant.now().toString(); + manifest.sealedAt = manifest.closedAt; manifest.lastDurableSequence = manifest.lastSequence; writeManifest(); } finally { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java index d4d55ba7dac7a..7ef4cf1b52178 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java @@ -56,6 +56,8 @@ @TableModel public abstract class IoTDBNonDataRegionSource extends IoTDBSource { + public static final long LOGICAL_BACKUP_COMPLETED_INDEX = Long.MAX_VALUE - 1; + private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBNonDataRegionSource.class); protected IoTDBTreePatternOperations treePattern; @@ -66,6 +68,14 @@ public abstract class IoTDBNonDataRegionSource extends IoTDBSource { // filled. Used only for metric framework. private int historicalEventsCount = 0; + // Logical backup snapshots must acknowledge all metadata before the coordinator drops the pipe. + // Leave room for queue maintenance to calculate index + 1 without overflowing. + private boolean finiteSnapshot; + private boolean snapshotOnly; + private long captureUntil; + private volatile long captureEndIndex = Long.MAX_VALUE; + private boolean snapshotCompletionReported; + private ConcurrentIterableLinkedQueue.DynamicIterator iterator; // If close() is called, hasBeenClosed will be set to true even if the extractor is started again. @@ -96,6 +106,9 @@ public void customize( } treePattern = (IoTDBTreePatternOperations) pattern; tablePattern = TablePattern.parsePipePatternFromSourceParameters(parameters); + snapshotOnly = parameters.getBooleanOrDefault("source.logical-backup.snapshot", false); + captureUntil = parameters.getLongOrDefault("source.logical-backup.until", -1L); + finiteSnapshot = snapshotOnly || captureUntil > 0; } @Override @@ -107,6 +120,13 @@ public void start() throws Exception { final ProgressIndex progressIndex = pipeTaskMeta.getProgressIndex(); warnIfUnexpectedHybridProgressIndex(progressIndex); final MetaProgressIndex metaProgressIndex = extractMetaProgressIndex(progressIndex); + if (finiteSnapshot + && metaProgressIndex != null + && metaProgressIndex.getIndex() == LOGICAL_BACKUP_COMPLETED_INDEX) { + snapshotCompletionReported = true; + super.start(); + return; + } final long nextIndex = Objects.isNull(metaProgressIndex) // If the index is invalid, the queue is seen as cleared before and thus @@ -115,6 +135,12 @@ public void start() throws Exception { ? getNextIndexAfterSnapshot() : metaProgressIndex.getIndex() + 1; iterator = getListeningQueue().newIterator(nextIndex); + if (captureUntil > 0) { + java.util.concurrent.CompletableFuture.delayedExecutor( + Math.max(0L, captureUntil - System.currentTimeMillis()), + java.util.concurrent.TimeUnit.MILLISECONDS) + .execute(() -> captureEndIndex = getListeningQueue().getTailIndex()); + } super.start(); } @@ -171,6 +197,9 @@ public EnrichedEvent supply() throws Exception { return null; } } + if (snapshotCompletionReported) { + return null; + } // Check whether snapshot being parsed exists PipeWritePlanEvent realtimeEvent = lastEvent; @@ -197,7 +226,9 @@ public EnrichedEvent supply() throws Exception { Long.MIN_VALUE, Long.MAX_VALUE); - if (canSkipSnapshotPrivilegeCheck(historicalEvent)) { + // Portable backups contain logical metadata plans, even when the user can read the whole + // snapshot. Snapshot file transfer remains available to ordinary replication pipes. + if (!finiteSnapshot && canSkipSnapshotPrivilegeCheck(historicalEvent)) { if (historicalEvents.isEmpty()) { // We only report progress for the last snapshot event. historicalEvent.bindProgressIndex(new MetaProgressIndex(iterator.getNextIndex() - 1)); @@ -221,7 +252,20 @@ public EnrichedEvent supply() throws Exception { // Realtime if (Objects.isNull(realtimeEvent)) { + if (finiteSnapshot + && (snapshotOnly || captureUntil <= 0 || iterator.getNextIndex() >= captureEndIndex)) { + final ProgressReportEvent completion = + new ProgressReportEvent(pipeName, creationTime, pipeTaskMeta); + completion.bindProgressIndex(new MetaProgressIndex(LOGICAL_BACKUP_COMPLETED_INDEX)); + completion.increaseReferenceCount(IoTDBNonDataRegionSource.class.getName()); + snapshotCompletionReported = true; + return completion; + } realtimeEvent = (PipeWritePlanEvent) iterator.next(getMaxBlockingTimeMs()); + if (captureUntil > 0 && iterator.getNextIndex() > captureEndIndex) { + // A blocking read may wake with an event appended after the boundary was captured. + realtimeEvent = null; + } } if (Objects.isNull(realtimeEvent)) { return null; @@ -305,6 +349,11 @@ public long getUnTransferredEventCount() { final ProgressIndex progressIndex = pipeTaskMeta.getProgressIndex(); warnIfUnexpectedHybridProgressIndex(progressIndex); final MetaProgressIndex metaProgressIndex = extractMetaProgressIndex(progressIndex); + if (finiteSnapshot + && metaProgressIndex != null + && metaProgressIndex.getIndex() == LOGICAL_BACKUP_COMPLETED_INDEX) { + return 0L; + } return Objects.nonNull(metaProgressIndex) ? getListeningQueue().getTailIndex() - metaProgressIndex.getIndex() - 1 : getListeningQueue().getSize() + historicalEventsCount; diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java index 10db588f2c3c9..3702850597d9e 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupArchiveReaderTest.java @@ -91,7 +91,7 @@ public void testUnlistedSegmentIsRejected() throws Exception { } @Test - public void testIncompleteLastSegmentRequiresExplicitOptIn() throws Exception { + public void testIncompleteOptInCannotBypassMissingFooter() throws Exception { final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); writeStream(directory, "stream", "data", 4096); final Path segment = onlySegment(directory); @@ -100,9 +100,7 @@ public void testIncompleteLastSegmentRequiresExplicitOptIn() throws Exception { } assertReadFails(directory, false); - final List streams = new LogicalBackupArchiveReader().read(directory, true); - Assert.assertEquals(1, streams.size()); - Assert.assertEquals(1, streams.get(0).getEventGroups().size()); + assertReadFails(directory, true); } @Test @@ -216,6 +214,103 @@ public void testDuplicateEventGroupIdIsRejected() throws Exception { assertReadFails(directory, false); } + @Test + public void testRootManifestIsPortableAndOrdersAllStreams() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + writeStream(root.resolve("data-7"), "data-7", "data", 4096); + writeStream(root.resolve("schema-3"), "schema-3", "schema", 4096); + writeStream(root.resolve("config-0"), "config-0", "config", 4096); + final LogicalBackupCatalog catalog = sealCatalog(root); + + final List streams = + new LogicalBackupArchiveReader().read(root.resolve("manifest.json"), false); + Assert.assertEquals("config", streams.get(0).getManifest().streamType); + Assert.assertEquals("schema", streams.get(1).getManifest().streamType); + Assert.assertEquals("data", streams.get(2).getManifest().streamType); + Assert.assertEquals(3, catalog.streams.size()); + Assert.assertEquals("SEALED", streams.get(0).getCatalog().status); + for (final LogicalBackupCatalog.StreamEntry entry : catalog.streams) { + Assert.assertFalse(Path.of(entry.manifest.file).isAbsolute()); + Assert.assertEquals(64, entry.manifest.sha256.length()); + Assert.assertEquals(64, entry.segments.get(0).sha256.length()); + } + } + + @Test + public void testRootManifestRejectsUnlistedFile() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + writeStream(root.resolve("data"), "data", "data", 4096); + sealCatalog(root); + Files.writeString(root.resolve("unlisted.txt"), "not part of the backup"); + assertReadFails(root, false); + assertReadFails(root, true); + } + + @Test + public void testIncompleteOptInCannotBypassRootChecksum() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + writeStream(root.resolve("data"), "data", "data", 4096); + sealCatalog(root); + final Path segment = onlySegment(root.resolve("data")); + try (final RandomAccessFile file = new RandomAccessFile(segment.toFile(), "rw")) { + file.seek(LogicalBackupFormat.SEGMENT_HEADER_SIZE + LogicalBackupFormat.RECORD_HEADER_SIZE); + file.writeByte(0); + } + assertReadFails(root, false); + assertReadFails(root, true); + } + + @Test + public void testRootManifestRejectsTraversalAndDuplicateFiles() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + writeStream(root.resolve("data"), "data", "data", 4096); + final LogicalBackupCatalog catalog = sealCatalog(root); + final String file = catalog.streams.get(0).manifest.file; + catalog.streams.get(0).manifest.file = "../outside/manifest.json"; + catalog.write(root); + assertReadFails(root, true); + catalog.streams.get(0).manifest.file = file; + catalog.streams.add(catalog.streams.get(0)); + catalog.write(root); + assertReadFails(root, true); + } + + @Test + public void testIncompleteStatusRequiresExplicitOptIn() throws Exception { + final Path root = temporaryFolder.newFolder().toPath(); + final Path directory = root.resolve("data"); + try (final LogicalBackupWriter writer = writer(directory, manifest("data", "data"), 4096)) { + writer.recordSkippedEvent(1, "unsupported"); + } + final LogicalBackupCatalog catalog = sealCatalog(root); + Assert.assertEquals("INCOMPLETE", catalog.status); + Assert.assertEquals(1, catalog.skippedEventCount); + assertReadFails(root, false); + Assert.assertEquals(1, new LogicalBackupArchiveReader().read(root, true).size()); + } + + @Test + public void testAllowIncompleteStillChecksSealedCounters() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + writeStream(directory, "data", "data", 4096); + final Path path = directory.resolve("manifest.json"); + final LogicalBackupManifest manifest = + GSON.fromJson(Files.readString(path), LogicalBackupManifest.class); + manifest.operationCounts.clear(); + Files.writeString(path, GSON.toJson(manifest)); + assertReadFails(directory, true); + } + + private static LogicalBackupCatalog sealCatalog(final Path root) throws IOException { + final LogicalBackupCatalog catalog = new LogicalBackupCatalog(); + catalog.backupId = "backup"; + catalog.pipeName = "pipe"; + catalog.scope = Arrays.asList("config", "schema", "data"); + catalog.model = Arrays.asList("tree", "table"); + catalog.seal(root, new LogicalBackupArchiveReader().read(root, true)); + return catalog; + } + private static void writeStream( final Path directory, final String streamId, final String streamType, final long segmentSize) throws Exception { @@ -247,7 +342,7 @@ private static LogicalBackupWriter writer( directory, manifest, segmentSize, - MAX_RECORD_BYTES, + (int) Math.min(MAX_RECORD_BYTES, segmentSize), LogicalBackupWriter.FsyncPolicy.ALWAYS, 1, 1, diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettingsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettingsTest.java new file mode 100644 index 0000000000000..cdef5ccafd8c0 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupSettingsTest.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.logicalbackup; + +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +public class LogicalBackupSettingsTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testInvalidSettingsFailBeforeCreatingDirectory() throws Exception { + final Path root = temporaryFolder.getRoot().toPath().resolve("backup"); + for (final Map invalid : + Arrays.asList( + Map.of("sink.backup-id", "../outside"), + Map.of("sink.backup-id", "bad\\path"), + Map.of("sink.backup-id", "CON"), + Map.of("sink.segment-size-bytes", "1024"), + Map.of("sink.max-record-bytes", "268435457"), + Map.of("sink.fsync-policy", "batch", "sink.fsync-batch-operations", "0"), + Map.of("sink.fsync-period-ms", "1000"), + Map.of("sink.include-heartbeat", "maybe"), + Map.of("sink.on-unsupported-event", "ignore"))) { + final Map values = new HashMap<>(invalid); + values.put("sink.dir", root.toString()); + try { + new LogicalBackupSettings(new PipeParameters(values), "backup"); + Assert.fail(invalid.toString()); + } catch (final IOException expected) { + Assert.assertFalse(Files.exists(root)); + } + } + } + + @Test + public void testAllStreamsUseTheSameWriterSettingsAndAppendIdentity() throws Exception { + final Path root = temporaryFolder.getRoot().toPath().resolve("backup"); + final Map values = new HashMap<>(); + values.put("sink.dir", root.toString()); + values.put("sink.resume", "append"); + values.put("sink.backup-id", "backup-id"); + values.put("sink.segment-size-bytes", "8192"); + values.put("sink.max-record-bytes", "1024"); + values.put("sink.fsync-policy", "periodic"); + values.put("sink.fsync-period-ms", "12"); + values.put("sink.capture-options", "{\"scope\":\"config,schema,data\"}"); + final LogicalBackupSettings settings = + new LogicalBackupSettings(new PipeParameters(values), "backup"); + for (final String type : Arrays.asList("config", "schema", "data")) { + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = "backup-id"; + manifest.streamId = type; + manifest.streamType = type; + manifest.timestampPrecision = "ms"; + try (final LogicalBackupWriter writer = settings.open(type, 1, manifest)) { + Assert.assertEquals(8192, writer.getManifest().segmentSizeBytes); + Assert.assertEquals(1024, writer.getManifest().maxRecordBytes); + Assert.assertEquals("periodic", writer.getManifest().fsyncPolicy); + } + values.put("sink.capture-options", "{\"scope\":\"data\"}"); + try { + new LogicalBackupSettings(new PipeParameters(values), "backup").open(type, 1, manifest); + Assert.fail(); + } catch (final IOException expected) { + // A different scope cannot silently append to an existing stream. + } + } + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java index ec7d5d63ed9fc..b4b97c233fb8c 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java @@ -326,14 +326,7 @@ public void testSealedSegmentAdvancesDurableSequenceWithFsyncNone() throws Excep final LogicalBackupManifest manifest = manifest(); try (final LogicalBackupWriter writer = new LogicalBackupWriter( - directory, - manifest, - 500, - MAX_RECORD_BYTES, - LogicalBackupWriter.FsyncPolicy.NONE, - 1, - 1, - false)) { + directory, manifest, 500, 500, LogicalBackupWriter.FsyncPolicy.NONE, 1, 1, false)) { writer.writeEvent( UUID.randomUUID(), 1, Collections.singletonList(request(new byte[20])), "m"); writer.writeEvent( @@ -533,7 +526,7 @@ private static LogicalBackupWriter writer( directory, manifest(), segmentSize, - MAX_RECORD_BYTES, + (int) Math.min(MAX_RECORD_BYTES, segmentSize), LogicalBackupWriter.FsyncPolicy.ALWAYS, 1, 1, diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSourceTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSourceTest.java index 3bfbd6dcefddb..bcfce467d3ff8 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSourceTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSourceTest.java @@ -28,8 +28,10 @@ import org.apache.iotdb.commons.exception.auth.AccessDeniedException; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.datastructure.queue.listening.AbstractPipeListeningQueue; +import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.event.PipeSnapshotEvent; import org.apache.iotdb.commons.pipe.event.PipeWritePlanEvent; +import org.apache.iotdb.commons.pipe.event.ProgressReportEvent; import org.junit.Assert; import org.junit.Test; @@ -39,10 +41,62 @@ import java.io.IOException; import java.util.Collections; +import java.util.LinkedList; import java.util.Optional; public class IoTDBNonDataRegionSourceTest { + @Test + public void testLogicalBackupParsesMetadataInsteadOfTransferringSnapshotFiles() throws Exception { + final AbstractPipeListeningQueue queue = Mockito.mock(AbstractPipeListeningQueue.class); + final TestNonDataRegionSource source = + new TestNonDataRegionSource(queue, new PipeTaskMeta(new MetaProgressIndex(10L), 0)); + final java.lang.reflect.Field finite = + IoTDBNonDataRegionSource.class.getDeclaredField("finiteSnapshot"); + finite.setAccessible(true); + finite.set(source, true); + final java.lang.reflect.Field snapshots = + IoTDBNonDataRegionSource.class.getDeclaredField("historicalEvents"); + snapshots.setAccessible(true); + snapshots.set( + source, + new LinkedList<>( + Collections.singletonList( + Mockito.mock(PipeSnapshotEvent.class, Mockito.RETURNS_SELF)))); + source.skipSnapshotPrivilegeCheck = true; + source.hasBeenStarted.set(true); + Assert.assertTrue(source.supply() instanceof ProgressReportEvent); + Assert.assertTrue(source.snapshotParsed); + Mockito.verifyZeroInteractions(queue); + } + + @Test + public void testFiniteSnapshotReportsOrderedCompletionAndNeverReadsRealtime() throws Exception { + final AbstractPipeListeningQueue queue = Mockito.mock(AbstractPipeListeningQueue.class); + final PipeTaskMeta task = new PipeTaskMeta(new MetaProgressIndex(10L), 0); + final TestNonDataRegionSource source = new TestNonDataRegionSource(queue, task); + final java.lang.reflect.Field finite = + IoTDBNonDataRegionSource.class.getDeclaredField("finiteSnapshot"); + finite.setAccessible(true); + finite.set(source, true); + source.hasBeenStarted.set(true); + final EnrichedEvent completion = source.supply(); + Assert.assertTrue(completion instanceof ProgressReportEvent); + Assert.assertEquals( + new MetaProgressIndex(IoTDBNonDataRegionSource.LOGICAL_BACKUP_COMPLETED_INDEX), + completion.getProgressIndex()); + Assert.assertNull(source.supply()); + task.updateProgressIndex(completion.getProgressIndex()); + Assert.assertEquals(0L, source.getUnTransferredEventCount()); + Mockito.verifyZeroInteractions(queue); + + final TestNonDataRegionSource restarted = new TestNonDataRegionSource(queue, task); + finite.set(restarted, true); + restarted.start(); + Assert.assertNull(restarted.supply()); + Mockito.verifyZeroInteractions(queue); + } + @Test public void testStartWithStateWrappedHybridProgressIndex() throws Exception { final AbstractPipeListeningQueue listeningQueue = @@ -94,6 +148,8 @@ public void testGetUnTransferredEventCountWithHybridProgressIndexWithoutMetaInde private static final class TestNonDataRegionSource extends IoTDBNonDataRegionSource { private final AbstractPipeListeningQueue listeningQueue; + private boolean skipSnapshotPrivilegeCheck; + private boolean snapshotParsed; private TestNonDataRegionSource( final AbstractPipeListeningQueue listeningQueue, final PipeTaskMeta pipeTaskMeta) { @@ -123,13 +179,13 @@ protected long getMaxBlockingTimeMs() { @Override protected boolean canSkipSnapshotPrivilegeCheck(final PipeSnapshotEvent event) { - return false; + return skipSnapshotPrivilegeCheck; } @Override protected void initSnapshotGenerator(final PipeSnapshotEvent event) throws IOException, IllegalPathException { - // Do nothing + snapshotParsed = true; } @Override From c3bf2028f6adf9f757b77d2565a4ba728e8afb4a Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:02:51 +0800 Subject: [PATCH 09/17] Secure logical backup temporary extraction --- .../tool/pipe/PipeLogicalBackupTool.java | 28 +++++++++++++++++-- .../tool/pipe/PipeLogicalBackupToolTest.java | 25 +++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java index e73067ed69ea9..69399d791aaae 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java @@ -61,10 +61,14 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; import java.security.MessageDigest; import java.security.SecureRandom; import java.util.ArrayList; import java.util.Comparator; +import java.util.EnumSet; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -80,6 +84,11 @@ public final class PipeLogicalBackupTool { private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + + private static final class SecureRandomHolder { + private static final SecureRandom INSTANCE = new SecureRandom(); + } + private static final String COMMAND_DUMP = "dump"; private static final String COMMAND_PACK = "pack"; private static final String COMMAND_RESTORE = "restore"; @@ -1435,7 +1444,7 @@ static void writeCheckpoint(final Path checkpoint, final ImportCheckpoint checkp private static String getClusterId() { final byte[] bytes = new byte[16]; - new SecureRandom().nextBytes(bytes); + SecureRandomHolder.INSTANCE.nextBytes(bytes); return "PIPE-LOGICAL-BACKUP-" + UUID.nameUUIDFromBytes(bytes); } @@ -1445,6 +1454,21 @@ private static void printUsage() { .LOG_LOGICAL_BACKUP_DUMP_VERIFY_INSPECT_PACK_RESTORE_OPTIONS_DUMP_CONNECT_TO_A_SOURCE_AND_SEAL_A_BACKUP_DIRECTORY_DOES_NOT_CREATE_A_ZIP_INSPECT_SHOW_BACKUP_IDENTITY_SCOPE_STREAMS_SEGMENTS_COUNTS_AND_STATUS_VERIFY_CHECK_STRUCTURE_CHECKSUMS_AND_SEMANTICS_WITHOUT_CHANGING_THE_INPUT_PACK_VERIFY_AND_PACKAGE_AN_EXISTING_DIRECTORY_AS_ZIP_ACCEPTS_NO_CONNECTION_OPTIONS_RESTORE_PREFLIGHT_AND_REPLAY_A_DIRECTORY_ROOT_MANIFEST_JSON_OR_ZIP_INSPECT_VERIFY_INPUT_PATH_JSON_ALLOW_INCOMPLETE_PACK_INPUT_DIR_OUTPUT_ZIP_JSON_ALLOW_INCOMPLETE_DUMP_HOST_HOST_PORT_PORT_USER_ROOT_PASSWORD_STDIN_PASSWORD_ENV_NAME_BACKUP_DIR_DIR_BACKUP_ID_ID_SCOPE_CONFIG_SCHEMA_DATA_MODEL_TREE_TABLE_INCLUDE_PATTERN_EXCLUDE_PATTERN_START_TIME_TIME_END_TIME_TIME_INCLUDE_AUTH_TRUE_FALSE_MODE_SNAPSHOT_CONTINUOUS_UNTIL_ISO_8601_RESUME_FAIL_IF_EXISTS_APPEND_NEW_PIPE_NAME_NAME_SEGMENT_SIZE_BYTES_268435456_MAX_RECORD_BYTES_67108864_FSYNC_POLICY_ALWAYS_BATCH_PERIODIC_NONE_FSYNC_BATCH_OPERATIONS_N_FSYNC_PERIOD_MS_N_ON_UNSUPPORTED_FAIL_SKIP_INCLUDE_HEARTBEAT_TRUE_FALSE_TIMEOUT_SECONDS_3600_POLL_INTERVAL_SECONDS_2_JSON_ALLOW_INCOMPLETE_CONTINUOUS_REQUIRES_UNTIL_STOP_SEQUENCE_IS_RESERVED_AND_CURRENTLY_REJECTED_OBJECT_FILTERS_REQUIRE_ONE_MODEL_TREE_USES_PATH_LISTS_TABLE_USES_DATABASE_REGEX_TABLE_REGEX_TREE_CONFIG_SCHEMA_CAPTURE_ACCEPTS_A_SINGLE_PREFIX_FULL_PATH_WITHOUT_EXCLUSIONS_TABLE_EXCLUSIONS_MUST_SELECT_WHOLE_DATABASES_OR_WHOLE_TABLE_NAME_PATTERNS_RESTORE_INPUT_PATH_HOST_HOST_PORT_PORT_USER_USER_PASSWORD_STDIN_PASSWORD_ENV_NAME_DRY_RUN_CHECKPOINT_FILE_RESUME_CONFLICT_FAIL_INCLUDE_AUTH_TRUE_FALSE_JSON_ALLOW_INCOMPLETE_RESTORING_AUTHORIZATION_OBJECTS_REQUIRES_INCLUDE_AUTH_TRUE_LEGACY_CONFIG_IS_CONSERVATIVE_A_TARGET_CONNECTION_IS_REQUIRED_FOR_REPLAY_LOCAL_DRY_RUN_NEEDS_NO_CONNECTION_TARGET_PREFLIGHT_CHECKS_VERSION_PRECISION_AUTHENTICATION_AND_HANDSHAKE_SUPPORT_OBJECT_CONFLICTS_AND_PER_OBJECT_PRIVILEGES_ARE_CHECKED_DURING_REPLAY_DEPRECATED_ALIASES_EXPORT_INPUT_DIR_OUTPUT_ZIP_PACK_EXPORT_WITH_SOURCE_CONNECTION_OPTIONS_DUMP_THEN_PACK_IMPORT_RESTORE_EXIT_CODE_0_SUCCESS_1_FAILURE_ERRORS_AND_DEPRECATION_NOTICES_GO_TO_STDERR_B1983BE8); } + static Path createPrivateBackupTempDirectory() throws IOException { + final Path userHome = Path.of(System.getProperty("user.home")).toAbsolutePath().normalize(); + if (Files.getFileAttributeView(userHome, PosixFileAttributeView.class) != null) { + return Files.createTempDirectory( + userHome, + "iotdb-logical-backup-", + PosixFilePermissions.asFileAttribute( + EnumSet.of( + PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, + PosixFilePermission.OWNER_EXECUTE))); + } + return Files.createTempDirectory(userHome, "iotdb-logical-backup-"); + } + static class ImportCheckpoint { String formatName; String formatVersion; @@ -1515,7 +1539,7 @@ private static BackupInput open(final Path source) throws IOException { || "manifest.json".equals(normalizedSource.getFileName().toString())) { return new BackupInput(normalizedSource, null); } - final Path temporaryDirectory = Files.createTempDirectory("iotdb-logical-backup-"); + final Path temporaryDirectory = createPrivateBackupTempDirectory(); try { extract(normalizedSource, temporaryDirectory); return new BackupInput(temporaryDirectory, temporaryDirectory); diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java index 4fee78c4b15b0..f039f275caedd 100644 --- a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java @@ -46,9 +46,12 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFilePermission; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.UUID; @@ -61,6 +64,28 @@ public class PipeLogicalBackupToolTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test + public void testBackupTemporaryDirectoryIsPrivate() throws Exception { + final Path directory = PipeLogicalBackupTool.createPrivateBackupTempDirectory(); + try { + Assert.assertEquals( + Path.of(System.getProperty("user.home")).toAbsolutePath().normalize(), + directory.getParent()); + final PosixFileAttributeView posixView = + Files.getFileAttributeView(directory, PosixFileAttributeView.class); + if (posixView != null) { + Assert.assertEquals( + EnumSet.of( + PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, + PosixFilePermission.OWNER_EXECUTE), + posixView.readAttributes().permissions()); + } + } finally { + Files.deleteIfExists(directory); + } + } + @Test public void testDirectoryPackZipVerifyAndRestoreDryRun() throws Exception { final Path root = temporaryFolder.getRoot().toPath(); From cd15321de564b63a270ad7e5cc517c7c3a98c544 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:28:46 +0800 Subject: [PATCH 10/17] Wait for logical backup Pipe metadata on every DataNode Poll each DataNode result independently while Pipe metadata propagates, then retain the existing capture and restore assertions. Use the test framework default credentials and a policy-compliant test-user password. --- .../manual/IoTDBPipeLogicalBackupIT.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeLogicalBackupIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeLogicalBackupIT.java index 4629bcb887506..e0cc1b1b0642d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeLogicalBackupIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeLogicalBackupIT.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupCatalog; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.itbase.env.BaseEnv; @@ -53,6 +54,8 @@ import java.util.concurrent.CompletionException; import java.util.concurrent.TimeUnit; +import static org.apache.iotdb.itbase.runtime.ClusterTestResultSet.getNodeResultSets; + @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeManual.class}) public class IoTDBPipeLogicalBackupIT extends AbstractPipeDualTreeModelManualIT { @@ -93,7 +96,12 @@ public void testBoundedContinuousTableCapture() throws Exception { final Statement statement = connection.createStatement(); final ResultSet result = statement.executeQuery("show pipe " + pipeName)) { - return result.next(); + // Pipe metadata propagates asynchronously between DataNodes. + boolean existsOnAllNodes = true; + for (final ResultSet nodeResult : getNodeResultSets(result)) { + existsOnAllNodes &= nodeResult.next(); + } + return existsOnAllNodes; } }); try (final Connection connection = senderEnv.getTableConnection(); @@ -178,7 +186,7 @@ public void testDumpPackPreflightRestoreWithScopeAndTimeFilters() throws Excepti "create database root.unselected", "create timeseries root.unselected.d.s with datatype=INT32,encoding=PLAIN", "insert into root.unselected.d(time,s) values(2,100)", - "create user logical_backup_excluded_user 'test_password'", + "create user logical_backup_excluded_user 'LogicalBackup!2026'", "flush"), null); final Path parent = temporaryFolder.getRoot().toPath().toAbsolutePath(); @@ -284,12 +292,14 @@ private static void runConnected(final BaseEnv env, final String command, final "--port", env.getPort(), "--user", - "root", + SessionConfig.DEFAULT_USER, "--password-stdin")); args.addAll(Arrays.asList(options)); final InputStream previous = System.in; try { - System.setIn(new ByteArrayInputStream("root\n".getBytes(StandardCharsets.UTF_8))); + System.setIn( + new ByteArrayInputStream( + (SessionConfig.DEFAULT_PASSWORD + "\n").getBytes(StandardCharsets.UTF_8))); Assert.assertEquals(0, PipeLogicalBackupTool.run(args.toArray(new String[0]))); } finally { System.setIn(previous); From bc38d803580d1bf73ec15231bafdab746d88ba2e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:58:05 +0800 Subject: [PATCH 11/17] Quote Windows logical backup paths --- scripts/tools/windows/logical-backup.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/tools/windows/logical-backup.bat b/scripts/tools/windows/logical-backup.bat index 14e3180e46994..967718977e1a5 100644 --- a/scripts/tools/windows/logical-backup.bat +++ b/scripts/tools/windows/logical-backup.bat @@ -20,8 +20,8 @@ @echo off if "%OS%" == "Windows_NT" setlocal -pushd %~dp0..\.. -if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD% +pushd "%~dp0..\.." +if NOT DEFINED IOTDB_HOME set "IOTDB_HOME=%CD%" popd if NOT DEFINED JAVA_HOME goto :no_java From 0b6b0060f6b74bb8edb6bdcadd89b4d18c409aff Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:21:06 +0800 Subject: [PATCH 12/17] Honor table exclude-all filters in logical backups --- .../tool/pipe/LogicalBackupDumpOptions.java | 2 +- .../tool/pipe/PipeLogicalBackupToolTest.java | 55 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/LogicalBackupDumpOptions.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/LogicalBackupDumpOptions.java index e28e288df0b42..f0a5039f90512 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/LogicalBackupDumpOptions.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/LogicalBackupDumpOptions.java @@ -212,7 +212,7 @@ private void applyObjectFilters(final CommandLine line) throws IOException { } for (int index = 0; index < 2; index++) { String regex = include[index]; - if (exclude != null && !".*".equals(exclude[index])) { + if (exclude != null && ".*".equals(exclude[1 - index])) { regex = "(?!(?:" + exclude[index] + ")$)(?:" + regex + ")"; } source.put(index == 0 ? "source.database-name" : "source.table-name", regex); diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java index f039f275caedd..859e5cb555f97 100644 --- a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java @@ -20,6 +20,7 @@ package org.apache.iotdb.tool.pipe; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; import org.apache.iotdb.commons.pipe.sink.client.IoTDBSyncClient; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader.BackupStream; @@ -28,6 +29,7 @@ import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; @@ -377,6 +379,59 @@ public void testDumpScopeAuthAndWriterOptionsAreExplicit() throws Exception { Assert.assertTrue(auth.source.get("source.inclusion").contains("auth")); } + @Test + public void testDumpTableExclusionsDoNotBroadenCapture() throws Exception { + final Map> cases = + Map.of( + "backup_skip/.*", + List.of("backup_keep/measurements", "backup_keep/other_table"), + ".*/other_table", + List.of("backup_keep/measurements", "backup_skip/measurements"), + ".*/.*", + List.of()); + for (final Map.Entry> testCase : cases.entrySet()) { + final LogicalBackupDumpOptions options = + new LogicalBackupDumpOptions( + PipeLogicalBackupTool.parse( + "dump", + new String[] { + "--scope", + "schema,data", + "--model", + "table", + "--include", + "backup_.*/.*", + "--exclude", + testCase.getKey() + }), + false); + final TablePattern pattern = + TablePattern.parsePipePatternFromSourceParameters(new PipeParameters(options.source)); + final List captured = new ArrayList<>(); + for (final String database : List.of("backup_keep", "backup_skip", "unselected")) { + for (final String table : List.of("measurements", "other_table")) { + if (pattern.matchesDatabase(database) && pattern.matchesTable(table)) { + captured.add(database + "/" + table); + } + } + } + Assert.assertEquals(testCase.getKey(), testCase.getValue(), captured); + } + } + + @Test + public void testDumpTableExcludeAllWithoutInclusion() throws Exception { + final LogicalBackupDumpOptions options = + new LogicalBackupDumpOptions( + PipeLogicalBackupTool.parse( + "dump", new String[] {"--scope", "data", "--model", "table", "--exclude", ".*/.*"}), + false); + final TablePattern pattern = + TablePattern.parsePipePatternFromSourceParameters(new PipeParameters(options.source)); + Assert.assertFalse( + pattern.matchesDatabase("backup_db") && pattern.matchesTable("measurements")); + } + @Test public void testDumpRejectsAmbiguousOrUnboundedCaptureBeforeCreatingTask() throws Exception { for (final String[] args : From 09bd6c2bf03c25d9824aad8e1a455948450e58ad Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:21:34 +0800 Subject: [PATCH 13/17] Scope logical backup event IDs to task lifetimes --- .../logicalbackup/LogicalBackupSink.java | 8 ++ .../logicalbackup/LogicalBackupSinkTest.java | 103 ++++++++++++++++++ .../logicalbackup/LogicalBackupSink.java | 8 ++ .../logicalbackup/LogicalBackupSinkTest.java | 72 ++++++++++++ 4 files changed, 191 insertions(+) create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java index 789953a05d4da..5fff064e0f89f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java @@ -58,6 +58,10 @@ @TableModel public class LogicalBackupSink implements PipeConnector { private static final int FILE_PIECE_BYTES = 64 * 1024; + // Commit IDs restart with each process and committer generation. Keep the process namespace + // stable across sink instances so a retry of the same delivery still has the same ID. + private static final UUID PROCESS_ID = UUID.randomUUID(); + private LogicalBackupWriter writer; private LogicalBackupSettings settings; @@ -177,11 +181,15 @@ private void unsupported(final Event event) throws Exception { private static UUID eventId(final EnrichedEvent event) { String identity = event.getClass().getName() + + "\0" + + PROCESS_ID + "\0" + event.getPipeName() + "\0" + event.getCreationTime() + "\0" + + (event.getCommitterKey() == null ? -1 : event.getCommitterKey().getRestartTimes()) + + "\0" + event.getCommitId(); if (event instanceof PipeConfigRegionSnapshotEvent) { final PipeConfigRegionSnapshotEvent snapshot = (PipeConfigRegionSnapshotEvent) event; diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java new file mode 100644 index 0000000000000..6f58b9fbd691c --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.pipe.sink.protocol.logicalbackup; + +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter; +import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; +import org.apache.iotdb.confignode.manager.pipe.event.PipeConfigRegionWritePlanEvent; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.lang.reflect.Field; +import java.nio.file.Path; +import java.util.List; + +public class LogicalBackupSinkTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testRecreatedTaskDoesNotDeduplicateANewEvent() throws Exception { + assertTaskRestartPreservesEvents(10L); + } + + @Test + public void testRecreatedTaskDoesNotRejectADifferentEvent() throws Exception { + assertTaskRestartPreservesEvents(20L); + } + + private void assertTaskRestartPreservesEvents(final long ttlAfterRestart) throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final Field writerField = LogicalBackupSink.class.getDeclaredField("writer"); + writerField.setAccessible(true); + for (int generation = 0; generation < 2; generation++) { + final PipeConfigRegionWritePlanEvent event = + new PipeConfigRegionWritePlanEvent( + new SetTTLPlan(generation == 0 ? 10L : ttlAfterRestart, "root", "backup"), + "pipe", + 1L, + null, + null, + null, + null, + null, + null, + true, + false); + event.setCommitterKeyAndCommitId(new CommitterKey("pipe", 1L, -1, generation), 1L); + Assert.assertTrue(event.increaseReferenceCount(getClass().getName())); + try (final LogicalBackupSink sink = new LogicalBackupSink()) { + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = "backup"; + manifest.pipeName = "pipe"; + manifest.pipeCreationTime = 1L; + manifest.streamId = "stream"; + manifest.streamType = "config"; + manifest.regionId = -1; + manifest.timestampPrecision = "ms"; + writerField.set( + sink, + new LogicalBackupWriter( + directory, + manifest, + 4096, + 1024, + LogicalBackupWriter.FsyncPolicy.ALWAYS, + 1, + 1, + generation > 0)); + sink.transfer(event); + sink.transfer(event); + } finally { + event.decreaseReferenceCount(getClass().getName(), false); + } + } + final List streams = + new LogicalBackupArchiveReader().read(directory, false); + Assert.assertEquals(1, streams.size()); + Assert.assertEquals(2, streams.get(0).getEventGroups().size()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java index a1ef953f4a007..69b1219e38e70 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSink.java @@ -82,6 +82,10 @@ public class LogicalBackupSink implements PipeConnector { private static final int FILE_PIECE_BYTES = 64 * 1024; + // Commit IDs restart with each process and committer generation. Keep the process namespace + // stable across sink instances so a retry of the same delivery still has the same ID. + private static final UUID PROCESS_ID = UUID.randomUUID(); + protected LogicalBackupWriter writer; private String pipeName; private long creationTime; @@ -514,6 +518,8 @@ protected void handleUnsupported(final Event event) throws Exception { protected UUID eventId(final EnrichedEvent event) { return UUID.nameUUIDFromBytes( (event.getClass().getName() + + '\0' + + PROCESS_ID + '\0' + event.getPipeName() + '\0' @@ -521,6 +527,8 @@ protected UUID eventId(final EnrichedEvent event) { + '\0' + event.getRegionId() + '\0' + + (event.getCommitterKey() == null ? -1 : event.getCommitterKey().getRestartTimes()) + + '\0' + commitIds(event)) .getBytes(StandardCharsets.UTF_8)); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java index f2f0407d9b9e3..4bad78378fbbb 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/logicalbackup/LogicalBackupSinkTest.java @@ -21,11 +21,16 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; import org.apache.iotdb.commons.pipe.config.plugin.configuraion.PipeTaskRuntimeConfiguration; import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskTemporaryRuntimeEnvironment; import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupArchiveReader; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; +import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.pipe.event.common.deletion.PipeDeleteDataNodeEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletRawReqV2; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; @@ -54,6 +59,73 @@ public class LogicalBackupSinkTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test + public void testRecreatedTaskDoesNotDeduplicateANewEvent() throws Exception { + assertTaskRestartPreservesEvents(10L); + } + + @Test + public void testRecreatedTaskDoesNotRejectADifferentEvent() throws Exception { + assertTaskRestartPreservesEvents(20L); + } + + private void assertTaskRestartPreservesEvents(final long endAfterRestart) throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + for (int generation = 0; generation < 2; generation++) { + final PipeDeleteDataNodeEvent event = + new PipeDeleteDataNodeEvent( + new DeleteDataNode( + new PlanNodeId("delete"), + Collections.singletonList(new MeasurementPath("root.backup.d.s1")), + 1L, + generation == 0 ? 10L : endAfterRestart), + "pipe", + 1L, + null, + null, + null, + null, + null, + null, + true, + false); + event.setCommitterKeyAndCommitId(new CommitterKey("pipe", 1L, 1, generation), 1L); + Assert.assertTrue(event.increaseReferenceCount(getClass().getName())); + try (final LogicalBackupSink sink = new LogicalBackupSink("data")) { + try (final LogicalBackupSink anotherSink = new LogicalBackupSink("data")) { + Assert.assertEquals(sink.eventId(event), anotherSink.eventId(event)); + } + final LogicalBackupManifest manifest = new LogicalBackupManifest(); + manifest.backupId = "backup"; + manifest.pipeName = "pipe"; + manifest.pipeCreationTime = 1L; + manifest.streamId = "stream"; + manifest.streamType = "data"; + manifest.regionId = 1; + manifest.timestampPrecision = "ms"; + sink.writer = + new LogicalBackupWriter( + directory, + manifest, + 4096, + 1024, + LogicalBackupWriter.FsyncPolicy.ALWAYS, + 1, + 1, + generation > 0); + sink.transfer(event); + // Retrying the same delivery must still be idempotent after reopening the writer. + sink.transfer(event); + } finally { + event.decreaseReferenceCount(getClass().getName(), false); + } + } + final List streams = + new LogicalBackupArchiveReader().read(directory, false); + Assert.assertEquals(1, streams.size()); + Assert.assertEquals(2, streams.get(0).getEventGroups().size()); + } + @Test public void testTemporaryValidationDoesNotCreateBackupStreams() throws Exception { final Path directory = temporaryFolder.getRoot().toPath().resolve("backup"); From 5c9adc8a80c5ac176f8d780b840a28b31d3172b7 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 4 Nov 2026 10:12:43 +0800 Subject: [PATCH 14/17] Rollback logical backup records after fsync failure --- .../logicalbackup/LogicalBackupWriter.java | 56 +++++++++---- .../LogicalBackupWriterTest.java | 80 +++++++++++++++++++ 2 files changed, 122 insertions(+), 14 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java index 86536587efef9..b0771ca021a54 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java @@ -286,20 +286,38 @@ public synchronized long writeEvent( } throw e; } - if (forceIfNeeded()) { - manifest.lastDurableSequence = manifest.lastSequence; + try { + if (forceIfNeeded()) { + manifest.lastDurableSequence = manifest.lastSequence; + } + updateActiveSegmentManifest(); + manifest.lastEventGroupId = eventGroupId.toString(); + manifest.lastEventDigest = eventDigest; + manifest.lastEventFirstSequence = firstSequence; + writeManifest(); + } catch (final IOException | RuntimeException e) { + try { + rollbackTo(eventStartPosition); + } catch (final IOException rollbackException) { + e.addSuppressed(rollbackException); + } + throw e; } - updateActiveSegmentManifest(); - manifest.lastEventGroupId = eventGroupId.toString(); - manifest.lastEventDigest = eventDigest; - manifest.lastEventFirstSequence = firstSequence; - writeManifest(); return firstSequence; } public synchronized void writeControl( final LogicalBackupRecordType recordType, final long eventTime, final String metadata) throws IOException { + writeControl(recordType, eventTime, metadata, false); + } + + private void writeControl( + final LogicalBackupRecordType recordType, + final long eventTime, + final String metadata, + final boolean skippedEvent) + throws IOException { ensureOpen(); final String safeMetadata = metadata == null ? "" : metadata; rollBeforeControlIfNecessary(safeMetadata); @@ -314,18 +332,28 @@ public synchronized void writeControl( } throw e; } - if (forceIfNeeded()) { - manifest.lastDurableSequence = manifest.lastSequence; + try { + if (forceIfNeeded()) { + manifest.lastDurableSequence = manifest.lastSequence; + } + updateActiveSegmentManifest(); + if (skippedEvent) { + manifest.skippedEventCount++; + } + writeManifest(); + } catch (final IOException | RuntimeException e) { + try { + rollbackTo(recordStartPosition); + } catch (final IOException rollbackException) { + e.addSuppressed(rollbackException); + } + throw e; } - updateActiveSegmentManifest(); - writeManifest(); } public synchronized void recordSkippedEvent(final long eventTime, final String metadata) throws IOException { - writeControl(LogicalBackupRecordType.SKIPPED_EVENT, eventTime, metadata); - manifest.skippedEventCount++; - writeManifest(); + writeControl(LogicalBackupRecordType.SKIPPED_EVENT, eventTime, metadata, true); } public synchronized LogicalBackupManifest getManifest() { diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java index b4b97c233fb8c..6a1715a5db0d0 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java @@ -27,11 +27,15 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; import java.io.IOException; import java.io.RandomAccessFile; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; import java.util.AbstractList; @@ -91,6 +95,48 @@ public void testDuplicateEventWithDifferentDigestFails() throws Exception { } } + @Test + public void testRetryAfterEventFsyncFailureDoesNotDuplicateGroup() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final UUID eventId = UUID.randomUUID(); + final TPipeTransferReq request = replayableRequest(1); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + final IOException failure = failNextFsync(writer); + try { + writer.writeEvent(eventId, 1, Collections.singletonList(request), "metadata"); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertSame(failure, expected); + } + writer.writeEvent(eventId, 1, Collections.singletonList(request), "metadata"); + } + final List streams = + new LogicalBackupArchiveReader().read(directory, false); + Assert.assertEquals(1, streams.get(0).getEventGroups().size()); + Assert.assertEquals( + request, streams.get(0).getEventGroups().get(0).getRequests().get(0).toTPipeTransferReq()); + } + + @Test + public void testRetrySkippedEventAfterFsyncFailureKeepsManifestConsistent() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + final IOException failure = failNextFsync(writer); + try { + writer.recordSkippedEvent(1, "unsupported-event"); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertSame(failure, expected); + } + writer.recordSkippedEvent(1, "unsupported-event"); + } + final List streams = + new LogicalBackupArchiveReader().read(directory, true); + Assert.assertEquals(1, streams.get(0).getManifest().skippedEventCount); + Assert.assertEquals( + Long.valueOf(1), streams.get(0).getManifest().operationCounts.get("SKIPPED_EVENT")); + } + @Test public void testAppendContinuesAndDeduplicatesLastEventAfterRestart() throws Exception { final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); @@ -533,6 +579,40 @@ private static LogicalBackupWriter writer( append); } + private static IOException failNextFsync(final LogicalBackupWriter writer) throws Exception { + final Field field = LogicalBackupWriter.class.getDeclaredField("channel"); + field.setAccessible(true); + final FileChannel original = (FileChannel) field.get(writer); + final FileChannel failing = + Mockito.mock( + FileChannel.class, + Mockito.withSettings() + .useConstructor() + .defaultAnswer( + invocation -> { + if ("implCloseChannel".equals(invocation.getMethod().getName())) { + original.close(); + return null; + } + try { + return invocation.getMethod().invoke(original, invocation.getArguments()); + } catch (final InvocationTargetException e) { + throw e.getCause(); + } + })); + final IOException failure = new IOException("simulated fsync failure"); + Mockito.doThrow(failure) + .doAnswer( + invocation -> { + original.force(true); + return null; + }) + .when(failing) + .force(true); + field.set(writer, failing); + return failure; + } + private static LogicalBackupManifest manifest() { final LogicalBackupManifest manifest = new LogicalBackupManifest(); manifest.backupId = "backup"; From f7d21bae403eed07f9ccdb5f8b0f242b11f5e2a1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 4 Nov 2026 10:47:40 +0800 Subject: [PATCH 15/17] Make logical backup rollover and rollback retries recoverable --- .../logicalbackup/LogicalBackupWriter.java | 128 ++++++++---- .../LogicalBackupWriterTest.java | 186 +++++++++++++++++- 2 files changed, 269 insertions(+), 45 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java index b0771ca021a54..7505f733f3f14 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriter.java @@ -90,6 +90,8 @@ public enum FsyncPolicy { private boolean closed; private boolean streamEnded; private boolean recoveredDuringOpen; + private long pendingRollbackPosition = -1; + private boolean segmentTransitionPending; public LogicalBackupWriter( final Path directory, @@ -673,40 +675,50 @@ private void sealSegment() throws IOException { if (channel == null || segmentRecordCount == 0) { return; } - final byte[] digest = segmentDigest.digest(); - final ByteBuffer footer = - ByteBuffer.allocate(LogicalBackupFormat.SEGMENT_FOOTER_SIZE).order(ByteOrder.BIG_ENDIAN); - footer.putInt(LogicalBackupFormat.FOOTER_MAGIC); - footer.putLong(segmentId); - footer.putLong(segmentFirstSequence); - footer.putLong(segmentLastSequence); - footer.putLong(segmentRecordCount); - footer.putLong(channel.position()); - footer.putLong(segmentCreatedAt); - footer.putLong(System.currentTimeMillis()); - footer.put(digest); - footer.putInt(0); - final byte[] footerBytes = footer.array(); - ByteBuffer.wrap(footerBytes) - .order(ByteOrder.BIG_ENDIAN) - .putInt( - LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4, - LogicalBackupFormat.crc32c( - footerBytes, 0, LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4)); - writeFully(ByteBuffer.wrap(footerBytes)); - channel.force(true); - final LogicalBackupManifest.Segment segment = - manifest.segments.get(manifest.segments.size() - 1); - segment.firstSequence = segmentFirstSequence; - segment.lastSequence = segmentLastSequence; - segment.recordCount = segmentRecordCount; - segment.sizeBytes = channel.position(); - segment.sha256 = LogicalBackupFormat.toHex(digest); - segment.status = "SEALED"; - manifest.lastDurableSequence = segmentLastSequence; + final long footerStartPosition = channel.position(); + try { + final byte[] digest = segmentDigest.digest(); + final ByteBuffer footer = + ByteBuffer.allocate(LogicalBackupFormat.SEGMENT_FOOTER_SIZE).order(ByteOrder.BIG_ENDIAN); + footer.putInt(LogicalBackupFormat.FOOTER_MAGIC); + footer.putLong(segmentId); + footer.putLong(segmentFirstSequence); + footer.putLong(segmentLastSequence); + footer.putLong(segmentRecordCount); + footer.putLong(footerStartPosition); + footer.putLong(segmentCreatedAt); + footer.putLong(System.currentTimeMillis()); + footer.put(digest); + footer.putInt(0); + final byte[] footerBytes = footer.array(); + ByteBuffer.wrap(footerBytes) + .order(ByteOrder.BIG_ENDIAN) + .putInt( + LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4, + LogicalBackupFormat.crc32c( + footerBytes, 0, LogicalBackupFormat.SEGMENT_FOOTER_SIZE - 4)); + writeFully(ByteBuffer.wrap(footerBytes)); + channel.force(true); + final LogicalBackupManifest.Segment segment = + manifest.segments.get(manifest.segments.size() - 1); + segment.firstSequence = segmentFirstSequence; + segment.lastSequence = segmentLastSequence; + segment.recordCount = segmentRecordCount; + segment.sizeBytes = channel.position(); + segment.sha256 = LogicalBackupFormat.toHex(digest); + segment.status = "SEALED"; + manifest.lastDurableSequence = segmentLastSequence; + writeManifest(); + } catch (final IOException | RuntimeException e) { + try { + rollbackTo(footerStartPosition); + } catch (final IOException rollbackException) { + e.addSuppressed(rollbackException); + } + throw e; + } channel.close(); channel = null; - writeManifest(); } private boolean forceIfNeeded() throws IOException { @@ -932,6 +944,23 @@ private void writeFully(final ByteBuffer buffer) throws IOException { } private void ensureOpen() throws IOException { + if (!closed) { + // A failed rollback must finish before deduplication or another append can use writer state. + if (pendingRollbackPosition >= 0) { + rollbackTo(pendingRollbackPosition); + } + if (segmentTransitionPending) { + if (channel != null) { + channel.close(); + channel = null; + } + openOrCreateSegment(); + rebuildManifestFromSegments(); + updateActiveSegmentManifest(); + writeManifest(); + segmentTransitionPending = false; + } + } if (closed || channel == null) { throw new IOException( LogicalBackupMessages.EXCEPTION_LOGICAL_BACKUP_WRITER_IS_CLOSED_EE463BBB); @@ -944,12 +973,11 @@ public synchronized void close() throws IOException { return; } try { - if (channel != null) { - if (!streamEnded) { - writeControl(LogicalBackupRecordType.STREAM_END, System.currentTimeMillis(), ""); - } - sealSegment(); + ensureOpen(); + if (!streamEnded) { + writeControl(LogicalBackupRecordType.STREAM_END, System.currentTimeMillis(), ""); } + sealSegment(); manifest.status = manifest.skippedEventCount > 0 ? "INCOMPLETE" : "SEALED"; manifest.closedAt = Instant.now().toString(); manifest.sealedAt = manifest.closedAt; @@ -979,9 +1007,7 @@ private void rollBeforeEventIfNecessary( if (channel.position() > LogicalBackupFormat.SEGMENT_HEADER_SIZE && channel.position() + requiredBytes + LogicalBackupFormat.SEGMENT_FOOTER_SIZE > segmentSizeBytes) { - sealSegment(); - segmentId++; - openNewSegment(); + rollSegment(); } } @@ -990,12 +1016,19 @@ private void rollBeforeControlIfNecessary(final String metadata) throws IOExcept if (channel.position() > LogicalBackupFormat.SEGMENT_HEADER_SIZE && channel.position() + requiredBytes + LogicalBackupFormat.SEGMENT_FOOTER_SIZE > segmentSizeBytes) { - sealSegment(); - segmentId++; - openNewSegment(); + rollSegment(); } } + private void rollSegment() throws IOException { + // If either stage fails, reopen from the segment files before accepting another write. + segmentTransitionPending = true; + sealSegment(); + segmentId++; + openNewSegment(); + segmentTransitionPending = false; + } + private static long frameSize(final long metadataBytes, final long payloadBytes) { return LogicalBackupFormat.RECORD_HEADER_SIZE + metadataBytes + payloadBytes + Integer.BYTES; } @@ -1025,11 +1058,19 @@ private static void updateEventDigest( } private void rollbackTo(final long position) throws IOException { + pendingRollbackPosition = position; final Path segmentPath = currentSegmentPath(); + if (channel == null || !channel.isOpen()) { + channel = FileChannel.open(segmentPath, StandardOpenOption.WRITE, StandardOpenOption.READ); + } channel.truncate(position); channel.force(true); channel.close(); channel = null; + final LogicalBackupManifest.Segment segment = + manifest.segments.get(manifest.segments.size() - 1); + segment.status = "ACTIVE"; + segment.sha256 = null; final LogicalBackupSegmentReader.ScanResult scan = reader.scan(segmentPath, false); openExistingSegment(segmentPath, scan); rebuildManifestFromSegments(); @@ -1040,6 +1081,7 @@ private void rollbackTo(final long position) throws IOException { lastFsyncAt = System.currentTimeMillis(); updateActiveSegmentManifest(); writeManifest(); + pendingRollbackPosition = -1; } private void rebuildManifestFromSegments() throws IOException { diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java index 6a1715a5db0d0..4e22e49f11e9d 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/logicalbackup/LogicalBackupWriterTest.java @@ -43,6 +43,7 @@ import java.util.Collections; import java.util.List; import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; public class LogicalBackupWriterTest { @@ -137,6 +138,70 @@ public void testRetrySkippedEventAfterFsyncFailureKeepsManifestConsistent() thro Long.valueOf(1), streams.get(0).getManifest().operationCounts.get("SKIPPED_EVENT")); } + @Test + public void testRetryAfterRollbackFsyncFailure() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final UUID eventId = UUID.randomUUID(); + try (final LogicalBackupWriter writer = writer(directory, 4096, false)) { + final IOException failure = failFsyncs(writer, 2); + try { + writer.writeEvent(eventId, 1, Collections.singletonList(replayableRequest(1)), "metadata"); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertSame(failure, expected); + Assert.assertEquals(1, expected.getSuppressed().length); + } + Assert.assertEquals( + 0, + writer.writeEvent( + eventId, 1, Collections.singletonList(replayableRequest(1)), "metadata")); + } + Assert.assertEquals( + 1, new LogicalBackupArchiveReader().read(directory, false).get(0).getEventGroups().size()); + } + + @Test + public void testRetryAfterManifestAndRollbackFsyncFailure() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final UUID eventId = UUID.randomUUID(); + try (final LogicalBackupWriter writer = + new LogicalBackupWriter( + directory, + manifest(), + 4096, + MAX_RECORD_BYTES, + LogicalBackupWriter.FsyncPolicy.NONE, + 1, + 1, + false)) { + final IOException rollbackFailure = failNextFsync(writer); + final Path blockedManifest = + directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME + ".tmp"); + Files.createDirectory(blockedManifest); + final Path blocker = Files.writeString(blockedManifest.resolve("blocker"), "blocker"); + try { + writer.writeEvent(eventId, 1, Collections.singletonList(replayableRequest(1)), "metadata"); + Assert.fail(); + } catch (final java.nio.file.DirectoryNotEmptyException expected) { + Assert.assertEquals(blockedManifest, Path.of(expected.getFile())); + Assert.assertArrayEquals(new Throwable[] {rollbackFailure}, expected.getSuppressed()); + } finally { + Files.delete(blocker); + Files.delete(blockedManifest); + } + Assert.assertEquals( + 0, + writer.writeEvent( + eventId, 1, Collections.singletonList(replayableRequest(1)), "metadata")); + } + final LogicalBackupArchiveReader.BackupStream stream = + new LogicalBackupArchiveReader().read(directory, false).get(0); + Assert.assertEquals(1, stream.getEventGroups().size()); + Assert.assertEquals( + replayableRequest(1), + stream.getEventGroups().get(0).getRequests().get(0).toTPipeTransferReq()); + } + @Test public void testAppendContinuesAndDeduplicatesLastEventAfterRestart() throws Exception { final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); @@ -366,6 +431,114 @@ public void testSegmentRollover() throws Exception { new LogicalBackupSegmentReader(MAX_RECORD_BYTES).scan(segments.get(1), false).isSealed()); } + @Test + public void testRetryRolloverAfterFsyncFailure() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final UUID eventId = UUID.randomUUID(); + try (final LogicalBackupWriter writer = writer(directory, 500, false)) { + writer.writeEvent( + UUID.randomUUID(), 1, Collections.singletonList(replayableRequest(1)), "metadata"); + final IOException failure = failNextFsync(writer); + try { + writer.writeEvent(eventId, 2, Collections.singletonList(replayableRequest(2)), "metadata"); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertSame(failure, expected); + } + Assert.assertEquals( + 3, + writer.writeEvent( + eventId, 2, Collections.singletonList(replayableRequest(2)), "metadata")); + } + final LogicalBackupArchiveReader.BackupStream stream = + new LogicalBackupArchiveReader().read(directory, false).get(0); + Assert.assertEquals(2, stream.getEventGroups().size()); + Assert.assertEquals( + replayableRequest(2), + stream.getEventGroups().get(1).getRequests().get(0).toTPipeTransferReq()); + } + + @Test + public void testRetryRolloverAfterManifestFailure() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final UUID eventId = UUID.randomUUID(); + try (final LogicalBackupWriter writer = writer(directory, 500, false)) { + writer.writeEvent( + UUID.randomUUID(), 1, Collections.singletonList(replayableRequest(1)), "metadata"); + final Path blockedManifest = + directory.resolve(LogicalBackupFormat.MANIFEST_FILE_NAME + ".tmp"); + Files.createDirectory(blockedManifest); + final Path blocker = Files.writeString(blockedManifest.resolve("blocker"), "blocker"); + try { + writer.writeEvent(eventId, 2, Collections.singletonList(replayableRequest(2)), "metadata"); + Assert.fail(); + } catch (final java.nio.file.DirectoryNotEmptyException expected) { + Assert.assertEquals(blockedManifest, Path.of(expected.getFile())); + } finally { + Files.delete(blocker); + Files.delete(blockedManifest); + } + Assert.assertEquals( + 3, + writer.writeEvent( + eventId, 2, Collections.singletonList(replayableRequest(2)), "metadata")); + } + Assert.assertEquals( + 2, new LogicalBackupArchiveReader().read(directory, false).get(0).getEventGroups().size()); + } + + @Test + public void testRetryRolloverAfterSegmentCreationFailure() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + final UUID eventId = UUID.randomUUID(); + try (final LogicalBackupWriter writer = writer(directory, 500, false)) { + writer.writeEvent( + UUID.randomUUID(), 1, Collections.singletonList(replayableRequest(1)), "metadata"); + final Path blockedSegment = directory.resolve(segmentFileName(1) + ".tmp"); + Files.createDirectory(blockedSegment); + final Path blocker = Files.writeString(blockedSegment.resolve("blocker"), "blocker"); + try { + writer.writeEvent(eventId, 2, Collections.singletonList(replayableRequest(2)), "metadata"); + Assert.fail(); + } catch (final java.nio.file.DirectoryNotEmptyException expected) { + Assert.assertEquals(blockedSegment, Path.of(expected.getFile())); + } finally { + Files.delete(blocker); + Files.delete(blockedSegment); + } + Assert.assertEquals( + 3, + writer.writeEvent( + eventId, 2, Collections.singletonList(replayableRequest(2)), "metadata")); + } + Assert.assertEquals( + 2, new LogicalBackupArchiveReader().read(directory, false).get(0).getEventGroups().size()); + } + + @Test + public void testRetryControlRolloverAfterFsyncFailure() throws Exception { + final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); + try (final LogicalBackupWriter writer = writer(directory, 258, false)) { + writer.writeControl(LogicalBackupRecordType.HEARTBEAT, 1, "first"); + final IOException failure = failNextFsync(writer); + try { + writer.writeControl(LogicalBackupRecordType.HEARTBEAT, 2, "second"); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertSame(failure, expected); + } + writer.writeControl(LogicalBackupRecordType.HEARTBEAT, 2, "second"); + } + Assert.assertEquals( + Long.valueOf(2), + new LogicalBackupArchiveReader() + .read(directory, false) + .get(0) + .getManifest() + .operationCounts + .get("HEARTBEAT")); + } + @Test public void testSealedSegmentAdvancesDurableSequenceWithFsyncNone() throws Exception { final Path directory = temporaryFolder.newFolder().toPath().resolve("backup"); @@ -580,6 +753,11 @@ private static LogicalBackupWriter writer( } private static IOException failNextFsync(final LogicalBackupWriter writer) throws Exception { + return failFsyncs(writer, 1); + } + + private static IOException failFsyncs(final LogicalBackupWriter writer, final int count) + throws Exception { final Field field = LogicalBackupWriter.class.getDeclaredField("channel"); field.setAccessible(true); final FileChannel original = (FileChannel) field.get(writer); @@ -601,9 +779,13 @@ private static IOException failNextFsync(final LogicalBackupWriter writer) throw } })); final IOException failure = new IOException("simulated fsync failure"); - Mockito.doThrow(failure) - .doAnswer( + final AtomicInteger attempts = new AtomicInteger(); + Mockito.doAnswer( invocation -> { + final int attempt = attempts.getAndIncrement(); + if (attempt < count) { + throw attempt == 0 ? failure : new IOException("simulated rollback fsync failure"); + } original.force(true); return null; }) From e8f181cdf51e6bc9d70781d301cdd6ac16976202 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:27:51 +0800 Subject: [PATCH 16/17] Rebuild logical backup snapshot staging on resume --- .../tool/pipe/PipeLogicalBackupTool.java | 56 ++-- .../tool/pipe/PipeLogicalBackupToolTest.java | 256 +++++++++++++++++- 2 files changed, 293 insertions(+), 19 deletions(-) diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java index 69399d791aaae..19d403325b1bf 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupTool.java @@ -925,20 +925,6 @@ private static int importBackup(final CommandLine line) throws Exception { return 0; } (line.hasOption(OPTION_JSON) ? System.err : System.out).println(GSON.toJson(report)); - final java.util.concurrent.atomic.AtomicBoolean uncertainRequest = - new java.util.concurrent.atomic.AtomicBoolean( - state.inProgress != null - && state.inProgress.uncertain - && streams.stream() - .flatMap(stream -> stream.getEventGroups().stream()) - .anyMatch( - group -> - group - .getEventGroupId() - .toString() - .equals(state.inProgress.eventGroupId) - && state.inProgress.nextRequestIndex - < group.getRequests().size())); try { state.status = "RUNNING"; state.lastError = null; @@ -949,7 +935,8 @@ private static int importBackup(final CommandLine line) throws Exception { checkpoint, state, stream, - request -> transfer(client, request, uncertainRequest.getAndSet(false)), + (request, allowIdempotentReplay) -> + transfer(client, request, allowIdempotentReplay), checkpointState -> writeCheckpoint(checkpoint, checkpointState)); } state.status = "COMPLETED"; @@ -1032,11 +1019,15 @@ static long importStream( if (nextRequestIndex < 0 || nextRequestIndex > requests.size()) { throw invalidCheckpoint(checkpoint); } + final boolean uncertainRequest = checkpointState.inProgress.uncertain; + replaySnapshotPieces(requests, nextRequestIndex, requestTransfer); for (int index = nextRequestIndex; index < requests.size(); index++) { checkpointState.inProgress.uncertain = true; checkpointPersister.persist(checkpointState); try { - requestTransfer.transfer(requests.get(index).toTPipeTransferReq()); + requestTransfer.transfer( + requests.get(index).toTPipeTransferReq(), + uncertainRequest && index == nextRequestIndex); } catch (final RestoreRequestRejectedException e) { checkpointState.inProgress.uncertain = false; throw e; @@ -1054,6 +1045,37 @@ static long importStream( return importedGroups; } + private static void replaySnapshotPieces( + final List requests, + final int nextRequestIndex, + final RequestTransfer requestTransfer) + throws Exception { + if (nextRequestIndex == requests.size()) { + return; + } + final short nextType = requests.get(nextRequestIndex).getRequestType(); + final short pieceType; + if (nextType == PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_PIECE.getType() + || nextType == PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL.getType()) { + pieceType = PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_PIECE.getType(); + } else if (nextType == PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_PIECE.getType() + || nextType == PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_SEAL.getType()) { + pieceType = PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_PIECE.getType(); + } else { + return; + } + + // A new receiver session discards staged files. Recreate the preceding pieces without changing + // the durable checkpoint: another interruption must retain the original uncertain request. + int firstPieceIndex = nextRequestIndex; + while (firstPieceIndex > 0 && requests.get(firstPieceIndex - 1).getRequestType() == pieceType) { + firstPieceIndex--; + } + for (int index = firstPieceIndex; index < nextRequestIndex; index++) { + requestTransfer.transfer(requests.get(index).toTPipeTransferReq(), false); + } + } + private static List read(final CommandLine line) throws IOException { final String source = input(line); try (final BackupInput input = BackupInput.open(Path.of(source))) { @@ -1516,7 +1538,7 @@ private RestoreRequestRejectedException(final String message) { @FunctionalInterface interface RequestTransfer { - void transfer(TPipeTransferReq request) throws Exception; + void transfer(TPipeTransferReq request, boolean allowIdempotentReplay) throws Exception; } @FunctionalInterface diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java index 859e5cb555f97..b98e5c4a04f2d 100644 --- a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java @@ -29,6 +29,8 @@ import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferSchemaSnapshotPieceReq; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferSchemaSnapshotSealReq; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; @@ -54,6 +56,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -512,6 +515,7 @@ public void testImportCheckpointResumesAtRequestGranularity() throws Exception { final PipeLogicalBackupTool.ImportCheckpoint state = PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); final List transferredBodies = new ArrayList<>(); + final List replayPermissions = new ArrayList<>(); final AtomicBoolean failCheckpointOnce = new AtomicBoolean(true); try { @@ -519,7 +523,10 @@ public void testImportCheckpointResumesAtRequestGranularity() throws Exception { checkpoint, state, stream, - request -> transferredBodies.add((int) request.getBody()[0]), + (request, allowIdempotentReplay) -> { + transferredBodies.add((int) request.getBody()[0]); + replayPermissions.add(allowIdempotentReplay); + }, checkpointState -> { if (checkpointState.inProgress != null && checkpointState.inProgress.nextRequestIndex == 1 @@ -545,15 +552,260 @@ public void testImportCheckpointResumesAtRequestGranularity() throws Exception { checkpoint, durableState, stream, - request -> transferredBodies.add((int) request.getBody()[0]), + (request, allowIdempotentReplay) -> { + transferredBodies.add((int) request.getBody()[0]); + replayPermissions.add(allowIdempotentReplay); + }, checkpointState -> PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState))); Assert.assertEquals(Arrays.asList(1, 1, 2), transferredBodies); + Assert.assertEquals(Arrays.asList(false, true, false), replayPermissions); Assert.assertNull(durableState.inProgress); Assert.assertEquals( Long.valueOf(stream.getEventGroups().get(0).getLastSequence()), durableState.appliedSequences.get(stream.getManifest().streamId)); } + @Test + public void testSnapshotResumeRebuildsAcknowledgedPieces() throws Exception { + for (final boolean config : Arrays.asList(false, true)) { + final Path source = temporaryFolder.newFolder().toPath().resolve("backup"); + writeBackup(source, "snapshot", config ? "config" : "schema", snapshotRequests(config)); + final List streams = new LogicalBackupArchiveReader().read(source, false); + final Path checkpoint = temporaryFolder.getRoot().toPath().resolve(config + ".checkpoint"); + final PipeLogicalBackupTool.ImportCheckpoint state = + PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); + final IoTDBSyncClient firstConnection = snapshotClient(new AtomicBoolean()); + final List sent = new ArrayList<>(); + try { + PipeLogicalBackupTool.importStream( + checkpoint, + state, + streams.get(0), + (request, allowIdempotentReplay) -> { + if (sent.size() == 3) { + throw new IOException("connection interrupted between snapshot pieces"); + } + PipeLogicalBackupTool.transfer(firstConnection, request, allowIdempotentReplay); + sent.add(request); + }, + checkpointState -> PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState)); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertEquals(3, sent.size()); + } + + final PipeLogicalBackupTool.ImportCheckpoint durableState = + PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); + Assert.assertEquals(3, durableState.inProgress.nextRequestIndex); + final AtomicBoolean applied = new AtomicBoolean(); + // A fresh handshake has no staged files from the first connection. + final IoTDBSyncClient freshConnection = snapshotClient(applied); + final List replayPermissions = new ArrayList<>(); + Assert.assertEquals( + 1, + PipeLogicalBackupTool.importStream( + checkpoint, + durableState, + streams.get(0), + (request, allowIdempotentReplay) -> { + replayPermissions.add(allowIdempotentReplay); + PipeLogicalBackupTool.transfer(freshConnection, request, allowIdempotentReplay); + }, + checkpointState -> + PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState))); + Assert.assertTrue(applied.get()); + Assert.assertEquals(Arrays.asList(false, false, false, true, false), replayPermissions); + Mockito.verify(freshConnection, Mockito.times(5)).pipeTransfer(Mockito.any()); + } + } + + @Test + public void testSnapshotReconstructionInterruptionPreservesUncertainSeal() throws Exception { + for (final boolean config : Arrays.asList(false, true)) { + final Path source = temporaryFolder.newFolder().toPath().resolve("backup"); + writeBackup(source, "snapshot", config ? "config" : "schema", snapshotRequests(config)); + final List streams = new LogicalBackupArchiveReader().read(source, false); + final Path checkpoint = temporaryFolder.getRoot().toPath().resolve(config + ".checkpoint"); + final PipeLogicalBackupTool.ImportCheckpoint state = + PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); + final AtomicBoolean applied = new AtomicBoolean(); + final IoTDBSyncClient firstConnection = snapshotClient(applied); + try { + PipeLogicalBackupTool.importStream( + checkpoint, + state, + streams.get(0), + (request, allowIdempotentReplay) -> { + PipeLogicalBackupTool.transfer(firstConnection, request, allowIdempotentReplay); + if (applied.get()) { + throw new IOException("seal response lost after applying the snapshot"); + } + }, + checkpointState -> PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState)); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(applied.get()); + } + + final PipeLogicalBackupTool.ImportCheckpoint interruptedState = + PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); + Assert.assertEquals(4, interruptedState.inProgress.nextRequestIndex); + Assert.assertTrue(interruptedState.inProgress.uncertain); + final IoTDBSyncClient interruptedConnection = snapshotClient(applied); + try { + PipeLogicalBackupTool.importStream( + checkpoint, + interruptedState, + streams.get(0), + (request, allowIdempotentReplay) -> { + Assert.assertFalse(allowIdempotentReplay); + Assert.assertEquals( + config + ? PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_PIECE.getType() + : PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_PIECE.getType(), + request.getType()); + PipeLogicalBackupTool.transfer(interruptedConnection, request, allowIdempotentReplay); + throw new IOException("another interruption while reconstructing staged files"); + }, + checkpointState -> PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState)); + Assert.fail(); + } catch (final IOException expected) { + // Reconstructing temporary files must not replace the durable uncertain seal. + } + PipeLogicalBackupTool.writeCheckpoint(checkpoint, interruptedState); + final PipeLogicalBackupTool.ImportCheckpoint durableState = + PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); + Assert.assertEquals(4, durableState.inProgress.nextRequestIndex); + Assert.assertTrue(durableState.inProgress.uncertain); + final IoTDBSyncClient freshConnection = snapshotClient(applied); + final List replayPermissions = new ArrayList<>(); + Assert.assertEquals( + 1, + PipeLogicalBackupTool.importStream( + checkpoint, + durableState, + streams.get(0), + (request, allowIdempotentReplay) -> { + replayPermissions.add(allowIdempotentReplay); + PipeLogicalBackupTool.transfer(freshConnection, request, allowIdempotentReplay); + }, + checkpointState -> + PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState))); + Mockito.verify(freshConnection, Mockito.times(5)).pipeTransfer(Mockito.any()); + Assert.assertEquals(Arrays.asList(false, false, false, false, true), replayPermissions); + } + } + + @Test + public void testSnapshotResumeDoesNotReplayAcknowledgedSeal() throws Exception { + final Path source = temporaryFolder.newFolder().toPath().resolve("backup"); + writeBackup(source, "snapshot", "schema", snapshotRequests(false)); + final List streams = new LogicalBackupArchiveReader().read(source, false); + final Path checkpoint = temporaryFolder.getRoot().toPath().resolve("snapshot.checkpoint"); + final PipeLogicalBackupTool.ImportCheckpoint state = + PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); + final AtomicBoolean applied = new AtomicBoolean(); + final IoTDBSyncClient client = snapshotClient(applied); + try { + PipeLogicalBackupTool.importStream( + checkpoint, + state, + streams.get(0), + (request, allowIdempotentReplay) -> + PipeLogicalBackupTool.transfer(client, request, allowIdempotentReplay), + checkpointState -> { + if (checkpointState.inProgress == null) { + throw new IOException("group completion checkpoint interrupted"); + } + PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState); + }); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(applied.get()); + } + final PipeLogicalBackupTool.ImportCheckpoint durableState = + PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); + Assert.assertEquals(5, durableState.inProgress.nextRequestIndex); + Assert.assertFalse(durableState.inProgress.uncertain); + Assert.assertEquals( + 1, + PipeLogicalBackupTool.importStream( + checkpoint, + durableState, + streams.get(0), + (request, allowIdempotentReplay) -> + Assert.fail("an acknowledged seal must not be sent again"), + checkpointState -> PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState))); + Assert.assertNull(durableState.inProgress); + } + + private static List snapshotRequests(final boolean config) throws IOException { + final List requests = + Arrays.asList( + PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq("mtree", 0, new byte[] {1, 2}), + PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq("mtree", 2, new byte[] {3, 4}), + PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq("tags", 0, new byte[] {5}), + PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq("tags", 1, new byte[] {6}), + PipeTransferSchemaSnapshotSealReq.toTPipeTransferReq( + "root.**", ".*", ".*", true, false, "mtree", 4, "tags", 2, null, 0, "root.db", "")); + // Schema and config snapshots share the file-piece and file-seal wire formats. + if (config) { + for (int index = 0; index < requests.size(); index++) { + requests + .get(index) + .setType( + index == requests.size() - 1 + ? PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_SEAL.getType() + : PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_PIECE.getType()); + } + } + return requests; + } + + private static IoTDBSyncClient snapshotClient(final AtomicBoolean applied) throws Exception { + final Map stagedFiles = new LinkedHashMap<>(); + final IoTDBSyncClient client = Mockito.mock(IoTDBSyncClient.class); + Mockito.when(client.pipeTransfer(Mockito.any())) + .thenAnswer( + invocation -> { + final TPipeTransferReq request = invocation.getArgument(0); + if (request.getType() == PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_PIECE.getType() + || request.getType() + == PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_PIECE.getType()) { + final PipeTransferSchemaSnapshotPieceReq piece = + PipeTransferSchemaSnapshotPieceReq.fromTPipeTransferReq(request.deepCopy()); + final ByteArrayOutputStream file = + stagedFiles.computeIfAbsent( + piece.getFileName(), key -> new ByteArrayOutputStream()); + if (file.size() != piece.getStartWritingOffset()) { + return new TPipeTransferResp( + new TSStatus(TSStatusCode.PIPE_TRANSFER_FILE_OFFSET_RESET.getStatusCode())); + } + file.write(piece.getFilePiece()); + return new TPipeTransferResp( + new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + } + final PipeTransferSchemaSnapshotSealReq seal = + PipeTransferSchemaSnapshotSealReq.fromTPipeTransferReq(request.deepCopy()); + for (int index = 0; index < seal.getFileNames().size(); index++) { + final ByteArrayOutputStream file = stagedFiles.get(seal.getFileNames().get(index)); + if (file == null || file.size() != seal.getFileLengths().get(index)) { + return new TPipeTransferResp( + new TSStatus(TSStatusCode.PIPE_TRANSFER_FILE_ERROR.getStatusCode())); + } + } + Assert.assertArrayEquals( + new byte[] {1, 2, 3, 4}, stagedFiles.get("mtree").toByteArray()); + Assert.assertArrayEquals(new byte[] {5, 6}, stagedFiles.get("tags").toByteArray()); + return new TPipeTransferResp( + new TSStatus( + applied.getAndSet(true) + ? TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode() + : TSStatusCode.SUCCESS_STATUS.getStatusCode())); + }); + return client; + } + private static void writeBackup(final Path directory) throws Exception { writeBackup(directory, Collections.singletonList(request(3))); } From 8dc0f37c5d072284ef46bf9be431d3e7c7afa246 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 4 Nov 2026 12:52:05 +0800 Subject: [PATCH 17/17] Use valid Config snapshot seals in logical backup tests --- .../tool/pipe/PipeLogicalBackupToolTest.java | 204 ++++++++++++------ 1 file changed, 138 insertions(+), 66 deletions(-) diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java index b98e5c4a04f2d..1ee6fd6c75924 100644 --- a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/pipe/PipeLogicalBackupToolTest.java @@ -29,6 +29,8 @@ import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupManifest; import org.apache.iotdb.commons.pipe.sink.logicalbackup.LogicalBackupWriter; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferFileSealReqV2; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferSchemaSnapshotPieceReq; import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferSchemaSnapshotSealReq; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; @@ -67,6 +69,11 @@ public class PipeLogicalBackupToolTest { + // CNSnapshotFileType.SCHEMA; ConfigNode is not a CLI dependency. + private static final String CONFIG_SCHEMA_FILE_TYPE = "4"; + // ConfigPhysicalPlanType.CreateDatabase and CreateSchemaTemplate. + private static final String CONFIG_SCHEMA_PLAN_TYPES = "200,800"; + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @Test @@ -574,7 +581,7 @@ public void testSnapshotResumeRebuildsAcknowledgedPieces() throws Exception { final Path checkpoint = temporaryFolder.getRoot().toPath().resolve(config + ".checkpoint"); final PipeLogicalBackupTool.ImportCheckpoint state = PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); - final IoTDBSyncClient firstConnection = snapshotClient(new AtomicBoolean()); + final IoTDBSyncClient firstConnection = snapshotClient(new AtomicBoolean(), config); final List sent = new ArrayList<>(); try { PipeLogicalBackupTool.importStream( @@ -599,7 +606,7 @@ public void testSnapshotResumeRebuildsAcknowledgedPieces() throws Exception { Assert.assertEquals(3, durableState.inProgress.nextRequestIndex); final AtomicBoolean applied = new AtomicBoolean(); // A fresh handshake has no staged files from the first connection. - final IoTDBSyncClient freshConnection = snapshotClient(applied); + final IoTDBSyncClient freshConnection = snapshotClient(applied, config); final List replayPermissions = new ArrayList<>(); Assert.assertEquals( 1, @@ -629,7 +636,7 @@ public void testSnapshotReconstructionInterruptionPreservesUncertainSeal() throw final PipeLogicalBackupTool.ImportCheckpoint state = PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); final AtomicBoolean applied = new AtomicBoolean(); - final IoTDBSyncClient firstConnection = snapshotClient(applied); + final IoTDBSyncClient firstConnection = snapshotClient(applied, config); try { PipeLogicalBackupTool.importStream( checkpoint, @@ -651,7 +658,7 @@ public void testSnapshotReconstructionInterruptionPreservesUncertainSeal() throw PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); Assert.assertEquals(4, interruptedState.inProgress.nextRequestIndex); Assert.assertTrue(interruptedState.inProgress.uncertain); - final IoTDBSyncClient interruptedConnection = snapshotClient(applied); + final IoTDBSyncClient interruptedConnection = snapshotClient(applied, config); try { PipeLogicalBackupTool.importStream( checkpoint, @@ -677,7 +684,7 @@ public void testSnapshotReconstructionInterruptionPreservesUncertainSeal() throw PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); Assert.assertEquals(4, durableState.inProgress.nextRequestIndex); Assert.assertTrue(durableState.inProgress.uncertain); - final IoTDBSyncClient freshConnection = snapshotClient(applied); + final IoTDBSyncClient freshConnection = snapshotClient(applied, config); final List replayPermissions = new ArrayList<>(); Assert.assertEquals( 1, @@ -698,71 +705,76 @@ public void testSnapshotReconstructionInterruptionPreservesUncertainSeal() throw @Test public void testSnapshotResumeDoesNotReplayAcknowledgedSeal() throws Exception { - final Path source = temporaryFolder.newFolder().toPath().resolve("backup"); - writeBackup(source, "snapshot", "schema", snapshotRequests(false)); - final List streams = new LogicalBackupArchiveReader().read(source, false); - final Path checkpoint = temporaryFolder.getRoot().toPath().resolve("snapshot.checkpoint"); - final PipeLogicalBackupTool.ImportCheckpoint state = - PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); - final AtomicBoolean applied = new AtomicBoolean(); - final IoTDBSyncClient client = snapshotClient(applied); - try { - PipeLogicalBackupTool.importStream( - checkpoint, - state, - streams.get(0), - (request, allowIdempotentReplay) -> - PipeLogicalBackupTool.transfer(client, request, allowIdempotentReplay), - checkpointState -> { - if (checkpointState.inProgress == null) { - throw new IOException("group completion checkpoint interrupted"); - } - PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState); - }); - Assert.fail(); - } catch (final IOException expected) { - Assert.assertTrue(applied.get()); - } - final PipeLogicalBackupTool.ImportCheckpoint durableState = - PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); - Assert.assertEquals(5, durableState.inProgress.nextRequestIndex); - Assert.assertFalse(durableState.inProgress.uncertain); - Assert.assertEquals( - 1, + for (final boolean config : new boolean[] {false, true}) { + final Path source = temporaryFolder.newFolder().toPath().resolve("backup"); + writeBackup(source, "snapshot", config ? "config" : "schema", snapshotRequests(config)); + final List streams = new LogicalBackupArchiveReader().read(source, false); + final Path checkpoint = temporaryFolder.getRoot().toPath().resolve(config + ".checkpoint"); + final PipeLogicalBackupTool.ImportCheckpoint state = + PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); + final AtomicBoolean applied = new AtomicBoolean(); + final IoTDBSyncClient client = snapshotClient(applied, config); + try { PipeLogicalBackupTool.importStream( checkpoint, - durableState, + state, streams.get(0), (request, allowIdempotentReplay) -> - Assert.fail("an acknowledged seal must not be sent again"), - checkpointState -> PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState))); - Assert.assertNull(durableState.inProgress); + PipeLogicalBackupTool.transfer(client, request, allowIdempotentReplay), + checkpointState -> { + if (checkpointState.inProgress == null) { + throw new IOException("group completion checkpoint interrupted"); + } + PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState); + }); + Assert.fail(); + } catch (final IOException expected) { + Assert.assertTrue(applied.get()); + } + final PipeLogicalBackupTool.ImportCheckpoint durableState = + PipeLogicalBackupTool.readCheckpoint(checkpoint, streams, "localhost", 6667, "root"); + Assert.assertEquals(5, durableState.inProgress.nextRequestIndex); + Assert.assertFalse(durableState.inProgress.uncertain); + Assert.assertEquals( + 1, + PipeLogicalBackupTool.importStream( + checkpoint, + durableState, + streams.get(0), + (request, allowIdempotentReplay) -> + Assert.fail("an acknowledged seal must not be sent again"), + checkpointState -> + PipeLogicalBackupTool.writeCheckpoint(checkpoint, checkpointState))); + Assert.assertNull(durableState.inProgress); + } } private static List snapshotRequests(final boolean config) throws IOException { - final List requests = - Arrays.asList( - PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq("mtree", 0, new byte[] {1, 2}), - PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq("mtree", 2, new byte[] {3, 4}), - PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq("tags", 0, new byte[] {5}), - PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq("tags", 1, new byte[] {6}), - PipeTransferSchemaSnapshotSealReq.toTPipeTransferReq( - "root.**", ".*", ".*", true, false, "mtree", 4, "tags", 2, null, 0, "root.db", "")); - // Schema and config snapshots share the file-piece and file-seal wire formats. if (config) { - for (int index = 0; index < requests.size(); index++) { - requests - .get(index) - .setType( - index == requests.size() - 1 - ? PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_SEAL.getType() - : PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_PIECE.getType()); - } + return Arrays.asList( + configPiece("snapshot", 0, new byte[] {1, 2}), + configPiece("snapshot", 2, new byte[] {3, 4}), + configPiece("template", 0, new byte[] {5}), + configPiece("template", 1, new byte[] {6}), + ConfigSeal.create()); } - return requests; + return Arrays.asList( + PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq("mtree", 0, new byte[] {1, 2}), + PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq("mtree", 2, new byte[] {3, 4}), + PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq("tags", 0, new byte[] {5}), + PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq("tags", 1, new byte[] {6}), + PipeTransferSchemaSnapshotSealReq.toTPipeTransferReq( + "root.**", ".*", ".*", true, false, "mtree", 4, "tags", 2, null, 0, "root.db", "")); + } + + private static TPipeTransferReq configPiece( + final String fileName, final long offset, final byte[] bytes) throws IOException { + return PipeTransferSchemaSnapshotPieceReq.toTPipeTransferReq(fileName, offset, bytes) + .setType(PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_PIECE.getType()); } - private static IoTDBSyncClient snapshotClient(final AtomicBoolean applied) throws Exception { + private static IoTDBSyncClient snapshotClient(final AtomicBoolean applied, final boolean config) + throws Exception { final Map stagedFiles = new LinkedHashMap<>(); final IoTDBSyncClient client = Mockito.mock(IoTDBSyncClient.class); Mockito.when(client.pipeTransfer(Mockito.any())) @@ -772,6 +784,12 @@ private static IoTDBSyncClient snapshotClient(final AtomicBoolean applied) throw if (request.getType() == PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_PIECE.getType() || request.getType() == PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_PIECE.getType()) { + Assert.assertEquals( + config + ? PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_PIECE.getType() + : PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_PIECE.getType(), + request.getType()); + // Both piece types use the shared PipeTransferFilePieceReq encoding. final PipeTransferSchemaSnapshotPieceReq piece = PipeTransferSchemaSnapshotPieceReq.fromTPipeTransferReq(request.deepCopy()); final ByteArrayOutputStream file = @@ -785,18 +803,50 @@ private static IoTDBSyncClient snapshotClient(final AtomicBoolean applied) throw return new TPipeTransferResp( new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); } - final PipeTransferSchemaSnapshotSealReq seal = - PipeTransferSchemaSnapshotSealReq.fromTPipeTransferReq(request.deepCopy()); - for (int index = 0; index < seal.getFileNames().size(); index++) { - final ByteArrayOutputStream file = stagedFiles.get(seal.getFileNames().get(index)); - if (file == null || file.size() != seal.getFileLengths().get(index)) { + final List fileNames; + final List fileLengths; + if (config) { + Assert.assertEquals( + PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_SEAL.getType(), request.getType()); + final PipeTransferFileSealReqV2 seal = + new ConfigSeal().translateFromTPipeTransferReq(request.deepCopy()); + Assert.assertEquals(CONFIG_SCHEMA_FILE_TYPE, seal.getParameters().get("fileType")); + Assert.assertEquals( + CONFIG_SCHEMA_PLAN_TYPES, seal.getParameters().get(ColumnHeaderConstant.TYPE)); + Assert.assertEquals( + "root.**", seal.getParameters().get(ColumnHeaderConstant.PATH_PATTERN)); + Assert.assertEquals( + ".*", seal.getParameters().get(ColumnHeaderConstant.TABLE_NAME)); + Assert.assertEquals( + ".*", seal.getParameters().get(PipeTransferFileSealReqV2.DATABASE_PATTERN)); + Assert.assertEquals("", seal.getParameters().get(PipeTransferFileSealReqV2.TREE)); + Assert.assertFalse( + seal.getParameters().containsKey(PipeTransferFileSealReqV2.TABLE)); + Assert.assertEquals("", seal.getParameters().get("authUserName")); + fileNames = seal.getFileNames(); + fileLengths = seal.getFileLengths(); + Assert.assertEquals(Arrays.asList("snapshot", "template"), fileNames); + Assert.assertEquals(Arrays.asList(4L, 2L), fileLengths); + } else { + Assert.assertEquals( + PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL.getType(), request.getType()); + final PipeTransferSchemaSnapshotSealReq seal = + PipeTransferSchemaSnapshotSealReq.fromTPipeTransferReq(request.deepCopy()); + fileNames = seal.getFileNames(); + fileLengths = seal.getFileLengths(); + } + for (int index = 0; index < fileNames.size(); index++) { + final ByteArrayOutputStream file = stagedFiles.get(fileNames.get(index)); + if (file == null || file.size() != fileLengths.get(index)) { return new TPipeTransferResp( new TSStatus(TSStatusCode.PIPE_TRANSFER_FILE_ERROR.getStatusCode())); } } Assert.assertArrayEquals( - new byte[] {1, 2, 3, 4}, stagedFiles.get("mtree").toByteArray()); - Assert.assertArrayEquals(new byte[] {5, 6}, stagedFiles.get("tags").toByteArray()); + new byte[] {1, 2, 3, 4}, + stagedFiles.get(config ? "snapshot" : "mtree").toByteArray()); + Assert.assertArrayEquals( + new byte[] {5, 6}, stagedFiles.get(config ? "template" : "tags").toByteArray()); return new TPipeTransferResp( new TSStatus( applied.getAndSet(true) @@ -806,6 +856,28 @@ private static IoTDBSyncClient snapshotClient(final AtomicBoolean applied) throw return client; } + private static final class ConfigSeal extends PipeTransferFileSealReqV2 { + private static TPipeTransferReq create() throws IOException { + // Match PipeTransferConfigSnapshotSealReq without adding a ConfigNode dependency. + final Map parameters = new LinkedHashMap<>(); + parameters.put(ColumnHeaderConstant.PATH_PATTERN, "root.**"); + parameters.put(DATABASE_PATTERN, ".*"); + parameters.put(ColumnHeaderConstant.TABLE_NAME, ".*"); + parameters.put(TREE, ""); + parameters.put("fileType", CONFIG_SCHEMA_FILE_TYPE); + parameters.put(ColumnHeaderConstant.TYPE, CONFIG_SCHEMA_PLAN_TYPES); + parameters.put("authUserName", ""); + return new ConfigSeal() + .convertToTPipeTransferReq( + Arrays.asList("snapshot", "template"), Arrays.asList(4L, 2L), parameters); + } + + @Override + protected PipeRequestType getPlanType() { + return PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_SEAL; + } + } + private static void writeBackup(final Path directory) throws Exception { writeBackup(directory, Collections.singletonList(request(3))); }