diff --git a/.github/scripts/checkstyle-pr.sh b/.github/scripts/checkstyle-pr.sh index bd86f8c8..34cdf29a 100644 --- a/.github/scripts/checkstyle-pr.sh +++ b/.github/scripts/checkstyle-pr.sh @@ -114,7 +114,7 @@ for module in "${!module_files[@]}"; do echo " - 运行 Checkstyle 检查(增量扫描)..." echo "$file_list" set +e - PROJECT_ROOT=$(pwd) + PROJECT_ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) rm -f "$report_file" output=$(cd "$module" && \ echo " Current directory: $(pwd)" && \ @@ -129,7 +129,7 @@ for module in "${!module_files[@]}"; do echo " ⚠️ not found: $include"; \ fi; \ done && \ - mvn checkstyle:check \ + mvn -f "$PROJECT_ROOT/pom.xml" checkstyle:check \ -Dcheckstyle.config.location="$PROJECT_ROOT/checkstyle/code-check-checkstyle.xml" \ -Dcheckstyle.violationSeverity=warning \ -Dcheckstyle.outputFormat=xml \ @@ -158,7 +158,7 @@ for module in "${!module_files[@]}"; do # (可选)生成 HTML 报告供人工查看 echo " - 生成 HTML 报告(可选)..." set +e - (cd "$module" && mvn checkstyle:checkstyle \ + (cd "$module" && mvn -f "$PROJECT_ROOT/pom.xml" checkstyle:checkstyle \ -Dcheckstyle.config.location="$PROJECT_ROOT/checkstyle/code-check-checkstyle.xml" \ -Dcheckstyle.includes="$file_list" \ -Dcheckstyle.violationSeverity=warning) > /dev/null 2>&1 diff --git a/.github/scripts/codeql-matrix.sh b/.github/scripts/codeql-matrix.sh new file mode 100644 index 00000000..1da56115 --- /dev/null +++ b/.github/scripts/codeql-matrix.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -euo pipefail + +java_build_mode="${1:-autobuild}" + +matrix_entries="" + +has_files() { + git ls-files "$@" | grep . >/dev/null +} + +add_entry() { + local entry="$1" + if [ -n "$matrix_entries" ]; then + matrix_entries="$matrix_entries,$entry" + else + matrix_entries="$entry" + fi +} + +if has_files '.github/workflows/*.yml' '.github/workflows/*.yaml'; then + add_entry '{"language":"actions","build-mode":"none"}' +fi + +if has_files '*.java'; then + add_entry "{\"language\":\"java-kotlin\",\"build-mode\":\"$java_build_mode\"}" +fi + +if has_files '*.js' '*.jsx' '*.ts' '*.tsx' '*.mjs' '*.cjs' '*.vue' '*.html'; then + add_entry '{"language":"javascript-typescript","build-mode":"none"}' +fi + +printf '{"include":[%s]}\n' "$matrix_entries" diff --git a/.github/workflows/codeql-full.yml b/.github/workflows/codeql-full.yml new file mode 100644 index 00000000..aeeece0b --- /dev/null +++ b/.github/workflows/codeql-full.yml @@ -0,0 +1,221 @@ +name: CodeQL Full Scan + +on: + schedule: + - cron: '34 7 * * 1' + workflow_dispatch: + +permissions: + contents: read + security-events: write + packages: read + actions: read + +jobs: + detect: + name: Detect CodeQL languages + runs-on: ubuntu-latest + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Build matrix + id: matrix + shell: bash + run: | + matrix=$(bash .github/scripts/codeql-matrix.sh manual) + printf 'matrix=%s\n' "$matrix" >> "$GITHUB_OUTPUT" + + analyze: + needs: detect + name: Full scan (${{ matrix.language }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.detect.outputs.matrix) }} + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Set up JDK 17 + if: matrix.language == 'java-kotlin' + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'temurin' + cache: maven + + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + + - name: Build project + if: matrix.language == 'java-kotlin' + run: mvn -B clean test-compile -DskipTests -Dcheckstyle.skip=true -Dpmd.skip=true -Dspotbugs.skip=true -Dcpd.skip=true + + - name: Prepare CodeQL SARIF directory + shell: bash + run: | + rm -rf codeql-sarif + mkdir -p codeql-sarif + + - name: Perform CodeQL Analysis + id: codeql-analysis + uses: github/codeql-action/analyze@v4 + with: + output: ${{ github.workspace }}/codeql-sarif + upload: always + category: "/codeql-full:${{ matrix.language }}" + + - name: Summarize CodeQL SARIF report + id: sarif-summary + if: always() + shell: bash + run: | + set -euo pipefail + + mkdir -p codeql-sarif + report_index="codeql-sarif/scan-files.txt" + sarif_count=0 + invalid_sarif=0 + violations=0 + + { + printf 'language=%s\n' '${{ matrix.language }}' + printf 'codeql_output=%s\n' '${{ github.workspace }}/codeql-sarif' + printf '\nGenerated SARIF files:\n' + } > "$report_index" + + while IFS= read -r -d '' file; do + sarif_count=$((sarif_count + 1)) + result_count=$(jq '[.runs[]?.results[]?] | length' "$file" 2>/dev/null || true) + + if [[ "$result_count" =~ ^[0-9]+$ ]]; then + violations=$((violations + result_count)) + printf '%s results=%s\n' "$file" "$result_count" >> "$report_index" + else + invalid_sarif=$((invalid_sarif + 1)) + printf '%s results=invalid-sarif\n' "$file" >> "$report_index" + fi + done < <(find codeql-sarif -type f -name '*.sarif' -print0) + + { + printf '\nSummary:\n' + printf 'sarif_count=%s\n' "$sarif_count" + printf 'invalid_sarif=%s\n' "$invalid_sarif" + printf 'violations=%s\n' "$violations" + } >> "$report_index" + + printf 'sarif_count=%s\n' "$sarif_count" >> "$GITHUB_OUTPUT" + printf 'invalid_sarif=%s\n' "$invalid_sarif" >> "$GITHUB_OUTPUT" + printf 'violations=%s\n' "$violations" >> "$GITHUB_OUTPUT" + + - name: Install SARIF tools + if: ${{ always() && hashFiles('codeql-sarif/**/*.sarif') != '' }} + run: python -m pip install sarif-tools + + - name: Generate CodeQL HTML report + id: html-report + if: ${{ always() && hashFiles('codeql-sarif/**/*.sarif') != '' }} + shell: bash + run: | + set -euo pipefail + + html_dir="codeql-html-report" + rm -rf "$html_dir" + mkdir -p "$html_dir" + + html_count=0 + while IFS= read -r -d '' sarif_file; do + sarif html "$sarif_file" --output "$html_dir" + html_count=$((html_count + 1)) + printf '%s -> %s/\n' "$sarif_file" "$html_dir" + done < <(find codeql-sarif -type f -name '*.sarif' -print0) + + index_file="$html_dir/reports.html" + { + printf '\n' + printf '\n' + printf 'CodeQL HTML Reports\n' + printf '\n' + printf '

CodeQL HTML Reports - %s

\n' '${{ matrix.language }}' + printf '\n' + printf '\n' + printf '\n' + } > "$index_file" + + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + printf '## CodeQL HTML report\n\n' + printf '| Metric | Result |\n' + printf '|------|------|\n' + printf '| Language | %s |\n' '${{ matrix.language }}' + printf '| HTML files | %s |\n' "$html_count" + printf '| Artifact | codeql-full-html-%s |\n' '${{ matrix.language }}' + } >> "$GITHUB_STEP_SUMMARY" + fi + + printf 'html_count=%s\n' "$html_count" >> "$GITHUB_OUTPUT" + + - name: Upload CodeQL SARIF report + if: always() + uses: actions/upload-artifact@v7 + with: + name: codeql-full-sarif-${{ matrix.language }} + path: codeql-sarif + if-no-files-found: error + retention-days: 30 + + - name: Upload CodeQL HTML report + if: ${{ always() && hashFiles('codeql-html-report/**/*.html') != '' }} + uses: actions/upload-artifact@v7 + with: + name: codeql-full-html-${{ matrix.language }} + path: codeql-html-report + if-no-files-found: error + retention-days: 30 + + - name: Check CodeQL findings + if: always() + shell: bash + env: + SARIF_COUNT: ${{ steps.sarif-summary.outputs.sarif_count }} + INVALID_SARIF: ${{ steps.sarif-summary.outputs.invalid_sarif }} + VIOLATIONS: ${{ steps.sarif-summary.outputs.violations }} + run: | + sarif_count="${SARIF_COUNT:-0}" + invalid_sarif="${INVALID_SARIF:-0}" + violations="${VIOLATIONS:-0}" + + if [[ "$sarif_count" -eq 0 ]]; then + echo "::error::CodeQL did not produce a SARIF report." + exit 1 + fi + + if [[ "$invalid_sarif" -ne 0 ]]; then + echo "::error::CodeQL produced $invalid_sarif invalid SARIF report(s)." + exit 1 + fi + + if [[ "$violations" -ne 0 ]]; then + echo "::error::CodeQL found $violations result(s)." + exit 1 + fi + + echo "CodeQL found no results." diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 1a5d4ced..d022f03d 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,102 +1,66 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL Advanced" +name: CodeQL Incremental on: push: branches: [ "develop" ] pull_request: branches: [ "develop" ] - schedule: - - cron: '24 15 * * 1' -jobs: - analyze: - name: Analyze (${{ matrix.language }}) - # Runner size impacts CodeQL analysis time. To learn more, please see: - # - https://gh.io/recommended-hardware-resources-for-running-codeql - # - https://gh.io/supported-runners-and-hardware-resources - # - https://gh.io/using-larger-runners (GitHub.com only) - # Consider using larger runners or machines with greater resources for possible analysis time improvements. - runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} - permissions: - # required for all workflows - security-events: write +permissions: + contents: read + security-events: write + packages: read + actions: read - # required to fetch internal or private CodeQL packs - packages: read +jobs: + detect: + name: Detect CodeQL languages + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 - # only required for workflows in private repositories - actions: read - contents: read + - name: Build matrix + id: matrix + shell: bash + run: | + matrix=$(bash .github/scripts/codeql-matrix.sh autobuild) + printf 'matrix=%s\n' "$matrix" >> "$GITHUB_OUTPUT" + analyze: + needs: detect + name: Analyze (${{ matrix.language }}) + runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - include: - - language: actions - build-mode: none - - language: java-kotlin - build-mode: autobuild # This mode only analyzes Java. Set this to 'autobuild' or 'manual' to analyze Kotlin too. - # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'rust', 'swift' - # Use `c-cpp` to analyze code written in C, C++ or both - # Use 'java-kotlin' to analyze code written in Java, Kotlin or both - # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both - # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, - # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. - # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how - # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages - steps: - - name: Checkout repository - uses: actions/checkout@v4 + matrix: ${{ fromJSON(needs.detect.outputs.matrix) }} - # Add any setup steps before running the `github/codeql-action/init` action. - # This includes steps like installing compilers or runtimes (`actions/setup-node` - # or others). This is typically only required for manual builds. - # - name: Setup runtime (example) - # uses: actions/setup-example@v1 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v4 - with: - languages: ${{ matrix.language }} - build-mode: ${{ matrix.build-mode }} - config-file: ./.github/codeql/codeql-config.yml - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 - # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality + - name: Set up JDK 17 + if: matrix.language == 'java-kotlin' + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'temurin' + cache: maven - # If the analyze step fails for one of the languages you are analyzing with - # "We were unable to automatically build your code", modify the matrix above - # to set the build mode to "manual" for that language. Then modify this step - # to build your code. - # ℹ️ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - name: Run manual build steps - if: matrix.build-mode == 'manual' - shell: bash - run: | - echo 'If you are using a "manual" build mode for one or more of the' \ - 'languages you are analyzing, replace this with the commands to build' \ - 'your code, for example:' - echo ' make bootstrap' - echo ' make release' - exit 1 + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + config-file: ./.github/codeql/codeql-config.yml - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 - with: - category: "/language:${{matrix.language}}" + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 + with: + category: "/language:${{ matrix.language }}" diff --git a/app/src/main/java/com/tinyengine/it/task/DatabaseCleanupService.java b/app/src/main/java/com/tinyengine/it/task/DatabaseCleanupService.java index 4589e394..aef73b4d 100644 --- a/app/src/main/java/com/tinyengine/it/task/DatabaseCleanupService.java +++ b/app/src/main/java/com/tinyengine/it/task/DatabaseCleanupService.java @@ -1,327 +1,418 @@ -/** - * Copyright (c) 2023 - present TinyEngine Authors. - * Copyright (c) 2023 - present Huawei Cloud Computing Technologies Co., Ltd. - * - * Use of this source code is governed by an MIT-style license. - * - * THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, - * BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR - * A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. - * - */ - -package com.tinyengine.it.task; - -import jakarta.annotation.PostConstruct; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; - -@Service -public class DatabaseCleanupService { - - private static final Logger logger = LoggerFactory.getLogger(DatabaseCleanupService.class); - private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - - @Autowired - private JdbcTemplate jdbcTemplate; - - @Autowired - private CleanupProperties cleanupProperties; - - private final Map executionStats = new ConcurrentHashMap<>(); - - private final AtomicInteger totalExecutions = new AtomicInteger(0); - - // 默认白名单表(如果配置文件未设置) - private static final List DEFAULT_TABLES = Arrays.asList( - "t_resource", "t_resource_group", "r_resource_group_resource", "t_app_extension", - "t_block", "t_block_carriers_relation", "t_block_group", "t_block_history", - "r_material_block", "r_material_history_block", "r_block_group_block", "t_datasource", - "t_i18n_entry", "t_model", "t_page", "t_page_history", "t_page_template" - ); - - /** - * 每天24:00自动执行清空操作 - */ - @Scheduled(cron = "${cleanup.cron-expression:0 0 0 * * ?}") - public void autoCleanupAtMidnight() { - if (!cleanupProperties.isEnabled()) { - logger.info("⏸️ Clearing tasks is disabled, skipping execution"); - return; - } - - String executionId = UUID.randomUUID().toString().substring(0, 8); - String startTime = LocalDateTime.now().format(FORMATTER); - - logger.info("======= Start executing the database clearing task [{}] =======", executionId); - logger.info("⏰ Time: {}", startTime); - logger.info("📋 Tables: {}", getWhitelistTables()); - - ExecutionStats stats = new ExecutionStats(executionId, startTime); - executionStats.put(executionId, stats); - totalExecutions.incrementAndGet(); - - int successCount = 0; - int failedCount = 0; - long totalRowsCleaned = 0L; - - for (String tableName : getWhitelistTables()) { - try { - validateTableName(tableName); - - if (!tableExists(tableName)) { - logger.warn("⚠️ Table {} does not exist, skip", tableName); - stats.recordSkipped(tableName, "Table does not exist"); - continue; - } - - long beforeCount = getTableRecordCount(tableName); - long rowsCleaned; - - if (cleanupProperties.isUseTruncate()) { - truncateTable(tableName); - rowsCleaned = beforeCount; - } else { - rowsCleaned = clearTableData(tableName); - } - - totalRowsCleaned += rowsCleaned; - successCount++; - - logger.info("✅ Table {} cleared: {} records deleted", tableName, rowsCleaned); - stats.recordSuccess(tableName, rowsCleaned); - - } catch (Exception e) { - failedCount++; - logger.error("❌ Failed to clear table {}: {}", tableName, e.getMessage(), e); - stats.recordFailure(tableName, e.getMessage()); - } - } - - String endTime = LocalDateTime.now().format(FORMATTER); - stats.setEndTime(endTime); - stats.setTotalRowsCleaned(totalRowsCleaned); - - logger.info("📊 ======= Task Completion Statistics [{}] =======", executionId); - logger.info("✅ Successful table count: {}", successCount); - logger.info("❌ Failure count: {}", failedCount); - logger.info("📈 Total deleted records: {}", totalRowsCleaned); - logger.info("⏰ Time-consuming: {} second", stats.getDurationSeconds()); - logger.info("🕐 Start: {}, End: {}", startTime, endTime); - logger.info("🎉 ======= Task execution completed =======\n"); - } - - /** - * 每天23:55发送预警通知 - */ - @Scheduled(cron = "0 55 23 * * ?") - public void sendCleanupWarning() { - if (!cleanupProperties.isEnabled() || !cleanupProperties.isSendWarning()) { - return; - } - - logger.warn("⚠️ ⚠️ ⚠️ Important Notice: The database table will be automatically cleared in 5 minutes!"); - logger.warn("📋 Target table: {}", getWhitelistTables()); - logger.warn("⏰ Execution Time: 00:00:00"); - logger.warn("💡 If you need to cancel, please change the settings: cleanup.enabled=false"); - logger.warn("=========================================="); - } - - /** - * 应用启动时初始化 - */ - @PostConstruct - public void init() { - logger.info("🚀 Database auto-clear service initialization completed"); - logger.info("📋 Configuration table: {}", getWhitelistTables()); - logger.info("⏰ Execution time: {}", cleanupProperties.getCronExpression()); - logger.info("🔧 Mode in use: {}", cleanupProperties.isUseTruncate() ? "TRUNCATE" : "DELETE"); - logger.info("✅ Service status: {}", cleanupProperties.isEnabled() ? "Enabled" : "Disabled"); - logger.info("=========================================="); - - } - - /** - * 获取白名单表列表 - */ - public List getWhitelistTables() { - List tables = cleanupProperties.getWhitelistTables(); - return tables != null && !tables.isEmpty() ? tables : DEFAULT_TABLES; - } - - /** - * 清空表数据(DELETE方式) - */ - private long clearTableData(String tableName) { - validateTableName(tableName); - String sql = "DELETE FROM " + tableName; - int affectedRows = jdbcTemplate.update(sql); - return affectedRows; - } - - /** - * 清空表数据(TRUNCATE方式) - */ - private void truncateTable(String tableName) { - validateTableName(tableName); - String sql = "TRUNCATE TABLE " + tableName; - jdbcTemplate.execute(sql); - } - - /** - * 检查表是否存在 - */ - public boolean tableExists(String tableName) { - try { - String sql = "SELECT COUNT(*) FROM information_schema.tables " + - "WHERE table_schema = DATABASE() AND table_name = ?"; - Integer count = jdbcTemplate.queryForObject(sql, Integer.class, tableName.toUpperCase()); - return count != null && count > 0; - } catch (Exception e) { - logger.warn("The checklist has failed: {}", e.getMessage()); - return false; - } - } - - /** - * 获取表记录数量 - */ - public long getTableRecordCount(String tableName) { - try { - validateTableName(tableName); - String sql = "SELECT COUNT(*) FROM " + tableName; - Long count = jdbcTemplate.queryForObject(sql, Long.class); - return count != null ? count : 0; - } catch (Exception e) { - logger.error("获取表记录数失败: {}", e.getMessage()); - return -1; - } - } - - /** - * 验证表名安全性 - */ - private void validateTableName(String tableName) { - if (tableName == null || tableName.trim().isEmpty()) { - throw new IllegalArgumentException("Table name cannot be empty"); - } - if (!tableName.matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) { - throw new IllegalArgumentException("Invalid table name format: " + tableName); - } - } - - /** - * 获取执行统计 - */ - public Map getExecutionStats() { - return new LinkedHashMap<>(executionStats); - } - - public int getTotalExecutions() { - return totalExecutions.get(); - } - - /** - * 执行统计内部类 - */ - public static class ExecutionStats { - private final String executionId; - private final String startTime; - private String endTime; - private long totalRowsCleaned; - private final Map tableResults = new LinkedHashMap<>(); - - public ExecutionStats(String executionId, String startTime) { - this.executionId = executionId; - this.startTime = startTime; - } - - public void recordSuccess(String tableName, long rowsCleaned) { - tableResults.put(tableName, new TableResult("SUCCESS", rowsCleaned, null)); - } - - public void recordFailure(String tableName, String errorMessage) { - tableResults.put(tableName, new TableResult("FAILED", 0, errorMessage)); - } - - public void recordSkipped(String tableName, String reason) { - tableResults.put(tableName, new TableResult("SKIPPED", 0, reason)); - } - - // Getters and setters - public String getExecutionId() { - return executionId; - } - - public String getStartTime() { - return startTime; - } - - public String getEndTime() { - return endTime; - } - - public void setEndTime(String endTime) { - this.endTime = endTime; - } - - public long getTotalRowsCleaned() { - return totalRowsCleaned; - } - - public void setTotalRowsCleaned(long totalRowsCleaned) { - this.totalRowsCleaned = totalRowsCleaned; - } - - public Map getTableResults() { - return tableResults; - } - - public long getDurationSeconds() { - if (startTime != null && endTime != null) { - LocalDateTime start = LocalDateTime.parse(startTime, FORMATTER); - LocalDateTime end = LocalDateTime.parse(endTime, FORMATTER); - return java.time.Duration.between(start, end).getSeconds(); - } - return 0; - } - } - - /** - * 表结果内部类 - */ - public static class TableResult { - private final String status; - private final long rowsCleaned; - private final String message; - - public TableResult(String status, long rowsCleaned, String message) { - this.status = status; - this.rowsCleaned = rowsCleaned; - this.message = message; - } - - // Getters - public String getStatus() { - return status; - } - - public long getRowsCleaned() { - return rowsCleaned; - } - - public String getMessage() { - return message; - } - } -} +/** + * Copyright (c) 2023 - present TinyEngine Authors. Copyright (c) 2023 - present Huawei Cloud + * Computing Technologies Co., Ltd. + * + *

Use of this source code is governed by an MIT-style license. + * + *

THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, + * BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR A + * PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + */ +package com.tinyengine.it.task; + +import jakarta.annotation.PostConstruct; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.DataAccessException; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +@SuppressWarnings("PMD.TooManyMethods") +@Service +public class DatabaseCleanupService { + + private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseCleanupService.class); + private static final DateTimeFormatter FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + private static final int EXEC_ID_LENGTH = 8; + + @Autowired private JdbcTemplate jdbcTemplate; + + @Autowired private CleanupProperties cleanupProperties; + + private final Map executionStats = new ConcurrentHashMap<>(); + + private final AtomicInteger totalExecutions = new AtomicInteger(0); + + // 默认白名单表(如果配置文件未设置) + private static final List DEFAULT_TABLES = + Arrays.asList( + "t_resource", + "t_resource_group", + "r_resource_group_resource", + "t_app_extension", + "t_block", + "t_block_carriers_relation", + "t_block_group", + "t_block_history", + "r_material_block", + "r_material_history_block", + "r_block_group_block", + "t_datasource", + "t_i18n_entry", + "t_model", + "t_page", + "t_page_history", + "t_page_template"); + + /** 每天24:00自动执行清空操作 */ + @Scheduled(cron = "${cleanup.cron-expression:0 0 0 * * ?}") + public void autoCleanupAtMidnight() { + if (!cleanupProperties.isEnabled()) { + logInfo("⏸️ Clearing tasks is disabled, skipping execution"); + return; + } + + final String executionId = createExecutionId(); + final String startTime = currentTime(); + + logInfo("======= Start executing the database clearing task [{}] =======", executionId); + logInfo("⏰ Time: {}", startTime); + logInfo("📋 Tables: {}", getWhitelistTables()); + + final ExecutionStats stats = new ExecutionStats(executionId, startTime); + executionStats.put(executionId, stats); + totalExecutions.incrementAndGet(); + + final CleanupSummary cleanupSummary = new CleanupSummary(); + + for (final String tableName : getWhitelistTables()) { + cleanTable(tableName, stats, cleanupSummary); + } + + final String endTime = currentTime(); + stats.setEndTime(endTime); + stats.setTotalRowsCleaned(cleanupSummary.getTotalRowsCleaned()); + + logInfo("📊 ======= Task Completion Statistics [{}] =======", executionId); + logInfo("✅ Successful table count: {}", cleanupSummary.getSuccessCount()); + logInfo("❌ Failure count: {}", cleanupSummary.getFailedCount()); + logInfo("📈 Total deleted records: {}", cleanupSummary.getTotalRowsCleaned()); + logInfo("⏰ Time-consuming: {} second", stats.getDurationSeconds()); + logInfo("🕐 Start: {}, End: {}", startTime, endTime); + logInfo("🎉 ======= Task execution completed =======\n"); + } + + /** 每天23:55发送预警通知 */ + @Scheduled(cron = "0 55 23 * * ?") + public void sendCleanupWarning() { + if (!cleanupProperties.isEnabled() || !cleanupProperties.isSendWarning()) { + return; + } + + logWarn( + "⚠️ ⚠️ ⚠️ Important Notice: The database table will be automatically cleared in 5" + + " minutes!"); + logWarn("📋 Target table: {}", getWhitelistTables()); + logWarn("⏰ Execution Time: 00:00:00"); + logWarn("💡 If you need to cancel, please change the settings: cleanup.enabled=false"); + logWarn("=========================================="); + } + + /** 应用启动时初始化 */ + @PostConstruct + public void init() { + logInfo("🚀 Database auto-clear service initialization completed"); + logInfo("📋 Configuration table: {}", getWhitelistTables()); + logInfo("⏰ Execution time: {}", cleanupProperties.getCronExpression()); + logInfo( + "🔧 Mode in use: {}", cleanupProperties.isUseTruncate() ? "TRUNCATE" : "DELETE"); + logInfo("✅ Service status: {}", cleanupProperties.isEnabled() ? "Enabled" : "Disabled"); + logInfo("=========================================="); + } + + /** + * 获取白名单表列表. + * + * @return whitelist table names + */ + @SuppressWarnings("PMD.LawOfDemeter") + public List getWhitelistTables() { + final List tables = cleanupProperties.getWhitelistTables(); + return tables != null && !tables.isEmpty() ? tables : DEFAULT_TABLES; + } + + private static String createExecutionId() { + final String fullUuid = UUID.randomUUID().toString(); + return truncateUuid(fullUuid); + } + + private static String truncateUuid(String uuid) { + return uuid.substring(0, EXEC_ID_LENGTH); + } + + private static String currentTime() { + final ZoneId systemZone = ZoneId.systemDefault(); + final LocalDateTime currentDateTime = LocalDateTime.now(systemZone); + return FORMATTER.format(currentDateTime); // 调用静态常量,传入参数 + } + + private void cleanTable( + final String tableName, + final ExecutionStats stats, + final CleanupSummary cleanupSummary) { + try { + validateTableName(tableName); + if (!tableExists(tableName)) { + logWarn("⚠️ Table {} does not exist, skip", tableName); + stats.recordSkipped(tableName, "Table does not exist"); + return; + } + + final long rowsCleaned = clearTable(tableName); + cleanupSummary.recordSuccess(rowsCleaned); + logInfo("✅ Table {} cleared: {} records deleted", tableName, rowsCleaned); + stats.recordSuccess(tableName, rowsCleaned); + } catch (DataAccessException | IllegalArgumentException exception) { + cleanupSummary.recordFailure(); + logError( + "❌ Failed to clear table {}: {}", + tableName, + exception.getMessage(), + exception); + stats.recordFailure(tableName, exception.getMessage()); + } + } + + private long clearTable(final String tableName) { + long result; // 存储最终返回值 + if (!cleanupProperties.isUseTruncate()) { + result = clearTableData(tableName); + } else { + final long recordCount = getTableRecordCount(tableName); + truncateTable(tableName); + result = recordCount; + } + return result; // 唯一的返回语句 + } + + /** + * 清空表数据(DELETE方式). + * + * @return number of deleted rows + */ + private long clearTableData(final String tableName) { + validateTableName(tableName); + final String sql = "DELETE FROM " + tableName; + return jdbcTemplate.update(sql); + } + + /** 清空表数据(TRUNCATE方式) */ + private void truncateTable(final String tableName) { + validateTableName(tableName); + final String sql = "TRUNCATE TABLE " + tableName; + jdbcTemplate.execute(sql); + } + + /** + * 检查表是否存在. + * + * @return whether the table exists + */ + public boolean tableExists(final String tableName) { + boolean exists = false; + try { + final String sql = + "SELECT COUNT(*) FROM information_schema.tables " + + "WHERE table_schema = DATABASE() AND table_name = ?"; + final Integer count = + jdbcTemplate.queryForObject(sql, Integer.class, tableName.toUpperCase(Locale.ROOT)); + exists = count != null && count > 0; + } catch (DataAccessException | IllegalArgumentException exception) { + logWarn("The checklist has failed: {}", exception.getMessage()); + } + return exists; + } + + /** + * 获取表记录数量. + * + * @return record count in the table + */ + public long getTableRecordCount(final String tableName) { + long result; // 存储返回值 + try { + validateTableName(tableName); + final String sql = "SELECT COUNT(*) FROM " + tableName; + final Long count = jdbcTemplate.queryForObject(sql, Long.class); + result = (count != null) ? count : 0; + } catch (DataAccessException | IllegalArgumentException exception) { + logError("获取表记录数失败: {}", exception.getMessage()); + result = -1; // 异常时返回 -1 + } + return result; // 唯一的退出点 + } + + /** 验证表名安全性 */ + private void validateTableName(final String tableName) { + if (tableName == null || tableName.isBlank()) { + throw new IllegalArgumentException("Table name cannot be empty"); + } + if (!tableName.matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) { + throw new IllegalArgumentException("Invalid table name format: " + tableName); + } + } + + /** + * 获取执行统计. + * + * @return execution statistics + */ + public Map getExecutionStats() { + return new LinkedHashMap<>(executionStats); + } + + public int getTotalExecutions() { + return totalExecutions.get(); + } + + private static void logInfo(final String message, final Object... arguments) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info(message, arguments); + } + } + + private static void logWarn(final String message, final Object... arguments) { + if (LOGGER.isWarnEnabled()) { + LOGGER.warn(message, arguments); + } + } + + private static void logError(final String message, final Object... arguments) { + if (LOGGER.isErrorEnabled()) { + LOGGER.error(message, arguments); + } + } + + private static final class CleanupSummary { + private int successCount; + private int failedCount; + private long totalRowsCleaned; + + private void recordSuccess(final long rowsCleaned) { + successCount++; + totalRowsCleaned += rowsCleaned; + } + + private void recordFailure() { + failedCount++; + } + + private int getSuccessCount() { + return successCount; + } + + private int getFailedCount() { + return failedCount; + } + + private long getTotalRowsCleaned() { + return totalRowsCleaned; + } + } + + /** 执行统计内部类 */ + public static class ExecutionStats { + private final String executionId; + private final String startTime; + private String endTime; + private long totalRowsCleaned; + private final Map tableResults = new LinkedHashMap<>(); + + public ExecutionStats(final String executionId, final String startTime) { + this.executionId = executionId; + this.startTime = startTime; + } + + public void recordSuccess(final String tableName, final long rowsCleaned) { + tableResults.put(tableName, new TableResult("SUCCESS", rowsCleaned, null)); + } + + public void recordFailure(final String tableName, final String errorMessage) { + tableResults.put(tableName, new TableResult("FAILED", 0, errorMessage)); + } + + public void recordSkipped(final String tableName, final String reason) { + tableResults.put(tableName, new TableResult("SKIPPED", 0, reason)); + } + + // Getters and setters + public String getExecutionId() { + return executionId; + } + + public String getStartTime() { + return startTime; + } + + public String getEndTime() { + return endTime; + } + + public void setEndTime(final String endTime) { + this.endTime = endTime; + } + + public long getTotalRowsCleaned() { + return totalRowsCleaned; + } + + public void setTotalRowsCleaned(final long totalRowsCleaned) { + this.totalRowsCleaned = totalRowsCleaned; + } + + public Map getTableResults() { + return tableResults; + } + + public long getDurationSeconds() { + long result = 0; // 默认值对应 startTime 或 endTime 为空的情况 + if (startTime != null && endTime != null) { + final LocalDateTime start = LocalDateTime.parse(startTime, FORMATTER); + final LocalDateTime end = LocalDateTime.parse(endTime, FORMATTER); + result = java.time.Duration.between(start, end).getSeconds(); + } + return result; // 唯一的退出点 + } + } + + /** 表结果内部类 */ + public static class TableResult { + private final String status; + private final long rowsCleaned; + private final String message; + + public TableResult(final String status, final long rowsCleaned, final String message) { + this.status = status; + this.rowsCleaned = rowsCleaned; + this.message = message; + } + + // Getters + public String getStatus() { + return status; + } + + public long getRowsCleaned() { + return rowsCleaned; + } + + public String getMessage() { + return message; + } + } +} diff --git a/base/src/main/java/com/tinyengine/it/common/exception/ServiceException.java b/base/src/main/java/com/tinyengine/it/common/exception/ServiceException.java index 987c8578..683134f8 100644 --- a/base/src/main/java/com/tinyengine/it/common/exception/ServiceException.java +++ b/base/src/main/java/com/tinyengine/it/common/exception/ServiceException.java @@ -37,4 +37,10 @@ public ServiceException(String code, String message) { this.code = code; this.message = message; } + + public ServiceException(String code, String message, Throwable cause) { + super(message, cause); + this.code = code; + this.message = message; + } } diff --git a/base/src/main/java/com/tinyengine/it/common/utils/SM4Utils.java b/base/src/main/java/com/tinyengine/it/common/utils/SM4Utils.java index 8c613b64..c8b382c8 100644 --- a/base/src/main/java/com/tinyengine/it/common/utils/SM4Utils.java +++ b/base/src/main/java/com/tinyengine/it/common/utils/SM4Utils.java @@ -1,71 +1,116 @@ -package com.tinyengine.it.common.utils; - -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import javax.crypto.Cipher; -import javax.crypto.KeyGenerator; -import javax.crypto.SecretKey; -import javax.crypto.spec.SecretKeySpec; -import java.security.SecureRandom; -import java.security.Security; -import java.util.Base64; - -public class SM4Utils { - - static { - Security.addProvider(new BouncyCastleProvider()); - } - - private static final String ALGORITHM = "SM4"; - private static final String TRANSFORMATION_ECB = "SM4/ECB/PKCS5Padding"; - private static final int KEY_SIZE = 128; - - /** - * 生成 SM4 密钥 - */ - public static String generateKeyBase64() throws Exception { - byte[] key = generateKey(); - return Base64.getEncoder().encodeToString(key); - } - - public static byte[] generateKey() throws Exception { - KeyGenerator kg = KeyGenerator.getInstance(ALGORITHM, "BC"); - kg.init(KEY_SIZE, new SecureRandom()); - SecretKey secretKey = kg.generateKey(); - return secretKey.getEncoded(); - } - - /** - * ECB 模式加密 - 只加密API密钥值 (Base64 结果) - */ - public static String encryptECB(String apiKey, String base64Key) throws Exception { - byte[] key = Base64.getDecoder().decode(base64Key); - byte[] encrypted = encryptECB(apiKey.getBytes("UTF-8"), key); - return Base64.getEncoder().encodeToString(encrypted); - } - - /** - * ECB 模式解密 - 直接返回API密钥 - */ - public static String decryptECB(String encryptedBase64, String base64Key) throws Exception { - byte[] key = Base64.getDecoder().decode(base64Key); - byte[] encrypted = Base64.getDecoder().decode(encryptedBase64); - byte[] decrypted = decryptECB(encrypted, key); - return new String(decrypted, "UTF-8"); - } - - // ECB 模式的底层方法保持不变 - private static byte[] encryptECB(byte[] data, byte[] key) throws Exception { - SecretKeySpec secretKeySpec = new SecretKeySpec(key, ALGORITHM); - Cipher cipher = Cipher.getInstance(TRANSFORMATION_ECB, "BC"); - cipher.init(Cipher.ENCRYPT_MODE, secretKeySpec); - return cipher.doFinal(data); - } - - private static byte[] decryptECB(byte[] encryptedData, byte[] key) throws Exception { - SecretKeySpec secretKeySpec = new SecretKeySpec(key, ALGORITHM); - Cipher cipher = Cipher.getInstance(TRANSFORMATION_ECB, "BC"); - cipher.init(Cipher.DECRYPT_MODE, secretKeySpec); - return cipher.doFinal(encryptedData); - } - -} +package com.tinyengine.it.common.utils; + +import org.bouncycastle.jce.provider.BouncyCastleProvider; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.security.SecureRandom; +import java.security.Security; +import java.util.Base64; + +import javax.crypto.Cipher; +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +public final class SM4Utils { + + private static final String ALGORITHM = "SM4"; + private static final String TRANSFORMATION = "SM4/GCM/NoPadding"; + private static final int KEY_SIZE = 128; + private static final int KEY_LENGTH_BYTES = KEY_SIZE / Byte.SIZE; + private static final int IV_LENGTH_BYTES = 12; + private static final int GCM_TAG_BITS = 128; + private static final SecureRandom SECURE_RANDOM = new SecureRandom(); + + static { + Security.addProvider(new BouncyCastleProvider()); + } + + private SM4Utils() { + // Utility class. + } + + /** + * 生成 SM4 密钥. + * + * @return generated SM4 key encoded as Base64 + */ + public static String generateKeyBase64() throws GeneralSecurityException { + final byte[] key = generateKey(); + final Base64.Encoder encoder = Base64.getEncoder(); + return encoder.encodeToString(key); + } + + public static byte[] generateKey() throws GeneralSecurityException { + final KeyGenerator keyGenerator = KeyGenerator.getInstance(ALGORITHM, "BC"); + keyGenerator.init(KEY_SIZE, SECURE_RANDOM); + final SecretKey secretKey = keyGenerator.generateKey(); + return secretKey.getEncoded(); + } + + public static String encrypt(final String apiKey, final String base64Key) + throws GeneralSecurityException { + final byte[] key = decodeKey(base64Key); + final byte[] nonce = new byte[IV_LENGTH_BYTES]; + SECURE_RANDOM.nextBytes(nonce); + + final byte[] encrypted = + doCipher( + Cipher.ENCRYPT_MODE, + apiKey.getBytes(StandardCharsets.UTF_8), + key, + nonce); + final ByteBuffer outputBuffer = + ByteBuffer.allocate(nonce.length + encrypted.length); + outputBuffer.put(nonce); + outputBuffer.put(encrypted); + final Base64.Encoder encoder = Base64.getEncoder(); + return encoder.encodeToString(outputBuffer.array()); + } + + public static String decrypt(final String encryptedBase64, final String base64Key) + throws GeneralSecurityException { + final byte[] key = decodeKey(base64Key); + final Base64.Decoder decoder = Base64.getDecoder(); + final byte[] encryptedWithIv = decoder.decode(encryptedBase64); + if (encryptedWithIv.length <= IV_LENGTH_BYTES) { + throw new IllegalArgumentException("Invalid encrypted payload"); + } + + final ByteBuffer buffer = ByteBuffer.wrap(encryptedWithIv); + final byte[] nonce = new byte[IV_LENGTH_BYTES]; + buffer.get(nonce); + final byte[] encrypted = new byte[buffer.remaining()]; + buffer.get(encrypted); + + final byte[] decrypted = + doCipher(Cipher.DECRYPT_MODE, encrypted, key, nonce); + return new String(decrypted, StandardCharsets.UTF_8); + } + + private static byte[] doCipher( + final int mode, + final byte[] data, + final byte[] key, + final byte[] nonce) + throws GeneralSecurityException { + final SecretKeySpec secretKeySpec = new SecretKeySpec(key, ALGORITHM); + final GCMParameterSpec parameterSpec = + new GCMParameterSpec(GCM_TAG_BITS, nonce); + final Cipher cipher = Cipher.getInstance(TRANSFORMATION, "BC"); + cipher.init(mode, secretKeySpec, parameterSpec); + return cipher.doFinal(data); + } + + private static byte[] decodeKey(final String base64Key) { + final Base64.Decoder decoder = Base64.getDecoder(); + final byte[] key = decoder.decode(base64Key); + if (key.length != KEY_LENGTH_BYTES) { + throw new IllegalArgumentException("SM4 key must be 128 bits"); + } + return key; + } +} diff --git a/base/src/main/java/com/tinyengine/it/common/utils/SqlIdentifierValidator.java b/base/src/main/java/com/tinyengine/it/common/utils/SqlIdentifierValidator.java index a9b5738d..96fa4279 100644 --- a/base/src/main/java/com/tinyengine/it/common/utils/SqlIdentifierValidator.java +++ b/base/src/main/java/com/tinyengine/it/common/utils/SqlIdentifierValidator.java @@ -1,35 +1,74 @@ package com.tinyengine.it.common.utils; import java.util.List; -import java.util.regex.Pattern; -public class SqlIdentifierValidator { - - private static final Pattern IDENTIFIER_PATTERN = - Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_]*$"); - - private static final Pattern ORDER_TYPE_PATTERN = - Pattern.compile("^(ASC|DESC)$", Pattern.CASE_INSENSITIVE); +public final class SqlIdentifierValidator { private SqlIdentifierValidator() { + // Utility class. } - public static void validate(String identifier) { - if (identifier == null || !IDENTIFIER_PATTERN.matcher(identifier).matches()) { + public static void validate(final String identifier) { + if (!isValidIdentifier(identifier)) { throw new IllegalArgumentException("Invalid SQL identifier: " + identifier); } } - public static void validateAll(List identifiers) { + public static String requireValidIdentifier(final String identifier) { + validate(identifier); + return identifier; + } + + public static void validateAll(final List identifiers) { if (identifiers == null) { return; } identifiers.forEach(SqlIdentifierValidator::validate); } - public static void validateOrderType(String orderType) { - if (orderType == null || !ORDER_TYPE_PATTERN.matcher(orderType).matches()) { + public static void validateOrderType(final String orderType) { + if (!isValidOrderType(orderType)) { throw new IllegalArgumentException("Invalid order type: " + orderType); } } + + public static String requireValidOrderType(final String orderType) { + validateOrderType(orderType); + return orderType.toUpperCase(java.util.Locale.ROOT); + } + + public static boolean isValidIdentifier(final String identifier) { + boolean valid = identifier != null && !identifier.isEmpty(); + if (valid) { + valid = isIdentifierStart(identifier.charAt(0)); + } + + for (int index = 1; valid && index < identifier.length(); index++) { + if (!isIdentifierPart(identifier.charAt(index))) { + valid = false; + } + } + return valid; + } + + public static boolean isValidOrderType(final String orderType) { + return "ASC".equalsIgnoreCase(orderType) || "DESC".equalsIgnoreCase(orderType); + } + + public static String escapeSqlLiteral(final Object value) { + final String stringValue = value == null ? null : value.toString(); + return stringValue == null + ? null + : stringValue.replace("\\", "\\\\").replace("'", "''"); + } + + private static boolean isIdentifierStart(final char character) { + return character == '_' + || character >= 'A' && character <= 'Z' + || character >= 'a' && character <= 'z'; + } + + private static boolean isIdentifierPart(final char character) { + return isIdentifierStart(character) || character >= '0' && character <= '9'; + } } diff --git a/base/src/main/java/com/tinyengine/it/dynamic/dao/DynamicSqlProvider.java b/base/src/main/java/com/tinyengine/it/dynamic/dao/DynamicSqlProvider.java index fba8eac1..3c63bef5 100644 --- a/base/src/main/java/com/tinyengine/it/dynamic/dao/DynamicSqlProvider.java +++ b/base/src/main/java/com/tinyengine/it/dynamic/dao/DynamicSqlProvider.java @@ -1,107 +1,226 @@ package com.tinyengine.it.dynamic.dao; +import com.tinyengine.it.common.utils.SqlIdentifierValidator; + import org.apache.ibatis.jdbc.SQL; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; - +@SuppressWarnings("PMD.TooManyMethods") public class DynamicSqlProvider { - public String select(Map params) { - String tableName = (String) params.get("tableName"); - List fields = (List) params.get("fields"); - Map conditions = (Map) params.get("conditions"); - Integer pageNum = (Integer) params.get("pageNum"); - Integer pageSize = (Integer) params.get("pageSize"); - String orderBy = (String) params.get("orderBy"); - String orderType = (String) params.get("orderType"); - - SQL sql = new SQL(); - - // 选择字段 - if (fields != null && !fields.isEmpty()) { - for (String field : fields) { - sql.SELECT(field); - } - } else { - sql.SELECT("*"); - } - - sql.FROM(tableName); - - // 条件 - if (conditions != null && !conditions.isEmpty()) { - for (Map.Entry entry : conditions.entrySet()) { - if (entry.getValue() != null) { - sql.WHERE(entry.getKey() + " = #{conditions." + entry.getKey() + "}"); - } - } - } - // 排序 - if (orderBy != null && !orderBy.isEmpty()) { - sql.ORDER_BY(orderBy + " " + orderType); - } - - // 分页 - if (pageNum != null && pageSize != null) { - return sql.toString() + " LIMIT " + (pageNum - 1) * pageSize + ", " + pageSize; - } - - return sql.toString(); - } - - public String insert(Map params) { - String tableName = (String) params.get("tableName"); - Map data = (Map) params.get("data"); - - SQL sql = new SQL(); - sql.INSERT_INTO(tableName); - - if (data != null && !data.isEmpty()) { - for (Map.Entry entry : data.entrySet()) { - sql.VALUES(entry.getKey(), "#{data." + entry.getKey() + "}"); - } - } - - return sql.toString(); - } - - public String update(Map params) { - String tableName = (String) params.get("tableName"); - Map data = (Map) params.get("data"); - Map conditions = (Map) params.get("conditions"); - - SQL sql = new SQL(); - sql.UPDATE(tableName); - - if (data != null && !data.isEmpty()) { - for (Map.Entry entry : data.entrySet()) { - sql.SET(entry.getKey() + " = #{data." + entry.getKey() + "}"); - } - } - - if (conditions != null && !conditions.isEmpty()) { - for (Map.Entry entry : conditions.entrySet()) { - sql.WHERE(entry.getKey() + " = #{conditions." + entry.getKey() + "}"); - } - } - - return sql.toString(); - } - - public String delete(Map params) { - String tableName = (String) params.get("tableName"); - Map conditions = (Map) params.get("conditions"); - - SQL sql = new SQL(); - sql.DELETE_FROM(tableName); - - if (conditions != null && !conditions.isEmpty()) { - for (Map.Entry entry : conditions.entrySet()) { - sql.WHERE(entry.getKey() + " = #{conditions." + entry.getKey() + "}"); - } - } - - return sql.toString(); - } + private static final String COUNT_SELECT = "COUNT(*) AS count"; + private static final String LEGACY_COUNT = "COUNT(*) as count"; + private static final String TABLE_NAME_PARAM = "tableName"; + private static final String CONDITIONS_PARAM = "conditions"; + private static final String DATA_PARAM = "data"; + + @SuppressWarnings("PMD.UnnecessaryConstructor") + public DynamicSqlProvider() { + // MyBatis instantiates providers through the default constructor. + } + + public String select(final Map params) { + final String tableName = requireIdentifier(params.get(TABLE_NAME_PARAM), TABLE_NAME_PARAM); + final List fields = getList(params.get("fields")); + final Map conditions = getMap(params.get(CONDITIONS_PARAM)); + final Integer pageNum = (Integer) params.get("pageNum"); + final Integer pageSize = (Integer) params.get("pageSize"); + final String orderBy = getOptionalIdentifier(params.get("orderBy"), "orderBy"); + final String orderType = getOrderType(params.get("orderType")); + + final SQL sql = new SQL(); + + if (fields != null && !fields.isEmpty()) { + for (final Object field : fields) { + sql.SELECT(getSelectField(field)); + } + } else { + sql.SELECT("*"); + } + + sql.FROM(tableName); + + if (conditions != null && !conditions.isEmpty()) { + final List conditionValues = new ArrayList<>(); + int index = 0; + for (final Map.Entry entry : conditions.entrySet()) { + if (entry.getValue() != null) { + final String columnName = requireIdentifier(entry.getKey(), "condition key"); + conditionValues.add(entry.getValue()); + sql.WHERE(columnName + " = #{conditionValues[" + index + "]}"); + index++; + } + } + params.put("conditionValues", conditionValues); + } + + if (orderBy != null && !orderBy.isEmpty()) { + sql.ORDER_BY(orderBy + " " + orderType); + } + + if (pageNum != null && pageSize != null) { + final int safePageNum = requirePositiveInt(pageNum, "pageNum"); + final int safePageSize = requirePositiveInt(pageSize, "pageSize"); + params.put("offset", (safePageNum - 1) * safePageSize); + params.put("limit", safePageSize); + return sql + " LIMIT #{offset}, #{limit}"; + } + + return sql.toString(); + } + + public String insert(final Map params) { + final String tableName = requireIdentifier(params.get(TABLE_NAME_PARAM), TABLE_NAME_PARAM); + final Map data = getRequiredMap(params.get(DATA_PARAM), DATA_PARAM); + final List dataValues = new ArrayList<>(); + + final SQL sql = new SQL(); + sql.INSERT_INTO(tableName); + + int index = 0; + for (final Map.Entry entry : data.entrySet()) { + final String columnName = requireIdentifier(entry.getKey(), "data key"); + dataValues.add(entry.getValue()); + sql.VALUES(columnName, "#{dataValues[" + index + "]}"); + index++; + } + params.put("dataValues", dataValues); + + return sql.toString(); + } + + public String update(final Map params) { + final String tableName = requireIdentifier(params.get(TABLE_NAME_PARAM), TABLE_NAME_PARAM); + final Map data = getRequiredMap(params.get(DATA_PARAM), DATA_PARAM); + final Map conditions = getRequiredMap(params.get(CONDITIONS_PARAM), CONDITIONS_PARAM); + final List dataValues = new ArrayList<>(); + final List conditionValues = new ArrayList<>(); + + final SQL sql = new SQL(); + sql.UPDATE(tableName); + + int dataIndex = 0; + for (final Map.Entry entry : data.entrySet()) { + final String columnName = requireIdentifier(entry.getKey(), "data key"); + dataValues.add(entry.getValue()); + sql.SET(columnName + " = #{dataValues[" + dataIndex + "]}"); + dataIndex++; + } + params.put("dataValues", dataValues); + + int conditionIndex = 0; + for (final Map.Entry entry : conditions.entrySet()) { + if (entry.getValue() != null) { + final String columnName = requireIdentifier(entry.getKey(), "condition key"); + conditionValues.add(entry.getValue()); + sql.WHERE(columnName + " = #{conditionValues[" + conditionIndex + "]}"); + conditionIndex++; + } + } + if (conditionValues.isEmpty()) { + throw new IllegalArgumentException("conditions cannot be empty"); + } + params.put("conditionValues", conditionValues); + + return sql.toString(); + } + + public String delete(final Map params) { + final String tableName = requireIdentifier(params.get(TABLE_NAME_PARAM), TABLE_NAME_PARAM); + final Map conditions = getRequiredMap(params.get(CONDITIONS_PARAM), CONDITIONS_PARAM); + final List conditionValues = new ArrayList<>(); + + final SQL sql = new SQL(); + sql.DELETE_FROM(tableName); + + int index = 0; + for (final Map.Entry entry : conditions.entrySet()) { + if (entry.getValue() != null) { + final String columnName = requireIdentifier(entry.getKey(), "condition key"); + conditionValues.add(entry.getValue()); + sql.WHERE(columnName + " = #{conditionValues[" + index + "]}"); + index++; + } + } + if (conditionValues.isEmpty()) { + throw new IllegalArgumentException("conditions cannot be empty"); + } + params.put("conditionValues", conditionValues); + + return sql.toString(); + } + + private String getSelectField(final Object field) { + if (field instanceof String + && LEGACY_COUNT.equalsIgnoreCase(((String) field).trim())) { + return COUNT_SELECT; + } + return requireIdentifier(field, "field"); + } + + private String getOptionalIdentifier(final Object value, final String name) { + if (value == null) { + return null; + } + final String identifier = requireString(value, name); + if (identifier.isEmpty()) { + return null; + } + return SqlIdentifierValidator.requireValidIdentifier(identifier); + } + + private String requireIdentifier(final Object value, final String name) { + return SqlIdentifierValidator.requireValidIdentifier(requireString(value, name)); + } + + private String requireString(final Object value, final String name) { + if (!(value instanceof String)) { + throw new IllegalArgumentException(name + " must be a string"); + } + return (String) value; + } + + private String getOrderType(final Object value) { + if (value == null || value instanceof String stringValue && stringValue.isEmpty()) { + return "ASC"; + } + return SqlIdentifierValidator.requireValidOrderType(requireString(value, "orderType")); + } + + private int requirePositiveInt(final Integer value, final String name) { + if (value == null || value <= 0) { + throw new IllegalArgumentException(name + " must be positive"); + } + return value; + } + + private List getList(final Object value) { + if (value == null) { + return Collections.emptyList(); + } + if (!(value instanceof List)) { + throw new IllegalArgumentException("fields must be a list"); + } + return (List) value; + } + + private Map getMap(final Object value) { + if (value == null) { + return Collections.emptyMap(); + } + if (!(value instanceof Map)) { + throw new IllegalArgumentException("conditions must be a map"); + } + return (Map) value; + } + + private Map getRequiredMap(final Object value, final String name) { + if (!(value instanceof Map) || ((Map) value).isEmpty()) { + throw new IllegalArgumentException(name + " cannot be empty"); + } + return (Map) value; + } } diff --git a/base/src/main/java/com/tinyengine/it/dynamic/service/DynamicModelService.java b/base/src/main/java/com/tinyengine/it/dynamic/service/DynamicModelService.java index 3e7cdb94..ab41f2a2 100644 --- a/base/src/main/java/com/tinyengine/it/dynamic/service/DynamicModelService.java +++ b/base/src/main/java/com/tinyengine/it/dynamic/service/DynamicModelService.java @@ -2,20 +2,23 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONException; import com.tinyengine.it.common.context.LoginUserContext; import com.tinyengine.it.common.exception.ExceptionEnum; import com.tinyengine.it.common.exception.ServiceException; +import com.tinyengine.it.common.utils.SqlIdentifierValidator; import com.tinyengine.it.dynamic.dto.DynamicDelete; import com.tinyengine.it.dynamic.dto.DynamicInsert; import com.tinyengine.it.dynamic.dto.DynamicQuery; import com.tinyengine.it.dynamic.dto.DynamicUpdate; import com.tinyengine.it.model.dto.ParametersDto; import com.tinyengine.it.model.entity.Model; +import com.tinyengine.it.service.material.ModelService; + import lombok.extern.slf4j.Slf4j; -import com.tinyengine.it.common.utils.SqlIdentifierValidator; -import com.tinyengine.it.service.material.ModelService; import org.springframework.context.annotation.Lazy; +import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.PreparedStatementCreator; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; @@ -25,716 +28,753 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; -import java.sql.*; +import java.sql.Connection; import java.sql.Date; -import java.util.*; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; - @Service @Slf4j +@SuppressWarnings("PMD.TooManyMethods") public class DynamicModelService { - private static final Set SYSTEM_FIELDS = Set.of( - "id", "created_at", "updated_at", "deleted_at", "created_by", "updated_by" - ); - - private final JdbcTemplate jdbcTemplate; - private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; - private final LoginUserContext loginUserContext; - private final ModelService modelService; - - public DynamicModelService(JdbcTemplate jdbcTemplate, - NamedParameterJdbcTemplate namedParameterJdbcTemplate, - LoginUserContext loginUserContext, - @Lazy ModelService modelService) { - this.jdbcTemplate = jdbcTemplate; - this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; - this.loginUserContext = loginUserContext; - this.modelService = modelService; - } - - - /** - * 创建动态表 - */ - @Transactional - public void createDynamicTable(Model modelMetadata) { - if(modelMetadata.getParameters()==null || modelMetadata.getParameters().isEmpty()){ - throw new ServiceException(ExceptionEnum.CM001.getResultCode(), "Model parameters cannot be null or empty"); - - } - String tableName = getTableName(modelMetadata.getNameEn()); - String sql = generateCreateTableSQL(tableName, modelMetadata); - - log.info("createDynamicTable SQL: \n{}", sql); - - try { - jdbcTemplate.execute(sql); - log.info("createDynamicTable ok: {}", tableName); - - } catch (Exception e) { - log.error("createDynamicTable failed: {}", tableName, e); - throw new ServiceException(ExceptionEnum.CM001.getResultCode(), ExceptionEnum.CM001.getResultCode()); - - } - } - private String generateDropTableSQL(String tableName) { - if (tableName == null || tableName.isEmpty()) { - throw new IllegalArgumentException("Table name cannot be null or empty"); - } - - // Validate table name to prevent SQL injection - if (!tableName.matches("^[a-zA-Z0-9_]+$")) { - throw new IllegalArgumentException("Invalid table name: " + tableName); - } - StringBuilder sql = new StringBuilder(); - sql.append("DROP TABLE IF EXISTS ").append(tableName).append(";"); - return sql.toString(); - } - - public void dropDynamicTable(Model modelMetadata) { - if (modelMetadata == null || modelMetadata.getNameEn() == null || modelMetadata.getNameEn().isEmpty()) { - throw new IllegalArgumentException("Model metadata or table name cannot be null or empty"); - } - String tableName = getTableName(modelMetadata.getNameEn()); - - String sql = generateDropTableSQL(tableName); - try { - jdbcTemplate.execute(sql); - log.info("Successfully dropped table: {}", tableName); - } catch (Exception e) { - log.error("Failed to drop table: {}", tableName, e); - throw new ServiceException(ExceptionEnum.CM001.getResultCode(), ExceptionEnum.CM001.getResultCode()); - - } - } - /** - * 生成创建表的SQL - */ - private String generateCreateTableSQL(String tableName, Model model) { - StringBuilder sql = new StringBuilder(); - sql.append("CREATE TABLE IF NOT EXISTS ").append(tableName).append(" (\n"); - - // 基础字段 - List columns = new ArrayList<>(); - columns.add("id INT PRIMARY KEY AUTO_INCREMENT"); - - - // 用户定义字段 - for (ParametersDto field : model.getParameters()) { - if(!Objects.equals(field.getProp(), "id")){ - String columnDef = generateColumnDefinition(field,"init"); - columns.add(columnDef); - } - } - // 基础字段 - columns.add("created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP"); - columns.add("updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"); - columns.add("deleted_at TIMESTAMP NULL"); - columns.add("created_by INT NOT NULL"); - columns.add("updated_by INT NOT NULL"); - - - sql.append(String.join(",\n", columns)); - sql.append("\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"); - - return sql.toString(); - } - public void initializeDynamicTable(Model model, Long userId) { - if (model == null || CollectionUtils.isEmpty(model.getParameters())) { - throw new IllegalArgumentException("Model or parameters cannot be null or empty"); - } - - String tableName = getTableName(model.getNameEn()); - List parameters = model.getParameters(); - - // Prepare columns and values - List columns = new ArrayList<>(); - List values = new ArrayList<>(); - - for (ParametersDto param : parameters) { - String columnName = param.getProp(); - String fieldType = param.getType(); - param.setDefaultValue("1"); - String value = param.getDefaultValue(); - - if (value == null && Boolean.TRUE.equals(param.getRequired())) { - throw new IllegalArgumentException("Missing required parameter defaultValue: " + columnName); - } - - columns.add(columnName); - values.add(convertValueByType(value, fieldType, columnName)); - } - - // Add system fields - columns.add("created_by"); - columns.add("updated_by"); - values.add(userId); - values.add(userId); - - // Construct SQL - String sql = String.format( - "INSERT INTO %s (%s) VALUES (%s)", - tableName, - String.join(", ", columns), - columns.stream().map(c -> "?").collect(Collectors.joining(", ")) - ); - - // Execute SQL - jdbcTemplate.update(sql, values.toArray()); - } - - - - - - /** - * 通用查询方法,避免TypeHandler冲突 - */ - public List> dynamicQuery(String tableName, - List fields, - Map conditions, - String orderBy, - Integer limit) { - - SqlIdentifierValidator.validate(tableName); - SqlIdentifierValidator.validateAll(fields); - if (conditions != null && !conditions.isEmpty()) { - for (String key : conditions.keySet()) { - SqlIdentifierValidator.validate(key); - } - } - if (orderBy != null && !orderBy.isEmpty()) { - SqlIdentifierValidator.validate(orderBy.replaceAll("(?i)\\s+(ASC|DESC)$", "")); - } - - // 1. 构建SQL - StringBuilder sql = new StringBuilder("SELECT "); - - if (fields != null && !fields.isEmpty()) { - sql.append(String.join(", ", fields)); - } else { - sql.append("*"); - } - - sql.append(" FROM ").append(tableName); - - // 2. 构建WHERE条件 - if (conditions != null && !conditions.isEmpty()) { - List whereClauses = new ArrayList<>(); - boolean whereAdded = false; - getWhereCondition(conditions, sql, whereAdded, whereClauses); - } - - // 3. 排序 - if (orderBy != null && !orderBy.isEmpty()) { - sql.append(" ORDER BY ").append(orderBy); - } - - // 4. 分页 - if (limit != null && limit > 0) { - sql.append(" LIMIT ").append(limit); - } - - // 5. 执行查询 - if (conditions != null && !conditions.isEmpty()) { - return namedParameterJdbcTemplate.queryForList(sql.toString(), conditions); - } else { - return jdbcTemplate.queryForList(sql.toString()); - } - } - - public List> dynamicCount(String tableName, Map conditions) { - - // 1. 构建SQL - StringBuilder sql = new StringBuilder("SELECT COUNT(*) as count"); - - - - sql.append(" FROM ").append(tableName); - - // 2. 构建WHERE条件 - if (conditions != null && !conditions.isEmpty()) { - boolean whereAdded = false; - List whereClauses = new ArrayList<>(); - getWhereCondition(conditions, sql, whereAdded, whereClauses); - } - - // 5. 执行查询 - if (conditions != null && !conditions.isEmpty()) { - return namedParameterJdbcTemplate.queryForList(sql.toString(), conditions); - } else { - return jdbcTemplate.queryForList(sql.toString()); - } - } - - private void getWhereCondition(Map conditions, StringBuilder sql, boolean whereAdded, List whereClauses) { - for (Map.Entry entry : conditions.entrySet()) { - if (entry.getValue() != null) { - whereAdded = true; - whereClauses.add(entry.getKey() + " = :" + entry.getKey()); - } - } - if(whereAdded){ - sql.append(" WHERE "); - sql.append(String.join(" AND ", whereClauses)); - - } - } - - /** - * 查询总数 - */ - public Long count(String tableName, Map conditions) { - List> result = dynamicCount(tableName, conditions); - return Long.parseLong(result.get(0).get("count").toString()); - } - /** - * 分页查询 - */ - public Map queryWithPage(DynamicQuery dto) { - String tableName = getTableName(dto.getNameEn()); - List fields = dto.getFields(); - Map conditions = dto.getParams(); - String orderBy = dto.getOrderBy(); - Integer pageNum = dto.getCurrentPage(); - Integer pageSize = dto.getPageSize(); - - validateQueryFields(dto); - - // 计算分页 - Integer limit = null; - if (pageNum != null && pageSize != null) { - limit = pageSize; - } - - // 执行查询 - List> data = dynamicQuery( - tableName, fields, conditions, orderBy, limit); - Long count = count(tableName, conditions); - Map result = new HashMap<>(); - result.put("success", true); - result.put("data", data); - result.put("total", count); - - return result; - } - - private Set getAllowedFields(String nameEn) { - List modelList = modelService.getModelByEnName(nameEn); - if (modelList == null || modelList.isEmpty()) { - return Collections.emptySet(); - } - Model model = modelList.get(0); - Set allowed = new HashSet<>(SYSTEM_FIELDS); - if (model.getParameters() != null) { - for (Object param : model.getParameters()) { - String prop = extractProp(param); - if (prop != null) { - allowed.add(prop); - } - } - } - return allowed; - } - - @SuppressWarnings("unchecked") - private String extractProp(Object param) { - if (param instanceof ParametersDto) { - return ((ParametersDto) param).getProp(); - } - if (param instanceof Map) { - Object value = ((Map) param).get("prop"); - return value != null ? value.toString() : null; - } - return null; - } - - private void validateQueryFields(DynamicQuery dto) { - Set allowedFields = getAllowedFields(dto.getNameEn()); - - if (dto.getFields() != null && !dto.getFields().isEmpty()) { - for (String field : dto.getFields()) { - SqlIdentifierValidator.validate(field); - if (!allowedFields.contains(field)) { - throw new IllegalArgumentException("不允许的字段: " + field); - } - } - } - - if (dto.getOrderBy() != null && !dto.getOrderBy().isEmpty()) { - SqlIdentifierValidator.validate(dto.getOrderBy()); - if (!allowedFields.contains(dto.getOrderBy())) { - throw new IllegalArgumentException("不允许的排序字段: " + dto.getOrderBy()); - } - } - - if (dto.getOrderType() != null) { - SqlIdentifierValidator.validateOrderType(dto.getOrderType()); - } - } - private Object convertValueByType(Object value, String fieldType, String columnName) { - try { - switch (fieldType) { - case "String": - return value != null ? value.toString() : null; - case "Number": - return value != null ? Integer.parseInt(value.toString()) : null; - case "Boolean": - return value != null ? Boolean.parseBoolean(value.toString()) : null; - case "Date": - return value !=null ? Date.valueOf(value.toString()):null; // Assume proper date formatting is handled elsewhere - case "DateTime": - return value; // Assume proper date formatting is handled elsewhere - case "Enum": - return value; // Validation for enums should be handled before this - default: - return value; - } - } catch (Exception e) { - throw new IllegalArgumentException("Invalid value for field: " + columnName, e); - } - } - @Transactional - public void modifyTableStructure(Model model) { - String tableName = getTableName(model.getNameEn()); - List parameters = model.getParameters(); - if(parameters == null || parameters.isEmpty()){ - throw new IllegalArgumentException("Model parameters cannot be null or empty"); - } - - // Fetch existing table structure - String fetchColumnsSql = "SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = ?"; - List> existingColumns = jdbcTemplate.query(fetchColumnsSql, new Object[]{tableName}, (rs, rowNum) -> { - Map column = new HashMap<>(); - column.put("COLUMN_NAME", rs.getString("COLUMN_NAME")); - column.put("DATA_TYPE", rs.getString("DATA_TYPE")); - return column; - }); - - Map existingColumnMap = existingColumns.stream() - .collect(Collectors.toMap(col -> col.get("COLUMN_NAME"), col -> col.get("DATA_TYPE"))); - - // Generate ALTER TABLE statements - List alterStatements = new ArrayList<>(); - // Add or modify columns based on parameters - for (int i = 0; i < parameters.size(); i++) { - String afterColumn = null; - if(i>0){ - afterColumn = parameters.get(i - 1).getProp(); - } - ParametersDto param = parameters.get(i); - String columnName = param.getProp(); - String columnType = mapJavaTypeToSQL(param.getType()); - - if (!existingColumnMap.containsKey(columnName)) { - // Add new column - String addSql = generateColumnDefinition(param,"add"); - if(afterColumn != null){ - addSql += " AFTER " + afterColumn; - } - alterStatements.add(addSql); - } else if (!existingColumnMap.get(columnName).equalsIgnoreCase(columnType)) { - // Modify existing column - alterStatements.add(String.format("MODIFY COLUMN %s %s", columnName, columnType)); - } - } - - addCommonFields(parameters); - - // Drop columns that are not in the parameters - for (String existingColumn : existingColumnMap.keySet()) { - if (parameters.stream().noneMatch(param -> param.getProp().equals(existingColumn))) { - alterStatements.add(String.format("DROP COLUMN %s", existingColumn)); - } - } - - // Execute ALTER TABLE statements - for (String alterStatement : alterStatements) { - String sql = String.format("ALTER TABLE %s %s", tableName, alterStatement); - jdbcTemplate.execute(sql); - } - } - private void addCommonFields(List parameters) { - ParametersDto id = new ParametersDto(); - id.setProp("id"); - parameters.add(id); - ParametersDto createdAt = new ParametersDto(); - createdAt.setProp("created_at"); - parameters.add(createdAt); - ParametersDto updatedAt = new ParametersDto(); - updatedAt.setProp("updated_at"); - parameters.add(updatedAt); - ParametersDto deletedAt = new ParametersDto(); - deletedAt.setProp("deleted_at"); - parameters.add(deletedAt); - ParametersDto createdBy = new ParametersDto(); - createdBy.setProp("created_by"); - parameters.add(createdBy); - ParametersDto updatedBy = new ParametersDto(); - updatedBy.setProp("updated_by"); - parameters.add(updatedBy); - } - private static String mapJavaTypeToSQL(String javaType) { - if (javaType == null) { - return "VARCHAR(255)"; // 默认处理 - } - switch (javaType) { - case "String", "ModelRef": - return "VARCHAR"; - case "Number": - return "INT"; - case "Boolean": - return "TINYINT"; - case "Date": - return "TIMESTAMP"; - case "Enum": - return "Enum"; - default: - return "TEXT"; // 默认处理 - } - } - - - /** - * 生成字段定义 - */ - private String generateColumnDefinition(ParametersDto field,String type) { - StringBuilder sb = new StringBuilder(); - if(type.equals("add")) { - sb.append("ADD COLUMN "); - } else if(type.equals("modify")){ - sb.append("MODIFY COLUMN "); - } - sb.append(field.getProp()).append(" "); - - // 映射数据类型 - switch (field.getType()) { - case "String": - int maxLength = field.getMaxLength() != null ? field.getMaxLength() : 255; - sb.append("VARCHAR(").append(maxLength).append(")"); - break; - case "Integer", "Number": - sb.append("INT"); - break; - case "Boolean": - sb.append("TINYINT(1)"); - break; - case "Date": - sb.append("DATE"); - break; - case "DateTime": - sb.append("DATETIME"); - break; - case "Enum": - sb.append("ENUM").append("(").append(getEnumOptions(field.getOptions())).append(")"); - break; - case "ModelRef": - sb.append("VARCHAR(255)"); // 存储JSON字符串,长度可根据实际需求调整 - break; - default: - sb.append("TEXT"); - } - - if (Boolean.TRUE.equals(field.getRequired())) { - sb.append(" NOT NULL"); - } - - if (field.getDefaultValue() != null) { - sb.append(" DEFAULT '").append(field.getDefaultValue()).append("'"); - } - if(field.getDescription()!=null && !field.getDescription().isEmpty()){ - sb.append(" COMMENT '").append(field.getDescription()).append("'"); - } - - return sb.toString(); - } - - private String getEnumOptions(String optionStr) { - List options= new ArrayList<>(); - if(optionStr == null || optionStr.trim().isEmpty()){ - throw new IllegalArgumentException("Enum options cannot be null or empty"); - } - JSONArray jsonList; - try { - jsonList = JSON.parseArray(optionStr); - } catch (Exception e) { - throw new IllegalArgumentException("Invalid enum options format, expected JSON array string", e); - } - for (int i = 0; i < jsonList.size(); i++) { - String value = jsonList.getJSONObject(i).getString("value"); - options.add(value); - } - - return options.stream() - .map(opt -> "'" + opt + "'") - .collect(Collectors.joining(", ")); - } - - - /** - * 验证表和数据 - */ - private void validateTableAndData(String tableName, Map data) { - if (tableName == null || tableName.trim().isEmpty()) { - throw new IllegalArgumentException("表名不能为空"); - } - - // 防止SQL注入,验证表名格式 - if (!tableName.matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) { - throw new IllegalArgumentException("表名格式不正确"); - } - - if (data == null || data.isEmpty()) { - throw new IllegalArgumentException("数据不能为空"); - } - - // 验证字段名格式 - for (String field : data.keySet()) { - if (!field.matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) { - throw new IllegalArgumentException("字段名格式不正确: " + field); - } - } - } - - - /** - * 创建数据 - */ - public Map createData(DynamicInsert dataDto) { - - - String tableName = getTableName(dataDto.getNameEn()); - Map record = new HashMap<>(dataDto.getParams()); - for (String col : record.keySet()) { - SqlIdentifierValidator.validate(col); - } - String userId = loginUserContext.getLoginUserId(); - // 添加系统字段 - record.put("created_by",userId); - record.put("updated_by", userId); - - // 构建SQL - String columns = String.join(", ", record.keySet()); - String placeholders = record.keySet().stream() - .map(k -> "?") - .collect(Collectors.joining(", ")); - - String sql = String.format( - "INSERT INTO %s (%s) VALUES (%s)", - tableName, columns, placeholders - ); - - KeyHolder keyHolder = new GeneratedKeyHolder(); - - jdbcTemplate.update(new PreparedStatementCreator() { - @Override - public PreparedStatement createPreparedStatement(Connection con) throws SQLException { - PreparedStatement ps = con.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS); - - int index = 1; - for (Object value : record.values()) { - ps.setObject(index++, value); - } - - return ps; - } - }, keyHolder); - - Long generatedId = keyHolder.getKey() != null ? keyHolder.getKey().longValue() : null; - - if (generatedId != null) { - record.put("id", generatedId); - } - - return record; - } - - - - - - /** - * 获取表名 - */ - private String getTableName(String modelId) { - return "dynamic_" + modelId.toLowerCase(Locale.ROOT); - } - - - - public Map getDataById(String modelId, Long id) { - String tableName = getTableName(modelId); - String sql = "SELECT * FROM " + tableName + " WHERE id = ?"; - - List> results = jdbcTemplate.queryForList(sql, id); - - if (results.isEmpty()) { - return null; - } else { - return results.get(0); - } - } - - public Map updateDateById(DynamicUpdate dto) { - String modelId = dto.getNameEn(); - Map params1 = dto.getParams(); - if(params1 == null || !params1.containsKey("id")) { - throw new IllegalArgumentException("更新操作必须指定ID"); - } - if(dto.getData() == null || dto.getData().isEmpty()) { - throw new IllegalArgumentException("更新操作必须指定更新数据"); - } - if(modelId == null || modelId.trim().isEmpty()) { - throw new IllegalArgumentException("模型ID不能为空"); - } - Long id = Long.parseLong(params1.get("id").toString()); - Map updateFields = dto.getData(); - for (String col : updateFields.keySet()) { - SqlIdentifierValidator.validate(col); - } - String tableName = getTableName(modelId); - StringBuilder sql = new StringBuilder("UPDATE " + tableName + " SET "); - List params = new ArrayList<>(); - - for (Map.Entry entry : updateFields.entrySet()) { - sql.append(entry.getKey()).append(" = ?, "); - params.add(entry.getValue()); - } - - // 去掉最后的逗号和空格 - sql.setLength(sql.length() - 2); - sql.append(" WHERE id = ?"); - params.add(id); - - int rowsAffected = jdbcTemplate.update(sql.toString(), params.toArray()); - - Map result = new HashMap<>(); - result.put("rowsAffected", rowsAffected); - return result; - } - - public Map deleteDataById(DynamicDelete dto) { - String modelId = dto.getNameEn(); - if(modelId == null || modelId.trim().isEmpty()) { - throw new IllegalArgumentException("模型ID不能为空"); - } - if(dto.getId() == null) { - throw new IllegalArgumentException("删除操作必须指定ID"); - } - Long id = Long.valueOf(dto.getId()); - - String tableName = getTableName(modelId); - String sql = "DELETE FROM " + tableName + " WHERE id = ?"; - int update = jdbcTemplate.update(sql, id); - Map result = new HashMap<>(); - result.put("rowsAffected", update); - return result; - } - - - - + private static final Set SYSTEM_FIELDS = + Set.of("id", "created_at", "updated_at", "deleted_at", "created_by", "updated_by"); + private static final int DEFAULT_VARCHAR = 255; + private static final int ASC_SUFFIX_LEN = 4; + private static final int DESC_SUFFIX_LEN = 5; + + private final JdbcTemplate jdbcTemplate; + private final NamedParameterJdbcTemplate namedJdbcTemplate; + private final LoginUserContext loginUserContext; + private final ModelService modelService; + + public DynamicModelService( + JdbcTemplate jdbcTemplate, + NamedParameterJdbcTemplate namedJdbcTemplate, + LoginUserContext loginUserContext, + @Lazy ModelService modelService) { + this.jdbcTemplate = jdbcTemplate; + this.namedJdbcTemplate = namedJdbcTemplate; + this.loginUserContext = loginUserContext; + this.modelService = modelService; + } + + /** 创建动态表 */ + @Transactional + public void createDynamicTable(Model modelMetadata) { + if (modelMetadata.getParameters() == null || modelMetadata.getParameters().isEmpty()) { + throw new ServiceException( + ExceptionEnum.CM001.getResultCode(), + "Model parameters cannot be null or empty"); + } + String tableName = getTableName(modelMetadata.getNameEn()); + String sql = generateCreateTableSQL(tableName, modelMetadata); + + log.info("createDynamicTable SQL: \n{}", sql); + + try { + jdbcTemplate.execute(sql); + log.info("createDynamicTable ok: {}", tableName); + + } catch (DataAccessException exception) { + log.error("createDynamicTable failed: {}", tableName, exception); + throw new ServiceException( + ExceptionEnum.CM001.getResultCode(), + ExceptionEnum.CM001.getResultCode(), + exception); + } + } + private String generateDropTableSQL(String tableName) { + SqlIdentifierValidator.validate(tableName); + StringBuilder sql = new StringBuilder(); + sql.append("DROP TABLE IF EXISTS ").append(tableName).append(";"); + return sql.toString(); + } + + public void dropDynamicTable(Model modelMetadata) { + if (modelMetadata == null + || modelMetadata.getNameEn() == null + || modelMetadata.getNameEn().isEmpty()) { + throw new IllegalArgumentException( + "Model metadata or table name cannot be null or empty"); + } + String tableName = getTableName(modelMetadata.getNameEn()); + + String sql = generateDropTableSQL(tableName); + try { + jdbcTemplate.execute(sql); + log.info("Successfully dropped table: {}", tableName); + } catch (DataAccessException exception) { + log.error("Failed to drop table: {}", tableName, exception); + throw new ServiceException( + ExceptionEnum.CM001.getResultCode(), + ExceptionEnum.CM001.getResultCode(), + exception); + } + } + + /** + * 生成创建表的SQL. + * + * @return SQL used to create the dynamic table + */ + private String generateCreateTableSQL(String tableName, Model model) { + SqlIdentifierValidator.validate(tableName); + StringBuilder sql = new StringBuilder(); + sql.append("CREATE TABLE IF NOT EXISTS ").append(tableName).append(" (\n"); + + // 基础字段 + List columns = new ArrayList<>(); + columns.add("id INT PRIMARY KEY AUTO_INCREMENT"); + + // 用户定义字段 + for (ParametersDto field : model.getParameters()) { + if (!Objects.equals(field.getProp(), "id")) { + SqlIdentifierValidator.validate(field.getProp()); + String columnDef = generateColumnDefinition(field, "init"); + columns.add(columnDef); + } + } + // 基础字段 + columns.add("created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP"); + columns.add("updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"); + columns.add("deleted_at TIMESTAMP NULL"); + columns.add("created_by INT NOT NULL"); + columns.add("updated_by INT NOT NULL"); + + sql.append(String.join(",\n", columns)); + sql.append("\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"); + + return sql.toString(); + } + + public void initializeDynamicTable(Model model, Long userId) { + if (model == null || CollectionUtils.isEmpty(model.getParameters())) { + throw new IllegalArgumentException("Model or parameters cannot be null or empty"); + } + + String tableName = getTableName(model.getNameEn()); + List parameters = model.getParameters(); + + // Prepare columns and values + List columns = new ArrayList<>(); + List values = new ArrayList<>(); + + for (ParametersDto param : parameters) { + String columnName = param.getProp(); + SqlIdentifierValidator.validate(columnName); + String fieldType = param.getType(); + param.setDefaultValue("1"); + String value = param.getDefaultValue(); + + if (value == null && Boolean.TRUE.equals(param.getRequired())) { + throw new IllegalArgumentException( + "Missing required parameter defaultValue: " + columnName); + } + + columns.add(columnName); + values.add(convertValueByType(value, fieldType, columnName)); + } + + // Add system fields + columns.add("created_by"); + columns.add("updated_by"); + values.add(userId); + values.add(userId); + + // Construct SQL + String sql = + String.format( + "INSERT INTO %s (%s) VALUES (%s)", + tableName, + String.join(", ", columns), + columns.stream().map(c -> "?").collect(Collectors.joining(", "))); + + // Execute SQL + jdbcTemplate.update(sql, values.toArray()); + } + + /** + * 通用查询方法,避免TypeHandler冲突. + * + * @return query results + */ + public List> dynamicQuery( + String tableName, + List fields, + Map conditions, + String orderBy, + Integer limit) { + + SqlIdentifierValidator.validate(tableName); + SqlIdentifierValidator.validateAll(fields); + if (conditions != null && !conditions.isEmpty()) { + for (String key : conditions.keySet()) { + SqlIdentifierValidator.validate(key); + } + } + if (orderBy != null && !orderBy.isEmpty()) { + validateOrderBy(orderBy); + } + + // 1. 构建SQL + StringBuilder sql = new StringBuilder("SELECT "); + + if (fields != null && !fields.isEmpty()) { + sql.append(String.join(", ", fields)); + } else { + sql.append("*"); + } + + sql.append(" FROM ").append(tableName); + + // 2. 构建WHERE条件 + if (conditions != null && !conditions.isEmpty()) { + List whereClauses = new ArrayList<>(); + boolean whereAdded = false; + getWhereCondition(conditions, sql, whereAdded, whereClauses); + } + + // 3. 排序 + if (orderBy != null && !orderBy.isEmpty()) { + sql.append(" ORDER BY ").append(orderBy); + } + + // 4. 分页 + if (limit != null && limit > 0) { + sql.append(" LIMIT ").append(limit); + } + + // 5. 执行查询 + if (conditions != null && !conditions.isEmpty()) { + return namedJdbcTemplate.queryForList(sql.toString(), conditions); + } else { + return jdbcTemplate.queryForList(sql.toString()); + } + } + + public List> dynamicCount( + String tableName, Map conditions) { + + // 1. 构建SQL + StringBuilder sql = new StringBuilder("SELECT COUNT(*) as count"); + + sql.append(" FROM ").append(tableName); + + // 2. 构建WHERE条件 + if (conditions != null && !conditions.isEmpty()) { + boolean whereAdded = false; + List whereClauses = new ArrayList<>(); + getWhereCondition(conditions, sql, whereAdded, whereClauses); + } + + // 5. 执行查询 + if (conditions != null && !conditions.isEmpty()) { + return namedJdbcTemplate.queryForList(sql.toString(), conditions); + } else { + return jdbcTemplate.queryForList(sql.toString()); + } + } + + private void getWhereCondition( + Map conditions, + StringBuilder sql, + boolean whereAdded, + List whereClauses) { + for (Map.Entry entry : conditions.entrySet()) { + if (entry.getValue() != null) { + whereAdded = true; + whereClauses.add(entry.getKey() + " = :" + entry.getKey()); + } + } + if (whereAdded) { + sql.append(" WHERE "); + sql.append(String.join(" AND ", whereClauses)); + } + } + + /** + * 查询总数. + * + * @return record count + */ + public Long count(String tableName, Map conditions) { + List> result = dynamicCount(tableName, conditions); + return Long.parseLong(result.get(0).get("count").toString()); + } + + /** + * 分页查询. + * + * @return paginated query result + */ + public Map queryWithPage(DynamicQuery dto) { + String tableName = getTableName(dto.getNameEn()); + List fields = dto.getFields(); + Map conditions = dto.getParams(); + String orderBy = dto.getOrderBy(); + Integer pageNum = dto.getCurrentPage(); + Integer pageSize = dto.getPageSize(); + + validateQueryFields(dto); + + // 计算分页 + Integer limit = null; + if (pageNum != null && pageSize != null) { + limit = pageSize; + } + + // 执行查询 + List> data = + dynamicQuery(tableName, fields, conditions, orderBy, limit); + Long count = count(tableName, conditions); + Map result = new HashMap<>(); + result.put("success", true); + result.put("data", data); + result.put("total", count); + + return result; + } + + private Set getAllowedFields(String nameEn) { + List modelList = modelService.getModelByEnName(nameEn); + if (modelList == null || modelList.isEmpty()) { + return Collections.emptySet(); + } + Model model = modelList.get(0); + Set allowed = new HashSet<>(SYSTEM_FIELDS); + if (model.getParameters() != null) { + for (Object param : model.getParameters()) { + String prop = extractProp(param); + if (prop != null) { + allowed.add(prop); + } + } + } + return allowed; + } + + @SuppressWarnings("unchecked") + private String extractProp(Object param) { + if (param instanceof ParametersDto) { + return ((ParametersDto) param).getProp(); + } + if (param instanceof Map) { + Object value = ((Map) param).get("prop"); + return value != null ? value.toString() : null; + } + return null; + } + + private void validateQueryFields(DynamicQuery dto) { + Set allowedFields = getAllowedFields(dto.getNameEn()); + + if (dto.getFields() != null && !dto.getFields().isEmpty()) { + for (String field : dto.getFields()) { + SqlIdentifierValidator.validate(field); + if (!allowedFields.contains(field)) { + throw new IllegalArgumentException("不允许的字段: " + field); + } + } + } + + if (dto.getOrderBy() != null && !dto.getOrderBy().isEmpty()) { + SqlIdentifierValidator.validate(dto.getOrderBy()); + if (!allowedFields.contains(dto.getOrderBy())) { + throw new IllegalArgumentException("不允许的排序字段: " + dto.getOrderBy()); + } + } + + if (dto.getOrderType() != null) { + SqlIdentifierValidator.validateOrderType(dto.getOrderType()); + } + } + + private Object convertValueByType(Object value, String fieldType, String columnName) { + try { + return switch (fieldType) { + case "String" -> value != null ? value.toString() : null; + case "Number" -> value != null ? Integer.parseInt(value.toString()) : null; + case "Boolean" -> value != null ? Boolean.parseBoolean(value.toString()) : null; + case "Date" -> + value != null + ? Date.valueOf(value.toString()) + : null; // Assume proper date formatting is handled elsewhere + case "DateTime" -> value; // Assume proper date formatting is handled elsewhere + case "Enum" -> value; // Validation for enums should be handled before this + default -> value; + }; + } catch (IllegalArgumentException exception) { + throw new IllegalArgumentException("Invalid value for field: " + columnName, exception); + } + } + + @Transactional + public void modifyTableStructure(Model model) { + String tableName = getTableName(model.getNameEn()); + SqlIdentifierValidator.validate(tableName); + List parameters = model.getParameters(); + if (parameters == null || parameters.isEmpty()) { + throw new IllegalArgumentException("Model parameters cannot be null or empty"); + } + + // Fetch existing table structure + String fetchColumnsSql = + "SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME =" + + " ?"; + List> existingColumns = + jdbcTemplate.query( + fetchColumnsSql, + new Object[] {tableName}, + (rs, rowNum) -> { + Map column = new HashMap<>(); + column.put("COLUMN_NAME", rs.getString("COLUMN_NAME")); + column.put("DATA_TYPE", rs.getString("DATA_TYPE")); + return column; + }); + + Map existingColumnMap = + existingColumns.stream() + .collect( + Collectors.toMap( + col -> col.get("COLUMN_NAME"), + col -> col.get("DATA_TYPE"))); + + // Generate ALTER TABLE statements + List alterStatements = new ArrayList<>(); + // Add or modify columns based on parameters + for (int i = 0; i < parameters.size(); i++) { + String afterColumn = null; + if (i > 0) { + afterColumn = parameters.get(i - 1).getProp(); + SqlIdentifierValidator.validate(afterColumn); + } + ParametersDto param = parameters.get(i); + String columnName = param.getProp(); + SqlIdentifierValidator.validate(columnName); + String columnType = mapJavaTypeToSQL(param.getType()); + + if (!existingColumnMap.containsKey(columnName)) { + // Add new column + String addSql = generateColumnDefinition(param, "add"); + if (afterColumn != null) { + addSql += " AFTER " + afterColumn; + } + alterStatements.add(addSql); + } else if (!existingColumnMap.get(columnName).equalsIgnoreCase(columnType)) { + // Modify existing column + alterStatements.add(String.format("MODIFY COLUMN %s %s", columnName, columnType)); + } + } + + addCommonFields(parameters); + + // Drop columns that are not in the parameters + for (String existingColumn : existingColumnMap.keySet()) { + SqlIdentifierValidator.validate(existingColumn); + if (parameters.stream().noneMatch(param -> param.getProp().equals(existingColumn))) { + alterStatements.add(String.format("DROP COLUMN %s", existingColumn)); + } + } + + // Execute ALTER TABLE statements + for (String alterStatement : alterStatements) { + String sql = String.format("ALTER TABLE %s %s", tableName, alterStatement); + jdbcTemplate.execute(sql); + } + } + + private void addCommonFields(List parameters) { + ParametersDto id = new ParametersDto(); + id.setProp("id"); + parameters.add(id); + ParametersDto createdAt = new ParametersDto(); + createdAt.setProp("created_at"); + parameters.add(createdAt); + ParametersDto updatedAt = new ParametersDto(); + updatedAt.setProp("updated_at"); + parameters.add(updatedAt); + ParametersDto deletedAt = new ParametersDto(); + deletedAt.setProp("deleted_at"); + parameters.add(deletedAt); + ParametersDto createdBy = new ParametersDto(); + createdBy.setProp("created_by"); + parameters.add(createdBy); + ParametersDto updatedBy = new ParametersDto(); + updatedBy.setProp("updated_by"); + parameters.add(updatedBy); + } + + private static String mapJavaTypeToSQL(String javaType) { + if (javaType == null) { + return "VARCHAR(" + DEFAULT_VARCHAR + ")"; // 默认处理 + } + return switch (javaType) { + case "String", "ModelRef" -> "VARCHAR"; + case "Number" -> "INT"; + case "Boolean" -> "TINYINT"; + case "Date" -> "TIMESTAMP"; + case "Enum" -> "Enum"; + default -> "TEXT"; // 默认处理 + }; + } + + /** + * 生成字段定义. + * + * @return column definition SQL + */ + private String generateColumnDefinition(ParametersDto field, String type) { + StringBuilder sb = new StringBuilder(); + if (type.equals("add")) { + sb.append("ADD COLUMN "); + } else if (type.equals("modify")) { + sb.append("MODIFY COLUMN "); + } + String columnName = SqlIdentifierValidator.requireValidIdentifier(field.getProp()); + sb.append(columnName).append(" "); + + // 映射数据类型 + switch (field.getType() == null ? "" : field.getType()) { + case "String": + int maxLength = + field.getMaxLength() != null ? field.getMaxLength() : DEFAULT_VARCHAR; + sb.append("VARCHAR(").append(maxLength).append(")"); + break; + case "Integer", "Number": + sb.append("INT"); + break; + case "Boolean": + sb.append("TINYINT(1)"); + break; + case "Date": + sb.append("DATE"); + break; + case "DateTime": + sb.append("DATETIME"); + break; + case "Enum": + sb.append("ENUM") + .append("(") + .append(getEnumOptions(field.getOptions())) + .append(")"); + break; + case "ModelRef": + sb.append("VARCHAR(255)"); // 存储JSON字符串,长度可根据实际需求调整 + break; + default: + sb.append("TEXT"); + } + + if (Boolean.TRUE.equals(field.getRequired())) { + sb.append(" NOT NULL"); + } + + if (field.getDefaultValue() != null) { + sb.append(" DEFAULT '") + .append(SqlIdentifierValidator.escapeSqlLiteral(field.getDefaultValue())) + .append("'"); + } + if (field.getDescription() != null && !field.getDescription().isEmpty()) { + sb.append(" COMMENT '") + .append(SqlIdentifierValidator.escapeSqlLiteral(field.getDescription())) + .append("'"); + } + + return sb.toString(); + } + + private String getEnumOptions(String optionStr) { + List options = new ArrayList<>(); + if (optionStr == null || optionStr.isBlank()) { + throw new IllegalArgumentException("Enum options cannot be null or empty"); + } + JSONArray jsonList; + try { + jsonList = JSON.parseArray(optionStr); + } catch (JSONException exception) { + throw new IllegalArgumentException( + "Invalid enum options format, expected JSON array string", exception); + } + for (int i = 0; i < jsonList.size(); i++) { + String value = jsonList.getJSONObject(i).getString("value"); + options.add(SqlIdentifierValidator.escapeSqlLiteral(value)); + } + + return options.stream().map(opt -> "'" + opt + "'").collect(Collectors.joining(", ")); + } + + /** 验证表和数据 */ + private void validateTableAndData(String tableName, Map data) { + SqlIdentifierValidator.validate(tableName); + + if (data == null || data.isEmpty()) { + throw new IllegalArgumentException("数据不能为空"); + } + + // 验证字段名格式 + for (String field : data.keySet()) { + SqlIdentifierValidator.validate(field); + } + } + + /** + * 创建数据. + * + * @return created record data + */ + public Map createData(DynamicInsert dataDto) { + + String tableName = getTableName(dataDto.getNameEn()); + Map record = new HashMap<>(dataDto.getParams()); + for (String col : record.keySet()) { + SqlIdentifierValidator.validate(col); + } + String userId = loginUserContext.getLoginUserId(); + // 添加系统字段 + record.put("created_by", userId); + record.put("updated_by", userId); + + // 构建SQL + String columns = String.join(", ", record.keySet()); + String placeholders = + record.keySet().stream().map(k -> "?").collect(Collectors.joining(", ")); + + String sql = + String.format("INSERT INTO %s (%s) VALUES (%s)", tableName, columns, placeholders); + + KeyHolder keyHolder = new GeneratedKeyHolder(); + + jdbcTemplate.update( + new PreparedStatementCreator() { + @Override + public PreparedStatement createPreparedStatement(Connection con) + throws SQLException { + PreparedStatement ps = + con.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS); + + int index = 1; + for (Object value : record.values()) { + ps.setObject(index++, value); + } + + return ps; + } + }, + keyHolder); + + Long generatedId = keyHolder.getKey() != null ? keyHolder.getKey().longValue() : null; + + if (generatedId != null) { + record.put("id", generatedId); + } + + return record; + } + + /** + * 获取表名. + * + * @return validated dynamic table name + */ + private String getTableName(String modelId) { + if (modelId == null || modelId.isBlank()) { + throw new IllegalArgumentException("Model name cannot be null or empty"); + } + String tableName = "dynamic_" + modelId.toLowerCase(Locale.ROOT); + return SqlIdentifierValidator.requireValidIdentifier(tableName); + } + + private void validateOrderBy(String orderBy) { + String trimmed = orderBy.trim(); + if (trimmed.isEmpty()) { + throw new IllegalArgumentException("Invalid order by: " + orderBy); + } + String upper = trimmed.toUpperCase(Locale.ROOT); + String identifier = trimmed; + if (upper.endsWith(" ASC")) { + identifier = trimmed.substring(0, trimmed.length() - ASC_SUFFIX_LEN).trim(); + } else if (upper.endsWith(" DESC")) { + identifier = trimmed.substring(0, trimmed.length() - DESC_SUFFIX_LEN).trim(); + } + SqlIdentifierValidator.validate(identifier); + } + + public Map getDataById(String modelId, Long id) { + String tableName = getTableName(modelId); + String sql = "SELECT * FROM " + tableName + " WHERE id = ?"; + + List> results = jdbcTemplate.queryForList(sql, id); + + return results.isEmpty() ? Collections.emptyMap() : results.get(0); + } + + public Map updateDateById(DynamicUpdate dto) { + String modelId = dto.getNameEn(); + Map params1 = dto.getParams(); + if (params1 == null || !params1.containsKey("id")) { + throw new IllegalArgumentException("更新操作必须指定ID"); + } + if (dto.getData() == null || dto.getData().isEmpty()) { + throw new IllegalArgumentException("更新操作必须指定更新数据"); + } + if (modelId == null || modelId.isBlank()) { + throw new IllegalArgumentException("模型ID不能为空"); + } + Long id = Long.parseLong(params1.get("id").toString()); + Map updateFields = dto.getData(); + for (String col : updateFields.keySet()) { + SqlIdentifierValidator.validate(col); + } + String tableName = getTableName(modelId); + StringBuilder sql = new StringBuilder("UPDATE " + tableName + " SET "); + List params = new ArrayList<>(); + + for (Map.Entry entry : updateFields.entrySet()) { + sql.append(entry.getKey()).append(" = ?, "); + params.add(entry.getValue()); + } + + // 去掉最后的逗号和空格 + sql.setLength(sql.length() - 2); + sql.append(" WHERE id = ?"); + params.add(id); + + int rowsAffected = jdbcTemplate.update(sql.toString(), params.toArray()); + + Map result = new HashMap<>(); + result.put("rowsAffected", rowsAffected); + return result; + } + + public Map deleteDataById(DynamicDelete dto) { + String modelId = dto.getNameEn(); + if (modelId == null || modelId.isBlank()) { + throw new IllegalArgumentException("模型ID不能为空"); + } + if (dto.getId() == null) { + throw new IllegalArgumentException("删除操作必须指定ID"); + } + Long id = Long.valueOf(dto.getId()); + + String tableName = getTableName(modelId); + String sql = "DELETE FROM " + tableName + " WHERE id = ?"; + int update = jdbcTemplate.update(sql, id); + Map result = new HashMap<>(); + result.put("rowsAffected", update); + return result; + } } diff --git a/base/src/main/java/com/tinyengine/it/dynamic/service/DynamicService.java b/base/src/main/java/com/tinyengine/it/dynamic/service/DynamicService.java index f33cb184..c8ad64b1 100644 --- a/base/src/main/java/com/tinyengine/it/dynamic/service/DynamicService.java +++ b/base/src/main/java/com/tinyengine/it/dynamic/service/DynamicService.java @@ -4,36 +4,48 @@ import com.tinyengine.it.common.context.LoginUserContext; import com.tinyengine.it.common.utils.SqlIdentifierValidator; import com.tinyengine.it.dynamic.dao.ModelDataDao; -import com.tinyengine.it.dynamic.dto.*; +import com.tinyengine.it.dynamic.dto.DynamicDelete; +import com.tinyengine.it.dynamic.dto.DynamicInsert; +import com.tinyengine.it.dynamic.dto.DynamicQuery; +import com.tinyengine.it.dynamic.dto.DynamicUpdate; import com.tinyengine.it.model.dto.ParametersDto; import com.tinyengine.it.model.entity.Model; import com.tinyengine.it.service.material.ModelService; + import jakarta.transaction.Transactional; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.math.BigInteger; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; @Service +@SuppressWarnings("PMD.TooManyMethods") public class DynamicService { - @Autowired - private ModelDataDao dynamicDao; - @Autowired - private ModelService modelService; - @Autowired - private LoginUserContext loginUserContext; + @Autowired private ModelDataDao dynamicDao; + @Autowired private ModelService modelService; + @Autowired private LoginUserContext loginUserContext; - private static final Set SYSTEM_FIELDS = Set.of( - "id", "created_at", "updated_at", "deleted_at", "created_by", "updated_by" - ); + private static final Set SYSTEM_FIELDS = + Set.of("id", "created_at", "updated_at", "deleted_at", "created_by", "updated_by"); + private static final int DEFAULT_PAGE_SIZE = 10; + private static final String TABLE_NAME_KEY = "tableName"; + private static final String CONDITIONS_KEY = "conditions"; public List query(DynamicQuery dto) { String tableName = getTableName(dto.getNameEn()); Map params = new HashMap<>(); - params.put("tableName", tableName); + params.put(TABLE_NAME_KEY, tableName); params.put("fields", dto.getFields()); - params.put("conditions", dto.getParams()); + params.put(CONDITIONS_KEY, dto.getParams()); params.put("pageNum", dto.getCurrentPage()); params.put("pageSize", dto.getPageSize()); params.put("orderBy", dto.getOrderBy()); @@ -44,23 +56,23 @@ public List query(DynamicQuery dto) { public Long count(String tableName, Map conditions) { Map params = new HashMap<>(); - params.put("tableName", tableName); + params.put(TABLE_NAME_KEY, tableName); params.put("fields", Arrays.asList("COUNT(*) as count")); - params.put("conditions", conditions); + params.put(CONDITIONS_KEY, conditions); List result = dynamicDao.select(params); return Long.parseLong(result.get(0).get("count").toString()); } public Map queryWithPage(DynamicQuery dto) { - if (dto.getNameEn() == null || dto.getNameEn().trim().isEmpty()) { + if (dto.getNameEn() == null || dto.getNameEn().isBlank()) { throw new IllegalArgumentException("查询操作必须指定模型名称"); } if (dto.getCurrentPage() == null || dto.getCurrentPage() <= 0) { dto.setCurrentPage(1); } if (dto.getPageSize() == null || dto.getPageSize() <= 0) { - dto.setPageSize(10); + dto.setPageSize(DEFAULT_PAGE_SIZE); } validateTableExists(dto.getNameEn()); validateConditionKeys(dto.getParams()); @@ -81,7 +93,7 @@ public Map queryWithPage(DynamicQuery dto) { @Transactional public Map insert(DynamicInsert dto) { - if (dto.getNameEn() == null || dto.getNameEn().trim().isEmpty()) { + if (dto.getNameEn() == null || dto.getNameEn().isBlank()) { throw new IllegalArgumentException("插入操作必须指定模型名称"); } if (dto.getParams() == null || dto.getParams().isEmpty()) { @@ -91,11 +103,11 @@ public Map insert(DynamicInsert dto) { validateTableAndData(dto.getNameEn(), dto.getParams()); String tableName = getTableName(dto.getNameEn()); Map params = new HashMap<>(); - params.put("tableName", tableName); + params.put(TABLE_NAME_KEY, tableName); params.put("data", dto.getParams()); String userId = loginUserContext.getLoginUserId(); - if (userId == null || userId.trim().isEmpty()) { + if (userId == null || userId.isBlank()) { List modelList = modelService.getModelByEnName(dto.getNameEn()); if (modelList.isEmpty()) { throw new IllegalArgumentException("模型不存在: " + dto.getNameEn()); @@ -117,7 +129,7 @@ public Map insert(DynamicInsert dto) { @Transactional public Map update(DynamicUpdate dto) { - if (dto.getNameEn() == null || dto.getNameEn().trim().isEmpty()) { + if (dto.getNameEn() == null || dto.getNameEn().isBlank()) { throw new IllegalArgumentException("更新操作必须指定模型名称"); } if (dto.getParams() == null || dto.getParams().isEmpty()) { @@ -131,9 +143,9 @@ public Map update(DynamicUpdate dto) { validateTableAndData(dto.getNameEn(), dto.getParams()); String tableName = getTableName(dto.getNameEn()); Map params = new HashMap<>(); - params.put("tableName", tableName); + params.put(TABLE_NAME_KEY, tableName); params.put("data", dto.getData()); - params.put("conditions", dto.getParams()); + params.put(CONDITIONS_KEY, dto.getParams()); Map result = new HashMap<>(); Integer update = dynamicDao.update(params); result.put("update", update); @@ -142,7 +154,7 @@ public Map update(DynamicUpdate dto) { @Transactional public Map delete(DynamicDelete dto) { - if (dto.getNameEn() == null || dto.getNameEn().trim().isEmpty()) { + if (dto.getNameEn() == null || dto.getNameEn().isBlank()) { throw new IllegalArgumentException("删除操作必须指定模型名称"); } if (dto.getId() == null) { @@ -154,8 +166,8 @@ public Map delete(DynamicDelete dto) { Map params = new HashMap<>(); Map conditions = new HashMap<>(); conditions.put("id", dto.getId()); - params.put("tableName", tableName); - params.put("conditions", conditions); + params.put(TABLE_NAME_KEY, tableName); + params.put(CONDITIONS_KEY, conditions); Map result = new HashMap<>(); Integer delete = dynamicDao.delete(params); result.put("delete", delete); @@ -222,13 +234,7 @@ private void validateQueryFields(DynamicQuery dto) { } private void validateTableAndData(String tableName, Map data) { - if (tableName == null || tableName.trim().isEmpty()) { - throw new IllegalArgumentException("表名不能为空"); - } - - if (!tableName.matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) { - throw new IllegalArgumentException("表名格式不正确"); - } + SqlIdentifierValidator.validate(tableName); if (data == null || data.isEmpty()) { throw new IllegalArgumentException("数据不能为空"); @@ -256,6 +262,10 @@ public void validateTableExists(String tableName) { } private String getTableName(String modelId) { - return "dynamic_" + modelId.toLowerCase(Locale.ROOT); + if (modelId == null || modelId.isBlank()) { + throw new IllegalArgumentException("模型名称不能为空"); + } + String tableName = "dynamic_" + modelId.toLowerCase(Locale.ROOT); + return SqlIdentifierValidator.requireValidIdentifier(tableName); } } diff --git a/base/src/main/java/com/tinyengine/it/rag/config/RAGConfig.java b/base/src/main/java/com/tinyengine/it/rag/config/RAGConfig.java index 354c4757..91d869f6 100644 --- a/base/src/main/java/com/tinyengine/it/rag/config/RAGConfig.java +++ b/base/src/main/java/com/tinyengine/it/rag/config/RAGConfig.java @@ -1,50 +1,57 @@ /** - * Copyright (c) 2023 - present TinyEngine Authors. - * Copyright (c) 2023 - present Huawei Cloud Computing Technologies Co., Ltd. + * Copyright (c) 2023 - present TinyEngine Authors. Copyright (c) 2023 - present Huawei Cloud + * Computing Technologies Co., Ltd. * - * Use of this source code is governed by an MIT-style license. - * - * THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, - * BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR - * A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + *

Use of this source code is governed by an MIT-style license. * + *

THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, + * BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR A + * PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. */ - package com.tinyengine.it.rag.config; import lombok.Data; + import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; -/** - * RAG config - */ +/** RAG config */ @Component @ConfigurationProperties(prefix = "rag") @Data public class RAGConfig { + private static final int DEF_CHUNK_SIZE = 1000; + private static final int DEF_CHUNK_OVER = 200; + private static final int DEF_MAX_RESULTS = 10; + private static final double DEF_MIN_SCORE = 0.7; + private static final int DEF_TIMEOUT_SEC = 30; + private static final int DEF_MAX_RETRIES = 5; + private static final int DEF_RETRY_MS = 1000; + private static final int DEF_BATCH_SIZE = 50; // 文档处理配置 - private int chunkSize = 1000; - private int chunkOverlap = 200; - private int maxResults = 10; - private double minScore = 0.7; + private int chunkSize = DEF_CHUNK_SIZE; + private int chunkOverlap = DEF_CHUNK_OVER; + private int maxResults = DEF_MAX_RESULTS; + private double minScore = DEF_MIN_SCORE; // Chroma 配置 private String chromaBaseUrl = System.getenv("CHROMA_BASE_URL"); + @SuppressWarnings("PMD.LongVariable") private String chromaCollectionName = "tinyengine_documents"; private String modelPath = System.getenv("MODEL_PATH"); private String tokenizerPath = System.getenv("TOKENIZER_PATH"); + private String documentRoot = System.getenv("FOLDER_PATH"); // 连接配置 - private int timeoutSeconds = 30; - private int maxRetries = 5; - private int retryIntervalMs = 1000; + private int timeoutSeconds = DEF_TIMEOUT_SEC; + private int maxRetries = DEF_MAX_RETRIES; + private int retryIntervalMs = DEF_RETRY_MS; // 嵌入模型配置 private String embeddingModel = "all-minilm-l6-v2"; - private int batchSize = 50; + private int batchSize = DEF_BATCH_SIZE; // 其他配置 - private boolean debugMode = false; + private boolean debugMode; } diff --git a/base/src/main/java/com/tinyengine/it/rag/config/VectorStoreConfig.java b/base/src/main/java/com/tinyengine/it/rag/config/VectorStoreConfig.java index 50ff5006..689b2305 100644 --- a/base/src/main/java/com/tinyengine/it/rag/config/VectorStoreConfig.java +++ b/base/src/main/java/com/tinyengine/it/rag/config/VectorStoreConfig.java @@ -1,233 +1,279 @@ -/** - * Copyright (c) 2023 - present TinyEngine Authors. - * Copyright (c) 2023 - present Huawei Cloud Computing Technologies Co., Ltd. - *

- * Use of this source code is governed by an MIT-style license. - *

- * THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, - * BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR - * A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. - */ - -package com.tinyengine.it.rag.config; - -import com.tinyengine.it.rag.service.StorageService; -import dev.langchain4j.data.embedding.Embedding; -import dev.langchain4j.model.embedding.EmbeddingModel; -import dev.langchain4j.model.embedding.onnx.OnnxEmbeddingModel; -import dev.langchain4j.model.embedding.onnx.PoolingMode; -import dev.langchain4j.model.output.Response; -import dev.langchain4j.store.embedding.EmbeddingSearchRequest; -import dev.langchain4j.store.embedding.EmbeddingSearchResult; -import dev.langchain4j.store.embedding.EmbeddingStore; -import dev.langchain4j.store.embedding.chroma.ChromaEmbeddingStore; -import dev.langchain4j.data.segment.TextSegment; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import okhttp3.OkHttpClient; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import java.time.Duration; -import java.util.Collections; -import java.util.List; - -/** - * Vector store config - */ -@Configuration -@RequiredArgsConstructor -@Slf4j -public class VectorStoreConfig { - - private final RAGConfig ragConfig; - - /** - * 嵌入模型 Bean - 尝试创建,失败时返回降级实现 - */ - @Bean - public EmbeddingModel embeddingModel() { - try { - // 检查必要的配置参数 - if (ragConfig.getModelPath() == null || ragConfig.getTokenizerPath() == null) { - log.warn("ONNX model configuration is incomplete, using fallback embedding model"); - return createFallbackEmbeddingModel(); - } - - log.info("Initializing ONNX embedding model..."); - - EmbeddingModel model = new OnnxEmbeddingModel( - ragConfig.getModelPath(), - ragConfig.getTokenizerPath(), - PoolingMode.MEAN - ); - - log.info("✅ ONNX embedding model initialization successful"); - return model; - - } catch (Exception e) { - log.warn("❌ ONNX embedding model initialization failed, using fallback implementation", e); - return createFallbackEmbeddingModel(); - } - } - - /** - * 嵌入存储 Bean - 尝试创建,失败时返回降级实现 - */ - @Bean - public EmbeddingStore embeddingStore() { - try { - // 检查必要的配置参数 - if (ragConfig.getChromaBaseUrl() == null) { - log.warn("ChromaDB configuration is incomplete, using fallback embedding store"); - return createFallbackEmbeddingStore(); - } - - log.info("Attempting to initialize ChromaDB connection: {}", ragConfig.getChromaBaseUrl()); - - // 测试连接 - if (!testChromaConnection(ragConfig.getChromaBaseUrl())) { - log.warn("ChromaDB connection test failed, using fallback embedding store"); - return createFallbackEmbeddingStore(); - } - - ChromaEmbeddingStore embeddingStore = ChromaEmbeddingStore.builder() - .baseUrl(ragConfig.getChromaBaseUrl()) - .collectionName(ragConfig.getChromaCollectionName() != null ? - ragConfig.getChromaCollectionName() : "documents") - .timeout(Duration.ofSeconds(30)) - .build(); - - log.info("✅ ChromaDB embeddingStore initialization successful"); - return embeddingStore; - - } catch (Exception e) { - log.warn("❌ ChromaDB initialization failed, using fallback embedding store", e); - return createFallbackEmbeddingStore(); - } - } - - /** - * 存储服务 Bean - 总是创建,依赖降级实现 - */ - @Bean - public StorageService vectorStorageService(EmbeddingModel embeddingModel, EmbeddingStore embeddingStore) { - try { - StorageService service = new StorageService(embeddingModel, embeddingStore); - - // 检查服务状态 - boolean modelAvailable = !(embeddingModel instanceof FallbackEmbeddingModel); - boolean storeAvailable = !(embeddingStore instanceof FallbackEmbeddingStore); - - if (modelAvailable && storeAvailable) { - log.info("✅ StorageService initialization completed - RAG features are fully available"); - } else { - log.warn("⚠️ StorageService initialization completed - RAG features are limited: " + - "Model available: {}, Store available: {}", modelAvailable, storeAvailable); - } - - return service; - - } catch (Exception e) { - log.error("❌ StorageService initialization failed, creating fallback instance", e); - // 创建完全降级的实例 - return new StorageService(createFallbackEmbeddingModel(), createFallbackEmbeddingStore()); - } - } - - /** - * 测试 ChromaDB 连接 - 返回布尔值而不是抛出异常 - */ - private boolean testChromaConnection(String baseUrl) { - try { - okhttp3.Request request = new okhttp3.Request.Builder() - .url(baseUrl + "/api/v1/heartbeat") - .get() - .build(); - - OkHttpClient client = new OkHttpClient.Builder() - .connectTimeout(Duration.ofSeconds(5)) - .readTimeout(Duration.ofSeconds(5)) - .build(); - - try (okhttp3.Response response = client.newCall(request).execute()) { - if (response.isSuccessful()) { - log.info("✅ ChromaDB connection test successful"); - return true; - } else { - log.warn("ChromaDB connection test failed with status: {}", response.code()); - return false; - } - } - } catch (Exception e) { - log.warn("ChromaDB connection test failed: {}", e.getMessage()); - return false; - } - } - - /** - * 创建降级嵌入模型 - */ - private EmbeddingModel createFallbackEmbeddingModel() { - return new FallbackEmbeddingModel(); - } - - /** - * 创建降级嵌入存储 - */ - private EmbeddingStore createFallbackEmbeddingStore() { - return new FallbackEmbeddingStore(); - } - - /** - * 降级嵌入模型实现 - */ - private static class FallbackEmbeddingModel implements EmbeddingModel { - @Override - public Response> embedAll(List textSegments) { - log.warn("RAG features are disabled - using fallback embedding model"); - // 返回空的嵌入列表 - return Response.from(Collections.emptyList()); - } - } - - /** - * 降级嵌入存储实现 - */ - private static class FallbackEmbeddingStore implements EmbeddingStore { - @Override - public String add(Embedding embedding) { - log.warn("RAG features are disabled - using fallback embedding store"); - return "fallback-id"; - } - - @Override - public void add(String id, Embedding embedding) { - log.warn("RAG features are disabled - using fallback embedding store"); - } - - @Override - public String add(Embedding embedding, TextSegment embedded) { - log.warn("RAG features are disabled - using fallback embedding store"); - return "fallback-id"; - } - - @Override - public List addAll(List embeddings) { - log.warn("RAG features are disabled - using fallback embedding store"); - return Collections.emptyList(); - } - - @Override - public List addAll(List embeddings, List embedded) { - log.warn("RAG features are disabled - using fallback embedding store"); - return Collections.emptyList(); - } - - @Override - public EmbeddingSearchResult search(EmbeddingSearchRequest request) { - log.warn("RAG features are disabled - using fallback embedding store"); - return new EmbeddingSearchResult<>(Collections.emptyList()); - } - } -} +/** + * Copyright (c) 2023 - present TinyEngine Authors. Copyright (c) 2023 - present Huawei Cloud + * Computing Technologies Co., Ltd. + * + *

Use of this source code is governed by an MIT-style license. + * + *

THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, + * BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR A + * PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + */ +package com.tinyengine.it.rag.config; + +import com.tinyengine.it.rag.service.StorageService; + +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.model.embedding.onnx.OnnxEmbeddingModel; +import dev.langchain4j.model.embedding.onnx.PoolingMode; +import dev.langchain4j.model.output.Response; +import dev.langchain4j.store.embedding.EmbeddingSearchRequest; +import dev.langchain4j.store.embedding.EmbeddingSearchResult; +import dev.langchain4j.store.embedding.EmbeddingStore; +import dev.langchain4j.store.embedding.chroma.ChromaEmbeddingStore; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import okhttp3.OkHttpClient; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.io.IOException; +import java.time.Duration; +import java.util.Collections; +import java.util.List; + +/** Vector store config */ +@Configuration +@RequiredArgsConstructor +@Slf4j +public class VectorStoreConfig { + private static final int CHROMA_TIMEOUT = 30; + private static final int HEALTH_TIMEOUT = 5; + private static final String FALLBACK_WARNING = + "RAG features are disabled - using fallback embedding store"; + + private final RAGConfig ragConfig; + + /** + * 嵌入模型 Bean - 尝试创建,失败时返回降级实现. + * + * @return embedding model bean + */ + @Bean + @SuppressWarnings("PMD.AvoidCatchingGenericException") + public EmbeddingModel embeddingModel() { + EmbeddingModel embeddingModel; + try { + // 检查必要的配置参数 + if (ragConfig.getModelPath() == null || ragConfig.getTokenizerPath() == null) { + logWarn("ONNX model configuration is incomplete, using fallback embedding model"); + embeddingModel = createFallbackEmbeddingModel(); + } else { + logInfo("Initializing ONNX embedding model..."); + embeddingModel = + new OnnxEmbeddingModel( + ragConfig.getModelPath(), + ragConfig.getTokenizerPath(), + PoolingMode.MEAN); + logInfo("ONNX embedding model initialization successful"); + } + } catch (RuntimeException exception) { + logWarn( + "ONNX embedding model initialization failed, using fallback implementation", + exception); + embeddingModel = createFallbackEmbeddingModel(); + } + return embeddingModel; + } + + /** + * 嵌入存储 Bean - 尝试创建,失败时返回降级实现. + * + * @return embedding store bean + */ + @Bean + @SuppressWarnings({"PMD.AvoidCatchingGenericException", "PMD.LawOfDemeter"}) + public EmbeddingStore embeddingStore() { + EmbeddingStore embeddingStore; + try { + // 检查必要的配置参数 + if (ragConfig.getChromaBaseUrl() == null) { + logWarn("ChromaDB configuration is incomplete, using fallback embedding store"); + embeddingStore = createFallbackEmbeddingStore(); + } else if (!testChromaConnection(ragConfig.getChromaBaseUrl())) { + logWarn("ChromaDB connection test failed, using fallback embedding store"); + embeddingStore = createFallbackEmbeddingStore(); + } else { + logInfo( + "Attempting to initialize ChromaDB connection: {}", + ragConfig.getChromaBaseUrl()); + String collectionName = ragConfig.getChromaCollectionName(); + if (collectionName == null) { + collectionName = "documents"; + } + embeddingStore = + ChromaEmbeddingStore.builder() + .baseUrl(ragConfig.getChromaBaseUrl()) + .collectionName(collectionName) + .timeout(Duration.ofSeconds(CHROMA_TIMEOUT)) + .build(); + logInfo("ChromaDB embeddingStore initialization successful"); + } + } catch (RuntimeException exception) { + logWarn("ChromaDB initialization failed, using fallback embedding store", exception); + embeddingStore = createFallbackEmbeddingStore(); + } + return embeddingStore; + } + + /** + * 存储服务 Bean - 总是创建,依赖降级实现. + * + * @return storage service bean + */ + @Bean + @SuppressWarnings("PMD.AvoidCatchingGenericException") + public StorageService vectorStorageService( + final EmbeddingModel embeddingModel, + final EmbeddingStore embeddingStore) { + StorageService storageService; + try { + storageService = new StorageService(embeddingModel, embeddingStore, ragConfig); + + // 检查服务状态 + final boolean modelAvailable = !(embeddingModel instanceof FallbackEmbeddingModel); + final boolean storeAvailable = !(embeddingStore instanceof FallbackEmbeddingStore); + + if (modelAvailable && storeAvailable) { + logInfo("StorageService initialization completed - RAG features are fully available"); + } else { + logWarn( + "StorageService initialization completed - RAG features are limited: " + + "Model available: {}, Store available: {}", + modelAvailable, + storeAvailable); + } + + } catch (RuntimeException exception) { + logError("StorageService initialization failed, creating fallback instance", exception); + // 创建完全降级的实例 + storageService = new StorageService( + createFallbackEmbeddingModel(), createFallbackEmbeddingStore(), ragConfig); + } + return storageService; + } + + /** + * 测试 ChromaDB 连接 - 返回布尔值而不是抛出异常. + * + * @return whether ChromaDB can be reached + */ + private boolean testChromaConnection(final String baseUrl) { + boolean connected = false; + try { + final okhttp3.Request request = + new okhttp3.Request.Builder().url(baseUrl + "/api/v1/heartbeat").get().build(); + + final OkHttpClient client = + new OkHttpClient.Builder() + .connectTimeout(Duration.ofSeconds(HEALTH_TIMEOUT)) + .readTimeout(Duration.ofSeconds(HEALTH_TIMEOUT)) + .build(); + + try (okhttp3.Response response = client.newCall(request).execute()) { + if (response.isSuccessful()) { + logInfo("ChromaDB connection test successful"); + connected = true; + } else { + logWarn("ChromaDB connection test failed with status: {}", response.code()); + } + } + } catch (IOException exception) { + logWarn("ChromaDB connection test failed: {}", exception.getMessage()); + } + return connected; + } + + /** + * 创建降级嵌入模型. + * + * @return fallback embedding model + */ + private EmbeddingModel createFallbackEmbeddingModel() { + return new FallbackEmbeddingModel(); + } + + /** + * 创建降级嵌入存储. + * + * @return fallback embedding store + */ + private EmbeddingStore createFallbackEmbeddingStore() { + return new FallbackEmbeddingStore(); + } + + private static void logInfo(final String message, final Object... arguments) { + if (log.isInfoEnabled()) { + log.info(message, arguments); + } + } + + private static void logWarn(final String message, final Object... arguments) { + if (log.isWarnEnabled()) { + log.warn(message, arguments); + } + } + + private static void logError(final String message, final Object... arguments) { + if (log.isErrorEnabled()) { + log.error(message, arguments); + } + } + + private static void logFallbackEmbeddingStoreWarning() { + logWarn(FALLBACK_WARNING); + } + + /** 降级嵌入模型实现 */ + private static class FallbackEmbeddingModel implements EmbeddingModel { + @Override + public Response> embedAll(final List textSegments) { + logWarn("RAG features are disabled - using fallback embedding model"); + // 返回空的嵌入列表 + return Response.from(Collections.emptyList()); + } + } + + /** 降级嵌入存储实现 */ + private static class FallbackEmbeddingStore implements EmbeddingStore { + @Override + public String add(final Embedding embedding) { + logFallbackEmbeddingStoreWarning(); + return "fallback-id"; + } + + @Override + public void add(final String identifier, final Embedding embedding) { + logFallbackEmbeddingStoreWarning(); + } + + @Override + public String add(final Embedding embedding, final TextSegment embedded) { + logFallbackEmbeddingStoreWarning(); + return "fallback-id"; + } + + @Override + public List addAll(final List embeddings) { + logFallbackEmbeddingStoreWarning(); + return Collections.emptyList(); + } + + @Override + public List addAll( + final List embeddings, final List embedded) { + logFallbackEmbeddingStoreWarning(); + return Collections.emptyList(); + } + + @Override + public EmbeddingSearchResult search(final EmbeddingSearchRequest request) { + logFallbackEmbeddingStoreWarning(); + return new EmbeddingSearchResult<>(Collections.emptyList()); + } + } +} diff --git a/base/src/main/java/com/tinyengine/it/rag/service/StorageService.java b/base/src/main/java/com/tinyengine/it/rag/service/StorageService.java index acdb725e..1d12423a 100644 --- a/base/src/main/java/com/tinyengine/it/rag/service/StorageService.java +++ b/base/src/main/java/com/tinyengine/it/rag/service/StorageService.java @@ -1,897 +1,1161 @@ -/** - * Copyright (c) 2023 - present TinyEngine Authors. - * Copyright (c) 2023 - present Huawei Cloud Computing Technologies Co., Ltd. - * - * Use of this source code is governed by an MIT-style license. - * - * THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, - * BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR - * A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. - * - */ - -package com.tinyengine.it.rag.service; - -import com.tinyengine.it.common.exception.ExceptionEnum; -import com.tinyengine.it.common.exception.ServiceException; -import com.tinyengine.it.rag.entity.BatchDeleteResult; -import com.tinyengine.it.rag.entity.BatchResult; -import com.tinyengine.it.rag.entity.DeleteResult; -import com.tinyengine.it.rag.entity.EmbeddingMatchDto; -import com.tinyengine.it.rag.config.RAGConfig; -import com.tinyengine.it.rag.entity.SearchRequest; -import com.tinyengine.it.rag.entity.VectorDocument; -import dev.langchain4j.data.document.Document; -import dev.langchain4j.data.document.DocumentSplitter; -import dev.langchain4j.data.document.loader.FileSystemDocumentLoader; -import dev.langchain4j.data.document.parser.TextDocumentParser; -import dev.langchain4j.data.document.parser.apache.pdfbox.ApachePdfBoxDocumentParser; -import dev.langchain4j.data.document.splitter.DocumentSplitters; -import dev.langchain4j.data.embedding.Embedding; -import dev.langchain4j.data.segment.TextSegment; -import dev.langchain4j.model.embedding.EmbeddingModel; -import dev.langchain4j.store.embedding.EmbeddingMatch; -import dev.langchain4j.store.embedding.EmbeddingSearchRequest; -import dev.langchain4j.store.embedding.EmbeddingStore; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * 存储服务 - 支持动态集合管理 - */ -@Slf4j -@Service -public class StorageService { - private final EmbeddingModel embeddingModel; - private final EmbeddingStore embeddingStore; - - private final RAGConfig ragConfig = new RAGConfig(); - - // 支持的集合列表 - private static final List SUPPORTED_COLLECTIONS = List.of( - "tinyengine_documents", - "agent_documents" - ); - - // 默认集合 - private static final String DEFAULT_COLLECTION = "tinyengine_documents"; - - // 集合映射配置 - private final Map collectionMapping = new HashMap<>(); - - /** - * 支持的文档格式 - */ - private static final List SUPPORTED_FORMATS = List.of( - ".pdf", ".txt", ".md", ".sql", ".java", ".py", ".js", ".ts", - ".html", ".css", ".xml", ".json", ".yaml", ".yml", ".properties", - ".sh", ".bat", ".cmd", ".c", ".cpp", ".h", ".hpp" - ); - - /** - * 文本文件格式(使用 TextDocumentParser) - */ - private static final List TEXT_FORMATS = List.of( - ".txt", ".md", ".sql", ".java", ".py", ".js", ".ts", - ".html", ".css", ".xml", ".json", ".yaml", ".yml", ".properties", - ".sh", ".bat", ".cmd", ".c", ".cpp", ".h", ".hpp" - ); - - /** - * Check if file format is supported - */ - private boolean isSupportedFormat(Path filePath) { - String fileName = filePath.getFileName().toString().toLowerCase(Locale.ROOT); - return SUPPORTED_FORMATS.stream().anyMatch(format -> fileName.endsWith(format)); - } - - /** - * 构造函数 - */ - public StorageService(EmbeddingModel embeddingModel, EmbeddingStore embeddingStore) { - this.embeddingModel = embeddingModel; - this.embeddingStore = embeddingStore; - log.info("StorageService initialized with support for {} file formats", SUPPORTED_FORMATS.size()); - - // 初始化集合映射 - initializeCollectionMapping(); - } - - /** - * 初始化集合映射配置 - */ - private void initializeCollectionMapping() { - // 配置特定文件类型到集合的映射 - collectionMapping.put("agent", "agent_documents"); - collectionMapping.put("tinyengine", "tinyengine_documents"); - - log.info("Collection mapping initialized: {}", collectionMapping); - } - - /** - * 检查集合名称是否有效 - */ - private boolean isValidCollection(String collectionName) { - return SUPPORTED_COLLECTIONS.contains(collectionName); - } - - /** - * 自动扫描文件夹并添加文档到知识库 - */ - public VectorDocument autoAddFolderToKnowledgeBase() { - try { - String folderPath = System.getenv("FOLDER_PATH"); - if (folderPath == null || folderPath.isBlank()) { - throw new ServiceException(ExceptionEnum.CM329.getResultCode(), "FOLDER_PATH does not exist: " + folderPath); - } - // 验证文件夹路径 - Path folder = Paths.get(folderPath); - if (!Files.exists(folder) || !Files.isDirectory(folder)) { - throw new ServiceException(ExceptionEnum.CM329.getResultCode(), "Folder does not exist: " + folderPath); - } - - // 扫描文件夹中的所有支持的文件 - List filePaths = scanSupportedFiles(folderPath); - - if (filePaths.isEmpty()) { - throw new ServiceException(ExceptionEnum.CM329.getResultCode(), - "No supported file formats found in folder. Supported formats: " + String.join(", ", - SUPPORTED_FORMATS)); - } - - log.info("Found {} supported files in folder: {}", filePaths.size(), folderPath); - - return initializeKnowledgeBase(filePaths); - - } catch (ServiceException e) { - throw e; - } catch (Exception e) { - log.error("Failed to auto add folder to knowledge base", e); - throw new ServiceException(ExceptionEnum.CM330.getResultCode(), "Auto add folder failed: " + e.getMessage()); - } - } - - /** - * 扫描文件夹中所有支持的文件 - */ - private List scanSupportedFiles(String folderPath) { - Path folder = Paths.get(folderPath); - - try (Stream pathStream = Files.walk(folder)) { - return pathStream - .filter(Files::isRegularFile) - .filter(this::isSupportedFormat) - .peek(filePath -> log.debug("Found supported file: {}", filePath)) - .map(Path::toString) - .sorted() - .collect(Collectors.toList()); - - } catch (IOException e) { - log.error("Failed to scan folder: {}", folderPath, e); - throw new ServiceException(ExceptionEnum.CM333.getResultCode(), ExceptionEnum.CM333.getResultMsg()); - } - } - - - - /** - * 根据文档路径和自定义集合确定目标集合 - */ - private String determineCollectionName(String filePath, String customCollection) { - // 如果指定了自定义集合,优先使用 - if (customCollection != null && !customCollection.trim().isEmpty()) { - if (!isValidCollection(customCollection)) { - log.warn("Invalid collection specified: {}, using default: {}", customCollection, DEFAULT_COLLECTION); - return DEFAULT_COLLECTION; - } - return customCollection; - } - - // 根据文件路径自动判断集合 - if (filePath != null) { - String lowerPath = filePath.toLowerCase(Locale.ROOT); - // 如果路径包含特定关键词,映射到对应集合 - for (Map.Entry entry : collectionMapping.entrySet()) { - if (lowerPath.contains(entry.getKey())) { - log.info("Auto-mapped file {} to collection: {}", filePath, entry.getValue()); - return entry.getValue(); - } - } - } - - // 默认集合 - return DEFAULT_COLLECTION; - } - - /** - * 检查文件格式是否支持 - */ - private boolean isSupportedFormat(String filePath) { - if (filePath == null) { - return false; - } - - String lowerPath = filePath.toLowerCase(Locale.ROOT); - return SUPPORTED_FORMATS.stream().anyMatch(lowerPath::endsWith); - } - - /** - * 检查是否为文本格式 - */ - private boolean isTextFormat(String filePath) { - if (filePath == null) { - return false; - } - - String lowerPath = filePath.toLowerCase(Locale.ROOT); - return TEXT_FORMATS.stream().anyMatch(lowerPath::endsWith); - } - - /** - * 获取文件格式描述 - */ - private String getFileFormatDescription(String filePath) { - if (filePath == null) { - return "unknown"; - } - - String lowerPath = filePath.toLowerCase(Locale.ROOT); - if (lowerPath.endsWith(".pdf")) { - return "PDF Document"; - } - if (lowerPath.endsWith(".sql")) { - return "SQL Script"; - } - if (lowerPath.endsWith(".java")) { - return "Java Source"; - } - if (lowerPath.endsWith(".py")) { - return "Python Script"; - } - if (lowerPath.endsWith(".js")) { - return "JavaScript"; - } - if (lowerPath.endsWith(".ts")) { - return "TypeScript"; - } - if (lowerPath.endsWith(".html")) { - return "HTML Document"; - } - if (lowerPath.endsWith(".css")) { - return "CSS Stylesheet"; - } - if (lowerPath.endsWith(".xml")) { - return "XML Document"; - } - if (lowerPath.endsWith(".json")) { - return "JSON Data"; - } - if (lowerPath.endsWith(".yaml") || lowerPath.endsWith(".yml")) { - return "YAML Configuration"; - } - if (lowerPath.endsWith(".properties")) { - return "Properties File"; - } - if (lowerPath.endsWith(".sh")) { - return "Shell Script"; - } - if (lowerPath.endsWith(".bat") || lowerPath.endsWith(".cmd")) { - return "Batch File"; - } - if (lowerPath.endsWith(".c")) { - return "C Source"; - } - if (lowerPath.endsWith(".cpp") || lowerPath.endsWith(".h") || lowerPath.endsWith(".hpp")) { - return "C++ Source"; - } - if (lowerPath.endsWith(".txt")) { - return "Text Document"; - } - if (lowerPath.endsWith(".md")) { - return "Markdown Document"; - } - - return "Unknown Format"; - } - - /** - * 添加文档到知识库(默认集合) - */ - public VectorDocument initializeKnowledgeBase(List documentPaths) { - return initializeKnowledgeBase(documentPaths, null, null); - } - - /** - * 添加文档到知识库(指定集合) - */ - public VectorDocument initializeKnowledgeBase(List documentPaths, String documentSetId, String collectionName) { - try { - // 确定目标集合 - String targetCollection = determineCollectionName( - documentPaths.isEmpty() ? null : documentPaths.get(0), - collectionName - ); - - log.info("Using collection: {} for document storage", targetCollection); - - List documents = loadDocuments(documentPaths, documentSetId, targetCollection); - - if (documents.isEmpty()) { - throw new ServiceException(ExceptionEnum.CM329.getResultCode(), ExceptionEnum.CM329.getResultMsg()); - } - - log.info("Successfully loaded {} documents for collection: {}", documents.size(), targetCollection); - - // 文档切分 - List segments = splitDocuments(documents); - log.info("Generated {} text segments for collection: {}", segments.size(), targetCollection); - - // 向量化并存储到指定集合 - return embedAndStore(segments, targetCollection); - - } catch (ServiceException e) { - throw e; - } catch (Exception e) { - log.error("Failed to add the document to the knowledge base", e); - throw new ServiceException(ExceptionEnum.CM330.getResultCode(), ExceptionEnum.CM330.getResultMsg()); - } - } - - /** - * 加载文档 - */ - private List loadDocuments(List documentPaths, String documentSetId, String collectionName) { - List documents = new ArrayList<>(); - - int loadedCount = 0; - int skippedCount = 0; - - for (String path : documentPaths) { - try { - // 检查文件是否存在 - if (!Files.exists(Paths.get(path))) { - log.warn("✗ File not found: {}", path); - skippedCount++; - continue; - } - - // 检查文件格式是否支持 - if (!isSupportedFormat(path)) { - log.warn("✗ Unsupported document format: {} ({})", path, getFileFormatDescription(path)); - skippedCount++; - continue; - } - - Path filePath = Paths.get(path); - Document document; - - if (path.toLowerCase(Locale.ROOT).endsWith(".pdf")) { - // PDF 文件使用 PDF 解析器 - ApachePdfBoxDocumentParser pdfParser = new ApachePdfBoxDocumentParser(); - document = FileSystemDocumentLoader.loadDocument(filePath, pdfParser); - } else if (isTextFormat(path)) { - // 所有文本文件使用 TextDocumentParser - document = FileSystemDocumentLoader.loadDocument(filePath, new TextDocumentParser()); - } else { - log.warn("✗ Unhandled document format: {} ({})", path, getFileFormatDescription(path)); - skippedCount++; - continue; - } - - // 添加元数据 - if (documentSetId != null) { - document.metadata().put("documentSetId", documentSetId); - } - document.metadata().put("source", path); - document.metadata().put("format", getFileFormatDescription(path)); - document.metadata().put("timestamp", String.valueOf(System.currentTimeMillis())); - document.metadata().put("collection", collectionName); // 添加集合信息 - - documents.add(document); - loadedCount++; - log.info("✓ Loaded document: {} ({}) to collection: {}", - path, getFileFormatDescription(path), collectionName); - - } catch (Exception e) { - log.error("✗ Failed to load the document: {} - {}", path, e.getMessage()); - skippedCount++; - } - } - - log.info("Document loading summary: {} loaded, {} skipped, {} total paths for collection: {}", - loadedCount, skippedCount, documentPaths.size(), collectionName); - - return documents; - } - - /** - * 文档切分 - */ - private List splitDocuments(List documents) { - DocumentSplitter splitter = DocumentSplitters.recursive( - ragConfig.getChunkSize(), - ragConfig.getChunkOverlap() - ); - return splitter.splitAll(documents); - } - - /** - * 向量化并存储到指定集合 - */ - private VectorDocument embedAndStore(List segments, String collectionName) { - log.info("Begin vectorized storage to collection: {}...", collectionName); - long startTime = System.currentTimeMillis(); - - int successCount = 0; - int errorCount = 0; - - // 批量处理,提高性能 - int batchSize = 50; - for (int i = 0; i < segments.size(); i += batchSize) { - int end = Math.min(i + batchSize, segments.size()); - List batch = segments.subList(i, end); - - BatchResult result = processBatch(batch, i, segments.size(), collectionName); - successCount += result.getSuccessCount(); - errorCount += result.getErrorCount(); - } - - long endTime = System.currentTimeMillis(); - log.info("Vectorization completed in collection {}: {} successful, {} failed, time taken: {} ms", - collectionName, successCount, errorCount, (endTime - startTime)); - - return new VectorDocument(successCount, errorCount, null, collectionName); - } - - /** - * 处理批次数据 - */ - private BatchResult processBatch(List batch, int startIndex, int totalSize, String collectionName) { - int successCount = 0; - int errorCount = 0; - - List embeddings = new ArrayList<>(); - List segmentsToStore = new ArrayList<>(); - - for (int i = 0; i < batch.size(); i++) { - TextSegment segment = batch.get(i); - try { - Embedding embedding = embeddingModel.embed(segment.text()).content(); - embeddings.add(embedding); - segmentsToStore.add(segment); - successCount++; - - if ((startIndex + i + 1) % 100 == 0) { - log.info("Processed {}/{} text segments for collection: {}", - (startIndex + i + 1), totalSize, collectionName); - } - } catch (Exception e) { - errorCount++; - log.error("Vectorization failed [{}] in collection {}: {}", - (startIndex + i + 1), collectionName, - segment.text().substring(0, Math.min(100, segment.text().length()))); - } - } - - if (!embeddings.isEmpty()) { - try { - embeddingStore.addAll(embeddings, segmentsToStore); - log.debug("Successfully stored {} text segments to vector database in collection: {}", - embeddings.size(), collectionName); - } catch (Exception e) { - log.error("Batch storage to vector database failed in collection: {}", collectionName, e); - errorCount += embeddings.size(); - successCount -= embeddings.size(); - } - } - BatchResult result = new BatchResult(); - result.setSuccessCount(successCount); - result.setErrorCount(errorCount); - return result; - } - - /** - * 在指定集合中检索 - */ - public List search(SearchRequest searchDto) { - try { - Embedding queryEmbedding = embeddingModel.embed(searchDto.getContent()).content(); - - EmbeddingSearchRequest searchRequest = EmbeddingSearchRequest.builder() - .queryEmbedding(queryEmbedding) - .maxResults(searchDto.getMaxResults()) - .minScore(searchDto.getMinScore()) - .build(); - - List> matches = embeddingStore.search(searchRequest).matches(); - - // 如果指定了集合名称,进行过滤 - if (searchDto.getCollection() != null) { - matches = filterByCollection(matches, searchDto.getCollection()); - } - - // 转换为 DTO - List results = matches.stream() - .map(EmbeddingMatchDto::from) - .collect(Collectors.toList()); - - log.info("Retrieved {} related documents from collection: {}", results.size(), - searchDto.getCollection() != null ? searchDto.getCollection() : "all collections"); - return results; - - } catch (Exception e) { - log.error("Retrieval failed", e); - throw new ServiceException(ExceptionEnum.CM331.getResultCode(), ExceptionEnum.CM331.getResultMsg()); - } - } - - /** - * 根据集合名称过滤结果 - */ - private static List> filterByCollection( - List> results, String collectionName) { - - List> filteredResults = new ArrayList<>(); - - for (EmbeddingMatch match : results) { - String collection = match.embedded().metadata().getString("collection"); - if (collectionName.equals(collection)) { - filteredResults.add(match); - } - } - - return filteredResults; - } - - /** - * 跨集合搜索(在所有集合中搜索) - */ - public Map> searchAcrossCollections(SearchRequest searchDto) { - Map> results = new HashMap<>(); - - for (String collection : SUPPORTED_COLLECTIONS) { - try { - searchDto.setCollection(collection); - List collectionResults = search(searchDto); - results.put(collection, collectionResults); - log.info("Found {} results in collection: {}", collectionResults.size(), collection); - } catch (Exception e) { - log.warn("Search failed in collection: {}", collection, e); - results.put(collection, new ArrayList<>()); - } - } - - return results; - } - - /** - * 根据文件路径删除指定集合中的文档 - */ - public DeleteResult deleteByFilePath(String filePath, String collectionName) { - try { - log.info("Deleting documents by file path: {} from collection: {}", - filePath, collectionName != null ? collectionName : "all collections"); - long startTime = System.currentTimeMillis(); - - // 搜索包含该文件路径的所有向量 - List> matches = searchBySource(filePath, collectionName); - - if (matches.isEmpty()) { - log.warn("No documents found for file path: {} in collection: {}", - filePath, collectionName != null ? collectionName : "any collection"); - return new DeleteResult(0, 0, filePath); - } - - // 提取要删除的向量ID - List idsToRemove = matches.stream() - .map(EmbeddingMatch::embeddingId) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - - int deletedCount = 0; - if (!idsToRemove.isEmpty()) { - // 批量删除向量 - embeddingStore.removeAll(idsToRemove); - deletedCount = idsToRemove.size(); - } - - long endTime = System.currentTimeMillis(); - log.info("Deleted {} vectors for file: {} from collection: {}, time taken: {} ms", - deletedCount, filePath, collectionName != null ? collectionName : "all collections", - (endTime - startTime)); - - return new DeleteResult(deletedCount, 0, filePath); - - } catch (Exception e) { - log.error("Failed to delete documents by file path: {} from collection: {}", - filePath, collectionName, e); - throw new ServiceException(ExceptionEnum.CM332.getResultCode(), "Delete document failed"); - } - } - - /** - * 根据源文件路径搜索向量(支持集合过滤) - */ - private List> searchBySource(String sourcePath, String collectionName) { - try { - // 使用更合理的查询文本 - String queryText = "document content analysis"; - - EmbeddingSearchRequest searchRequest = EmbeddingSearchRequest.builder() - .queryEmbedding(embeddingModel.embed(queryText).content()) - .maxResults(1000) - .minScore(0.1) - .build(); - - List> allMatches = embeddingStore.search(searchRequest).matches(); - - // 在应用层过滤 - return allMatches.stream() - .filter(match -> { - String source = match.embedded().metadata().getString("source"); - String collection = match.embedded().metadata().getString("collection"); - - boolean sourceMatch = source != null && source.equals(sourcePath); - boolean collectionMatch = collectionName == null || - (collection != null && collection.equals(collectionName)); - - return sourceMatch && collectionMatch; - }) - .collect(Collectors.toList()); - - } catch (Exception e) { - log.error("Failed to search vectors by source: {} in collection: {}", sourcePath, collectionName, e); - return new ArrayList<>(); - } - } - - /** - * 批量删除多个文件(默认集合) - */ - public BatchDeleteResult deleteMultipleFiles(List filePaths) { - return deleteMultipleFiles(filePaths, null); - } - - /** - * 批量删除多个文件(指定集合) - */ - public BatchDeleteResult deleteMultipleFiles(List filePaths, String collectionName) { - try { - log.info("Deleting multiple files: {} from collection: {}", filePaths, - collectionName != null ? collectionName : "all collections"); - long startTime = System.currentTimeMillis(); - - int totalDeleted = 0; - int totalFailed = 0; - List results = new ArrayList<>(); - - for (String filePath : filePaths) { - try { - DeleteResult result = deleteByFilePath(filePath, collectionName); - results.add(result); - totalDeleted += result.getDeletedCount(); - if (result.getFailedCount() > 0) { - totalFailed += result.getFailedCount(); - } - } catch (Exception e) { - log.error("Failed to delete file: {} from collection: {}", filePath, collectionName, e); - totalFailed++; - results.add(new DeleteResult(0, 1, filePath)); - } - } - - long endTime = System.currentTimeMillis(); - log.info("Batch deletion completed: {} deleted, {} failed, time taken: {} ms from collection: {}", - totalDeleted, totalFailed, (endTime - startTime), - collectionName != null ? collectionName : "all collections"); - - return new BatchDeleteResult(totalDeleted, totalFailed, results); - - } catch (Exception e) { - log.error("Failed to delete multiple files from collection: {}", collectionName, e); - throw new ServiceException(ExceptionEnum.CM332.getResultCode(), "Batch delete files failed"); - } - } - - /** - * 获取所有集合及其包含的文档路径 - */ - public Map> getAllCollectionDocuments() { - Map> collectionDocuments = new HashMap<>(); - try { - // 遍历所有支持的集合 - for (String collection : SUPPORTED_COLLECTIONS) { - List documents = getStoredFiles(collection); - if (!documents.isEmpty()) { - collectionDocuments.put(collection, documents); - } - } - - log.info("Retrieved documents from {} collections", collectionDocuments.size()); - return collectionDocuments; - - } catch (Exception e) { - log.error("Failed to get collection documents", e); - return collectionDocuments; - } - } - - /** - * 获取指定集合的文档路径 - */ - public Map> getCollectionDocuments(String collectionName) { - Map> result = new HashMap<>(); - try { - if (!isValidCollection(collectionName)) { - throw new ServiceException(ExceptionEnum.CM001.getResultCode(), "Invalid collection name: " + collectionName); - } - - List documents = getStoredFiles(collectionName); - - result.put(collectionName, documents); - - log.info("Retrieved {} documents from collection: {}", documents.size(), collectionName); - return result; - - } catch (Exception e) { - log.error("Failed to get documents from collection: {}", collectionName, e); - return result; - } - } - - /** - * 获取所有已存储的文件列表(支持集合过滤) - */ - public List getStoredFiles() { - return getStoredFiles(null); - } - - /** - * 获取指定集合中已存储的文件列表 - */ - public List getStoredFiles(String collectionName) { - try { - EmbeddingSearchRequest searchRequest = EmbeddingSearchRequest.builder() - .queryEmbedding(embeddingModel.embed("test").content()) - .maxResults(10000) - .minScore(0.0) - .build(); - - List> allMatches = embeddingStore.search(searchRequest).matches(); - - // 提取所有唯一的源文件路径(支持集合过滤) - return allMatches.stream() - .filter(match -> { - String collection = match.embedded().metadata().getString("collection"); - return collectionName == null || - (collection != null && collection.equals(collectionName)); - }) - .map(match -> match.embedded().metadata().getString("source")) - .filter(Objects::nonNull) - .distinct() - .collect(Collectors.toList()); - - } catch (Exception e) { - log.error("Failed to get stored files list for collection: {}", collectionName, e); - return new ArrayList<>(); - } - } - - - /** - * 获取文档集列表(支持集合过滤) - */ - public List getDocumentSets() { - return getDocumentSets(null); - } - - /** - * 获取指定集合中的文档集列表 - */ - public List getDocumentSets(String collectionName) { - try { - EmbeddingSearchRequest searchRequest = EmbeddingSearchRequest.builder() - .queryEmbedding(embeddingModel.embed("test").content()) - .maxResults(10000) - .minScore(0.0) - .build(); - - List> allMatches = embeddingStore.search(searchRequest).matches(); - - // 提取所有唯一的文档集ID(支持集合过滤) - return allMatches.stream() - .filter(match -> { - String collection = match.embedded().metadata().getString("collection"); - return collectionName == null || - (collection != null && collection.equals(collectionName)); - }) - .map(match -> match.embedded().metadata().getString("documentSetId")) - .filter(Objects::nonNull) - .distinct() - .collect(Collectors.toList()); - - } catch (Exception e) { - log.error("Failed to get document sets list for collection: {}", collectionName, e); - return new ArrayList<>(); - } - } - - /** - * 获取支持的集合列表 - */ - public List getSupportedCollections() { - return new ArrayList<>(SUPPORTED_COLLECTIONS); - } - - /** - * 获取集合统计信息 - */ - public Map getCollectionStats() { - Map stats = new HashMap<>(); - - for (String collection : SUPPORTED_COLLECTIONS) { - try { - List files = getStoredFiles(collection); - stats.put(collection, files.size()); - log.debug("Collection {} has {} files", collection, files.size()); - } catch (Exception e) { - log.warn("Failed to get stats for collection: {}", collection, e); - stats.put(collection, 0); - } - } - - return stats; - } - - /** - * 清空指定集合 - */ - public void clearCollection(String collectionName) { - try { - if (!isValidCollection(collectionName)) { - throw new ServiceException(ExceptionEnum.CM001.getResultCode(), "Invalid collection name: " + collectionName); - } - - // 获取该集合中的所有文档并删除 - List files = getStoredFiles(collectionName); - if (!files.isEmpty()) { - deleteMultipleFiles(files, collectionName); - } - - log.info("Collection cleared successfully: {}", collectionName); - } catch (Exception e) { - log.error("Failed to clear collection: {}", collectionName, e); - throw new ServiceException(ExceptionEnum.CM001.getResultCode(), "Clear collection failed: " + collectionName); - } - } - - /** - * 清空向量库(所有集合) - */ - public void clearVectorStore() { - try { - embeddingStore.removeAll(); - log.info("Vector store cleared successfully (all collections)"); - - log.info("Vector store cleared successfully (all collections)"); - } catch (Exception e) { - log.error("Failed to clear vector library", e); - throw new ServiceException(ExceptionEnum.CM001.getResultCode(), "Clear vector store failed"); - } - } - -} - +/** + * Copyright (c) 2023 - present TinyEngine Authors. Copyright (c) 2023 - present Huawei Cloud + * Computing Technologies Co., Ltd. + * + *

Use of this source code is governed by an MIT-style license. + * + *

THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, + * BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR A + * PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + */ +package com.tinyengine.it.rag.service; + +import com.tinyengine.it.common.exception.ExceptionEnum; +import com.tinyengine.it.common.exception.ServiceException; +import com.tinyengine.it.rag.config.RAGConfig; +import com.tinyengine.it.rag.entity.BatchDeleteResult; +import com.tinyengine.it.rag.entity.BatchResult; +import com.tinyengine.it.rag.entity.DeleteResult; +import com.tinyengine.it.rag.entity.EmbeddingMatchDto; +import com.tinyengine.it.rag.entity.SearchRequest; +import com.tinyengine.it.rag.entity.VectorDocument; + +import dev.langchain4j.data.document.Document; +import dev.langchain4j.data.document.DocumentSplitter; +import dev.langchain4j.data.document.loader.FileSystemDocumentLoader; +import dev.langchain4j.data.document.parser.TextDocumentParser; +import dev.langchain4j.data.document.parser.apache.pdfbox.ApachePdfBoxDocumentParser; +import dev.langchain4j.data.document.splitter.DocumentSplitters; +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.store.embedding.EmbeddingMatch; +import dev.langchain4j.store.embedding.EmbeddingSearchRequest; +import dev.langchain4j.store.embedding.EmbeddingStore; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** 存储服务 - 支持动态集合管理 */ +@Slf4j +@Service +@SuppressWarnings("PMD.TooManyMethods") +public class StorageService { + private final EmbeddingModel embeddingModel; + private final EmbeddingStore embeddingStore; + + private final RAGConfig ragConfig; + + // 支持的集合列表 + private static final List VALID_COLLECTIONS = + List.of("tinyengine_documents", "agent_documents"); + private static final int STORE_BATCH_SIZE = 50; + private static final int LOG_INTERVAL = 100; + private static final int PREVIEW_LEN = 100; + private static final int SOURCE_SCAN_LIMIT = 1000; + private static final double SOURCE_MIN_SCORE = 0.1; + private static final int LIST_SCAN_LIMIT = 10_000; + + // 默认集合 + private static final String DEFAULT_COLL = "tinyengine_documents"; + + // 集合映射配置 + private final Map collectionMapping = new HashMap<>(); + + private static void logDebug(final String message, final Object... arguments) { + if (log.isDebugEnabled()) { + log.debug(message, arguments); + } + } + + private static void logInfo(final String message, final Object... arguments) { + if (log.isInfoEnabled()) { + log.info(message, arguments); + } + } + + private static void logWarn(final String message, final Object... arguments) { + if (log.isWarnEnabled()) { + log.warn(message, arguments); + } + } + + private static void logError(final String message, final Object... arguments) { + if (log.isErrorEnabled()) { + log.error(message, arguments); + } + } + + /** 支持的文档格式 */ + private static final List SUPPORTED_FORMATS = + List.of( + ".pdf", + ".txt", + ".md", + ".sql", + ".java", + ".py", + ".js", + ".ts", + ".html", + ".css", + ".xml", + ".json", + ".yaml", + ".yml", + ".properties", + ".sh", + ".bat", + ".cmd", + ".c", + ".cpp", + ".h", + ".hpp"); + + /** 文本文件格式(使用 TextDocumentParser) */ + private static final List TEXT_FORMATS = + List.of( + ".txt", + ".md", + ".sql", + ".java", + ".py", + ".js", + ".ts", + ".html", + ".css", + ".xml", + ".json", + ".yaml", + ".yml", + ".properties", + ".sh", + ".bat", + ".cmd", + ".c", + ".cpp", + ".h", + ".hpp"); + + private static final Map FORMAT_DESC = + Map.ofEntries( + Map.entry(".pdf", "PDF Document"), + Map.entry(".sql", "SQL Script"), + Map.entry(".java", "Java Source"), + Map.entry(".py", "Python Script"), + Map.entry(".js", "JavaScript"), + Map.entry(".ts", "TypeScript"), + Map.entry(".html", "HTML Document"), + Map.entry(".css", "CSS Stylesheet"), + Map.entry(".xml", "XML Document"), + Map.entry(".json", "JSON Data"), + Map.entry(".yaml", "YAML Configuration"), + Map.entry(".yml", "YAML Configuration"), + Map.entry(".properties", "Properties File"), + Map.entry(".sh", "Shell Script"), + Map.entry(".bat", "Batch File"), + Map.entry(".cmd", "Batch File"), + Map.entry(".c", "C Source"), + Map.entry(".cpp", "C++ Source"), + Map.entry(".h", "C++ Source"), + Map.entry(".hpp", "C++ Source"), + Map.entry(".txt", "Text Document"), + Map.entry(".md", "Markdown Document")); + + /** + * Check if file format is supported. + * + * @return whether the file format is supported + */ + private boolean isSupportedFormat(Path filePath) { + String fileName = filePath.getFileName().toString().toLowerCase(Locale.ROOT); + return SUPPORTED_FORMATS.stream().anyMatch(format -> fileName.endsWith(format)); + } + + Path getDocumentRoot() { + String rootPath = ragConfig.getDocumentRoot(); + if (rootPath == null || rootPath.isBlank()) { + throw new ServiceException( + ExceptionEnum.CM329.getResultCode(), "Document root is not configured"); + } + + try { + Path root = Paths.get(rootPath).toAbsolutePath().normalize(); + if (!Files.exists(root) || !Files.isDirectory(root)) { + throw new ServiceException( + ExceptionEnum.CM329.getResultCode(), + "Document root does not exist: " + root); + } + return root.toRealPath(); + } catch (InvalidPathException | IOException e) { + throw new ServiceException( + ExceptionEnum.CM329.getResultCode(), "Invalid document root: " + rootPath); + } + } + + Path resolveDocumentPath(String rawPath) { + return resolveDocumentPath(rawPath, getDocumentRoot()); + } + + Path resolveDocumentPath(String rawPath, Path documentRoot) { + if (rawPath == null || rawPath.isBlank()) { + throw new ServiceException( + ExceptionEnum.CM329.getResultCode(), "Document path cannot be empty"); + } + + try { + Path root = documentRoot.toAbsolutePath().normalize(); + Path requested = Paths.get(rawPath); + Path resolved = + requested.isAbsolute() + ? requested.toAbsolutePath().normalize() + : root.resolve(requested).normalize(); + if (!resolved.startsWith(root)) { + throw new ServiceException( + ExceptionEnum.CM329.getResultCode(), + "Document path is outside the allowed root"); + } + return resolved; + } catch (InvalidPathException e) { + throw new ServiceException( + ExceptionEnum.CM329.getResultCode(), "Invalid document path"); + } + } + + private Path resolveRealDocumentPath(Path filePath, Path documentRoot) throws IOException { + Path root = documentRoot.toRealPath(); + Path realPath = filePath.toRealPath(); + if (!realPath.startsWith(root)) { + throw new ServiceException( + ExceptionEnum.CM329.getResultCode(), + "Document path is outside the allowed root"); + } + return realPath; + } + + /** 构造函数 */ + public StorageService( + EmbeddingModel embeddingModel, EmbeddingStore embeddingStore) { + this(embeddingModel, embeddingStore, new RAGConfig()); + } + + @Autowired + public StorageService( + EmbeddingModel embeddingModel, + EmbeddingStore embeddingStore, + RAGConfig ragConfig) { + this.embeddingModel = embeddingModel; + this.embeddingStore = embeddingStore; + this.ragConfig = ragConfig == null ? new RAGConfig() : ragConfig; + logInfo( + "StorageService initialized with support for {} file formats", + SUPPORTED_FORMATS.size()); + + // 初始化集合映射 + initializeCollectionMapping(); + } + + /** 初始化集合映射配置 */ + private void initializeCollectionMapping() { + // 配置特定文件类型到集合的映射 + collectionMapping.put("agent", "agent_documents"); + collectionMapping.put("tinyengine", "tinyengine_documents"); + + logInfo("Collection mapping initialized: {}", collectionMapping); + } + + /** + * 检查集合名称是否有效. + * + * @return whether the collection name is valid + */ + private boolean isValidCollection(String collectionName) { + return VALID_COLLECTIONS.contains(collectionName); + } + + /** + * 自动扫描文件夹并添加文档到知识库. + * + * @return vector storage result + */ + public VectorDocument autoAddFolderToKnowledgeBase() { + try { + Path folder = getDocumentRoot(); + + // 扫描文件夹中的所有支持的文件 + List filePaths = scanSupportedFiles(folder); + + if (filePaths.isEmpty()) { + throw new ServiceException( + ExceptionEnum.CM329.getResultCode(), + "No supported file formats found in folder. Supported formats: " + + String.join(", ", SUPPORTED_FORMATS)); + } + + logInfo("Found {} supported files in folder: {}", filePaths.size(), folder); + + return initializeKnowledgeBase(filePaths); + + } catch (ServiceException e) { + throw e; + } catch (Exception e) { + logError("Failed to auto add folder to knowledge base", e); + throw new ServiceException( + ExceptionEnum.CM330.getResultCode(), + "Auto add folder failed: " + e.getMessage()); + } + } + + /** + * 扫描文件夹中所有支持的文件. + * + * @return supported file paths + */ + private List scanSupportedFiles(Path folder) { + try (Stream pathStream = Files.walk(folder)) { + return pathStream + .filter(Files::isRegularFile) + .filter(this::isSupportedFormat) + .peek(filePath -> logDebug("Found supported file: {}", filePath)) + .map(filePath -> filePath.toAbsolutePath().normalize().toString()) + .sorted() + .collect(Collectors.toList()); + + } catch (IOException e) { + logError("Failed to scan folder: {}", folder, e); + throw new ServiceException( + ExceptionEnum.CM333.getResultCode(), ExceptionEnum.CM333.getResultMsg()); + } + } + + /** + * 根据文档路径和自定义集合确定目标集合. + * + * @return target collection name + */ + private String determineCollectionName(String filePath, String customCollection) { + // 如果指定了自定义集合,优先使用 + if (customCollection != null && !customCollection.isBlank()) { + if (!isValidCollection(customCollection)) { + logWarn( + "Invalid collection specified: {}, using default: {}", + customCollection, + DEFAULT_COLL); + return DEFAULT_COLL; + } + return customCollection; + } + + // 根据文件路径自动判断集合 + if (filePath != null) { + String lowerPath = filePath.toLowerCase(Locale.ROOT); + // 如果路径包含特定关键词,映射到对应集合 + for (Map.Entry entry : collectionMapping.entrySet()) { + if (lowerPath.contains(entry.getKey())) { + logInfo("Auto-mapped file {} to collection: {}", filePath, entry.getValue()); + return entry.getValue(); + } + } + } + + // 默认集合 + return DEFAULT_COLL; + } + + /** + * 检查文件格式是否支持. + * + * @return whether the file format is supported + */ + private boolean isSupportedFormat(String filePath) { + if (filePath == null) { + return false; + } + + String lowerPath = filePath.toLowerCase(Locale.ROOT); + return SUPPORTED_FORMATS.stream().anyMatch(lowerPath::endsWith); + } + + /** + * 检查是否为文本格式. + * + * @return whether the file is text format + */ + private boolean isTextFormat(String filePath) { + if (filePath == null) { + return false; + } + + String lowerPath = filePath.toLowerCase(Locale.ROOT); + return TEXT_FORMATS.stream().anyMatch(lowerPath::endsWith); + } + + /** + * 获取文件格式描述. + * + * @return file format description + */ + private String getFileFormatDescription(String filePath) { + String description = "unknown"; + if (filePath != null) { + description = "Unknown Format"; + String lowerPath = filePath.toLowerCase(Locale.ROOT); + for (Map.Entry entry : FORMAT_DESC.entrySet()) { + if (lowerPath.endsWith(entry.getKey())) { + description = entry.getValue(); + break; + } + } + } + return description; + } + + /** + * 添加文档到知识库(默认集合). + * + * @return vector storage result + */ + public VectorDocument initializeKnowledgeBase(List documentPaths) { + return initializeKnowledgeBase(documentPaths, null, null); + } + + /** + * 添加文档到知识库(指定集合). + * + * @return vector storage result + */ + public VectorDocument initializeKnowledgeBase( + List documentPaths, String documentSetId, String collectionName) { + try { + if (documentPaths == null || documentPaths.isEmpty()) { + throw new ServiceException( + ExceptionEnum.CM329.getResultCode(), ExceptionEnum.CM329.getResultMsg()); + } + // 确定目标集合 + String targetCollection = + determineCollectionName( + documentPaths.isEmpty() ? null : documentPaths.get(0), collectionName); + + logInfo("Using collection: {} for document storage", targetCollection); + + List documents = + loadDocuments(documentPaths, documentSetId, targetCollection); + + if (documents.isEmpty()) { + throw new ServiceException( + ExceptionEnum.CM329.getResultCode(), ExceptionEnum.CM329.getResultMsg()); + } + + logInfo( + "Successfully loaded {} documents for collection: {}", + documents.size(), + targetCollection); + + // 文档切分 + List segments = splitDocuments(documents); + logInfo( + "Generated {} text segments for collection: {}", + segments.size(), + targetCollection); + + // 向量化并存储到指定集合 + return embedAndStore(segments, targetCollection); + + } catch (ServiceException e) { + throw e; + } catch (Exception e) { + logError("Failed to add the document to the knowledge base", e); + throw new ServiceException( + ExceptionEnum.CM330.getResultCode(), ExceptionEnum.CM330.getResultMsg()); + } + } + + /** + * 加载文档. + * + * @return loaded documents + */ + private List loadDocuments( + List documentPaths, String documentSetId, String collectionName) { + List documents = new ArrayList<>(); + Path documentRoot = getDocumentRoot(); + + int loadedCount = 0; + int skippedCount = 0; + + for (String path : documentPaths) { + try { + Path filePath = resolveDocumentPath(path, documentRoot); + // 检查文件是否存在 + if (!Files.exists(filePath)) { + logWarn("✗ File not found: {}", path); + skippedCount++; + continue; + } + filePath = resolveRealDocumentPath(filePath, documentRoot); + + // 检查文件格式是否支持 + if (!isSupportedFormat(filePath)) { + logWarn( + "✗ Unsupported document format: {} ({})", + path, + getFileFormatDescription(path)); + skippedCount++; + continue; + } + + Document document; + + if (filePath.toString().toLowerCase(Locale.ROOT).endsWith(".pdf")) { + // PDF 文件使用 PDF 解析器 + ApachePdfBoxDocumentParser pdfParser = new ApachePdfBoxDocumentParser(); + document = FileSystemDocumentLoader.loadDocument(filePath, pdfParser); + } else if (isTextFormat(filePath.toString())) { + // 所有文本文件使用 TextDocumentParser + document = + FileSystemDocumentLoader.loadDocument( + filePath, new TextDocumentParser()); + } else { + logWarn( + "✗ Unhandled document format: {} ({})", + path, + getFileFormatDescription(path)); + skippedCount++; + continue; + } + + // 添加元数据 + if (documentSetId != null) { + document.metadata().put("documentSetId", documentSetId); + } + document.metadata().put("source", filePath.toString()); + document.metadata().put("format", getFileFormatDescription(filePath.toString())); + document.metadata().put("timestamp", String.valueOf(System.currentTimeMillis())); + document.metadata().put("collection", collectionName); // 添加集合信息 + + documents.add(document); + loadedCount++; + logInfo( + "✓ Loaded document: {} ({}) to collection: {}", + filePath, + getFileFormatDescription(filePath.toString()), + collectionName); + + } catch (Exception e) { + logError("✗ Failed to load the document: {} - {}", path, e.getMessage()); + skippedCount++; + } + } + + logInfo( + "Document loading summary: {} loaded, {} skipped, {} total paths for collection:" + + " {}", + loadedCount, + skippedCount, + documentPaths.size(), + collectionName); + + return documents; + } + + /** + * 文档切分. + * + * @return text segments + */ + private List splitDocuments(List documents) { + DocumentSplitter splitter = + DocumentSplitters.recursive(ragConfig.getChunkSize(), ragConfig.getChunkOverlap()); + return splitter.splitAll(documents); + } + + /** + * 向量化并存储到指定集合. + * + * @return vector storage result + */ + private VectorDocument embedAndStore(List segments, String collectionName) { + logInfo("Begin vectorized storage to collection: {}...", collectionName); + long startTime = System.currentTimeMillis(); + + int successCount = 0; + int errorCount = 0; + + // 批量处理,提高性能 + int batchSize = STORE_BATCH_SIZE; + for (int i = 0; i < segments.size(); i += batchSize) { + int end = Math.min(i + batchSize, segments.size()); + List batch = segments.subList(i, end); + + BatchResult result = processBatch(batch, i, segments.size(), collectionName); + successCount += result.getSuccessCount(); + errorCount += result.getErrorCount(); + } + + long endTime = System.currentTimeMillis(); + logInfo( + "Vectorization completed in collection {}: {} successful, {} failed, time taken: {}" + + " ms", + collectionName, + successCount, + errorCount, + (endTime - startTime)); + + return new VectorDocument(successCount, errorCount, null, collectionName); + } + + /** + * 处理批次数据. + * + * @return batch processing result + */ + private BatchResult processBatch( + List batch, int startIndex, int totalSize, String collectionName) { + int successCount = 0; + int errorCount = 0; + + List embeddings = new ArrayList<>(); + List segmentsToStore = new ArrayList<>(); + + for (int i = 0; i < batch.size(); i++) { + TextSegment segment = batch.get(i); + try { + Embedding embedding = embeddingModel.embed(segment.text()).content(); + embeddings.add(embedding); + segmentsToStore.add(segment); + successCount++; + + if ((startIndex + i + 1) % LOG_INTERVAL == 0) { + logInfo( + "Processed {}/{} text segments for collection: {}", + (startIndex + i + 1), + totalSize, + collectionName); + } + } catch (Exception e) { + errorCount++; + logError( + "Vectorization failed [{}] in collection {}: {}", + (startIndex + i + 1), + collectionName, + segment.text() + .substring(0, Math.min(PREVIEW_LEN, segment.text().length()))); + } + } + + if (!embeddings.isEmpty()) { + try { + embeddingStore.addAll(embeddings, segmentsToStore); + logDebug( + "Successfully stored {} text segments to vector database in collection: {}", + embeddings.size(), + collectionName); + } catch (Exception e) { + logError( + "Batch storage to vector database failed in collection: {}", + collectionName, + e); + errorCount += embeddings.size(); + successCount -= embeddings.size(); + } + } + BatchResult result = new BatchResult(); + result.setSuccessCount(successCount); + result.setErrorCount(errorCount); + return result; + } + + /** + * 在指定集合中检索. + * + * @return matched embedding results + */ + public List search(SearchRequest searchDto) { + try { + Embedding queryEmbedding = embeddingModel.embed(searchDto.getContent()).content(); + + EmbeddingSearchRequest searchRequest = + EmbeddingSearchRequest.builder() + .queryEmbedding(queryEmbedding) + .maxResults(searchDto.getMaxResults()) + .minScore(searchDto.getMinScore()) + .build(); + + List> matches = + embeddingStore.search(searchRequest).matches(); + + // 如果指定了集合名称,进行过滤 + if (searchDto.getCollection() != null) { + matches = filterByCollection(matches, searchDto.getCollection()); + } + + // 转换为 DTO + List results = + matches.stream().map(EmbeddingMatchDto::from).collect(Collectors.toList()); + + logInfo( + "Retrieved {} related documents from collection: {}", + results.size(), + searchDto.getCollection() != null + ? searchDto.getCollection() + : "all collections"); + return results; + + } catch (Exception e) { + logError("Retrieval failed", e); + throw new ServiceException( + ExceptionEnum.CM331.getResultCode(), ExceptionEnum.CM331.getResultMsg()); + } + } + + /** + * 根据集合名称过滤结果. + * + * @return filtered embedding results + */ + private static List> filterByCollection( + List> results, String collectionName) { + + List> filteredResults = new ArrayList<>(); + + for (EmbeddingMatch match : results) { + String collection = match.embedded().metadata().getString("collection"); + if (collectionName.equals(collection)) { + filteredResults.add(match); + } + } + + return filteredResults; + } + + /** + * 跨集合搜索(在所有集合中搜索). + * + * @return search results grouped by collection + */ + public Map> searchAcrossCollections(SearchRequest searchDto) { + Map> results = new HashMap<>(); + + for (String collection : VALID_COLLECTIONS) { + try { + searchDto.setCollection(collection); + List collectionResults = search(searchDto); + results.put(collection, collectionResults); + logInfo( + "Found {} results in collection: {}", collectionResults.size(), collection); + } catch (Exception e) { + logWarn("Search failed in collection: {}", collection, e); + results.put(collection, new ArrayList<>()); + } + } + + return results; + } + + /** + * 根据文件路径删除指定集合中的文档. + * + * @return delete result + */ + public DeleteResult deleteByFilePath(String filePath, String collectionName) { + try { + String safeFilePath = resolveDocumentPath(filePath).toString(); + logInfo( + "Deleting documents by file path: {} from collection: {}", + safeFilePath, + collectionName != null ? collectionName : "all collections"); + long startTime = System.currentTimeMillis(); + + // 搜索包含该文件路径的所有向量 + List> matches = + searchBySource(safeFilePath, collectionName); + + if (matches.isEmpty()) { + logWarn( + "No documents found for file path: {} in collection: {}", + safeFilePath, + collectionName != null ? collectionName : "any collection"); + return new DeleteResult(0, 0, safeFilePath); + } + + // 提取要删除的向量ID + List idsToRemove = + matches.stream() + .map(EmbeddingMatch::embeddingId) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + int deletedCount = 0; + if (!idsToRemove.isEmpty()) { + // 批量删除向量 + embeddingStore.removeAll(idsToRemove); + deletedCount = idsToRemove.size(); + } + + long endTime = System.currentTimeMillis(); + logInfo( + "Deleted {} vectors for file: {} from collection: {}, time taken: {} ms", + deletedCount, + safeFilePath, + collectionName != null ? collectionName : "all collections", + (endTime - startTime)); + + return new DeleteResult(deletedCount, 0, safeFilePath); + + } catch (ServiceException e) { + throw e; + } catch (Exception e) { + logError( + "Failed to delete documents by file path: {} from collection: {}", + filePath, + collectionName, + e); + throw new ServiceException( + ExceptionEnum.CM332.getResultCode(), "Delete document failed"); + } + } + + /** + * 根据源文件路径搜索向量(支持集合过滤). + * + * @return matched vectors for the source path + */ + private List> searchBySource( + String sourcePath, String collectionName) { + try { + // 使用更合理的查询文本 + String queryText = "document content analysis"; + + EmbeddingSearchRequest searchRequest = + EmbeddingSearchRequest.builder() + .queryEmbedding(embeddingModel.embed(queryText).content()) + .maxResults(SOURCE_SCAN_LIMIT) + .minScore(SOURCE_MIN_SCORE) + .build(); + + List> allMatches = + embeddingStore.search(searchRequest).matches(); + + // 在应用层过滤 + return allMatches.stream() + .filter( + match -> { + String source = match.embedded().metadata().getString("source"); + String collection = + match.embedded().metadata().getString("collection"); + + boolean sourceMatch = source != null && source.equals(sourcePath); + boolean collectionMatch = + collectionName == null + || (collection != null + && collection.equals(collectionName)); + + return sourceMatch && collectionMatch; + }) + .collect(Collectors.toList()); + + } catch (Exception e) { + logError( + "Failed to search vectors by source: {} in collection: {}", + sourcePath, + collectionName, + e); + return new ArrayList<>(); + } + } + + /** + * 批量删除多个文件(默认集合). + * + * @return batch delete result + */ + public BatchDeleteResult deleteMultipleFiles(List filePaths) { + return deleteMultipleFiles(filePaths, null); + } + + /** + * 批量删除多个文件(指定集合). + * + * @return batch delete result + */ + public BatchDeleteResult deleteMultipleFiles(List filePaths, String collectionName) { + try { + logInfo( + "Deleting multiple files: {} from collection: {}", + filePaths, + collectionName != null ? collectionName : "all collections"); + long startTime = System.currentTimeMillis(); + + int totalDeleted = 0; + int totalFailed = 0; + List results = new ArrayList<>(); + + for (String filePath : filePaths) { + try { + DeleteResult result = deleteByFilePath(filePath, collectionName); + results.add(result); + totalDeleted += result.getDeletedCount(); + if (result.getFailedCount() > 0) { + totalFailed += result.getFailedCount(); + } + } catch (Exception e) { + logError( + "Failed to delete file: {} from collection: {}", + filePath, + collectionName, + e); + totalFailed++; + results.add(new DeleteResult(0, 1, filePath)); + } + } + + long endTime = System.currentTimeMillis(); + logInfo( + "Batch deletion completed: {} deleted, {} failed, time taken: {} ms from" + + " collection: {}", + totalDeleted, + totalFailed, + (endTime - startTime), + collectionName != null ? collectionName : "all collections"); + + return new BatchDeleteResult(totalDeleted, totalFailed, results); + + } catch (Exception e) { + logError("Failed to delete multiple files from collection: {}", collectionName, e); + throw new ServiceException( + ExceptionEnum.CM332.getResultCode(), "Batch delete files failed"); + } + } + + /** + * 获取所有集合及其包含的文档路径. + * + * @return documents grouped by collection + */ + public Map> getAllCollectionDocuments() { + Map> collectionDocuments = new HashMap<>(); + try { + // 遍历所有支持的集合 + for (String collection : VALID_COLLECTIONS) { + List documents = getStoredFiles(collection); + if (!documents.isEmpty()) { + collectionDocuments.put(collection, documents); + } + } + + logInfo("Retrieved documents from {} collections", collectionDocuments.size()); + return collectionDocuments; + + } catch (Exception e) { + logError("Failed to get collection documents", e); + return collectionDocuments; + } + } + + /** + * 获取指定集合的文档路径. + * + * @return documents in the collection + */ + public Map> getCollectionDocuments(String collectionName) { + Map> result = new HashMap<>(); + try { + if (!isValidCollection(collectionName)) { + throw new ServiceException( + ExceptionEnum.CM001.getResultCode(), + "Invalid collection name: " + collectionName); + } + + List documents = getStoredFiles(collectionName); + + result.put(collectionName, documents); + + logInfo( + "Retrieved {} documents from collection: {}", documents.size(), collectionName); + return result; + + } catch (Exception e) { + logError("Failed to get documents from collection: {}", collectionName, e); + return result; + } + } + + /** + * 获取所有已存储的文件列表(支持集合过滤). + * + * @return stored file paths + */ + public List getStoredFiles() { + return getStoredFiles(null); + } + + /** + * 获取指定集合中已存储的文件列表. + * + * @return stored file paths + */ + public List getStoredFiles(String collectionName) { + try { + EmbeddingSearchRequest searchRequest = + EmbeddingSearchRequest.builder() + .queryEmbedding(embeddingModel.embed("test").content()) + .maxResults(LIST_SCAN_LIMIT) + .minScore(0.0) + .build(); + + List> allMatches = + embeddingStore.search(searchRequest).matches(); + + // 提取所有唯一的源文件路径(支持集合过滤) + return allMatches.stream() + .filter( + match -> { + String collection = + match.embedded().metadata().getString("collection"); + return collectionName == null + || (collection != null + && collection.equals(collectionName)); + }) + .map(match -> match.embedded().metadata().getString("source")) + .filter(Objects::nonNull) + .distinct() + .collect(Collectors.toList()); + + } catch (Exception e) { + logError("Failed to get stored files list for collection: {}", collectionName, e); + return new ArrayList<>(); + } + } + + /** + * 获取文档集列表(支持集合过滤). + * + * @return stored document set identifiers + */ + public List getDocumentSets() { + return getDocumentSets(null); + } + + /** + * 获取指定集合中的文档集列表. + * + * @return stored document set identifiers + */ + public List getDocumentSets(String collectionName) { + try { + EmbeddingSearchRequest searchRequest = + EmbeddingSearchRequest.builder() + .queryEmbedding(embeddingModel.embed("test").content()) + .maxResults(LIST_SCAN_LIMIT) + .minScore(0.0) + .build(); + + List> allMatches = + embeddingStore.search(searchRequest).matches(); + + // 提取所有唯一的文档集ID(支持集合过滤) + return allMatches.stream() + .filter( + match -> { + String collection = + match.embedded().metadata().getString("collection"); + return collectionName == null + || (collection != null + && collection.equals(collectionName)); + }) + .map(match -> match.embedded().metadata().getString("documentSetId")) + .filter(Objects::nonNull) + .distinct() + .collect(Collectors.toList()); + + } catch (Exception e) { + logError("Failed to get document sets list for collection: {}", collectionName, e); + return new ArrayList<>(); + } + } + + /** + * 获取支持的集合列表. + * + * @return supported collection names + */ + public List getSupportedCollections() { + return new ArrayList<>(VALID_COLLECTIONS); + } + + /** + * 获取集合统计信息. + * + * @return collection statistics + */ + public Map getCollectionStats() { + Map stats = new HashMap<>(); + + for (String collection : VALID_COLLECTIONS) { + try { + List files = getStoredFiles(collection); + stats.put(collection, files.size()); + logDebug("Collection {} has {} files", collection, files.size()); + } catch (Exception e) { + logWarn("Failed to get stats for collection: {}", collection, e); + stats.put(collection, 0); + } + } + + return stats; + } + + /** 清空指定集合 */ + public void clearCollection(String collectionName) { + try { + if (!isValidCollection(collectionName)) { + throw new ServiceException( + ExceptionEnum.CM001.getResultCode(), + "Invalid collection name: " + collectionName); + } + + // 获取该集合中的所有文档并删除 + List files = getStoredFiles(collectionName); + if (!files.isEmpty()) { + deleteMultipleFiles(files, collectionName); + } + + logInfo("Collection cleared successfully: {}", collectionName); + } catch (Exception e) { + logError("Failed to clear collection: {}", collectionName, e); + throw new ServiceException( + ExceptionEnum.CM001.getResultCode(), + "Clear collection failed: " + collectionName); + } + } + + /** 清空向量库(所有集合) */ + public void clearVectorStore() { + try { + embeddingStore.removeAll(); + logInfo("Vector store cleared successfully (all collections)"); + + logInfo("Vector store cleared successfully (all collections)"); + } catch (Exception e) { + logError("Failed to clear vector library", e); + throw new ServiceException( + ExceptionEnum.CM001.getResultCode(), "Clear vector store failed"); + } + } +} diff --git a/base/src/main/java/com/tinyengine/it/service/app/impl/AiChatServiceImpl.java b/base/src/main/java/com/tinyengine/it/service/app/impl/AiChatServiceImpl.java index 804257ca..cf122c38 100644 --- a/base/src/main/java/com/tinyengine/it/service/app/impl/AiChatServiceImpl.java +++ b/base/src/main/java/com/tinyengine/it/service/app/impl/AiChatServiceImpl.java @@ -1,21 +1,18 @@ /** - * Copyright (c) 2023 - present TinyEngine Authors. - * Copyright (c) 2023 - present Huawei Cloud Computing Technologies Co., Ltd. + * Copyright (c) 2023 - present TinyEngine Authors. Copyright (c) 2023 - present Huawei Cloud + * Computing Technologies Co., Ltd. * - * Use of this source code is governed by an MIT-style license. - * - * THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, - * BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR - * A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + *

Use of this source code is governed by an MIT-style license. * + *

THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, + * BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR A + * PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. */ - package com.tinyengine.it.service.app.impl; import com.tinyengine.it.common.base.Result; import com.tinyengine.it.common.enums.Enums; -import com.tinyengine.it.common.exception.ExceptionEnum; -import com.tinyengine.it.common.exception.ServiceException; + import com.tinyengine.it.common.log.SystemServiceLog; import com.tinyengine.it.gateway.ai.AiChatClient; import com.tinyengine.it.model.dto.AiMessages; @@ -42,8 +39,9 @@ @Slf4j public class AiChatServiceImpl implements AiChatService { private static final Pattern PATTERN_TAG_START = Pattern.compile("```javascript|||"); - private static final Pattern PATTERN_MESSAGE = Pattern.compile(".*编码时遵从以下几条要求.*"); + private static final Pattern PATTERN_TAG_END = + Pattern.compile("```|||"); + private static final String REQ_MARKER = "编码时遵从以下几条要求"; /** * Get start and end int [ ]. @@ -69,7 +67,7 @@ public static int[] getStartAndEnd(String str) { } } - return new int[]{start, end}; + return new int[] {start, end}; } @SystemServiceLog(description = "getAnswerFromAi 获取ai回答") @@ -87,23 +85,18 @@ public Result> getAnswerFromAi(AiParam aiParam) { List> choices = (List>) data.get("choices"); Map message = (Map) choices.get(0).get("message"); - String answerContent = ""; - String isFinish = ""; + String finishReasonValue = ""; + StringBuilder answerContent = new StringBuilder(); Object finishReason = choices.get(0).get("finish_reason"); if (finishReason instanceof String) { - isFinish = (String) finishReason; - } - if (!"length".equals(isFinish)) { - answerContent = message.get("content"); + finishReasonValue = (String) finishReason; } - // 若内容被截断,继续请求AI - while ("length".equals(isFinish)) { + while ("length".equals(finishReasonValue)) { String prefix = message.get("content"); - answerContent = answerContent + prefix; + answerContent.append(prefix); // 将此部分内容加入消息列表 - Map partialMessage = new HashMap<>(); AiMessages aiMessages = new AiMessages(); List messagesList = aiParam.getMessages(); aiMessages.setRole("assistant"); @@ -113,23 +106,18 @@ public Result> getAnswerFromAi(AiParam aiParam) { aiParam.setMessages(messagesList); // 再次请求AI - try { - data = requestAnswerFromAi(aiParam.getMessages(), aiParam.getFoundationModel()).getData(); - } catch (Exception e) { - throw new ServiceException(ExceptionEnum.CM001.getResultCode(), e.getMessage()); - } + data = + requestAnswerFromAi(aiParam.getMessages(), aiParam.getFoundationModel()) + .getData(); choices = (List>) data.get("choices"); message = (Map) choices.get(0).get("message"); - StringBuilder sb = new StringBuilder(); - answerContent = String.valueOf(sb.append(message.get("content"))); finishReason = choices.get(0).get("finish_reason"); if (finishReason instanceof String) { - isFinish = (String) finishReason; + finishReasonValue = (String) finishReason; } } - // 通过二方包将页面转成schema - String codes = extractCode(answerContent); - Map result = buildResult(answerContent, message); + answerContent.append(message.get("content")); + Map result = buildResult(answerContent.toString(), message); return Result.success(result); } @@ -138,7 +126,7 @@ private Result> checkParam(AiParam aiParam) { if (token == null || token.isEmpty()) { return Result.failed("The token cannot be empty"); } - if (!Pattern.matches("^[A-Za-z0-9_.-]+$", token)) { + if (!isSafeToken(token)) { return Result.failed("Invalid token format"); } @@ -152,7 +140,8 @@ private Result> checkParam(AiParam aiParam) { } foundationModel.put("model", model); aiParam.setFoundationModel(foundationModel); - Result> resultData = requestAnswerFromAi(aiParam.getMessages(), aiParam.getFoundationModel()); + Result> resultData = + requestAnswerFromAi(aiParam.getMessages(), aiParam.getFoundationModel()); // 调用接口失败时且data为null if (!resultData.isSuccess() && resultData.getData() == null) { return Result.failed(resultData.getCode(), resultData.getMessage()); @@ -174,12 +163,13 @@ private Map buildResult(String answerContent, Map> requestAnswerFromAi(List messages, - Map foundationModel) { + private Result> requestAnswerFromAi( + List messages, Map foundationModel) { List aiMessages = formatMessage(messages); AiParam aiParam = new AiParam(foundationModel, aiMessages); - AiChatClient aiChatClient = new AiChatClient(foundationModel.get("model"), foundationModel.get("token")); + AiChatClient aiChatClient = + new AiChatClient(foundationModel.get("model"), foundationModel.get("token")); Map response = aiChatClient.executeChatRequest(aiParam); // 适配文心一言的响应数据结构,文心的部分异常情况status也是200,需要转为400,以免前端无所适从 if (response.get("error_code") != null) { @@ -189,7 +179,8 @@ private Result> requestAnswerFromAi(List message } if (response.get("error") != null) { String code = (response.get("code") != null) ? response.get("code").toString() : ""; - String message = (response.get("message") != null) ? response.get("message").toString() : ""; + String message = + (response.get("message") != null) ? response.get("message").toString() : ""; return Result.failed(code, message); } if (Enums.FoundationModel.ERNIBOT_TURBO.getValue().equals(foundationModel.get("model"))) { @@ -200,8 +191,8 @@ private Result> requestAnswerFromAi(List message /** * 转换模型返回格式 - *

- * 暂且只满足回复中只包括一个代码块的场景 + * + *

暂且只满足回复中只包括一个代码块的场景 * * @param response ai返回内容 * @return result 返回结果 @@ -228,8 +219,8 @@ private Result> modelResultConvet(Map respon /** * 提取回复中的代码 - *

- * 暂且只满足回复中只包括一个代码块的场景 + * + *

暂且只满足回复中只包括一个代码块的场景 * * @param content ai回复的内容 * @return 提取的文本 string @@ -246,8 +237,8 @@ public String extractCode(String content) { /** * 去除回复中的代码 - *

- * 暂且只满足回复中只包括一个代码块的场景 + * + *

暂且只满足回复中只包括一个代码块的场景 * * @param content ai回复的内容 * @return 去除代码后的回复内容 string @@ -266,18 +257,25 @@ private List formatMessage(List messages) { AiMessages defaultWords = new AiMessages(); defaultWords.setRole("user"); defaultWords.setContent( - "你是一名前端开发专家,编码时遵从以下几条要求:\n" + "###\n" + "1. 只使用 element-ui组件库的el-button 和 el-table组件\n" - + "2. el-table表格组件的使用方式为 " - + "columns的columnData表示列数据,其中用title表示列名,field表示表格数据字段; data的tableData表示表格展示的数据。 " - + "el-table标签内不得出现子元素\n" + "3. 使用vue2技术栈\n" + "4. 回复中只能有一个代码块\n" - + "5. 不要加任何注释\n" + "6. el-table标签内不得出现el-table-column\n" + "###"); + "你是一名前端开发专家,编码时遵从以下几条要求:\n" + + "###\n" + + "1. 只使用 element-ui组件库的el-button 和 el-table组件\n" + + "2. el-table表格组件的使用方式为 " + + " columns的columnData表示列数据,其中用title表示列名,field表示表格数据字段;" + + " data的tableData表示表格展示的数据。 el-table标签内不得出现子元素\n" + + "3. 使用vue2技术栈\n" + + "4. 回复中只能有一个代码块\n" + + "5. 不要加任何注释\n" + + "6. el-table标签内不得出现el-table-column\n" + + "###"); defaultWords.setName(messages.get(0).getName()); String role = messages.get(0).getRole(); String content = messages.get(0).getContent(); List aiMessages = new ArrayList<>(); - if (!PATTERN_MESSAGE.matcher(content).matches()) { + if (content == null || !content.contains(REQ_MARKER)) { AiMessages aiMessagesResult = messages.get(0); aiMessagesResult.setContent(defaultWords.getContent() + "\n" + content); } @@ -287,4 +285,19 @@ private List formatMessage(List messages) { return messages; } + + private boolean isSafeToken(String token) { + for (int i = 0; i < token.length(); i++) { + char c = token.charAt(i); + if (!((c >= 'A' && c <= 'Z') + || (c >= 'a' && c <= 'z') + || (c >= '0' && c <= '9') + || c == '_' + || c == '.' + || c == '-')) { + return false; + } + } + return true; + } } diff --git a/base/src/main/java/com/tinyengine/it/service/app/impl/v1/AiChatV1ServiceImpl.java b/base/src/main/java/com/tinyengine/it/service/app/impl/v1/AiChatV1ServiceImpl.java index 71ea4760..29c1a9c5 100644 --- a/base/src/main/java/com/tinyengine/it/service/app/impl/v1/AiChatV1ServiceImpl.java +++ b/base/src/main/java/com/tinyengine/it/service/app/impl/v1/AiChatV1ServiceImpl.java @@ -1,271 +1,320 @@ -/** - * Copyright (c) 2023 - present TinyEngine Authors. - * Copyright (c) 2023 - present Huawei Cloud Computing Technologies Co., Ltd. - * - * Use of this source code is governed by an MIT-style license. - * - * THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, - * BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR - * A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. - * - */ - -package com.tinyengine.it.service.app.impl.v1; - -import com.fasterxml.jackson.databind.JsonNode; -import com.tinyengine.it.common.exception.ServiceException; -import com.tinyengine.it.common.log.SystemServiceLog; -import com.tinyengine.it.common.utils.JsonUtils; -import com.tinyengine.it.common.utils.SM4Utils; -import com.tinyengine.it.config.OpenAIConfig; -import com.tinyengine.it.model.dto.ChatRequest; -import com.tinyengine.it.service.app.v1.AiChatV1Service; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; -import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; - +/** + * Copyright (c) 2023 - present TinyEngine Authors. Copyright (c) 2023 - present Huawei Cloud + * Computing Technologies Co., Ltd. + * + *

Use of this source code is governed by an MIT-style license. + * + *

THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, + * BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR A + * PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + */ +package com.tinyengine.it.service.app.impl.v1; + +import com.fasterxml.jackson.databind.JsonNode; +import com.tinyengine.it.common.exception.ServiceException; +import com.tinyengine.it.common.log.SystemServiceLog; +import com.tinyengine.it.common.utils.JsonUtils; +import com.tinyengine.it.common.utils.SM4Utils; +import com.tinyengine.it.config.OpenAIConfig; +import com.tinyengine.it.model.dto.ChatRequest; +import com.tinyengine.it.service.app.v1.AiChatV1Service; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Service; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + import java.io.IOException; import java.io.InputStream; import java.net.Inet4Address; -import java.net.InetAddress; import java.net.Inet6Address; +import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; import java.net.UnknownHostException; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.nio.charset.StandardCharsets; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; import java.time.Duration; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Set; - -/** - * The type AiChat v1 service. - * - * @since 2025-08-06 - */ -@Slf4j -@Service -public class AiChatV1ServiceImpl implements AiChatV1Service { - private final OpenAIConfig config; - private final HttpClient httpClient; - - public AiChatV1ServiceImpl(OpenAIConfig config) { - this.config = config; - this.httpClient = HttpClient.newBuilder() - .connectTimeout(Duration.ofSeconds(config.getTimeoutSeconds())) - .followRedirects(HttpClient.Redirect.NEVER) - .build(); - } - - /** - * chatCompletion. - * - * @param request the request - * @return Object the Object - */ - @Override - @SystemServiceLog(description = "chatCompletion") - public Object chatCompletion(ChatRequest request) throws Exception { - String requestBody = buildRequestBody(request); - String encryptApiKey = request.getApiKey() != null ? request.getApiKey() : config.getApiKey(); - String apiKey = getApiKey(encryptApiKey); - String baseUrl = request.getBaseUrl(); - - // 规范化URL处理 - String normalizedUrl = normalizeApiUrl(baseUrl); - - // 对最终请求 URL 做安全校验(在 normalize 之后,确保校验的是真正发出的地址) - validateFinalUrl(normalizedUrl); - - HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() - .uri(URI.create(normalizedUrl)) - .header("Content-Type", "application/json") - .header("Authorization", "Bearer " + apiKey) - .POST(HttpRequest.BodyPublishers.ofString(requestBody)); - if (request.isStream()) { - requestBuilder.header("Accept", "text/event-stream"); - return processStreamResponse(requestBuilder); - } else { - return processStandardResponse(requestBuilder); - } - } - - /** - * get token. - * - * @param apiKey the apiKey - * @return token the token - */ - @Override - public String getToken(String apiKey) throws Exception { - String sm4Key = System.getenv("SM4KEY"); - String encrypt = SM4Utils.encryptECB(apiKey, sm4Key); - return "EKEY_"+ encrypt; - } - - /** - * 规范化API URL,兼容不同厂商 - */ - private String normalizeApiUrl(String baseUrl) { - if (baseUrl == null || baseUrl.trim().isEmpty()) { - baseUrl = config.getBaseUrl(); - } - baseUrl = baseUrl.trim(); - - if (baseUrl.contains("/chat/completions") || baseUrl.contains("/v1/chat/completions")) { - return ensureUrlProtocol(baseUrl); - } - - if (baseUrl.contains("v1")) { - return ensureUrlProtocol(baseUrl) + "/chat/completions"; - } - if (baseUrl.endsWith("#")) { - return ensureUrlProtocol(baseUrl); - } else { - return ensureUrlProtocol(baseUrl) + "/v1/chat/completions"; - } - } - - /** - * 确保URL有正确的协议前缀 - */ - private String ensureUrlProtocol(String url) { - if (url.startsWith("http://") || url.startsWith("https://")) { - return url; - } - // 默认使用https - return "https://" + url; - } - - private String buildRequestBody(ChatRequest request) { - Map body = new HashMap<>(); - body.put("model", request.getModel() != null ? request.getModel() : config.getDefaultModel()); - body.put("messages", request.getMessages()); - body.put("stream", request.isStream()); - body.put("tools", request.getTools()); - if (request.getMaxTokens() != null) { - body.put("max_tokens", request.getMaxTokens()); - } - body.put("temperature", request.getTemperature()); - if (request.getTemperature() != null) { - body.put("temperature", request.getTemperature()); - } - if (request.getSearchOptions() != null) { - body.put("stream_options", request.getSearchOptions()); - } - if (request.getPresencePenalty() != null) { - body.put("presence_penalty", request.getPresencePenalty()); - } - if (request.getResponseFormat() != null) { - body.put("response_format", request.getResponseFormat()); - } - if (request.getMaxInputTokens() != null) { - body.put("max_input_tokens", request.getMaxInputTokens()); - } - if (request.getMaxInputTokens() != null) { - body.put("vl_high_resolution_images", request.getVlHighResolutionImages()); - } - if (request.getEnableThinking() != null) { - body.put("enable_thinking", request.getEnableThinking()); - } - if (request.getToolChoice() != null) { - body.put("tool_choice", request.getToolChoice()); - } - if (request.getStop() != null) { - body.put("stop", request.getStop()); - } - if (request.getParallelToolCalls() != null) { - body.put("parallel_tool_calls", request.getParallelToolCalls()); - } - if (request.getEnableSearch() != null) { - body.put("enable_search", request.getEnableSearch()); - } - if (request.getFrequencyPenalty() != null) { - body.put("frequency_penalty", request.getFrequencyPenalty()); - } - - return JsonUtils.encode(body); - } - - private JsonNode processStandardResponse(HttpRequest.Builder requestBuilder) { - HttpResponse response = null; - String code = null; - String message = null; - try { - response = httpClient.send( - requestBuilder.build(), HttpResponse.BodyHandlers.ofString()); - code = String.valueOf(response.statusCode()); - if (response.statusCode() != 200) { - String errorBody = response.body(); - - // 尝试解析错误JSON - JsonNode errorNode = JsonUtils.MAPPER.readTree(errorBody); - message = errorNode.get("error").get("message").asText(); - throw new ServiceException(code, message); - } - return JsonUtils.MAPPER.readTree(response.body()); - } catch (IOException | InterruptedException e) { - throw new ServiceException(code, message); - } - - - } - - private StreamingResponseBody processStreamResponse(HttpRequest.Builder requestBuilder) { - return outputStream -> { - HttpResponse response = null; - try { - response = httpClient.send( - requestBuilder.build(), HttpResponse.BodyHandlers.ofInputStream() - ); - } catch (InterruptedException e) { - throw new ServiceException("500", e.getMessage()); - } - - log.info("Received AI API response, status code {}", response.statusCode()); - - if (response.statusCode() != 200) { - String errorBody = new String(response.body().readAllBytes(), StandardCharsets.UTF_8); - - log.info("errorBody: {}", errorBody); - - JsonNode errorNode = JsonUtils.MAPPER.readTree(errorBody); - throw new ServiceException(String.valueOf(response.statusCode()), errorNode.get("error").get("message").asText()); - } - - // 正常流处理逻辑 - try (InputStream inputStream = response.body()) { - byte[] buffer = new byte[8192]; - int bytesRead; - while ((bytesRead = inputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - outputStream.flush(); - } - } - }; - } - - private static final Set LOOPBACK_HOSTS = Set.of("localhost", "127.0.0.1", "::1", "[::1]"); - - void validateFinalUrl(String finalUrl) { - URI uri; - try { - uri = new URI(finalUrl); - } catch (URISyntaxException e) { - throw new ServiceException("400", "Invalid baseUrl format"); - } - - String host = uri.getHost(); - if (host == null || host.isEmpty()) { - throw new ServiceException("400", "Invalid baseUrl: missing host"); - } - +import java.util.Set; + +/** + * The type AiChat v1 service. + * + * @since 2025-08-06 + */ +@Slf4j +@Service +@SuppressWarnings("PMD.TooManyMethods") +public class AiChatV1ServiceImpl implements AiChatV1Service { + private static final int HTTP_OK = 200; + private static final int STREAM_BUF_SIZE = 8192; + private static final int BYTE_MASK = 0xFF; + private static final int IPV4_CG_FIRST = 100; + private static final int IPV4_CG_MIN = 64; + private static final int IPV4_CG_MAX = 127; + private static final int IPV4_192 = 192; + private static final int IPV4_198 = 198; + private static final int IPV4_BENCH_A = 18; + private static final int IPV4_BENCH_B = 19; + private static final int IPV4_DOC_51 = 51; + private static final int IPV4_DOC_100 = 100; + private static final int IPV4_DOC_203 = 203; + private static final int IPV4_DOC_113 = 113; + private static final int IPV4_RESERVED = 240; + private static final int IPV6_UNIQUE = 0xFE; + private static final int IPV6_UNIQUE_MASK = 0xFC; + private static final int IPV6_DOC_FIRST = 0x20; + private static final int IPV6_DOC_SECOND = 0x01; + private static final int IPV6_DOC_THIRD = 0x0D; + private static final int IPV6_DOC_FOURTH = 0xB8; + private static final int IPV6_MULTICAST = 0xFF; + private static final int IPV6_FOURTH_IDX = 3; + private static final String EKEY_PREFIX = "EKEY_"; + private static final Set LOOPBACK_HOSTS = + Set.of("localhost", "127.0.0.1", "::1", "[::1]"); + + private final OpenAIConfig config; + private final HttpClient httpClient; + + public AiChatV1ServiceImpl(OpenAIConfig config) { + this.config = config; + this.httpClient = + HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(config.getTimeoutSeconds())) + .followRedirects(HttpClient.Redirect.NEVER) + .build(); + } + + /** + * chatCompletion. + * + * @param request the request + * @return Object the Object + */ + @Override + @SystemServiceLog(description = "chatCompletion") + public Object chatCompletion(ChatRequest request) throws Exception { + String requestBody = buildRequestBody(request); + String encryptApiKey = + request.getApiKey() != null ? request.getApiKey() : config.getApiKey(); + String apiKey = getApiKey(encryptApiKey); + String baseUrl = request.getBaseUrl(); + + // 规范化URL处理 + String normalizedUrl = normalizeApiUrl(baseUrl); + + // 对最终请求 URL 做安全校验(在 normalize 之后,确保校验的是真正发出的地址) + URI requestUri = validateFinalUrl(normalizedUrl); + + HttpRequest.Builder requestBuilder = + HttpRequest.newBuilder() + .uri(requestUri) + .header("Content-Type", "application/json") + .header("Authorization", "Bearer " + apiKey) + .POST(HttpRequest.BodyPublishers.ofString(requestBody)); + if (request.isStream()) { + requestBuilder.header("Accept", "text/event-stream"); + return processStreamResponse(requestBuilder); + } else { + return processStandardResponse(requestBuilder); + } + } + + /** + * get token. + * + * @param apiKey the apiKey + * @return token the token + */ + @Override + public String getToken(String apiKey) throws GeneralSecurityException { + String sm4Key = System.getenv("SM4KEY"); + String encrypt = SM4Utils.encrypt(apiKey, sm4Key); + return EKEY_PREFIX + encrypt; + } + + /** + * 规范化API URL,兼容不同厂商. + * + * @return normalized API URL + */ + private String normalizeApiUrl(String baseUrl) { + final String configuredUrl = + baseUrl == null || baseUrl.isBlank() ? config.getBaseUrl() : baseUrl; + final String normalizedUrl = configuredUrl.trim(); + + if (normalizedUrl.contains("/chat/completions") + || normalizedUrl.contains("/v1/chat/completions")) { + return ensureUrlProtocol(normalizedUrl); + } + + if (normalizedUrl.contains("v1")) { + return ensureUrlProtocol(normalizedUrl) + "/chat/completions"; + } + if (normalizedUrl.endsWith("#")) { + return ensureUrlProtocol(normalizedUrl); + } else { + return ensureUrlProtocol(normalizedUrl) + "/v1/chat/completions"; + } + } + + /** + * 确保URL有正确的协议前缀. + * + * @return URL with protocol + */ + private String ensureUrlProtocol(String url) { + if (url.startsWith("http://") || url.startsWith("https://")) { + return url; + } + // 默认使用https + return "https://" + url; + } + + private String buildRequestBody(ChatRequest request) { + Map body = new HashMap<>(); + body.put( + "model", + request.getModel() != null ? request.getModel() : config.getDefaultModel()); + body.put("messages", request.getMessages()); + body.put("stream", request.isStream()); + body.put("tools", request.getTools()); + if (request.getMaxTokens() != null) { + body.put("max_tokens", request.getMaxTokens()); + } + body.put("temperature", request.getTemperature()); + if (request.getTemperature() != null) { + body.put("temperature", request.getTemperature()); + } + if (request.getSearchOptions() != null) { + body.put("stream_options", request.getSearchOptions()); + } + if (request.getPresencePenalty() != null) { + body.put("presence_penalty", request.getPresencePenalty()); + } + if (request.getResponseFormat() != null) { + body.put("response_format", request.getResponseFormat()); + } + if (request.getMaxInputTokens() != null) { + body.put("max_input_tokens", request.getMaxInputTokens()); + } + if (request.getMaxInputTokens() != null) { + body.put("vl_high_resolution_images", request.getVlHighResolutionImages()); + } + if (request.getEnableThinking() != null) { + body.put("enable_thinking", request.getEnableThinking()); + } + if (request.getToolChoice() != null) { + body.put("tool_choice", request.getToolChoice()); + } + if (request.getStop() != null) { + body.put("stop", request.getStop()); + } + if (request.getParallelToolCalls() != null) { + body.put("parallel_tool_calls", request.getParallelToolCalls()); + } + if (request.getEnableSearch() != null) { + body.put("enable_search", request.getEnableSearch()); + } + if (request.getFrequencyPenalty() != null) { + body.put("frequency_penalty", request.getFrequencyPenalty()); + } + + return JsonUtils.encode(body); + } + + private JsonNode processStandardResponse(HttpRequest.Builder requestBuilder) { + try { + final HttpResponse response = + httpClient.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()); + final String code = String.valueOf(response.statusCode()); + if (response.statusCode() != HTTP_OK) { + final String errorBody = response.body(); + + // 尝试解析错误JSON + final JsonNode errorNode = JsonUtils.MAPPER.readTree(errorBody); + final String message = errorNode.get("error").get("message").asText(); + throw new ServiceException(code, message); + } + return JsonUtils.MAPPER.readTree(response.body()); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new ServiceException("500", "AI request interrupted", exception); + } catch (IOException exception) { + throw new ServiceException("500", "AI request failed", exception); + } + } + + private StreamingResponseBody processStreamResponse(HttpRequest.Builder requestBuilder) { + return outputStream -> { + final HttpResponse response; + try { + response = + httpClient.send( + requestBuilder.build(), HttpResponse.BodyHandlers.ofInputStream()); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new ServiceException("500", "AI request interrupted", exception); + } catch (IOException exception) { + throw new ServiceException("500", "AI request failed", exception); + } + + if (log.isInfoEnabled()) { + log.info("Received AI API response, status code {}", response.statusCode()); + } + + if (response.statusCode() != HTTP_OK) { + String errorBody = + new String(response.body().readAllBytes(), StandardCharsets.UTF_8); + + if (log.isInfoEnabled()) { + log.info("errorBody: {}", errorBody); + } + + JsonNode errorNode = JsonUtils.MAPPER.readTree(errorBody); + throw new ServiceException( + String.valueOf(response.statusCode()), + errorNode.get("error").get("message").asText()); + } + + // 正常流处理逻辑 + try (InputStream inputStream = response.body()) { + byte[] buffer = new byte[STREAM_BUF_SIZE]; + int bytesRead = inputStream.read(buffer); + while (bytesRead != -1) { + outputStream.write(buffer, 0, bytesRead); + outputStream.flush(); + bytesRead = inputStream.read(buffer); + } + } + }; + } + + URI validateFinalUrl(String finalUrl) { + URI uri; + try { + uri = new URI(finalUrl); + } catch (URISyntaxException e) { + throw new ServiceException("400", "Invalid baseUrl format"); + } + + String host = uri.getHost(); + if (host == null || host.isEmpty()) { + throw new ServiceException("400", "Invalid baseUrl: missing host"); + } + if (uri.getUserInfo() != null || uri.getRawFragment() != null) { + throw new ServiceException( + "400", "Invalid baseUrl: user info and fragments are not allowed"); + } + boolean isLoopback = LOOPBACK_HOSTS.contains(host.toLowerCase(Locale.ROOT)); List allowedHosts = config.getAllowedHosts(); @@ -276,23 +325,23 @@ void validateFinalUrl(String finalUrl) { } enforceHttpsAndIpCheck(uri, host); - return; + return uri; } - boolean matched = allowedHosts.stream() - .anyMatch(allowed -> allowed.equalsIgnoreCase(host)); + boolean matched = allowedHosts.stream().anyMatch(allowed -> allowed.equalsIgnoreCase(host)); if (!matched) { - throw new ServiceException("400", - "Host not allowed: " + host + ". Allowed hosts: " + allowedHosts); + throw new ServiceException( + "400", "Host not allowed: " + host + ". Allowed hosts: " + allowedHosts); } if (isLoopback) { - return; + return uri; } enforceHttpsAndIpCheck(uri, host); + return uri; } - + void enforceHttpsAndIpCheck(URI uri, String host) { String scheme = uri.getScheme(); if (scheme == null || !"https".equalsIgnoreCase(scheme)) { @@ -316,10 +365,10 @@ InetAddress[] resolveHostAddresses(String host) throws UnknownHostException { boolean isBlockedAddress(InetAddress address) { if (address.isLoopbackAddress() - || address.isSiteLocalAddress() - || address.isLinkLocalAddress() - || address.isAnyLocalAddress() - || address.isMulticastAddress()) { + || address.isSiteLocalAddress() + || address.isLinkLocalAddress() + || address.isAnyLocalAddress() + || address.isMulticastAddress()) { return true; } @@ -334,59 +383,52 @@ boolean isBlockedAddress(InetAddress address) { private boolean isBlockedIpv4(Inet4Address address) { byte[] octets = address.getAddress(); - int first = octets[0] & 0xFF; - int second = octets[1] & 0xFF; - int third = octets[2] & 0xFF; + int first = octets[0] & BYTE_MASK; + int second = octets[1] & BYTE_MASK; + int third = octets[2] & BYTE_MASK; - if (first == 0) { - return true; - } - if (first == 100 && second >= 64 && second <= 127) { - return true; - } - if (first == 192 && second == 0 && third == 0) { - return true; - } - if (first == 192 && second == 0 && third == 2) { - return true; - } - if (first == 198 && (second == 18 || second == 19)) { - return true; - } - if (first == 198 && second == 51 && third == 100) { - return true; - } - if (first == 203 && second == 0 && third == 113) { - return true; - } - return first >= 240; + boolean currentNetwork = first == 0; + boolean sharedSpace = + first == IPV4_CG_FIRST && second >= IPV4_CG_MIN && second <= IPV4_CG_MAX; + boolean protocolAssign = first == IPV4_192 && second == 0 && third == 0; + boolean documentationA = first == IPV4_192 && second == 0 && third == 2; + boolean benchmarking = + first == IPV4_198 && (second == IPV4_BENCH_A || second == IPV4_BENCH_B); + boolean documentationB = + first == IPV4_198 && second == IPV4_DOC_51 && third == IPV4_DOC_100; + boolean documentationC = first == IPV4_DOC_203 && second == 0 && third == IPV4_DOC_113; + return currentNetwork + || sharedSpace + || protocolAssign + || documentationA + || benchmarking + || documentationB + || documentationC + || first >= IPV4_RESERVED; } private boolean isBlockedIpv6(Inet6Address address) { byte[] octets = address.getAddress(); - int first = octets[0] & 0xFF; - int second = octets[1] & 0xFF; + int first = octets[0] & BYTE_MASK; + int second = octets[1] & BYTE_MASK; - if ((first & 0xFE) == 0xFC) { - return true; + boolean uniqueLocal = (first & IPV6_UNIQUE) == IPV6_UNIQUE_MASK; + boolean documentation = false; + if (first == IPV6_DOC_FIRST && second == IPV6_DOC_SECOND) { + int third = octets[2] & BYTE_MASK; + int fourth = octets[IPV6_FOURTH_IDX] & BYTE_MASK; + documentation = third == IPV6_DOC_THIRD && fourth == IPV6_DOC_FOURTH; } - if (first == 0x20 && second == 0x01) { - int third = octets[2] & 0xFF; - int fourth = octets[3] & 0xFF; - if (third == 0x0D && fourth == 0xB8) { - return true; - } - } - return first == 0xFF; + return uniqueLocal || documentation || first == IPV6_MULTICAST; } - private String getApiKey(String encryptApiKey) throws Exception { + private String getApiKey(String encryptApiKey) throws GeneralSecurityException { String sm4Key = System.getenv("SM4KEY"); - - if (encryptApiKey.startsWith("EKEY_")) { - String encryptBase64ApiKey = encryptApiKey.substring(5); - return SM4Utils.decryptECB(encryptBase64ApiKey, sm4Key); - } - return encryptApiKey; - } -} + + if (encryptApiKey.startsWith(EKEY_PREFIX)) { + String encodedApiKey = encryptApiKey.substring(EKEY_PREFIX.length()); + return SM4Utils.decrypt(encodedApiKey, sm4Key); + } + return encryptApiKey; + } +} diff --git a/base/src/main/java/com/tinyengine/it/service/material/impl/BlockServiceImpl.java b/base/src/main/java/com/tinyengine/it/service/material/impl/BlockServiceImpl.java index d80a7bce..6ac4cf1b 100644 --- a/base/src/main/java/com/tinyengine/it/service/material/impl/BlockServiceImpl.java +++ b/base/src/main/java/com/tinyengine/it/service/material/impl/BlockServiceImpl.java @@ -1,17 +1,17 @@ /** - * Copyright (c) 2023 - present TinyEngine Authors. - * Copyright (c) 2023 - present Huawei Cloud Computing Technologies Co., Ltd. + * Copyright (c) 2023 - present TinyEngine Authors. Copyright (c) 2023 - present Huawei Cloud + * Computing Technologies Co., Ltd. * - * Use of this source code is governed by an MIT-style license. - * - * THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, - * BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR - * A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + *

Use of this source code is governed by an MIT-style license. * + *

THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, + * BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR A + * PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. */ - package com.tinyengine.it.service.material.impl; +import cn.hutool.core.bean.BeanUtil; + import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.core.toolkit.StringUtils; @@ -48,7 +48,6 @@ import com.tinyengine.it.service.app.I18nEntryService; import com.tinyengine.it.service.material.BlockService; -import cn.hutool.core.bean.BeanUtil; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; @@ -65,7 +64,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -76,30 +74,25 @@ */ @Service @Slf4j +@SuppressWarnings("PMD.TooManyMethods") public class BlockServiceImpl extends ServiceImpl implements BlockService { - @Autowired - private UserMapper userMapper; + private static final int DEFAULT_PAGE_SIZE = 10; - @Autowired - private AppMapper appMapper; + @Autowired private UserMapper userMapper; - @Autowired - private BlockHistoryMapper blockHistoryMapper; + @Autowired private AppMapper appMapper; - @Autowired - private I18nEntryService i18nEntryService; + @Autowired private BlockHistoryMapper blockHistoryMapper; - @Autowired - private I18nEntryMapper i18nEntryMapper; + @Autowired private I18nEntryService i18nEntryService; - @Autowired - private BlockGroupMapper blockGroupMapper; + @Autowired private I18nEntryMapper i18nEntryMapper; - @Autowired - private BlockGroupBlockMapper blockGroupBlockMapper; + @Autowired private BlockGroupMapper blockGroupMapper; - @Autowired - private LoginUserContext loginUserContext; + @Autowired private BlockGroupBlockMapper blockGroupBlockMapper; + + @Autowired private LoginUserContext loginUserContext; /** * 查询表t_block所有数据 @@ -124,12 +117,14 @@ public BlockDto queryBlockById(Integer id) { return blockDto; } boolean isPublished = - blockDto.getLastBuildInfo() != null && blockDto.getLastBuildInfo().get("result") instanceof Boolean - ? (Boolean) blockDto.getLastBuildInfo().get("result") - : Boolean.FALSE; + blockDto.getLastBuildInfo() != null + && blockDto.getLastBuildInfo().get("result") instanceof Boolean + ? (Boolean) blockDto.getLastBuildInfo().get("result") + : Boolean.FALSE; blockDto.setIsPublished(isPublished); - List groups = blockGroupMapper.findBlockGroupByBlockId(blockDto.getId(), - loginUserContext.getLoginUserId()); + List groups = + blockGroupMapper.findBlockGroupByBlockId( + blockDto.getId(), loginUserContext.getLoginUserId()); blockDto.setGroups(groups); return blockDto; } @@ -188,40 +183,40 @@ public Result updateBlockById(BlockParam blockParam) { blocks.setScreenshot(""); } + Result result; if (blockParam.getGroups() == null) { baseMapper.updateBlockById(blocks); BlockDto blockDtoResult = queryBlockById(blocks.getId()); - return Result.success(blockDtoResult); - } - - // 根据区块id获取区块所在分组 - List blockGroups = blockGroupMapper.findBlockGroupByBlockId(blocks.getId(), - loginUserContext.getLoginUserId()); - // 删除区块与分组关系 - if (blockGroups != null && !blockGroups.isEmpty()) { - List blockGroupIds = blockGroups.stream().map(BlockGroup::getId).collect(Collectors.toList()); - for (Integer id : blockGroupIds) { - blockGroupBlockMapper.deleteByGroupIdAndBlockId(id, blocks.getId()); + result = Result.success(blockDtoResult); + } else { + // 根据区块id获取区块所在分组 + List blockGroups = + blockGroupMapper.findBlockGroupByBlockId( + blocks.getId(), loginUserContext.getLoginUserId()); + // 删除区块与分组关系 + if (blockGroups != null && !blockGroups.isEmpty()) { + List blockGroupIds = + blockGroups.stream().map(BlockGroup::getId).collect(Collectors.toList()); + for (Integer id : blockGroupIds) { + blockGroupBlockMapper.deleteByGroupIdAndBlockId(id, blocks.getId()); + } } - } - // 更新区块 - baseMapper.updateBlockById(blocks); - BlockDto blockDtoResult = new BlockDto(); - // 参数存在区块分组且无值 - if (blockParam.getGroups().isEmpty()) { - blockDtoResult = queryBlockById(blocks.getId()); - return Result.success(blockDtoResult); - } - for (Integer groupId : blockParam.getGroups()) { - BlockGroupBlock blockGroupBlock = new BlockGroupBlock(); - blockGroupBlock.setBlockId(blockParam.getId()); - blockGroupBlock.setBlockGroupId(groupId); - blockGroupBlockMapper.createBlockGroupBlock(blockGroupBlock); + // 更新区块 + baseMapper.updateBlockById(blocks); + if (!blockParam.getGroups().isEmpty()) { + for (Integer groupId : blockParam.getGroups()) { + BlockGroupBlock blockGroupBlock = new BlockGroupBlock(); + blockGroupBlock.setBlockId(blockParam.getId()); + blockGroupBlock.setBlockGroupId(groupId); + blockGroupBlockMapper.createBlockGroupBlock(blockGroupBlock); + } + } + BlockDto blockDtoResult = queryBlockById(blocks.getId()); + result = Result.success(blockDtoResult); } - blockDtoResult = queryBlockById(blocks.getId()); - return Result.success(blockDtoResult); + return result; } /** @@ -275,7 +270,8 @@ public Result createBlock(BlockParam blockParam) { * @param framework the framework * @return the block assets */ - public Map> getBlockAssets(Map pageContent, String framework) { + public Map> getBlockAssets( + Map pageContent, String framework) { List block = new ArrayList<>(); try { @@ -295,23 +291,39 @@ public Map> getBlockAssets(Map pageContent, mergedAssets.put("styles", new ArrayList<>()); // Merge the assets using streams - return blocksList.stream().map(Block::getAssets).map(assetsMap -> { - Map> tempMap = new HashMap<>(); - tempMap.put("material", (List) assetsMap.getOrDefault("material", new ArrayList<>())); - tempMap.put("scripts", (List) assetsMap.getOrDefault("scripts", new ArrayList<>())); - tempMap.put("styles", (List) assetsMap.getOrDefault("styles", new ArrayList<>())); - return tempMap; - }).reduce(mergedAssets, (acc, curr) -> { - acc.get("material").addAll(curr.get("material")); - acc.get("scripts").addAll(curr.get("scripts")); - acc.get("styles").addAll(curr.get("styles")); - return acc; - }, (map1, map2) -> { - map1.get("material").addAll(map2.get("material")); - map1.get("scripts").addAll(map2.get("scripts")); - map1.get("styles").addAll(map2.get("styles")); - return map1; - }); + return blocksList.stream() + .map(Block::getAssets) + .map( + assetsMap -> { + Map> tempMap = new HashMap<>(); + tempMap.put( + "material", + (List) + assetsMap.getOrDefault("material", new ArrayList<>())); + tempMap.put( + "scripts", + (List) + assetsMap.getOrDefault("scripts", new ArrayList<>())); + tempMap.put( + "styles", + (List) + assetsMap.getOrDefault("styles", new ArrayList<>())); + return tempMap; + }) + .reduce( + mergedAssets, + (acc, curr) -> { + acc.get("material").addAll(curr.get("material")); + acc.get("scripts").addAll(curr.get("scripts")); + acc.get("styles").addAll(curr.get("styles")); + return acc; + }, + (map1, map2) -> { + map1.get("material").addAll(map2.get("material")); + map1.get("scripts").addAll(map2.get("scripts")); + map1.get("styles").addAll(map2.get("styles")); + return map1; + }); } /** @@ -326,9 +338,10 @@ public List getBlockInfo(List block, String framework) { QueryWrapper queryWrapper = new QueryWrapper<>(); if (block != null && !block.isEmpty()) { // 处理 blockLabelName 为数组的情况 - String labelsCondition = block.stream() - .map(name -> "label = '" + name + "'") - .collect(Collectors.joining(" OR ")); + String labelsCondition = + block.stream() + .map(name -> "label = '" + name + "'") + .collect(Collectors.joining(" OR ")); // 添加标签条件 queryWrapper.and(wrapper -> wrapper.apply(labelsCondition)); @@ -364,7 +377,8 @@ public void traverseBlocks(String content, List block) throws JsonProces } } if (schemaMap.containsKey("children") && schemaMap.get("children") instanceof List) { - traverseBlocks(JsonUtils.MAPPER.writeValueAsString(schemaMap.get("children")), block); + traverseBlocks( + JsonUtils.MAPPER.writeValueAsString(schemaMap.get("children")), block); } } } @@ -388,8 +402,8 @@ public boolean isBlock(Map schema) { @Override public IPage findBlocksByPagetionList(BlockParamDto blockParamDto) { String appId = blockParamDto.getAppId(); - // 如果 appId 存在并且不匹配指定的正则表达式,则删除它 - if (appId != null && !Pattern.matches("^[1-9]+[0-9]*$", appId)) { + // 如果 appId 不是正整数,则删除它 + if (appId != null && !isPositiveInteger(appId)) { blockParamDto.setAppId(null); // 设置成null达到map中remove的效果 } // 获取查询条件 @@ -422,7 +436,7 @@ public IPage findBlocksByPagetionList(BlockParamDto blockParamDto) { int limit = blockParamDto.getLimit() != null ? blockParamDto.getLimit() : 0; int start = blockParamDto.getStart() != null ? blockParamDto.getStart() : 0; int pageNum = start == 0 && limit == 0 ? 1 : (start / limit) + 1; - int pageSize = limit == 0 ? 10 : limit; + int pageSize = limit == 0 ? DEFAULT_PAGE_SIZE : limit; Page page = new Page<>(pageNum, pageSize); return baseMapper.selectPage(page, queryWrapper); } @@ -458,29 +472,40 @@ public List getNotInGroupBlocks(NotGroupDto notGroupDto) { } for (BlockDto blockDto : blocksList) { - List blockGroups = blockGroupMapper.findBlockGroupByBlockId(blockDto.getId(), - loginUserContext.getLoginUserId()); + List blockGroups = + blockGroupMapper.findBlockGroupByBlockId( + blockDto.getId(), loginUserContext.getLoginUserId()); blockDto.setGroups(blockGroups); } return blocksList.stream() - .filter(item -> { - // 过滤掉未发布的 - if (item.getLastBuildInfo() == null || item.getContent() == null || item.getAssets() == null) { - return false; - } - // 组过滤 - if (item.getGroups() != null && item.getGroups() - .stream() - .anyMatch(group -> group != null - && group.getId().equals(notGroupDto.getGroupId()))) { - return false; - } - // 公开范围过滤 - if (item.getPublicStatus() == Enums.Scope.FULL_PUBLIC.getValue()) { - return true; - } - return item.getPublicStatus() == Enums.Scope.PUBLIC_IN_TENANTS.getValue(); - }).collect(Collectors.toList()); + .filter( + item -> { + // 过滤掉未发布的 + if (item.getLastBuildInfo() == null + || item.getContent() == null + || item.getAssets() == null) { + return false; + } + // 组过滤 + if (item.getGroups() != null + && item.getGroups().stream() + .anyMatch( + group -> + group != null + && group.getId() + .equals( + notGroupDto + .getGroupId()))) { + return false; + } + // 公开范围过滤 + if (item.getPublicStatus() == Enums.Scope.FULL_PUBLIC.getValue()) { + return true; + } + return item.getPublicStatus() + == Enums.Scope.PUBLIC_IN_TENANTS.getValue(); + }) + .collect(Collectors.toList()); } /** @@ -529,7 +554,8 @@ public Result deploy(BlockBuildDto blockBuildDto) { try { BlockDto blockDto = blockBuildDto.getBlock(); - List i18nList = i18nEntryMapper.findI18nEntriesByHostandHostType(id, "block"); + List i18nList = + i18nEntryMapper.findI18nEntriesByHostandHostType(id, "block"); // 序列化国际化词条 SchemaI18n appEntries = i18nEntryService.formatEntriesList(i18nList); BlockHistory blockHistory = new BlockHistory(); @@ -552,18 +578,19 @@ public Result deploy(BlockBuildDto blockBuildDto) { Map buildInfo = createBuildInfo(blockBuildDto.getVersion(), now); blockHistory.setBuildInfo(buildInfo); blockHistory.setId(null); + Result result = Result.failed(ExceptionEnum.CM008); int blockHistoryResult = blockHistoryMapper.createBlockHistory(blockHistory); - if (blockHistoryResult < 1) { - return Result.failed(ExceptionEnum.CM008); + if (blockHistoryResult >= 1) { + BlockParam blockParam = new BlockParam(); + blockParam.setLastBuildInfo(buildInfo); + blockParam.setLatestHistoryId(blockHistory); + blockParam.setLatestVersion(blockHistory.getVersion()); + blockParam.setId(blockDto.getId()); + blockParam.setAppId(blockDto.getAppId()); + blockParam.setGroups(null); + result = updateBlockById(blockParam); } - BlockParam blockParam = new BlockParam(); - blockParam.setLastBuildInfo(buildInfo); - blockParam.setLatestHistoryId(blockHistory); - blockParam.setLatestVersion(blockHistory.getVersion()); - blockParam.setId(blockDto.getId()); - blockParam.setAppId(blockDto.getAppId()); - blockParam.setGroups(null); - return updateBlockById(blockParam); + return result; } catch (Exception e) { return Result.failed(ExceptionEnum.CM001); } @@ -587,10 +614,14 @@ public IPage findBlocksByConditionPagetion(Map request) { String description = request.get("description"); QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.and(wrapper -> wrapper.like(StringUtils.isNotEmpty(nameCn), "name", nameCn) - .or() - .like(StringUtils.isNotEmpty(description), "description", description) - ); + queryWrapper.and( + wrapper -> + wrapper.like(StringUtils.isNotEmpty(nameCn), "name", nameCn) + .or() + .like( + StringUtils.isNotEmpty(description), + "description", + description)); List blocksList = baseMapper.selectList(queryWrapper); Page page = new Page<>(1, blocksList.size()); return baseMapper.selectPage(page, queryWrapper); @@ -610,11 +641,12 @@ public List getUsers(List blocksList) { return users; } // 提取 createdBy 列表中的唯一值 - blocksList.forEach(item -> { - if (item.getCreatedBy() != null && !userSet.contains(item.getCreatedBy())) { - userSet.add(item.getCreatedBy()); - } - }); + blocksList.forEach( + item -> { + if (item.getCreatedBy() != null && !userSet.contains(item.getCreatedBy())) { + userSet.add(item.getCreatedBy()); + } + }); List userIdsList = new ArrayList<>(userSet); @@ -625,7 +657,7 @@ public List getUsers(List blocksList) { /** * 获取区块 * - * @param appId the appId + * @param appId the appId * @param groupId the groupId * @return the list */ @@ -648,7 +680,9 @@ public Result> listNew(String appId, String groupId) { List blocksList = new ArrayList<>(); // 如果有 groupId, 只查group下的block,以及自己创建的区块 if (groupIdTemp != 0) { - blocksList = baseMapper.findBlockByBlockGroupId(groupIdTemp, loginUserContext.getLoginUserId()); + blocksList = + baseMapper.findBlockByBlockGroupId( + groupIdTemp, loginUserContext.getLoginUserId()); return Result.success(blocksList); } // 如果没有 groupId @@ -664,23 +698,37 @@ public Result> listNew(String appId, String groupId) { List personalBlocks = queryBlockByCondition(blocks); List retBlocks = new ArrayList<>(); // 合并 personalBlocks 和 appBlocks 数组 - List combinedBlocks = Stream.concat(personalBlocks.stream(), appBlocks.stream()) - .collect(Collectors.toList()); + List combinedBlocks = + Stream.concat(personalBlocks.stream(), appBlocks.stream()) + .collect(Collectors.toList()); // 遍历合并后的数组,检查是否存在具有相同 id 的元素 - combinedBlocks.forEach(block -> { - boolean isFind = retBlocks.stream().anyMatch(retBlock -> Objects.equals(retBlock.getId(), block.getId())); - if (!isFind) { - retBlocks.add(block); - } - }); + combinedBlocks.forEach( + block -> { + boolean isFind = + retBlocks.stream() + .anyMatch( + retBlock -> + Objects.equals( + retBlock.getId(), block.getId())); + if (!isFind) { + retBlocks.add(block); + } + }); // 给is_published赋值 - List result = retBlocks.stream().map(b -> { - boolean isPublished = b.getLastBuildInfo() != null && b.getLastBuildInfo().get("result") instanceof Boolean - ? (Boolean) b.getLastBuildInfo().get("result") - : Boolean.FALSE; - b.setIsPublished(isPublished); - return b; - }).collect(Collectors.toList()); + List result = + retBlocks.stream() + .map( + b -> { + boolean isPublished = + b.getLastBuildInfo() != null + && b.getLastBuildInfo().get("result") + instanceof Boolean + ? (Boolean) b.getLastBuildInfo().get("result") + : Boolean.FALSE; + b.setIsPublished(isPublished); + return b; + }) + .collect(Collectors.toList()); return Result.success(result); } @@ -702,7 +750,8 @@ public int ensureBlockId(BlockDto blockDto) { queryBlock.setFramework(blockDto.getFramework()); queryBlock.setCreatedBy(loginUserContext.getLoginUserId()); List blockList = baseMapper.queryBlockByCondition(queryBlock); - List groups = blockDto.getGroups().stream().map(BlockGroup::getId).collect(Collectors.toList()); + List groups = + blockDto.getGroups().stream().map(BlockGroup::getId).collect(Collectors.toList()); ; blockDto.setGroups(null); BlockParam blockParam = new BlockParam(); @@ -749,4 +798,21 @@ private Map createBuildInfo(String version, LocalDateTime buildT buildInfo.put("endTime", buildTime.format(formatter)); return buildInfo; } + + private boolean isPositiveInteger(String value) { + if (value == null || value.isEmpty()) { + return false; + } + char first = value.charAt(0); + if (first < '1' || first > '9') { + return false; + } + for (int i = 1; i < value.length(); i++) { + char c = value.charAt(i); + if (c < '0' || c > '9') { + return false; + } + } + return true; + } } diff --git a/base/src/main/java/com/tinyengine/it/service/material/impl/ModelServiceImpl.java b/base/src/main/java/com/tinyengine/it/service/material/impl/ModelServiceImpl.java index 663511f7..73c0326d 100644 --- a/base/src/main/java/com/tinyengine/it/service/material/impl/ModelServiceImpl.java +++ b/base/src/main/java/com/tinyengine/it/service/material/impl/ModelServiceImpl.java @@ -1,410 +1,442 @@ -/** - * Copyright (c) 2023 - present TinyEngine Authors. - * Copyright (c) 2023 - present Huawei Cloud Computing Technologies Co., Ltd. - * - * Use of this source code is governed by an MIT-style license. - * - * THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, - * BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR - * A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. - * - */ - -package com.tinyengine.it.service.material.impl; - -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; -import com.tinyengine.it.common.context.LoginUserContext; -import com.tinyengine.it.common.enums.Enums; -import com.tinyengine.it.common.exception.ExceptionEnum; -import com.tinyengine.it.common.exception.ServiceException; -import com.tinyengine.it.common.log.SystemServiceLog; -import com.tinyengine.it.common.utils.JsonUtils; -import com.tinyengine.it.dynamic.service.DynamicModelService; -import com.tinyengine.it.mapper.ModelMapper; -import com.tinyengine.it.model.dto.MethodDto; -import com.tinyengine.it.model.dto.ParametersDto; -import com.tinyengine.it.model.dto.RequestParameter; -import com.tinyengine.it.model.dto.ResponseParameter; -import com.tinyengine.it.model.entity.Model; -import com.tinyengine.it.service.material.ModelService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import org.springframework.util.CollectionUtils; - -import java.io.IOException; -import java.util.*; -import java.util.stream.Collectors; - -@Service -@Slf4j -public class ModelServiceImpl extends ServiceImpl implements ModelService { - - @Autowired - private DynamicModelService dynamicModelService; - - @Autowired - private LoginUserContext loginUserContext; - /** - * 查询表t_model信息 - * - * @param id - * @return the Model - */ - @Override - @SystemServiceLog(description = "根据id查询model实现方法") - public Model queryModelById(Integer id) { - return this.baseMapper.selectById(id); - } - - /** - * 根据name查询表t_model信息 - * - * @param nameCn - * @return the model list - */ - @Override - @SystemServiceLog(description = "根据名称查询model实现方法") - public List getModelByName(String nameCn) { - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.like("name_cn", nameCn); - return this.baseMapper.selectList(queryWrapper); - } - - /** - * 根据name查询表t_model信息 - * - * @param nameEn - * @return the model list - */ - @Override - @SystemServiceLog(description = "根据名称查询model实现方法") - public List getModelByEnName(String nameEn) { - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("name_en", nameEn); - return this.baseMapper.selectList(queryWrapper); } - - /** - * 分页查询表t_model信息 - * - * @return the list - */ - @Override - @SystemServiceLog(description = "分页查询model实现方法") - public Page pageQuery(int currentPage, int pageSize, String nameCn, String nameEn) { - Page page = new Page<>(currentPage, pageSize); - QueryWrapper queryWrapper = new QueryWrapper<>(); - - // 判断 nameCn 是否存在 - if (nameCn != null && !nameCn.isEmpty()) { - queryWrapper.like("name_cn", nameCn); - } - - // 判断 nameEn 是否存在 - if (nameEn != null && !nameEn.isEmpty()) { - if (nameCn != null && !nameCn.isEmpty()) { - queryWrapper.or().like("name_en", nameEn); - } else { - queryWrapper.like("name_en", nameEn); - } - } - queryWrapper.eq("created_by", loginUserContext.getLoginUserId()); - queryWrapper.eq("tenant_id", loginUserContext.getTenantId()); - page(page, queryWrapper); - return page; - } - - /** - * 创建t_material - * - * @param model - * @return the model - * @ param the model - */ - @Override - @SystemServiceLog(description = "创建model实现方法") - @Transactional - public Model createModel(Model model) { - // 验证模型唯一性 - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("name_en", model.getNameEn()); - if (this.baseMapper.selectCount(queryWrapper) > 0) { - throw new ServiceException(ExceptionEnum.CM003.getResultCode(), "Model with the same name already exists"); - } - List methodDtos = new ArrayList<>(); - methodDtos.add(getMethodDto(Enums.methodName.CREATED.getValue(), Enums.methodName.INSERTAPI.getValue(), model)); - methodDtos.add(getMethodDto(Enums.methodName.UPDATE.getValue(), Enums.methodName.UPDATEAPI.getValue(), model)); - methodDtos.add(getMethodDto(Enums.methodName.QUERY.getValue(), Enums.methodName.QUERYAPI.getValue(), model)); - methodDtos.add(getMethodDto(Enums.methodName.DELETE.getValue(), Enums.methodName.DELETEAPI.getValue(), model)); - model.setMethod(methodDtos); - model.setTenantId(loginUserContext.getTenantId()); - int result = this.baseMapper.createModel(model); - if (result != 1) { - throw new ServiceException(ExceptionEnum.CM001.getResultCode(), ExceptionEnum.CM001.getResultCode()); - } - // 创建动态表 - dynamicModelService.createDynamicTable(model); - return model; - } - - /** - * 删除t_model - * - * @param id - * @return the Model - * @ param the id - */ - @Override - @SystemServiceLog(description = "根据id删除model实现方法") - @Transactional - public Model deleteModelById(Integer id) { - Model model = this.baseMapper.selectById(id); - int result = this.baseMapper.deleteById(id); - if (result != 1) { - throw new ServiceException(ExceptionEnum.CM001.getResultCode(), ExceptionEnum.CM001.getResultCode()); - } - try { - dynamicModelService.dropDynamicTable(model); - } catch (Exception e) { - log.error("deleteModelById", e); - throw new ServiceException(ExceptionEnum.CM001.getResultCode(), ExceptionEnum.CM001.getResultCode()); - - } - return model; - } - - /** - * 修改t_model - * - * @param model - * @return the model - * @ param the model - */ - @Override - @SystemServiceLog(description = "根据id修改model实现方法") - @Transactional - public Model updateModelById(Model model) { - List methodDtos = new ArrayList<>(); - methodDtos.add(getMethodDto(Enums.methodName.CREATED.getValue(), Enums.methodName.INSERTAPI.getValue(), model)); - methodDtos.add(getMethodDto(Enums.methodName.UPDATE.getValue(), Enums.methodName.UPDATEAPI.getValue(), model)); - methodDtos.add(getMethodDto(Enums.methodName.QUERY.getValue(), Enums.methodName.QUERYAPI.getValue(), model)); - methodDtos.add(getMethodDto(Enums.methodName.DELETE.getValue(), Enums.methodName.DELETEAPI.getValue(), model)); - model.setMethod(methodDtos); - if (model.getId() == null) { - throw new ServiceException(ExceptionEnum.CM002.getResultCode(), ExceptionEnum.CM002.getResultCode()); - } - int result = this.baseMapper.updateModelById(model); - if (result != 1) { - throw new ServiceException(ExceptionEnum.CM001.getResultCode(), ExceptionEnum.CM001.getResultCode()); - } - - - // 修改动态表 - try { - dynamicModelService.modifyTableStructure(model); - } catch (Exception e) { - log.error("updateModelById", e); - throw new ServiceException(ExceptionEnum.CM001.getResultCode(), ExceptionEnum.CM001.getResultCode()); - } - Model modelResult = this.baseMapper.selectById(model.getId()); - return modelResult; - } - - /** - * 获取Model建表sql - * - * @param id - * @return the String - * @ param the id - */ - @Override - public String getTableById(Integer id) { - Model model = this.baseMapper.selectById(id); - StringBuilder sql = new StringBuilder(getTableByModle(model)); - List rawList = model.getParameters(); - List fields = rawList.stream() - .map(item -> JsonUtils.MAPPER.convertValue(item, ParametersDto.class)) - .collect(Collectors.toList()); - fields.forEach(item -> { - if(item.getIsModel()) { - Model result = this.baseMapper.selectById(item.getDefaultValue()); - sql.append(getTableByModle(result)); - } - }); - return sql.toString(); - } - - /** - * 获取所有模型的建表SQL语句 - * @return 拼接好的SQL语句字符串,每个表的SQL用分号分隔并换行 - * @throws IOException 如果JSON解析失败 - */ - @Override - public String getAllTable() { - // 查询所有模型 - List modelList = this.baseMapper.selectList(null); - if (CollectionUtils.isEmpty(modelList)) { - return ""; - } - - StringJoiner sqlJoiner = new StringJoiner(" "); - - modelList.stream() - .map(this::getTableByModle) - .forEach(sqlJoiner::add); - - return sqlJoiner.toString(); - } - - /** - * 获取所有模型名称列表 - * - * @return 模型名称列表 - */ - @Override - public List getAllModelName() { - List modelList = this.baseMapper.selectList(null); - if (!CollectionUtils.isEmpty(modelList)) { - return modelList.stream() - .map(Model::getNameEn) - .collect(Collectors.toList()); - } - return null; - } - - private String getTableByModle(Model model) { - List rawList = model.getParameters(); - List fields = rawList.stream() - .map(item -> JsonUtils.MAPPER.convertValue(item, ParametersDto.class)) - .collect(Collectors.toList()); - - StringBuilder sql = new StringBuilder("CREATE TABLE " + model.getNameEn() + " ("); - - for (int i = 0; i < fields.size(); i++) { - ParametersDto field = fields.get(i); - - String prop = field.getProp(); - String type = field.getType(); - String defaultValue = field.getDefaultValue(); - - // 根据字段类型映射为 SQL 数据类型 - String sqlType = mapJavaTypeToSQL(type); - - sql.append(prop).append(" ").append(sqlType); - - if (defaultValue != null && !defaultValue.isEmpty()) { - sql.append(" DEFAULT ").append(defaultValue); - } - - // 如果不是最后一个字段,添加逗号 - if (i != fields.size() - 1) { - sql.append(", "); - } - } - - sql.append(");"); - return sql.toString(); - } - - private static String mapJavaTypeToSQL(String javaType) { - if (javaType == null) { - return "VARCHAR(255)"; // 默认处理 - } - switch (javaType) { - case "String": - return "VARCHAR(500)"; - case "Number": - return "INT"; - case "Boolean": - return "BOOLEAN"; - case "Date": - return "TIMESTAMP"; - case "Enum": - return "LONGTEXT"; - default: - return "LONGTEXT"; // 默认处理 - } - } - - private MethodDto getMethodDto(String name, String nameEn, Model model) { - MethodDto methodDto = new MethodDto(); - methodDto.setName(name); - methodDto.setNameEn(nameEn); - List responseParameterList = getResponseParameters(name); - RequestParameter requestParameter = new RequestParameter(); - requestParameter.setProp(Enums.methodParam.ID.getValue()); - requestParameter.setType(Enums.paramType.NUMBER.getValue()); - List parameterList = new ArrayList<>(); - RequestParameter requestNameEn = new RequestParameter(); - requestNameEn.setProp(Enums.methodParam.NAMEEN.getValue()); - requestNameEn.setType(Enums.paramType.STRING.getValue()); - parameterList.add(requestNameEn); - if (name.equals(Enums.methodName.QUERY.getValue())) { - RequestParameter currentPage = new RequestParameter(); - currentPage.setProp(Enums.methodParam.CURRENTPAGE.getValue()); - currentPage.setType(Enums.paramType.NUMBER.getValue()); - RequestParameter pageSize = new RequestParameter(); - pageSize.setProp(Enums.methodParam.PAGESIZE.getValue()); - pageSize.setType(Enums.paramType.NUMBER.getValue()); - RequestParameter nameCn = new RequestParameter(); - nameCn.setProp(Enums.methodParam.NAMECN.getValue()); - nameCn.setType(Enums.paramType.STRING.getValue()); - parameterList.add(currentPage); - parameterList.add(pageSize); - parameterList.add(nameCn); - - } - if( name.equals(Enums.methodName.UPDATE.getValue())) { - RequestParameter requestParameterData = new RequestParameter(); - requestParameterData.setProp(Enums.methodParam.DATA.getValue()); - requestParameterData.setType(Enums.paramType.OBJECT.getValue()); - requestParameterData.setChildren(model.getParameters()); - parameterList.add(requestParameterData); - } - if (!name.equals(Enums.methodName.DELETE.getValue())) { - RequestParameter requestParameterparams = new RequestParameter(); - requestParameterparams.setProp(Enums.methodParam.PARAMS.getValue()); - requestParameterparams.setType(Enums.paramType.OBJECT.getValue()); - requestParameterparams.setChildren(model.getParameters()); - parameterList.add(requestParameterparams); - methodDto.setRequestParameters(parameterList); - methodDto.setResponseParameters(responseParameterList); - return methodDto; - } - - parameterList.add(requestParameter); - methodDto.setRequestParameters(parameterList); - methodDto.setResponseParameters(responseParameterList); - return methodDto; - } - - private static List getResponseParameters(String name) { - ResponseParameter code = new ResponseParameter(); - code.setProp(Enums.methodParam.CODE.getValue()); - code.setType(Enums.paramType.NUMBER.getValue()); - ResponseParameter message = new ResponseParameter(); - message.setProp(Enums.methodParam.MESSAGE.getValue()); - message.setType(Enums.paramType.STRING.getValue()); - ResponseParameter data = new ResponseParameter(); - data.setProp(Enums.methodParam.DATA.getValue()); - data.setType(Enums.paramType.ENUM.getValue()); - - List responseParameterList = new ArrayList<>(); - if (name.equals(Enums.methodName.QUERY.getValue())) { - ResponseParameter total = new ResponseParameter(); - total.setProp(Enums.methodParam.TOTAL.getValue()); - total.setType(Enums.paramType.NUMBER.getValue()); - responseParameterList.add(total); - } - - responseParameterList.add(code); - responseParameterList.add(message); - responseParameterList.add(data); - return responseParameterList; - } - - -} +/** + * Copyright (c) 2023 - present TinyEngine Authors. Copyright (c) 2023 - present Huawei Cloud + * Computing Technologies Co., Ltd. + * + *

Use of this source code is governed by an MIT-style license. + * + *

THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, + * BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR A + * PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + */ +package com.tinyengine.it.service.material.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.tinyengine.it.common.context.LoginUserContext; +import com.tinyengine.it.common.enums.Enums; +import com.tinyengine.it.common.exception.ExceptionEnum; +import com.tinyengine.it.common.exception.ServiceException; +import com.tinyengine.it.common.log.SystemServiceLog; +import com.tinyengine.it.common.utils.JsonUtils; +import com.tinyengine.it.dynamic.service.DynamicModelService; +import com.tinyengine.it.mapper.ModelMapper; +import com.tinyengine.it.model.dto.MethodDto; +import com.tinyengine.it.model.dto.ParametersDto; +import com.tinyengine.it.model.dto.RequestParameter; +import com.tinyengine.it.model.dto.ResponseParameter; +import com.tinyengine.it.model.entity.Model; +import com.tinyengine.it.service.material.ModelService; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.CollectionUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.StringJoiner; +import java.util.stream.Collectors; + +@Service +@Slf4j +@SuppressWarnings("PMD.TooManyMethods") +public class ModelServiceImpl extends ServiceImpl implements ModelService { + + @Autowired private DynamicModelService dynamicModelService; + + @Autowired private LoginUserContext loginUserContext; + + /** + * 查询表t_model信息 + * + * @param id + * @return the Model + */ + @Override + @SystemServiceLog(description = "根据id查询model实现方法") + public Model queryModelById(Integer id) { + return this.baseMapper.selectById(id); + } + + /** + * 根据name查询表t_model信息 + * + * @param nameCn + * @return the model list + */ + @Override + @SystemServiceLog(description = "根据名称查询model实现方法") + public List getModelByName(String nameCn) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.like("name_cn", nameCn); + return this.baseMapper.selectList(queryWrapper); + } + + /** + * 根据name查询表t_model信息 + * + * @param nameEn + * @return the model list + */ + @Override + @SystemServiceLog(description = "根据名称查询model实现方法") + public List getModelByEnName(String nameEn) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("name_en", nameEn); + return this.baseMapper.selectList(queryWrapper); + } + + /** + * 分页查询表t_model信息 + * + * @return the list + */ + @Override + @SystemServiceLog(description = "分页查询model实现方法") + public Page pageQuery(int currentPage, int pageSize, String nameCn, String nameEn) { + Page page = new Page<>(currentPage, pageSize); + QueryWrapper queryWrapper = new QueryWrapper<>(); + + // 判断 nameCn 是否存在 + if (nameCn != null && !nameCn.isEmpty()) { + queryWrapper.like("name_cn", nameCn); + } + + // 判断 nameEn 是否存在 + if (nameEn != null && !nameEn.isEmpty()) { + if (nameCn != null && !nameCn.isEmpty()) { + queryWrapper.or().like("name_en", nameEn); + } else { + queryWrapper.like("name_en", nameEn); + } + } + queryWrapper.eq("created_by", loginUserContext.getLoginUserId()); + queryWrapper.eq("tenant_id", loginUserContext.getTenantId()); + page(page, queryWrapper); + return page; + } + + /** + * 创建t_material + * + * @param model + * @return the model @ param the model + */ + @Override + @SystemServiceLog(description = "创建model实现方法") + @Transactional + public Model createModel(Model model) { + // 验证模型唯一性 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("name_en", model.getNameEn()); + if (this.baseMapper.selectCount(queryWrapper) > 0) { + throw new ServiceException( + ExceptionEnum.CM003.getResultCode(), "Model with the same name already exists"); + } + List methodDtos = new ArrayList<>(); + methodDtos.add( + getMethodDto( + Enums.methodName.CREATED.getValue(), + Enums.methodName.INSERTAPI.getValue(), + model)); + methodDtos.add( + getMethodDto( + Enums.methodName.UPDATE.getValue(), + Enums.methodName.UPDATEAPI.getValue(), + model)); + methodDtos.add( + getMethodDto( + Enums.methodName.QUERY.getValue(), + Enums.methodName.QUERYAPI.getValue(), + model)); + methodDtos.add( + getMethodDto( + Enums.methodName.DELETE.getValue(), + Enums.methodName.DELETEAPI.getValue(), + model)); + model.setMethod(methodDtos); + model.setTenantId(loginUserContext.getTenantId()); + int result = this.baseMapper.createModel(model); + if (result != 1) { + throw new ServiceException( + ExceptionEnum.CM001.getResultCode(), ExceptionEnum.CM001.getResultCode()); + } + // 创建动态表 + dynamicModelService.createDynamicTable(model); + return model; + } + + /** + * 删除t_model + * + * @param id + * @return the Model @ param the id + */ + @Override + @SystemServiceLog(description = "根据id删除model实现方法") + @Transactional + public Model deleteModelById(Integer id) { + Model model = this.baseMapper.selectById(id); + int result = this.baseMapper.deleteById(id); + if (result != 1) { + throw new ServiceException( + ExceptionEnum.CM001.getResultCode(), ExceptionEnum.CM001.getResultCode()); + } + try { + dynamicModelService.dropDynamicTable(model); + } catch (ServiceException exception) { + log.error("deleteModelById", exception); + throw new ServiceException( + ExceptionEnum.CM001.getResultCode(), + ExceptionEnum.CM001.getResultCode(), + exception); + } + return model; + } + + /** + * 修改t_model + * + * @param model + * @return the model @ param the model + */ + @Override + @SystemServiceLog(description = "根据id修改model实现方法") + @Transactional + public Model updateModelById(Model model) { + List methodDtos = new ArrayList<>(); + methodDtos.add( + getMethodDto( + Enums.methodName.CREATED.getValue(), + Enums.methodName.INSERTAPI.getValue(), + model)); + methodDtos.add( + getMethodDto( + Enums.methodName.UPDATE.getValue(), + Enums.methodName.UPDATEAPI.getValue(), + model)); + methodDtos.add( + getMethodDto( + Enums.methodName.QUERY.getValue(), + Enums.methodName.QUERYAPI.getValue(), + model)); + methodDtos.add( + getMethodDto( + Enums.methodName.DELETE.getValue(), + Enums.methodName.DELETEAPI.getValue(), + model)); + model.setMethod(methodDtos); + if (model.getId() == null) { + throw new ServiceException( + ExceptionEnum.CM002.getResultCode(), ExceptionEnum.CM002.getResultCode()); + } + int result = this.baseMapper.updateModelById(model); + if (result != 1) { + throw new ServiceException( + ExceptionEnum.CM001.getResultCode(), ExceptionEnum.CM001.getResultCode()); + } + + // 修改动态表 + try { + dynamicModelService.modifyTableStructure(model); + } catch (ServiceException exception) { + log.error("updateModelById", exception); + throw new ServiceException( + ExceptionEnum.CM001.getResultCode(), + ExceptionEnum.CM001.getResultCode(), + exception); + } + Model modelResult = this.baseMapper.selectById(model.getId()); + return modelResult; + } + + /** + * 获取Model建表sql + * + * @param id + * @return the String @ param the id + */ + @Override + public String getTableById(Integer id) { + Model model = this.baseMapper.selectById(id); + StringBuilder sql = new StringBuilder(getTableByModle(model)); + List rawList = model.getParameters(); + List fields = + rawList.stream() + .map(item -> JsonUtils.MAPPER.convertValue(item, ParametersDto.class)) + .collect(Collectors.toList()); + fields.forEach( + item -> { + if (Boolean.TRUE.equals(item.getIsModel())) { + Model result = this.baseMapper.selectById(item.getDefaultValue()); + sql.append(getTableByModle(result)); + } + }); + return sql.toString(); + } + + /** + * 获取所有模型的建表SQL语句 + * + * @return 拼接好的SQL语句字符串,每个表的SQL用分号分隔并换行 + * @throws IOException 如果JSON解析失败 + */ + @Override + public String getAllTable() { + // 查询所有模型 + List modelList = this.baseMapper.selectList(null); + if (CollectionUtils.isEmpty(modelList)) { + return ""; + } + + StringJoiner sqlJoiner = new StringJoiner(" "); + + modelList.stream().map(this::getTableByModle).forEach(sqlJoiner::add); + + return sqlJoiner.toString(); + } + + /** + * 获取所有模型名称列表 + * + * @return 模型名称列表 + */ + @Override + public List getAllModelName() { + List modelList = this.baseMapper.selectList(null); + if (!CollectionUtils.isEmpty(modelList)) { + return modelList.stream().map(Model::getNameEn).collect(Collectors.toList()); + } + return Collections.emptyList(); + } + + private String getTableByModle(Model model) { + List rawList = model.getParameters(); + List fields = + rawList.stream() + .map(item -> JsonUtils.MAPPER.convertValue(item, ParametersDto.class)) + .collect(Collectors.toList()); + + StringBuilder sql = new StringBuilder("CREATE TABLE " + model.getNameEn() + " ("); + + for (int i = 0; i < fields.size(); i++) { + ParametersDto field = fields.get(i); + + String prop = field.getProp(); + String type = field.getType(); + String defaultValue = field.getDefaultValue(); + + // 根据字段类型映射为 SQL 数据类型 + String sqlType = mapJavaTypeToSQL(type); + + sql.append(prop).append(" ").append(sqlType); + + if (defaultValue != null && !defaultValue.isEmpty()) { + sql.append(" DEFAULT ").append(defaultValue); + } + + // 如果不是最后一个字段,添加逗号 + if (i != fields.size() - 1) { + sql.append(", "); + } + } + + sql.append(");"); + return sql.toString(); + } + + private static String mapJavaTypeToSQL(String javaType) { + if (javaType == null) { + return "VARCHAR(255)"; // 默认处理 + } + return switch (javaType) { + case "String" -> "VARCHAR(500)"; + case "Number" -> "INT"; + case "Boolean" -> "BOOLEAN"; + case "Date" -> "TIMESTAMP"; + case "Enum" -> "LONGTEXT"; + default -> "LONGTEXT"; // 默认处理 + }; + } + + private MethodDto getMethodDto(String name, String nameEn, Model model) { + MethodDto methodDto = new MethodDto(); + methodDto.setName(name); + methodDto.setNameEn(nameEn); + List responseParameterList = getResponseParameters(name); + RequestParameter requestParameter = new RequestParameter(); + requestParameter.setProp(Enums.methodParam.ID.getValue()); + requestParameter.setType(Enums.paramType.NUMBER.getValue()); + List parameterList = new ArrayList<>(); + RequestParameter requestNameEn = new RequestParameter(); + requestNameEn.setProp(Enums.methodParam.NAMEEN.getValue()); + requestNameEn.setType(Enums.paramType.STRING.getValue()); + parameterList.add(requestNameEn); + if (name.equals(Enums.methodName.QUERY.getValue())) { + RequestParameter currentPage = new RequestParameter(); + currentPage.setProp(Enums.methodParam.CURRENTPAGE.getValue()); + currentPage.setType(Enums.paramType.NUMBER.getValue()); + RequestParameter pageSize = new RequestParameter(); + pageSize.setProp(Enums.methodParam.PAGESIZE.getValue()); + pageSize.setType(Enums.paramType.NUMBER.getValue()); + RequestParameter nameCn = new RequestParameter(); + nameCn.setProp(Enums.methodParam.NAMECN.getValue()); + nameCn.setType(Enums.paramType.STRING.getValue()); + parameterList.add(currentPage); + parameterList.add(pageSize); + parameterList.add(nameCn); + } + if (name.equals(Enums.methodName.UPDATE.getValue())) { + RequestParameter requestParameterData = new RequestParameter(); + requestParameterData.setProp(Enums.methodParam.DATA.getValue()); + requestParameterData.setType(Enums.paramType.OBJECT.getValue()); + requestParameterData.setChildren(model.getParameters()); + parameterList.add(requestParameterData); + } + if (!name.equals(Enums.methodName.DELETE.getValue())) { + RequestParameter requestParameterparams = new RequestParameter(); + requestParameterparams.setProp(Enums.methodParam.PARAMS.getValue()); + requestParameterparams.setType(Enums.paramType.OBJECT.getValue()); + requestParameterparams.setChildren(model.getParameters()); + parameterList.add(requestParameterparams); + methodDto.setRequestParameters(parameterList); + methodDto.setResponseParameters(responseParameterList); + return methodDto; + } + + parameterList.add(requestParameter); + methodDto.setRequestParameters(parameterList); + methodDto.setResponseParameters(responseParameterList); + return methodDto; + } + + private static List getResponseParameters(String name) { + ResponseParameter code = new ResponseParameter(); + code.setProp(Enums.methodParam.CODE.getValue()); + code.setType(Enums.paramType.NUMBER.getValue()); + ResponseParameter message = new ResponseParameter(); + message.setProp(Enums.methodParam.MESSAGE.getValue()); + message.setType(Enums.paramType.STRING.getValue()); + ResponseParameter data = new ResponseParameter(); + data.setProp(Enums.methodParam.DATA.getValue()); + data.setType(Enums.paramType.ENUM.getValue()); + + List responseParameterList = new ArrayList<>(); + if (name.equals(Enums.methodName.QUERY.getValue())) { + ResponseParameter total = new ResponseParameter(); + total.setProp(Enums.methodParam.TOTAL.getValue()); + total.setType(Enums.paramType.NUMBER.getValue()); + responseParameterList.add(total); + } + + responseParameterList.add(code); + responseParameterList.add(message); + responseParameterList.add(data); + return responseParameterList; + } +} diff --git a/base/src/test/java/com/tinyengine/it/common/utils/SM4UtilsTest.java b/base/src/test/java/com/tinyengine/it/common/utils/SM4UtilsTest.java new file mode 100644 index 00000000..ff16d38f --- /dev/null +++ b/base/src/test/java/com/tinyengine/it/common/utils/SM4UtilsTest.java @@ -0,0 +1,39 @@ +package com.tinyengine.it.common.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +import java.security.GeneralSecurityException; +import java.util.Base64; + +class SM4UtilsTest { + private static final int SHORT_KEY_LEN = 8; + + @Test + void encryptAndDecryptRoundTrip() throws GeneralSecurityException { + String key = SM4Utils.generateKeyBase64(); + String encrypted = SM4Utils.encrypt("secret-api-key", key); + + assertEquals("secret-api-key", SM4Utils.decrypt(encrypted, key), "round trip should preserve the API key"); + } + + @Test + void encryptUsesRandomIv() throws GeneralSecurityException { + String key = SM4Utils.generateKeyBase64(); + + String first = SM4Utils.encrypt("same-plain-text", key); + String second = SM4Utils.encrypt("same-plain-text", key); + + assertNotEquals(first, second); + } + + @Test + void rejectsInvalidKeyLength() { + String invalidKey = Base64.getEncoder().encodeToString(new byte[SHORT_KEY_LEN]); + + assertThrows(IllegalArgumentException.class, () -> SM4Utils.encrypt("secret", invalidKey)); + } +} diff --git a/base/src/test/java/com/tinyengine/it/common/utils/SqlIdentifierValidatorTest.java b/base/src/test/java/com/tinyengine/it/common/utils/SqlIdentifierValidatorTest.java index bd5ec105..273a1bd6 100644 --- a/base/src/test/java/com/tinyengine/it/common/utils/SqlIdentifierValidatorTest.java +++ b/base/src/test/java/com/tinyengine/it/common/utils/SqlIdentifierValidatorTest.java @@ -1,13 +1,14 @@ package com.tinyengine.it.common.utils; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertThrows; - +@SuppressWarnings("PMD.TooManyMethods") class SqlIdentifierValidatorTest { @Test @@ -30,24 +31,39 @@ void rejectEmptyIdentifier() { @Test void rejectSqlInjectionInIdentifier() { - assertThrows(IllegalArgumentException.class, () -> SqlIdentifierValidator.validate("@@version")); - assertThrows(IllegalArgumentException.class, () -> SqlIdentifierValidator.validate("1; DROP TABLE users")); - assertThrows(IllegalArgumentException.class, () -> SqlIdentifierValidator.validate("id OR 1=1")); - assertThrows(IllegalArgumentException.class, () -> SqlIdentifierValidator.validate("(SELECT password FROM t_user)")); - assertThrows(IllegalArgumentException.class, () -> SqlIdentifierValidator.validate("name AS leaked")); - assertThrows(IllegalArgumentException.class, () -> SqlIdentifierValidator.validate("name'")); - assertThrows(IllegalArgumentException.class, () -> SqlIdentifierValidator.validate("name\"")); + assertThrows( + IllegalArgumentException.class, () -> SqlIdentifierValidator.validate("@@version")); + assertThrows( + IllegalArgumentException.class, + () -> SqlIdentifierValidator.validate("1; DROP TABLE users")); + assertThrows( + IllegalArgumentException.class, () -> SqlIdentifierValidator.validate("id OR 1=1")); + assertThrows( + IllegalArgumentException.class, + () -> SqlIdentifierValidator.validate("(SELECT password FROM t_user)")); + assertThrows( + IllegalArgumentException.class, + () -> SqlIdentifierValidator.validate("name AS leaked")); + assertThrows( + IllegalArgumentException.class, () -> SqlIdentifierValidator.validate("name'")); + assertThrows( + IllegalArgumentException.class, () -> SqlIdentifierValidator.validate("name\"")); } @Test void rejectSubqueryInIdentifier() { - assertThrows(IllegalArgumentException.class, - () -> SqlIdentifierValidator.validate("(SELECT GROUP_CONCAT(table_name) FROM information_schema.tables)")); + assertThrows( + IllegalArgumentException.class, + () -> + SqlIdentifierValidator.validate( + "(SELECT GROUP_CONCAT(table_name) FROM" + + " information_schema.tables)")); } @Test void rejectStartingWithDigit() { - assertThrows(IllegalArgumentException.class, () -> SqlIdentifierValidator.validate("1name")); + assertThrows( + IllegalArgumentException.class, () -> SqlIdentifierValidator.validate("1name")); } @Test @@ -59,7 +75,8 @@ void validateAllWithValidList() { @Test void validateAllRejectsInvalidEntry() { List fields = Arrays.asList("id", "@@version", "name"); - assertThrows(IllegalArgumentException.class, () -> SqlIdentifierValidator.validateAll(fields)); + assertThrows( + IllegalArgumentException.class, () -> SqlIdentifierValidator.validateAll(fields)); } @Test @@ -81,33 +98,45 @@ void validateOrderTypeDesc() { @Test void rejectInvalidOrderType() { - assertThrows(IllegalArgumentException.class, () -> SqlIdentifierValidator.validateOrderType("INVALID")); - assertThrows(IllegalArgumentException.class, () -> SqlIdentifierValidator.validateOrderType("; DROP TABLE")); - assertThrows(IllegalArgumentException.class, () -> SqlIdentifierValidator.validateOrderType(null)); + assertThrows( + IllegalArgumentException.class, + () -> SqlIdentifierValidator.validateOrderType("INVALID")); + assertThrows( + IllegalArgumentException.class, + () -> SqlIdentifierValidator.validateOrderType("; DROP TABLE")); + assertThrows( + IllegalArgumentException.class, + () -> SqlIdentifierValidator.validateOrderType(null)); } @Test void rejectSqliPatternsInIdentifier() { String[] sqliPayloads = { - "@@version", - "@@datadir", - "SLEEP(5)", - "BENCHMARK(10000000,SHA1('test'))", - "LOAD_FILE('/etc/passwd')", - "INTO OUTFILE '/tmp/shell.php'", - "UNION SELECT 1,2,3", - "information_schema.tables", - "1 OR 1=1", - "'; DROP TABLE users--", - "name AND 1=1", - "id; SELECT SLEEP(5)", - "COUNT(*)", - "GROUP_CONCAT(username)" + "@@version", + "@@datadir", + "SLEEP(5)", + "BENCHMARK(10000000,SHA1('test'))", + "LOAD_FILE('/etc/passwd')", + "INTO OUTFILE '/tmp/shell.php'", + "UNION SELECT 1,2,3", + "information_schema.tables", + "1 OR 1=1", + "'; DROP TABLE users--", + "name AND 1=1", + "id; SELECT SLEEP(5)", + "COUNT(*)", + "GROUP_CONCAT(username)" }; for (String payload : sqliPayloads) { - assertThrows(IllegalArgumentException.class, + assertThrows( + IllegalArgumentException.class, () -> SqlIdentifierValidator.validate(payload), "Should reject: " + payload); } } + + @Test + void escapeSqlLiteral() { + assertEquals("it''s \\\\ ok", SqlIdentifierValidator.escapeSqlLiteral("it's \\ ok")); + } } diff --git a/base/src/test/java/com/tinyengine/it/service/material/impl/ModelServiceImplTest.java b/base/src/test/java/com/tinyengine/it/service/material/impl/ModelServiceImplTest.java new file mode 100644 index 00000000..f267215f --- /dev/null +++ b/base/src/test/java/com/tinyengine/it/service/material/impl/ModelServiceImplTest.java @@ -0,0 +1,54 @@ +package com.tinyengine.it.service.material.impl; + +import static org.mockito.Mockito.when; + +import cn.hutool.core.util.ReflectUtil; + +import com.tinyengine.it.mapper.ModelMapper; +import com.tinyengine.it.model.dto.ParametersDto; +import com.tinyengine.it.model.entity.Model; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.util.ArrayList; +import java.util.List; + +class ModelServiceImplTest { + + @Mock private ModelMapper modelMapper; + + @InjectMocks private ModelServiceImpl modelServiceImpl; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + ReflectUtil.setFieldValue(modelServiceImpl, "baseMapper", modelMapper); + } + + @Test + void getTableByIdShouldIgnoreNullIsModel() { + Model model = new Model(); + model.setNameEn("main_model"); + + ParametersDto field = new ParametersDto(); + field.setProp("name"); + field.setType("String"); + field.setIsModel(null); + + List parameters = new ArrayList<>(); + parameters.add(field); + model.setParameters(parameters); + + when(modelMapper.selectById(1)).thenReturn(model); + + String sql = modelServiceImpl.getTableById(1); + + Assertions.assertTrue(sql.contains("CREATE TABLE main_model")); + Assertions.assertTrue(sql.contains("name VARCHAR(500)")); + } +} diff --git a/pmd/errorprone-ruleset.xml b/pmd/errorprone-ruleset.xml new file mode 100644 index 00000000..4b384eb5 --- /dev/null +++ b/pmd/errorprone-ruleset.xml @@ -0,0 +1,10 @@ + + + Error-prone rules excluding the unstable DFA rule. + + + + diff --git a/pom.xml b/pom.xml index 617b05fe..175a12c8 100644 --- a/pom.xml +++ b/pom.xml @@ -278,11 +278,13 @@ category/java/bestpractices.xml category/java/codestyle.xml category/java/design.xml - category/java/errorprone.xml + + ${maven.multiModuleProjectDirectory}/pmd/errorprone-ruleset.xml category/java/performance.xml category/java/security.xml true + false false 100 @@ -295,6 +297,8 @@ true + 2 + 51 @@ -305,7 +309,8 @@ cpd-check - true + + false