diff --git a/document/data/doc-last-modified.json b/document/data/doc-last-modified.json index b4dd3674166f..275eab0ecf71 100644 --- a/document/data/doc-last-modified.json +++ b/document/data/doc-last-modified.json @@ -355,10 +355,8 @@ "content/self-host/upgrading/4-16/4161.mdx": "2026-08-24T10:06:28+08:00", "content/self-host/upgrading/4-16/4162.en.mdx": "2026-09-03T21:50:14+08:00", "content/self-host/upgrading/4-16/4162.mdx": "2026-09-03T21:50:14+08:00", - "content/self-host/upgrading/4-17/4170.en.mdx": "2026-09-12T20:05:26+08:00", - "content/self-host/upgrading/4-17/4170.mdx": "2026-09-12T20:05:26+08:00", - "content/self-host/upgrading/4-17/4171.en.mdx": "2026-09-12T22:05:59+08:00", - "content/self-host/upgrading/4-17/4171.mdx": "2026-09-12T22:05:59+08:00", + "content/self-host/upgrading/4-17/4170.en.mdx": "2026-09-11T13:12:32+08:00", + "content/self-host/upgrading/4-17/4170.mdx": "2026-09-11T13:12:32+08:00", "content/self-host/upgrading/outdated/40.en.mdx": "2026-08-16T23:16:43+08:00", "content/self-host/upgrading/outdated/40.mdx": "2026-08-16T23:16:43+08:00", "content/self-host/upgrading/outdated/41.en.mdx": "2026-08-16T23:16:43+08:00", @@ -501,6 +499,6 @@ "content/self-host/upgrading/upgrade-instruction.mdx": "2026-09-08T22:44:33+08:00", "content/self-host/upgrading/version-timeline.en.mdx": "2026-08-16T23:16:43+08:00", "content/self-host/upgrading/version-timeline.mdx": "2026-08-16T23:16:43+08:00", - "content/toc.en.mdx": "2026-09-12T22:05:59+08:00", - "content/toc.mdx": "2026-09-12T22:05:59+08:00" + "content/toc.en.mdx": "2026-09-03T21:50:14+08:00", + "content/toc.mdx": "2026-09-03T21:50:14+08:00" } \ No newline at end of file diff --git a/packages/global/core/app/type.ts b/packages/global/core/app/type.ts index 8492bcf402b0..3cbb97d51e18 100644 --- a/packages/global/core/app/type.ts +++ b/packages/global/core/app/type.ts @@ -125,22 +125,24 @@ export const EntryPointItemTypeSchema = z.object({ }); export type EntryPointItemType = z.infer; -export const EntryPointItemsTypeSchema = z.array(EntryPointItemTypeSchema).superRefine((items, ctx) => { - const seenNames = new Map(); - items.forEach((item, index) => { - const normalizedName = item.name.trim(); - const previousIndex = seenNames.get(normalizedName); - if (previousIndex !== undefined) { - ctx.addIssue({ - code: 'custom', - path: [index, 'name'], - message: 'Entry point names must be unique' - }); - return; - } - seenNames.set(normalizedName, index); +export const EntryPointItemsTypeSchema = z + .array(EntryPointItemTypeSchema) + .superRefine((items, ctx) => { + const seenNames = new Map(); + items.forEach((item, index) => { + const normalizedName = item.name.trim(); + const previousIndex = seenNames.get(normalizedName); + if (previousIndex !== undefined) { + ctx.addIssue({ + code: 'custom', + path: [index, 'name'], + message: 'Entry point names must be unique' + }); + return; + } + seenNames.set(normalizedName, index); + }); }); -}); export const AppChatConfigTypeSchema = z.object({ welcomeText: optionalNullToUndefined(z.string()).meta({ diff --git a/packages/global/core/dataset/type.ts b/packages/global/core/dataset/type.ts index c3a165d6ca0d..04b27df3d996 100644 --- a/packages/global/core/dataset/type.ts +++ b/packages/global/core/dataset/type.ts @@ -119,6 +119,27 @@ export const ChunkSettingsSchema = z.object({ }); export type ChunkSettingsType = z.infer; +/* ===== Iultmzh file parse config ===== */ +// 外部文档解析服务开关,整体校验、整体存取;仅当 collection 的 customPdfParse=true +// 且系统配置了 customPdfParse.url 时生效。缺失字段由读取层用固定默认值补全。 +export const sangforFileParseConfigSchema = z + .object({ + keep_header_footer: z.boolean().optional().meta({ + description: '是否保留页眉页脚,默认删除' + }), + keep_appendix: z.boolean().optional().meta({ + description: '是否保留附录,默认删除首个附录标题后的内容' + }), + image_analysis: z.boolean().optional().meta({ + description: '是否保留图片识别(含印章)结果,默认关闭' + }), + chart_analysis: z.boolean().optional().meta({ + description: '是否保留图表转表格结果(同时保留原图片),默认关闭' + }) + }) + .meta({ description: '外部文档解析配置' }); +export type IultmzhFileParseConfigType = z.infer; + /* ===== Dataset ===== */ export const DatasetSchema = z .object({ @@ -154,6 +175,10 @@ export const DatasetSchema = z .meta({ description: '网站配置' }), chunkSettings: ChunkSettingsSchema.optional().meta({ description: '分块配置' }), + sangforFileParseConfig: sangforFileParseConfigSchema + .optional() + .meta({ description: '外部文档解析配置' }), + apiDatasetServer: ApiDatasetServerSchema.optional().meta({ description: 'API 服务器配置' }), deleteTime: z.coerce.date().nullish().meta({ description: '删除时间' }), diff --git a/packages/global/core/workflow/migration/legacy/systemConfig.ts b/packages/global/core/workflow/migration/legacy/systemConfig.ts index 2ec937737e8a..73bc85af59d1 100644 --- a/packages/global/core/workflow/migration/legacy/systemConfig.ts +++ b/packages/global/core/workflow/migration/legacy/systemConfig.ts @@ -147,7 +147,10 @@ export function migrateSystemConfigToChatConfig(input: LegacyWorkflowDataInput) ], [ 'entryPoints', - getSystemConfigInputValue(systemConfigNode, NodeInputKeyEnum.entryPoints) + getSystemConfigInputValue( + systemConfigNode, + NodeInputKeyEnum.entryPoints + ) ] ]; configValues.forEach(([key, value]) => { diff --git a/packages/global/openapi/core/dataset/api.ts b/packages/global/openapi/core/dataset/api.ts index caa3c4af1939..153d36b9a4ae 100644 --- a/packages/global/openapi/core/dataset/api.ts +++ b/packages/global/openapi/core/dataset/api.ts @@ -8,6 +8,7 @@ import { DatasetItemSchema, DatasetSchema, DatasetListItemSchema, + sangforFileParseConfigSchema, SearchDataResponseItemSchema } from '../../../core/dataset/type'; import { AppListSortEnum } from '../../../core/app/constants'; @@ -72,6 +73,9 @@ export const CreateDatasetBodySchema = z.object({ }), apiDatasetServer: ApiDatasetServerSchema.optional().meta({ description: '第三方知识库服务器配置(API/飞书/语雀/钉钉)' + }), + sangforFileParseConfig: sangforFileParseConfigSchema.optional().meta({ + description: '外部文档解析开关(页眉页脚/附录/图片识别/图转表),仅对 customPdfParse 解析路径生效' }) }); @@ -111,6 +115,10 @@ export const CreateDatasetWithFilesBodySchema = z.object({ vlmModelId: z.string().nullable().optional().meta({ description: '视觉语言模型 ID;未传沿用默认,null 或空字符串表示不设置', example: '' + }), + sangforFileParseConfig: sangforFileParseConfigSchema.optional().meta({ + description: + '外部文档解析开关(页眉页脚/附录/图片识别/图转表),仅对 customPdfParse 解析路径生效' }) }) .meta({ description: '知识库参数' }), @@ -443,6 +451,10 @@ export const UpdateDatasetBodySchema = z.object({ }), chunkSettings: ChunkSettingsSchema.optional().meta({ description: '分块配置' + }), + sangforFileParseConfig: sangforFileParseConfigSchema.optional().meta({ + description: + '外部文档解析开关(页眉页脚/附录/图片识别/图转表),仅对 customPdfParse 解析路径生效;编辑后仅对新解析的文件生效' }) }); export type UpdateDatasetBody = z.infer; diff --git a/packages/service/common/file/read/utils.ts b/packages/service/common/file/read/utils.ts index 50be7d2f6af9..1fb28dd9ae74 100644 --- a/packages/service/common/file/read/utils.ts +++ b/packages/service/common/file/read/utils.ts @@ -1,5 +1,6 @@ import FormData from 'form-data'; import type { ReadFileResponse } from '../../../worker/readFile/type'; +import type { IultmzhFileParseConfigType } from '@fastgpt/global/core/dataset/type'; import { axios } from '../../api/axios'; import { parseMarkdownBase64Images } from '@fastgpt/global/common/string/markdown'; import { createPdfParseUsage } from '../../../support/wallet/usage/controller'; @@ -7,6 +8,7 @@ import { useDoc2xServer } from '../../../thirdProvider/doc2x'; import { useTextinServer } from '../../../thirdProvider/textin'; import { useSomarkServer } from '../../../thirdProvider/somark'; import { parseFromSangfor, useSangforParse } from '../../../thirdProvider/sangfor'; +import { appendIultmzhFileParseFields } from '../../../thirdProvider/sangfor/parseConfig'; import { readRawContentFromBuffer, readRawContentFromSource } from '../../../worker/function'; import { getLogger, LogCategories } from '../../logger'; import { getImageBuffer } from '../image/utils'; @@ -33,6 +35,7 @@ export const readFileContentByBuffer = async ({ buffer, encoding, customPdfParse = false, + sangforFileParseConfig, usageId, getFormatText = true, imageKeyOptions, @@ -46,6 +49,7 @@ export const readFileContentByBuffer = async ({ encoding: string; customPdfParse?: boolean; + sangforFileParseConfig?: IultmzhFileParseConfigType; usageId?: string; getFormatText?: boolean; imageKeyOptions?: { @@ -62,6 +66,7 @@ export const readFileContentByBuffer = async ({ buffer, encoding, customPdfParse, + sangforFileParseConfig, usageId, getFormatText, imageKeyOptions, @@ -77,6 +82,7 @@ export const readFileContentBySource = async ({ tmbId, source, customPdfParse = false, + sangforFileParseConfig, usageId, getFormatText = true, imageKeyOptions, @@ -86,6 +92,7 @@ export const readFileContentBySource = async ({ tmbId: string; source: FileSource; customPdfParse?: boolean; + sangforFileParseConfig?: IultmzhFileParseConfigType; usageId?: string; getFormatText?: boolean; imageKeyOptions?: { @@ -101,6 +108,7 @@ export const readFileContentBySource = async ({ source, encoding: source.metadata.encoding ?? '', customPdfParse, + sangforFileParseConfig, usageId, getFormatText, imageKeyOptions, @@ -115,6 +123,7 @@ const readFileContent = async ({ source, encoding: initialEncoding, customPdfParse, + sangforFileParseConfig, usageId, getFormatText, imageKeyOptions, @@ -127,6 +136,7 @@ const readFileContent = async ({ source?: FileSource; encoding: string; customPdfParse: boolean; + sangforFileParseConfig?: IultmzhFileParseConfigType; usageId?: string; getFormatText: boolean; imageKeyOptions?: { @@ -235,6 +245,7 @@ const readFileContent = async ({ data.append('file', buffer, { filename: `file.${materializedExtension}` }); + appendIultmzhFileParseFields(data, sangforFileParseConfig); const { data: response } = await axios.post<{ pages: number; markdown: string; @@ -356,7 +367,8 @@ const readFileContent = async ({ const { text, pages } = await parseFromSangfor({ fileBuffer: buffer, extension: materializedExtension, - imageKeyOptions + imageKeyOptions, + sangforFileParseConfig }); reportPdfParseUsage(pages); diff --git a/packages/service/common/s3/sources/dataset/index.ts b/packages/service/common/s3/sources/dataset/index.ts index 049b504f271e..0b2f36c2dae3 100644 --- a/packages/service/common/s3/sources/dataset/index.ts +++ b/packages/service/common/s3/sources/dataset/index.ts @@ -140,8 +140,16 @@ export class S3DatasetSource extends S3PrivateBucket { } async getDatasetFileRawText(params: GetDatasetFileContentParams) { - const { fileId, teamId, tmbId, customPdfParse, getFormatText, usageId, datasetId } = - GetDatasetFileContentParamsSchema.parse(params); + const { + fileId, + teamId, + tmbId, + customPdfParse, + sangforFileParseConfig, + getFormatText, + usageId, + datasetId + } = GetDatasetFileContentParamsSchema.parse(params); if (!isAuthorizedDatasetFileS3Key({ key: fileId, datasetId })) { return Promise.reject('Invalid dataset file key'); @@ -149,6 +157,7 @@ export class S3DatasetSource extends S3PrivateBucket { const rawTextBuffer = await this.rawTextSource.getRawTextBuffer({ customPdfParse, + sangforFileParseConfig, sourceId: fileId }); if (rawTextBuffer) { @@ -166,6 +175,7 @@ export class S3DatasetSource extends S3PrivateBucket { tmbId, source, customPdfParse, + sangforFileParseConfig, usageId, getFormatText, imageKeyOptions: { @@ -177,7 +187,8 @@ export class S3DatasetSource extends S3PrivateBucket { sourceId: fileId, sourceName: filename, text: rawText, - customPdfParse + customPdfParse, + sangforFileParseConfig }); return { diff --git a/packages/service/common/s3/sources/dataset/type.ts b/packages/service/common/s3/sources/dataset/type.ts index 0b7a89cd9f7d..884554709e71 100644 --- a/packages/service/common/s3/sources/dataset/type.ts +++ b/packages/service/common/s3/sources/dataset/type.ts @@ -1,4 +1,5 @@ import { ObjectIdSchema } from '@fastgpt/global/common/type/mongo'; +import { sangforFileParseConfigSchema } from '@fastgpt/global/core/dataset/type'; import { ReadStream } from 'fs'; import z from 'zod'; @@ -29,6 +30,7 @@ export const GetDatasetFileContentParamsSchema = z.object({ tmbId: ObjectIdSchema, fileId: z.string().nonempty(), // 这是 ObjectKey customPdfParse: z.boolean().optional(), + sangforFileParseConfig: sangforFileParseConfigSchema.optional(), getFormatText: z.boolean().optional(), // 数据类型都尽可能转化成 markdown 格式 datasetId: ObjectIdSchema, usageId: ObjectIdSchema.optional() diff --git a/packages/service/common/s3/sources/rawText/index.ts b/packages/service/common/s3/sources/rawText/index.ts index dfe192fe05f7..6fadd7897adb 100644 --- a/packages/service/common/s3/sources/rawText/index.ts +++ b/packages/service/common/s3/sources/rawText/index.ts @@ -25,12 +25,12 @@ export class S3RawTextSource extends S3PrivateBucket { } async addRawTextBuffer(params: AddRawTextBufferParams) { - const { sourceId, sourceName, text, customPdfParse } = + const { sourceId, sourceName, text, customPdfParse, sangforFileParseConfig } = AddRawTextBufferParamsSchema.parse(params); // 因为 Key 唯一对应一个 Object 所以不需要根据文件内容计算 Hash 直接用 Key 计算 Hash 就行了 const hash = createHash('md5').update(sourceId).digest('hex'); - const key = getFileS3Key.rawText({ hash, customPdfParse }); + const key = getFileS3Key.rawText({ hash, customPdfParse, sangforFileParseConfig }); const buffer = Buffer.from(text); await MongoS3TTL.create({ @@ -57,10 +57,10 @@ export class S3RawTextSource extends S3PrivateBucket { } async getRawTextBuffer(params: GetRawTextBufferParams) { - const { customPdfParse, sourceId } = params; + const { customPdfParse, sangforFileParseConfig, sourceId } = params; const hash = createHash('md5').update(sourceId).digest('hex'); - const key = getFileS3Key.rawText({ hash, customPdfParse }); + const key = getFileS3Key.rawText({ hash, customPdfParse, sangforFileParseConfig }); if (!(await this.isObjectExists(key))) return null; diff --git a/packages/service/common/s3/sources/rawText/type.ts b/packages/service/common/s3/sources/rawText/type.ts index 4979abcc010b..f6ffb09f2dff 100644 --- a/packages/service/common/s3/sources/rawText/type.ts +++ b/packages/service/common/s3/sources/rawText/type.ts @@ -1,10 +1,15 @@ +import { sangforFileParseConfigSchema } from '@fastgpt/global/core/dataset/type'; import z from 'zod'; export const AddRawTextBufferParamsSchema = z.object({ customPdfParse: z.boolean().optional(), + sangforFileParseConfig: sangforFileParseConfigSchema.optional(), sourceId: z.string().nonempty(), sourceName: z.string().nonempty(), text: z.string() }); export type AddRawTextBufferParams = z.input; -export type GetRawTextBufferParams = Pick; +export type GetRawTextBufferParams = Pick< + AddRawTextBufferParams, + 'customPdfParse' | 'sangforFileParseConfig' | 'sourceId' +>; diff --git a/packages/service/common/s3/utils.ts b/packages/service/common/s3/utils.ts index 8aa4f0ec78aa..87ad97ceee9f 100644 --- a/packages/service/common/s3/utils.ts +++ b/packages/service/common/s3/utils.ts @@ -1,4 +1,5 @@ import { isAfter } from 'date-fns'; +import { createHash } from 'node:crypto'; import type { ClientSession } from 'mongoose'; import { buffer as consumeStreamToBuffer } from 'node:stream/consumers'; import type { Readable } from 'node:stream'; @@ -299,9 +300,29 @@ export const getFileS3Key = { }; }, - rawText: ({ hash, customPdfParse }: { hash: string; customPdfParse?: boolean }) => { + rawText: ({ + hash, + customPdfParse, + sangforFileParseConfig + }: { + hash: string; + customPdfParse?: boolean; + sangforFileParseConfig?: Record; + }) => { + // 解析配置纳入缓存 key,避免同文件不同开关共享缓存;未传配置保持旧 key 格式 + const configSuffix = sangforFileParseConfig + ? `-${createHash('md5') + .update( + JSON.stringify(sangforFileParseConfig, (_key, value) => + value instanceof Object && !Array.isArray(value) + ? Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b))) + : value + ) + ) + .digest('hex')}` + : ''; return encodeS3ObjectKey( - [S3Sources.rawText, `${hash}${customPdfParse ? '-true' : ''}`].join('/') + [S3Sources.rawText, `${hash}${customPdfParse ? '-true' : ''}${configSuffix}`].join('/') ); } }; diff --git a/packages/service/core/dataset/apiDataset/custom/api.ts b/packages/service/core/dataset/apiDataset/custom/api.ts index 7ce94d91977b..c1ce7a701db2 100644 --- a/packages/service/core/dataset/apiDataset/custom/api.ts +++ b/packages/service/core/dataset/apiDataset/custom/api.ts @@ -4,6 +4,7 @@ import type { ApiDatasetDetailResponse, APIFileServerType } from '@fastgpt/global/core/dataset/apiDataset/type'; +import type { IultmzhFileParseConfigType } from '@fastgpt/global/core/dataset/type'; import { type Method } from 'axios'; import { createProxyAxios } from '../../../../common/api/axios'; import { readFileRawTextByUrl } from '../../read'; @@ -170,6 +171,7 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServerTy tmbId, apiFileId, customPdfParse, + sangforFileParseConfig, datasetId, usageId }: { @@ -177,6 +179,7 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServerTy tmbId: string; apiFileId: string; customPdfParse?: boolean; + sangforFileParseConfig?: IultmzhFileParseConfigType; datasetId: string; usageId?: string; }): Promise => { @@ -204,7 +207,8 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServerTy // Get from buffer const rawTextBuffer = await getS3RawTextSource().getRawTextBuffer({ sourceId: previewUrl, - customPdfParse + customPdfParse, + sangforFileParseConfig }); if (rawTextBuffer) { return { @@ -220,6 +224,7 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServerTy relatedId: apiFileId, datasetId, customPdfParse, + sangforFileParseConfig, usageId, getFormatText: true }); @@ -230,7 +235,8 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServerTy sourceId: previewUrl, sourceName, text: rawText, - customPdfParse + customPdfParse, + sangforFileParseConfig }); return { diff --git a/packages/service/core/dataset/read.ts b/packages/service/core/dataset/read.ts index f887eaf51234..ea2e30ed3143 100644 --- a/packages/service/core/dataset/read.ts +++ b/packages/service/core/dataset/read.ts @@ -4,6 +4,7 @@ import { DatasetCollectionDataProcessModeEnum, DatasetSourceReadTypeEnum } from '@fastgpt/global/core/dataset/constants'; +import type { IultmzhFileParseConfigType } from '@fastgpt/global/core/dataset/type'; import { urlsFetch } from '../../common/string/cheerio'; import { type TextSplitProps } from '../../common/string/textSplitter'; import { readFileContentBySource } from '../../common/file/read/utils'; @@ -58,6 +59,7 @@ export const readFileRawTextByUrl = async ({ tmbId, url, customPdfParse, + sangforFileParseConfig, getFormatText, datasetId, usageId, @@ -67,6 +69,7 @@ export const readFileRawTextByUrl = async ({ tmbId: string; url: string; customPdfParse?: boolean; + sangforFileParseConfig?: IultmzhFileParseConfigType; getFormatText?: boolean; relatedId: string; // externalFileId / apiFileId datasetId: string; @@ -94,6 +97,7 @@ export const readFileRawTextByUrl = async ({ const { rawText } = await retryFn(() => readFileContentBySource({ customPdfParse, + sangforFileParseConfig, usageId, getFormatText, source, @@ -123,6 +127,7 @@ export const readDatasetSourceRawText = async ({ externalFileId, apiDatasetServer, customPdfParse, + sangforFileParseConfig, getFormatText, usageId, datasetId @@ -132,6 +137,8 @@ export const readDatasetSourceRawText = async ({ type: DatasetSourceReadTypeEnum; sourceId: string; customPdfParse?: boolean; + /** 外部文档解析开关;建议传入 getDatasetIultmzhFileParseConfig(dataset) 补全后的完整配置 */ + sangforFileParseConfig?: IultmzhFileParseConfigType; getFormatText?: boolean; selector?: string; // link selector @@ -158,6 +165,7 @@ export const readDatasetSourceRawText = async ({ fileId: sourceId, getFormatText, customPdfParse, + sangforFileParseConfig, usageId, datasetId }); @@ -190,6 +198,7 @@ export const readDatasetSourceRawText = async ({ relatedId: externalFileId, datasetId, customPdfParse, + sangforFileParseConfig, usageId }); return { @@ -202,6 +211,7 @@ export const readDatasetSourceRawText = async ({ teamId, tmbId, customPdfParse, + sangforFileParseConfig, datasetId, usageId }); @@ -222,6 +232,7 @@ export const readApiServerFileContent = async ({ teamId, tmbId, customPdfParse, + sangforFileParseConfig, datasetId, usageId }: { @@ -230,6 +241,7 @@ export const readApiServerFileContent = async ({ teamId: string; tmbId: string; customPdfParse?: boolean; + sangforFileParseConfig?: IultmzhFileParseConfigType; datasetId: string; usageId?: string; }): Promise<{ @@ -241,6 +253,7 @@ export const readApiServerFileContent = async ({ tmbId, apiFileId, customPdfParse, + sangforFileParseConfig, datasetId, usageId }); diff --git a/packages/service/core/dataset/schema.ts b/packages/service/core/dataset/schema.ts index 218e03b22355..1d8727669c6b 100644 --- a/packages/service/core/dataset/schema.ts +++ b/packages/service/core/dataset/schema.ts @@ -131,6 +131,15 @@ const DatasetSchema = new Schema({ chunkSettings: { type: ChunkSettings }, + // 外部文档解析服务开关,整体存取;缺失字段由读取层用固定默认值补全 + sangforFileParseConfig: { + type: { + keep_header_footer: Boolean, + keep_appendix: Boolean, + image_analysis: Boolean, + chart_analysis: Boolean + } + }, inheritPermission: { type: Boolean, default: true diff --git a/packages/service/test/common/s3/utils.test.ts b/packages/service/test/common/s3/utils.test.ts index 80c8178c30c7..315bea625cf9 100644 --- a/packages/service/test/common/s3/utils.test.ts +++ b/packages/service/test/common/s3/utils.test.ts @@ -873,3 +873,47 @@ describe('getFileS3Key', () => { }); }); }); + +describe('getFileS3Key.rawText', () => { + const hash = 'abc123def456'; + const sangforFileParseConfig = { + keep_header_footer: true, + keep_appendix: false, + image_analysis: true, + chart_analysis: false + }; + + it('keeps the legacy key format when no parse config is given', () => { + const noFlag = getFileS3Key.rawText({ hash }); + expect(noFlag).toBe(getFileS3Key.rawText({ hash, customPdfParse: false })); + expect(noFlag).toContain(hash); + expect(getFileS3Key.rawText({ hash, customPdfParse: true })).toContain(`${hash}-true`); + }); + + it('is stable for the same config regardless of key order', () => { + const reordered = { + chart_analysis: false, + image_analysis: true, + keep_appendix: false, + keep_header_footer: true + }; + + expect(getFileS3Key.rawText({ hash, customPdfParse: true, sangforFileParseConfig })).toBe( + getFileS3Key.rawText({ hash, customPdfParse: true, sangforFileParseConfig: reordered }) + ); + }); + + it('changes the key when config values differ', () => { + const flipped = { ...sangforFileParseConfig, keep_header_footer: false }; + + expect(getFileS3Key.rawText({ hash, customPdfParse: true, sangforFileParseConfig })).not.toBe( + getFileS3Key.rawText({ hash, customPdfParse: true, sangforFileParseConfig: flipped }) + ); + }); + + it('differs from the legacy key once a config is attached', () => { + expect(getFileS3Key.rawText({ hash, customPdfParse: true, sangforFileParseConfig })).not.toBe( + getFileS3Key.rawText({ hash, customPdfParse: true }) + ); + }); +}); diff --git a/packages/service/test/thirdProvider/sangfor.test.ts b/packages/service/test/thirdProvider/sangfor.test.ts index af16b07557b3..acfbf3489036 100644 --- a/packages/service/test/thirdProvider/sangfor.test.ts +++ b/packages/service/test/thirdProvider/sangfor.test.ts @@ -1,4 +1,5 @@ import FormData from 'form-data'; +import { getDatasetIultmzhFileParseConfig } from '@fastgpt/service/thirdProvider/sangfor/parseConfig'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const { postMock, parseMarkdownImagesMock, uploadParsedPdfImageMock, mockEnv } = vi.hoisted(() => ({ @@ -208,3 +209,29 @@ describe('Sangfor provider', () => { expect(postMock).not.toHaveBeenCalled(); }); }); + +describe('getDatasetIultmzhFileParseConfig', () => { + const allDisabled = { + keep_header_footer: false, + keep_appendix: false, + image_analysis: false, + chart_analysis: false + }; + + it('为空数据集/空配置补全四布尔默认值', () => { + expect(getDatasetIultmzhFileParseConfig(undefined)).toEqual(allDisabled); + expect(getDatasetIultmzhFileParseConfig(null)).toEqual(allDisabled); + expect(getDatasetIultmzhFileParseConfig({})).toEqual(allDisabled); + }); + + it('保留已配置字段,仅补全缺失字段', () => { + expect( + getDatasetIultmzhFileParseConfig({ sangforFileParseConfig: { keep_header_footer: true } }) + ).toEqual({ + keep_header_footer: true, + keep_appendix: false, + image_analysis: false, + chart_analysis: false + }); + }); +}); diff --git a/packages/service/thirdProvider/sangfor/index.ts b/packages/service/thirdProvider/sangfor/index.ts index 051cf191a0ff..7c09fb19419d 100644 --- a/packages/service/thirdProvider/sangfor/index.ts +++ b/packages/service/thirdProvider/sangfor/index.ts @@ -1,11 +1,13 @@ import FormData from 'form-data'; import { getErrText } from '@fastgpt/global/common/error/utils'; import { parseMarkdownBase64Images } from '@fastgpt/global/common/string/markdown'; +import type { IultmzhFileParseConfigType } from '@fastgpt/global/core/dataset/type'; import z from 'zod'; import { axios } from '../../common/api/axios'; import { getImageBuffer } from '../../common/file/image/utils'; import { uploadParsedPdfImage, type ParsedPdfImageKeyOptions } from '../../common/file/read/image'; import { serviceEnv } from '../../env'; +import { appendIultmzhFileParseFields } from './parseConfig'; const SangforParseResponseSchema = z.object({ pages: z.number().int().nonnegative(), @@ -31,11 +33,13 @@ export const useSangforParse = (extension: string): boolean => { export const parseFromSangfor = async ({ fileBuffer, extension, - imageKeyOptions + imageKeyOptions, + sangforFileParseConfig }: { fileBuffer: Buffer; extension: string; imageKeyOptions?: ParsedPdfImageKeyOptions; + sangforFileParseConfig?: IultmzhFileParseConfigType; }) => { const { url, key } = global.systemEnv?.customPdfParse ?? {}; if (!url) { @@ -45,6 +49,7 @@ export const parseFromSangfor = async ({ try { const form = new FormData(); form.append('file', fileBuffer, { filename: `file.${extension}` }); + appendIultmzhFileParseFields(form, sangforFileParseConfig); const { data } = await axios.post(url, form, { timeout: serviceEnv.SANGFOR_PARSE_TIMEOUT_SECONDS * 1000, headers: { diff --git a/packages/service/thirdProvider/sangfor/parseConfig.ts b/packages/service/thirdProvider/sangfor/parseConfig.ts new file mode 100644 index 000000000000..16859b776e74 --- /dev/null +++ b/packages/service/thirdProvider/sangfor/parseConfig.ts @@ -0,0 +1,31 @@ +import type FormData from 'form-data'; +import type { + DatasetSchemaType, + IultmzhFileParseConfigType +} from '@fastgpt/global/core/dataset/type'; + +/** + * 从 dataset 取 sangfor 文件解析配置,缺失字段用固定默认值补全为完整四个 boolean, + * 避免不同解析引擎对缺失字段的不同兜底行为。 + */ +export const getDatasetIultmzhFileParseConfig = ( + dataset?: Pick | null +): IultmzhFileParseConfigType => ({ + keep_header_footer: dataset?.sangforFileParseConfig?.keep_header_footer ?? false, + keep_appendix: dataset?.sangforFileParseConfig?.keep_appendix ?? false, + image_analysis: dataset?.sangforFileParseConfig?.image_analysis ?? false, + chart_analysis: dataset?.sangforFileParseConfig?.chart_analysis ?? false +}); + +// 外部解析服务契约:四个开关逐个以独立表单字段下发,值统一转字符串(服务端自行 str→bool) +export const appendIultmzhFileParseFields = ( + form: FormData, + config?: IultmzhFileParseConfigType +) => { + if (!config) return; + Object.entries(config).forEach(([fieldKey, value]) => { + if (value !== undefined && value !== null) { + form.append(fieldKey, String(value)); + } + }); +}; diff --git a/projects/app/src/pages/api/core/dataset/create.ts b/projects/app/src/pages/api/core/dataset/create.ts index 15eee3d8f3f2..155231df9864 100644 --- a/projects/app/src/pages/api/core/dataset/create.ts +++ b/projects/app/src/pages/api/core/dataset/create.ts @@ -40,7 +40,8 @@ async function handler(req: ApiRequestProps): Promise { agentModel, vlmModelId, vlmModel, - apiDatasetServer + apiDatasetServer, + sangforFileParseConfig } = parseApiInput({ req, bodySchema: CreateDatasetBodySchema }).body; // auth @@ -95,7 +96,8 @@ async function handler(req: ApiRequestProps): Promise { ...(vlmModelStore?.modelId && { vlmModelId: vlmModelStore.modelId }), avatar, type, - apiDatasetServer + apiDatasetServer, + ...(sangforFileParseConfig && { sangforFileParseConfig }) } ], { session, ordered: true } diff --git a/projects/app/src/pages/api/core/dataset/createWithFiles.ts b/projects/app/src/pages/api/core/dataset/createWithFiles.ts index f154f9c0b2e1..535e819b596e 100644 --- a/projects/app/src/pages/api/core/dataset/createWithFiles.ts +++ b/projects/app/src/pages/api/core/dataset/createWithFiles.ts @@ -42,7 +42,15 @@ async function handler(req: ApiRequestProps): Promise) { externalReadUrl, apiDatasetServer, autoSync, + sangforFileParseConfig, chunkSettings: rawChunkSettings } } = parseApiInput({ @@ -253,6 +254,8 @@ async function handler(req: ApiRequestProps) { ...(externalReadUrl !== undefined && { externalReadUrl }), ...(isMove && { inheritPermission: true }), ...(typeof autoSync === 'boolean' && { autoSync }), + // 传空对象等价于恢复全部开关默认值(读取层补全),旧文件已固化的解析结果不受影响 + ...(sangforFileParseConfig !== undefined && { sangforFileParseConfig }), ...apiDatasetParams, ...(!isMove && { updateTime: new Date() }) }, diff --git a/projects/app/src/service/core/dataset/queues/datasetParse.ts b/projects/app/src/service/core/dataset/queues/datasetParse.ts index d9598ed526db..6386dd88e65c 100644 --- a/projects/app/src/service/core/dataset/queues/datasetParse.ts +++ b/projects/app/src/service/core/dataset/queues/datasetParse.ts @@ -20,6 +20,7 @@ import { addMinutes } from 'date-fns'; import { checkTeamAiPointsAndLock } from './utils'; import { getErrText } from '@fastgpt/global/common/error/utils'; import { delay } from '@fastgpt/global/common/system/utils'; +import { getDatasetIultmzhFileParseConfig } from '@fastgpt/service/thirdProvider/sangfor/parseConfig'; import { rawText2Chunks, readDatasetSourceRawText } from '@fastgpt/service/core/dataset/read'; import { getLLMMaxChunkSize } from '@fastgpt/global/core/dataset/training/utils'; import { checkDatasetIndexLimit } from '@fastgpt/service/support/permission/teamLimit'; @@ -323,6 +324,10 @@ export const datasetParseQueue = async (): Promise => { teamId: data.teamId, tmbId: data.tmbId, customPdfParse: collection.customPdfParse, + // 解析开关仅对外部解析路径生效;非 customPdfParse 时传 undefined,rawText 缓存沿用旧 key + sangforFileParseConfig: collection.customPdfParse + ? getDatasetIultmzhFileParseConfig(dataset) + : undefined, usageId: data.billId, datasetId: data.datasetId, ...sourceReadType