From ac52dd880cb1de690b950cc755ffeb3a67038103 Mon Sep 17 00:00:00 2001 From: ohowandanliao Date: Fri, 11 Sep 2026 16:58:00 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat(dataset):=20=E7=9F=A5=E8=AF=86?= =?UTF-8?q?=E5=BA=93=E7=BA=A7=20pdfParseConfig=20=E6=96=87=E6=A1=A3?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E5=BC=80=E5=85=B3(#68624)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/global/core/dataset/type.ts | 23 ++++++++++ packages/global/openapi/core/dataset/api.ts | 12 +++++ packages/service/common/file/read/utils.ts | 20 ++++++++- .../common/s3/sources/dataset/index.ts | 17 +++++-- .../service/common/s3/sources/dataset/type.ts | 2 + .../common/s3/sources/rawText/index.ts | 8 ++-- .../service/common/s3/sources/rawText/type.ts | 7 ++- packages/service/common/s3/utils.ts | 25 ++++++++++- .../core/dataset/apiDataset/custom/api.ts | 10 ++++- packages/service/core/dataset/read.ts | 35 +++++++++++++++ packages/service/core/dataset/schema.ts | 9 ++++ packages/service/test/common/s3/utils.test.ts | 44 +++++++++++++++++++ .../service/test/core/dataset/read.test.ts | 30 ++++++++++++- .../service/thirdProvider/sangfor/index.ts | 13 +++++- .../app/src/pages/api/core/dataset/create.ts | 6 ++- .../pages/api/core/dataset/createWithFiles.ts | 6 ++- .../api/core/dataset/file/getPreviewChunks.ts | 8 +++- .../app/src/pages/api/core/dataset/update.ts | 3 ++ .../core/dataset/queues/datasetParse.ts | 8 +++- 19 files changed, 265 insertions(+), 21 deletions(-) diff --git a/packages/global/core/dataset/type.ts b/packages/global/core/dataset/type.ts index 1d2664bf95f6..32421d4eabc6 100644 --- a/packages/global/core/dataset/type.ts +++ b/packages/global/core/dataset/type.ts @@ -64,6 +64,27 @@ export const ChunkSettingsSchema = z.object({ }); export type ChunkSettingsType = z.infer; +/* ===== Pdf parse config ===== */ +// 外部文档解析(pdf2text)开关,整体校验、整体存取;仅当 collection 的 customPdfParse=true +// 且系统配置了 customPdfParse.url 时生效。缺失字段由读取层用固定默认值补全。 +export const PdfParseConfigSchema = z + .object({ + keep_header_footer: z.boolean().optional().meta({ + description: '是否保留页眉页脚,默认删除' + }), + keep_appendix: z.boolean().optional().meta({ + description: '是否保留附录,默认删除首个附录标题后的内容' + }), + image_analysis: z.boolean().optional().meta({ + description: '是否保留图片识别(含印章)结果,默认关闭' + }), + chart_analysis: z.boolean().optional().meta({ + description: '是否保留图表转表格结果(同时保留原图片),默认关闭' + }) + }) + .meta({ description: '外部文档解析配置' }); +export type PdfParseConfigType = z.infer; + /* ===== Dataset ===== */ export const DatasetSchema = z .object({ @@ -99,6 +120,8 @@ export const DatasetSchema = z .meta({ description: '网站配置' }), chunkSettings: ChunkSettingsSchema.optional().meta({ description: '分块配置' }), + pdfParseConfig: PdfParseConfigSchema.optional().meta({ description: '外部文档解析配置' }), + apiDatasetServer: ApiDatasetServerSchema.optional().meta({ description: 'API 服务器配置' }), deleteTime: z.coerce.date().nullish().meta({ description: '删除时间' }), diff --git a/packages/global/openapi/core/dataset/api.ts b/packages/global/openapi/core/dataset/api.ts index caa3c4af1939..75d2cb3a1988 100644 --- a/packages/global/openapi/core/dataset/api.ts +++ b/packages/global/openapi/core/dataset/api.ts @@ -8,6 +8,7 @@ import { DatasetItemSchema, DatasetSchema, DatasetListItemSchema, + PdfParseConfigSchema, SearchDataResponseItemSchema } from '../../../core/dataset/type'; import { AppListSortEnum } from '../../../core/app/constants'; @@ -72,6 +73,9 @@ export const CreateDatasetBodySchema = z.object({ }), apiDatasetServer: ApiDatasetServerSchema.optional().meta({ description: '第三方知识库服务器配置(API/飞书/语雀/钉钉)' + }), + pdfParseConfig: PdfParseConfigSchema.optional().meta({ + description: '外部文档解析开关(页眉页脚/附录/图片识别/图转表),仅对 customPdfParse 解析路径生效' }) }); @@ -111,6 +115,10 @@ export const CreateDatasetWithFilesBodySchema = z.object({ vlmModelId: z.string().nullable().optional().meta({ description: '视觉语言模型 ID；未传沿用默认，null 或空字符串表示不设置', example: '' + }), + pdfParseConfig: PdfParseConfigSchema.optional().meta({ + description: + '外部文档解析开关(页眉页脚/附录/图片识别/图转表),仅对 customPdfParse 解析路径生效' }) }) .meta({ description: '知识库参数' }), @@ -443,6 +451,10 @@ export const UpdateDatasetBodySchema = z.object({ }), chunkSettings: ChunkSettingsSchema.optional().meta({ description: '分块配置' + }), + pdfParseConfig: PdfParseConfigSchema.optional().meta({ + description: + '外部文档解析开关(页眉页脚/附录/图片识别/图转表),仅对 customPdfParse 解析路径生效;编辑后仅对新解析的文件生效' }) }); export type UpdateDatasetBody = z.infer; diff --git a/packages/service/common/file/read/utils.ts b/packages/service/common/file/read/utils.ts index 50be7d2f6af9..4a5f51a1d2a1 100644 --- a/packages/service/common/file/read/utils.ts +++ b/packages/service/common/file/read/utils.ts @@ -1,5 +1,6 @@ import FormData from 'form-data'; import type { ReadFileResponse } from '../../../worker/readFile/type'; +import type { PdfParseConfigType } from '@fastgpt/global/core/dataset/type'; import { axios } from '../../api/axios'; import { parseMarkdownBase64Images } from '@fastgpt/global/common/string/markdown'; import { createPdfParseUsage } from '../../../support/wallet/usage/controller'; @@ -33,6 +34,7 @@ export const readFileContentByBuffer = async ({ buffer, encoding, customPdfParse = false, + pdfParseConfig, usageId, getFormatText = true, imageKeyOptions, @@ -46,6 +48,7 @@ export const readFileContentByBuffer = async ({ encoding: string; customPdfParse?: boolean; + pdfParseConfig?: PdfParseConfigType; usageId?: string; getFormatText?: boolean; imageKeyOptions?: { @@ -62,6 +65,7 @@ export const readFileContentByBuffer = async ({ buffer, encoding, customPdfParse, + pdfParseConfig, usageId, getFormatText, imageKeyOptions, @@ -77,6 +81,7 @@ export const readFileContentBySource = async ({ tmbId, source, customPdfParse = false, + pdfParseConfig, usageId, getFormatText = true, imageKeyOptions, @@ -86,6 +91,7 @@ export const readFileContentBySource = async ({ tmbId: string; source: FileSource; customPdfParse?: boolean; + pdfParseConfig?: PdfParseConfigType; usageId?: string; getFormatText?: boolean; imageKeyOptions?: { @@ -101,6 +107,7 @@ export const readFileContentBySource = async ({ source, encoding: source.metadata.encoding ?? '', customPdfParse, + pdfParseConfig, usageId, getFormatText, imageKeyOptions, @@ -115,6 +122,7 @@ const readFileContent = async ({ source, encoding: initialEncoding, customPdfParse, + pdfParseConfig, usageId, getFormatText, imageKeyOptions, @@ -127,6 +135,7 @@ const readFileContent = async ({ source?: FileSource; encoding: string; customPdfParse: boolean; + pdfParseConfig?: PdfParseConfigType; usageId?: string; getFormatText: boolean; imageKeyOptions?: { @@ -235,6 +244,14 @@ const readFileContent = async ({ data.append('file', buffer, { filename: `file.${materializedExtension}` }); + // 外部解析服务契约:四个开关逐个以独立表单字段下发,值统一转字符串(服务端自行 str→bool) + if (pdfParseConfig) { + Object.entries(pdfParseConfig).forEach(([key, value]) => { + if (value !== undefined && value !== null) { + data.append(key, String(value)); + } + }); + } const { data: response } = await axios.post<{ pages: number; markdown: string; @@ -356,7 +373,8 @@ const readFileContent = async ({ const { text, pages } = await parseFromSangfor({ fileBuffer: buffer, extension: materializedExtension, - imageKeyOptions + imageKeyOptions, + pdfParseConfig }); reportPdfParseUsage(pages); diff --git a/packages/service/common/s3/sources/dataset/index.ts b/packages/service/common/s3/sources/dataset/index.ts index 049b504f271e..9b51c1fac0e5 100644 --- a/packages/service/common/s3/sources/dataset/index.ts +++ b/packages/service/common/s3/sources/dataset/index.ts @@ -140,8 +140,16 @@ export class S3DatasetSource extends S3PrivateBucket { } async getDatasetFileRawText(params: GetDatasetFileContentParams) { - const { fileId, teamId, tmbId, customPdfParse, getFormatText, usageId, datasetId } = - GetDatasetFileContentParamsSchema.parse(params); + const { + fileId, + teamId, + tmbId, + customPdfParse, + pdfParseConfig, + getFormatText, + usageId, + datasetId + } = GetDatasetFileContentParamsSchema.parse(params); if (!isAuthorizedDatasetFileS3Key({ key: fileId, datasetId })) { return Promise.reject('Invalid dataset file key'); @@ -149,6 +157,7 @@ export class S3DatasetSource extends S3PrivateBucket { const rawTextBuffer = await this.rawTextSource.getRawTextBuffer({ customPdfParse, + pdfParseConfig, sourceId: fileId }); if (rawTextBuffer) { @@ -166,6 +175,7 @@ export class S3DatasetSource extends S3PrivateBucket { tmbId, source, customPdfParse, + pdfParseConfig, usageId, getFormatText, imageKeyOptions: { @@ -177,7 +187,8 @@ export class S3DatasetSource extends S3PrivateBucket { sourceId: fileId, sourceName: filename, text: rawText, - customPdfParse + customPdfParse, + pdfParseConfig }); return { diff --git a/packages/service/common/s3/sources/dataset/type.ts b/packages/service/common/s3/sources/dataset/type.ts index 0b7a89cd9f7d..c59e79f8bb49 100644 --- a/packages/service/common/s3/sources/dataset/type.ts +++ b/packages/service/common/s3/sources/dataset/type.ts @@ -1,4 +1,5 @@ import { ObjectIdSchema } from '@fastgpt/global/common/type/mongo'; +import { PdfParseConfigSchema } from '@fastgpt/global/core/dataset/type'; import { ReadStream } from 'fs'; import z from 'zod'; @@ -29,6 +30,7 @@ export const GetDatasetFileContentParamsSchema = z.object({ tmbId: ObjectIdSchema, fileId: z.string().nonempty(), // 这是 ObjectKey customPdfParse: z.boolean().optional(), + pdfParseConfig: PdfParseConfigSchema.optional(), getFormatText: z.boolean().optional(), // 数据类型都尽可能转化成 markdown 格式 datasetId: ObjectIdSchema, usageId: ObjectIdSchema.optional() diff --git a/packages/service/common/s3/sources/rawText/index.ts b/packages/service/common/s3/sources/rawText/index.ts index dfe192fe05f7..1e84f88a960d 100644 --- a/packages/service/common/s3/sources/rawText/index.ts +++ b/packages/service/common/s3/sources/rawText/index.ts @@ -25,12 +25,12 @@ export class S3RawTextSource extends S3PrivateBucket { } async addRawTextBuffer(params: AddRawTextBufferParams) { - const { sourceId, sourceName, text, customPdfParse } = + const { sourceId, sourceName, text, customPdfParse, pdfParseConfig } = AddRawTextBufferParamsSchema.parse(params); // 因为 Key 唯一对应一个 Object 所以不需要根据文件内容计算 Hash 直接用 Key 计算 Hash 就行了 const hash = createHash('md5').update(sourceId).digest('hex'); - const key = getFileS3Key.rawText({ hash, customPdfParse }); + const key = getFileS3Key.rawText({ hash, customPdfParse, pdfParseConfig }); const buffer = Buffer.from(text); await MongoS3TTL.create({ @@ -57,10 +57,10 @@ export class S3RawTextSource extends S3PrivateBucket { } async getRawTextBuffer(params: GetRawTextBufferParams) { - const { customPdfParse, sourceId } = params; + const { customPdfParse, pdfParseConfig, sourceId } = params; const hash = createHash('md5').update(sourceId).digest('hex'); - const key = getFileS3Key.rawText({ hash, customPdfParse }); + const key = getFileS3Key.rawText({ hash, customPdfParse, pdfParseConfig }); if (!(await this.isObjectExists(key))) return null; diff --git a/packages/service/common/s3/sources/rawText/type.ts b/packages/service/common/s3/sources/rawText/type.ts index 4979abcc010b..97076052b321 100644 --- a/packages/service/common/s3/sources/rawText/type.ts +++ b/packages/service/common/s3/sources/rawText/type.ts @@ -1,10 +1,15 @@ +import { PdfParseConfigSchema } from '@fastgpt/global/core/dataset/type'; import z from 'zod'; export const AddRawTextBufferParamsSchema = z.object({ customPdfParse: z.boolean().optional(), + pdfParseConfig: PdfParseConfigSchema.optional(), sourceId: z.string().nonempty(), sourceName: z.string().nonempty(), text: z.string() }); export type AddRawTextBufferParams = z.input; -export type GetRawTextBufferParams = Pick; +export type GetRawTextBufferParams = Pick< + AddRawTextBufferParams, + 'customPdfParse' | 'pdfParseConfig' | 'sourceId' +>; diff --git a/packages/service/common/s3/utils.ts b/packages/service/common/s3/utils.ts index 8aa4f0ec78aa..e2a866aa7742 100644 --- a/packages/service/common/s3/utils.ts +++ b/packages/service/common/s3/utils.ts @@ -1,4 +1,5 @@ import { isAfter } from 'date-fns'; +import { createHash } from 'node:crypto'; import type { ClientSession } from 'mongoose'; import { buffer as consumeStreamToBuffer } from 'node:stream/consumers'; import type { Readable } from 'node:stream'; @@ -299,9 +300,29 @@ export const getFileS3Key = { }; }, - rawText: ({ hash, customPdfParse }: { hash: string; customPdfParse?: boolean }) => { + rawText: ({ + hash, + customPdfParse, + pdfParseConfig + }: { + hash: string; + customPdfParse?: boolean; + pdfParseConfig?: Record; + }) => { + // 解析配置纳入缓存 key,避免同文件不同开关共享缓存;未传配置保持旧 key 格式 + const configSuffix = pdfParseConfig + ? `-${createHash('md5') + .update( + JSON.stringify(pdfParseConfig, (_key, value) => + value instanceof Object && !Array.isArray(value) + ? Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b))) + : value + ) + ) + .digest('hex')}` + : ''; return encodeS3ObjectKey( - [S3Sources.rawText, `${hash}${customPdfParse ? '-true' : ''}`].join('/') + [S3Sources.rawText, `${hash}${customPdfParse ? '-true' : ''}${configSuffix}`].join('/') ); } }; diff --git a/packages/service/core/dataset/apiDataset/custom/api.ts b/packages/service/core/dataset/apiDataset/custom/api.ts index 7ce94d91977b..a4381b155b44 100644 --- a/packages/service/core/dataset/apiDataset/custom/api.ts +++ b/packages/service/core/dataset/apiDataset/custom/api.ts @@ -4,6 +4,7 @@ import type { ApiDatasetDetailResponse, APIFileServerType } from '@fastgpt/global/core/dataset/apiDataset/type'; +import type { PdfParseConfigType } from '@fastgpt/global/core/dataset/type'; import { type Method } from 'axios'; import { createProxyAxios } from '../../../../common/api/axios'; import { readFileRawTextByUrl } from '../../read'; @@ -170,6 +171,7 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServerTy tmbId, apiFileId, customPdfParse, + pdfParseConfig, datasetId, usageId }: { @@ -177,6 +179,7 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServerTy tmbId: string; apiFileId: string; customPdfParse?: boolean; + pdfParseConfig?: PdfParseConfigType; datasetId: string; usageId?: string; }): Promise => { @@ -204,7 +207,8 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServerTy // Get from buffer const rawTextBuffer = await getS3RawTextSource().getRawTextBuffer({ sourceId: previewUrl, - customPdfParse + customPdfParse, + pdfParseConfig }); if (rawTextBuffer) { return { @@ -220,6 +224,7 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServerTy relatedId: apiFileId, datasetId, customPdfParse, + pdfParseConfig, usageId, getFormatText: true }); @@ -230,7 +235,8 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServerTy sourceId: previewUrl, sourceName, text: rawText, - customPdfParse + customPdfParse, + pdfParseConfig }); return { diff --git a/packages/service/core/dataset/read.ts b/packages/service/core/dataset/read.ts index 89511c4ef5e6..ed1d717d1ca4 100644 --- a/packages/service/core/dataset/read.ts +++ b/packages/service/core/dataset/read.ts @@ -2,6 +2,7 @@ import { ChunkTriggerConfigTypeEnum, DatasetSourceReadTypeEnum } from '@fastgpt/global/core/dataset/constants'; +import type { PdfParseConfigType } from '@fastgpt/global/core/dataset/type'; import { urlsFetch } from '../../common/string/cheerio'; import { type TextSplitProps } from '../../common/string/textSplitter'; import { readFileContentBySource } from '../../common/file/read/utils'; @@ -18,6 +19,28 @@ import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; import { getBackendFileOperationTimeoutMs } from '../../common/file/parseTimeout'; import { createExternalHttpFileSource } from '../../common/file/read/source'; import { getTeamFileSizeLimitBytes } from '../../support/permission/fileLimit'; +import type { DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; + +// 外部文档解析开关的固定默认值(产品口径:页眉页脚/附录/图片识别/图转表默认均关闭) +export const DefaultPdfParseConfig: PdfParseConfigType = { + keep_header_footer: false, + keep_appendix: false, + image_analysis: false, + chart_analysis: false +}; + +/** + * 从 dataset 取解析配置,缺失字段用固定默认值补全为完整四个 boolean, + * 避免不同解析引擎对缺失字段的不同兜底行为。 + */ +export const getDatasetPdfParseConfig = ( + dataset?: Pick | null +): PdfParseConfigType => ({ + keep_header_footer: dataset?.pdfParseConfig?.keep_header_footer ?? false, + keep_appendix: dataset?.pdfParseConfig?.keep_appendix ?? false, + image_analysis: dataset?.pdfParseConfig?.image_analysis ?? false, + chart_analysis: dataset?.pdfParseConfig?.chart_analysis ?? false +}); const datasetCsvColumnTypes = new Set(['q', 'a', 'index', 'indexes', 'metadata']); @@ -54,6 +77,7 @@ export const readFileRawTextByUrl = async ({ tmbId, url, customPdfParse, + pdfParseConfig, getFormatText, datasetId, usageId, @@ -63,6 +87,7 @@ export const readFileRawTextByUrl = async ({ tmbId: string; url: string; customPdfParse?: boolean; + pdfParseConfig?: PdfParseConfigType; getFormatText?: boolean; relatedId: string; // externalFileId / apiFileId datasetId: string; @@ -90,6 +115,7 @@ export const readFileRawTextByUrl = async ({ const { rawText } = await retryFn(() => readFileContentBySource({ customPdfParse, + pdfParseConfig, usageId, getFormatText, source, @@ -119,6 +145,7 @@ export const readDatasetSourceRawText = async ({ externalFileId, apiDatasetServer, customPdfParse, + pdfParseConfig, getFormatText, usageId, datasetId @@ -128,6 +155,8 @@ export const readDatasetSourceRawText = async ({ type: DatasetSourceReadTypeEnum; sourceId: string; customPdfParse?: boolean; + /** 外部文档解析开关;建议传入 getDatasetPdfParseConfig(dataset) 补全后的完整配置 */ + pdfParseConfig?: PdfParseConfigType; getFormatText?: boolean; selector?: string; // link selector @@ -154,6 +183,7 @@ export const readDatasetSourceRawText = async ({ fileId: sourceId, getFormatText, customPdfParse, + pdfParseConfig, usageId, datasetId }); @@ -186,6 +216,7 @@ export const readDatasetSourceRawText = async ({ relatedId: externalFileId, datasetId, customPdfParse, + pdfParseConfig, usageId }); return { @@ -198,6 +229,7 @@ export const readDatasetSourceRawText = async ({ teamId, tmbId, customPdfParse, + pdfParseConfig, datasetId, usageId }); @@ -218,6 +250,7 @@ export const readApiServerFileContent = async ({ teamId, tmbId, customPdfParse, + pdfParseConfig, datasetId, usageId }: { @@ -226,6 +259,7 @@ export const readApiServerFileContent = async ({ teamId: string; tmbId: string; customPdfParse?: boolean; + pdfParseConfig?: PdfParseConfigType; datasetId: string; usageId?: string; }): Promise<{ @@ -237,6 +271,7 @@ export const readApiServerFileContent = async ({ tmbId, apiFileId, customPdfParse, + pdfParseConfig, datasetId, usageId }); diff --git a/packages/service/core/dataset/schema.ts b/packages/service/core/dataset/schema.ts index 218e03b22355..56d432691890 100644 --- a/packages/service/core/dataset/schema.ts +++ b/packages/service/core/dataset/schema.ts @@ -131,6 +131,15 @@ const DatasetSchema = new Schema({ chunkSettings: { type: ChunkSettings }, + // 外部文档解析(pdf2text)开关,整体存取;缺失字段由读取层用固定默认值补全 + pdfParseConfig: { + type: { + keep_header_footer: Boolean, + keep_appendix: Boolean, + image_analysis: Boolean, + chart_analysis: Boolean + } + }, inheritPermission: { type: Boolean, default: true diff --git a/packages/service/test/common/s3/utils.test.ts b/packages/service/test/common/s3/utils.test.ts index 80c8178c30c7..dcd6e31b7bf7 100644 --- a/packages/service/test/common/s3/utils.test.ts +++ b/packages/service/test/common/s3/utils.test.ts @@ -873,3 +873,47 @@ describe('getFileS3Key', () => { }); }); }); + +describe('getFileS3Key.rawText', () => { + const hash = 'abc123def456'; + const pdfParseConfig = { + keep_header_footer: true, + keep_appendix: false, + image_analysis: true, + chart_analysis: false + }; + + it('keeps the legacy key format when no parse config is given', () => { + const noFlag = getFileS3Key.rawText({ hash }); + expect(noFlag).toBe(getFileS3Key.rawText({ hash, customPdfParse: false })); + expect(noFlag).toContain(hash); + expect(getFileS3Key.rawText({ hash, customPdfParse: true })).toContain(`${hash}-true`); + }); + + it('is stable for the same config regardless of key order', () => { + const reordered = { + chart_analysis: false, + image_analysis: true, + keep_appendix: false, + keep_header_footer: true + }; + + expect(getFileS3Key.rawText({ hash, customPdfParse: true, pdfParseConfig })).toBe( + getFileS3Key.rawText({ hash, customPdfParse: true, pdfParseConfig: reordered }) + ); + }); + + it('changes the key when config values differ', () => { + const flipped = { ...pdfParseConfig, keep_header_footer: false }; + + expect(getFileS3Key.rawText({ hash, customPdfParse: true, pdfParseConfig })).not.toBe( + getFileS3Key.rawText({ hash, customPdfParse: true, pdfParseConfig: flipped }) + ); + }); + + it('differs from the legacy key once a config is attached', () => { + expect(getFileS3Key.rawText({ hash, customPdfParse: true, pdfParseConfig })).not.toBe( + getFileS3Key.rawText({ hash, customPdfParse: true }) + ); + }); +}); diff --git a/packages/service/test/core/dataset/read.test.ts b/packages/service/test/core/dataset/read.test.ts index 22de166874eb..74e3c1a69975 100644 --- a/packages/service/test/core/dataset/read.test.ts +++ b/packages/service/test/core/dataset/read.test.ts @@ -29,7 +29,11 @@ vi.mock('@fastgpt/service/core/dataset/apiDataset', () => ({ }) })); -import { readDatasetSourceRawText, readFileRawTextByUrl } from '@fastgpt/service/core/dataset/read'; +import { + getDatasetPdfParseConfig, + readDatasetSourceRawText, + readFileRawTextByUrl +} from '@fastgpt/service/core/dataset/read'; describe('readDatasetSourceRawText', () => { beforeEach(() => { @@ -169,3 +173,27 @@ describe('readFileRawTextByUrl', () => { ); }); }); + +describe('getDatasetPdfParseConfig', () => { + const allDisabled = { + keep_header_footer: false, + keep_appendix: false, + image_analysis: false, + chart_analysis: false + }; + + it('为空数据集/空配置补全四布尔默认值', () => { + expect(getDatasetPdfParseConfig(undefined)).toEqual(allDisabled); + expect(getDatasetPdfParseConfig(null)).toEqual(allDisabled); + expect(getDatasetPdfParseConfig({})).toEqual(allDisabled); + }); + + it('保留已配置字段,仅补全缺失字段', () => { + expect(getDatasetPdfParseConfig({ pdfParseConfig: { keep_header_footer: true } })).toEqual({ + keep_header_footer: true, + keep_appendix: false, + image_analysis: false, + chart_analysis: false + }); + }); +}); diff --git a/packages/service/thirdProvider/sangfor/index.ts b/packages/service/thirdProvider/sangfor/index.ts index 051cf191a0ff..80e9f7f37c85 100644 --- a/packages/service/thirdProvider/sangfor/index.ts +++ b/packages/service/thirdProvider/sangfor/index.ts @@ -1,6 +1,7 @@ import FormData from 'form-data'; import { getErrText } from '@fastgpt/global/common/error/utils'; import { parseMarkdownBase64Images } from '@fastgpt/global/common/string/markdown'; +import type { PdfParseConfigType } from '@fastgpt/global/core/dataset/type'; import z from 'zod'; import { axios } from '../../common/api/axios'; import { getImageBuffer } from '../../common/file/image/utils'; @@ -31,11 +32,13 @@ export const useSangforParse = (extension: string): boolean => { export const parseFromSangfor = async ({ fileBuffer, extension, - imageKeyOptions + imageKeyOptions, + pdfParseConfig }: { fileBuffer: Buffer; extension: string; imageKeyOptions?: ParsedPdfImageKeyOptions; + pdfParseConfig?: PdfParseConfigType; }) => { const { url, key } = global.systemEnv?.customPdfParse ?? {}; if (!url) { @@ -45,6 +48,14 @@ export const parseFromSangfor = async ({ try { const form = new FormData(); form.append('file', fileBuffer, { filename: `file.${extension}` }); + // 外部解析服务契约:四个开关逐个以独立表单字段下发,值统一转字符串(服务端自行 str→bool) + if (pdfParseConfig) { + Object.entries(pdfParseConfig).forEach(([configKey, value]) => { + if (value !== undefined && value !== null) { + form.append(configKey, String(value)); + } + }); + } const { data } = await axios.post(url, form, { timeout: serviceEnv.SANGFOR_PARSE_TIMEOUT_SECONDS * 1000, headers: { diff --git a/projects/app/src/pages/api/core/dataset/create.ts b/projects/app/src/pages/api/core/dataset/create.ts index 15eee3d8f3f2..4a211b151cbe 100644 --- a/projects/app/src/pages/api/core/dataset/create.ts +++ b/projects/app/src/pages/api/core/dataset/create.ts @@ -40,7 +40,8 @@ async function handler(req: ApiRequestProps): Promise { agentModel, vlmModelId, vlmModel, - apiDatasetServer + apiDatasetServer, + pdfParseConfig } = parseApiInput({ req, bodySchema: CreateDatasetBodySchema }).body; // auth @@ -95,7 +96,8 @@ async function handler(req: ApiRequestProps): Promise { ...(vlmModelStore?.modelId && { vlmModelId: vlmModelStore.modelId }), avatar, type, - apiDatasetServer + apiDatasetServer, + ...(pdfParseConfig && { pdfParseConfig }) } ], { session, ordered: true } diff --git a/projects/app/src/pages/api/core/dataset/createWithFiles.ts b/projects/app/src/pages/api/core/dataset/createWithFiles.ts index f154f9c0b2e1..733fe541b777 100644 --- a/projects/app/src/pages/api/core/dataset/createWithFiles.ts +++ b/projects/app/src/pages/api/core/dataset/createWithFiles.ts @@ -42,7 +42,8 @@ async function handler(req: ApiRequestProps): Promise) { externalReadUrl, apiDatasetServer, autoSync, + pdfParseConfig, chunkSettings: rawChunkSettings } } = parseApiInput({ @@ -253,6 +254,8 @@ async function handler(req: ApiRequestProps) { ...(externalReadUrl !== undefined && { externalReadUrl }), ...(isMove && { inheritPermission: true }), ...(typeof autoSync === 'boolean' && { autoSync }), + // 传空对象等价于恢复全部开关默认值(读取层补全),旧文件已固化的解析结果不受影响 + ...(pdfParseConfig !== undefined && { pdfParseConfig }), ...apiDatasetParams, ...(!isMove && { updateTime: new Date() }) }, diff --git a/projects/app/src/service/core/dataset/queues/datasetParse.ts b/projects/app/src/service/core/dataset/queues/datasetParse.ts index 50438f2a2a84..8ccb3253689f 100644 --- a/projects/app/src/service/core/dataset/queues/datasetParse.ts +++ b/projects/app/src/service/core/dataset/queues/datasetParse.ts @@ -20,7 +20,11 @@ import { addMinutes } from 'date-fns'; import { checkTeamAiPointsAndLock } from './utils'; import { getErrText } from '@fastgpt/global/common/error/utils'; import { delay } from '@fastgpt/global/common/system/utils'; -import { rawText2Chunks, readDatasetSourceRawText } from '@fastgpt/service/core/dataset/read'; +import { + getDatasetPdfParseConfig, + rawText2Chunks, + readDatasetSourceRawText +} from '@fastgpt/service/core/dataset/read'; import { getLLMMaxChunkSize } from '@fastgpt/global/core/dataset/training/utils'; import { checkDatasetIndexLimit } from '@fastgpt/service/support/permission/teamLimit'; import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils'; @@ -323,6 +327,8 @@ export const datasetParseQueue = async (): Promise => { teamId: data.teamId, tmbId: data.tmbId, customPdfParse: collection.customPdfParse, + // 解析开关仅对外部解析路径生效;非 customPdfParse 时传 undefined,rawText 缓存沿用旧 key + pdfParseConfig: collection.customPdfParse ? getDatasetPdfParseConfig(dataset) : undefined, usageId: data.billId, datasetId: data.datasetId, ...sourceReadType From bfbb3462f45f39e9349118364ad2e1ad537c1a16 Mon Sep 17 00:00:00 2001 From: ohowandanliao Date: Mon, 14 Sep 2026 09:49:47 +0800 Subject: [PATCH 2/2] =?UTF-8?q?chore(dataset):=20=E6=B3=A8=E9=87=8A?= =?UTF-8?q?=E6=B8=85=E7=90=86=E5=86=85=E9=83=A8=E6=9C=8D=E5=8A=A1=E5=91=BD?= =?UTF-8?q?=E5=90=8D(#68624)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/global/core/dataset/type.ts | 2 +- packages/service/core/dataset/schema.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/global/core/dataset/type.ts b/packages/global/core/dataset/type.ts index 32421d4eabc6..3da4e632752d 100644 --- a/packages/global/core/dataset/type.ts +++ b/packages/global/core/dataset/type.ts @@ -65,7 +65,7 @@ export const ChunkSettingsSchema = z.object({ export type ChunkSettingsType = z.infer; /* ===== Pdf parse config ===== */ -// 外部文档解析(pdf2text)开关,整体校验、整体存取;仅当 collection 的 customPdfParse=true +// 外部文档解析服务开关,整体校验、整体存取;仅当 collection 的 customPdfParse=true // 且系统配置了 customPdfParse.url 时生效。缺失字段由读取层用固定默认值补全。 export const PdfParseConfigSchema = z .object({ diff --git a/packages/service/core/dataset/schema.ts b/packages/service/core/dataset/schema.ts index 56d432691890..5cba08a481a3 100644 --- a/packages/service/core/dataset/schema.ts +++ b/packages/service/core/dataset/schema.ts @@ -131,7 +131,7 @@ const DatasetSchema = new Schema({ chunkSettings: { type: ChunkSettings }, - // 外部文档解析(pdf2text)开关,整体存取;缺失字段由读取层用固定默认值补全 + // 外部文档解析服务开关,整体存取;缺失字段由读取层用固定默认值补全 pdfParseConfig: { type: { keep_header_footer: Boolean,