diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index 9d6b430aeec..b556f5b8263 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -52338,6 +52338,10 @@ components: LLMObsCustomEvalConfigBedrockOptions: description: AWS Bedrock-specific options for LLM provider configuration. properties: + inference_profile: + description: Bedrock inference profile identifier, such as an application inference profile ARN. + example: "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123" + type: string region: description: AWS region for Bedrock. example: "us-east-1" @@ -52428,6 +52432,11 @@ components: properties: assessment_criteria: $ref: "#/components/schemas/LLMObsCustomEvalConfigAssessmentCriteria" + context_query: + description: Query used to extract additional context for the evaluation. + example: "@input.context" + nullable: true + type: string inference_params: $ref: "#/components/schemas/LLMObsCustomEvalConfigInferenceParams" last_used_library_prompt_template_name: @@ -52452,6 +52461,15 @@ components: items: $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptMessage" type: array + target_query: + description: Query used to extract the target value to evaluate. + example: "@output.value" + nullable: true + type: string + user_specified_json_post_processing_function: + description: User-provided function applied to post-process the JSON output of the LLM judge. + nullable: true + type: string required: - inference_params type: object @@ -52473,16 +52491,29 @@ components: vertex_ai: $ref: "#/components/schemas/LLMObsCustomEvalConfigVertexAIOptions" type: object + LLMObsCustomEvalConfigListResponse: + description: Response containing a list of custom LLM Observability evaluator configurations. + properties: + data: + description: List of custom evaluator configuration data objects. + items: + $ref: "#/components/schemas/LLMObsCustomEvalConfigData" + type: array + required: + - data + type: object LLMObsCustomEvalConfigParsingType: description: Output parsing type for a custom LLM judge evaluator. enum: - structured_output - json + - keyword_search example: "structured_output" type: string x-enum-varnames: - STRUCTURED_OUTPUT - JSON + - KEYWORD_SEARCH LLMObsCustomEvalConfigPromptContent: description: A content block within a prompt message. properties: @@ -52589,6 +52620,13 @@ components: eval_scope: $ref: "#/components/schemas/LLMObsCustomEvalConfigEvalScope" nullable: true + experiment_project_ids: + description: Experiment project IDs this evaluator is scoped to. + items: + description: An experiment project ID. + format: uuid + type: string + type: array filter: description: Filter expression to select which spans to evaluate. example: "@service:my-service" @@ -116425,6 +116463,67 @@ paths: x-unstable: |- This endpoint is in Preview and may introduce breaking changes. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/unstable/llm-obs/config/evaluators/custom: + get: + description: List all custom LLM Observability evaluator configurations for the organization. + operationId: ListLLMObsCustomEvalConfigs + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + - attributes: + category: "Custom" + created_at: "2024-01-15T10:30:00Z" + created_by: + email: "user@example.com" + eval_name: "my-custom-evaluator" + last_updated_by: + email: "user@example.com" + llm_judge_config: + inference_params: + max_tokens: 1024 + temperature: 0.7 + parsing_type: "structured_output" + llm_provider: + integration_provider: "openai" + model_name: "gpt-4o" + target: + application_name: "my-llm-app" + enabled: true + sampling_percentage: 50.0 + updated_at: "2024-01-15T10:30:00Z" + id: "my-custom-evaluator" + type: "evaluator_config" + schema: + $ref: "#/components/schemas/LLMObsCustomEvalConfigListResponse" + description: OK + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: List custom evaluator configurations + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). /api/unstable/llm-obs/config/evaluators/custom/{eval_name}: delete: description: Delete a custom LLM Observability evaluator configuration by its name. diff --git a/examples/v2/llm-observability/ListLLMObsCustomEvalConfigs.java b/examples/v2/llm-observability/ListLLMObsCustomEvalConfigs.java new file mode 100644 index 00000000000..45a837b9a25 --- /dev/null +++ b/examples/v2/llm-observability/ListLLMObsCustomEvalConfigs.java @@ -0,0 +1,25 @@ +// List custom evaluator configurations returns "OK" response + +import com.datadog.api.client.ApiClient; +import com.datadog.api.client.ApiException; +import com.datadog.api.client.v2.api.LlmObservabilityApi; +import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigListResponse; + +public class Example { + public static void main(String[] args) { + ApiClient defaultClient = ApiClient.getDefaultApiClient(); + defaultClient.setUnstableOperationEnabled("v2.listLLMObsCustomEvalConfigs", true); + LlmObservabilityApi apiInstance = new LlmObservabilityApi(defaultClient); + + try { + LLMObsCustomEvalConfigListResponse result = apiInstance.listLLMObsCustomEvalConfigs(); + System.out.println(result); + } catch (ApiException e) { + System.err.println("Exception when calling LlmObservabilityApi#listLLMObsCustomEvalConfigs"); + System.err.println("Status code: " + e.getCode()); + System.err.println("Reason: " + e.getResponseBody()); + System.err.println("Response headers: " + e.getResponseHeaders()); + e.printStackTrace(); + } + } +} diff --git a/examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.java b/examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.java index 6583874900c..854439313cf 100644 --- a/examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.java +++ b/examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.java @@ -47,6 +47,7 @@ public static void main(String[] args) { .minThreshold(0.7) .passValues(Arrays.asList("pass", "yes")) .passWhen(true)) + .contextQuery("@input.context") .inferenceParams( new LLMObsCustomEvalConfigInferenceParams() .maxTokens(1024L) @@ -87,11 +88,15 @@ public static void main(String[] args) { .result("sunny, 72F") .toolId("call_abc123") .type("function"))))) - .role("user")))) + .role("user"))) + .targetQuery("@output.value") + .userSpecifiedJsonPostProcessingFunction(null)) .llmProvider( new LLMObsCustomEvalConfigLLMProvider() .bedrock( new LLMObsCustomEvalConfigBedrockOptions() + .inferenceProfile( + "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123") .region("us-east-1")) .integrationAccountId("my-account-id") .integrationProvider( diff --git a/src/main/java/com/datadog/api/client/ApiClient.java b/src/main/java/com/datadog/api/client/ApiClient.java index 42f44708488..471a474d129 100644 --- a/src/main/java/com/datadog/api/client/ApiClient.java +++ b/src/main/java/com/datadog/api/client/ApiClient.java @@ -812,6 +812,7 @@ public class ApiClient { put("v2.getLLMObsPrompt", false); put("v2.getLLMObsPromptVersion", false); put("v2.listLLMObsAnnotationQueues", false); + put("v2.listLLMObsCustomEvalConfigs", false); put("v2.listLLMObsDatasetRecords", false); put("v2.listLLMObsDatasets", false); put("v2.listLLMObsDatasetVersions", false); diff --git a/src/main/java/com/datadog/api/client/v2/api/LlmObservabilityApi.java b/src/main/java/com/datadog/api/client/v2/api/LlmObservabilityApi.java index 7cd226335a9..e30851d31ca 100644 --- a/src/main/java/com/datadog/api/client/v2/api/LlmObservabilityApi.java +++ b/src/main/java/com/datadog/api/client/v2/api/LlmObservabilityApi.java @@ -18,6 +18,7 @@ import com.datadog.api.client.v2.model.LLMObsAnnotationsResponse; import com.datadog.api.client.v2.model.LLMObsCreatePromptRequest; import com.datadog.api.client.v2.model.LLMObsCreatePromptVersionRequest; +import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigListResponse; import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigResponse; import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigUpdateRequest; import com.datadog.api.client.v2.model.LLMObsDataDeletionRequest; @@ -6391,6 +6392,137 @@ public ApiResponse listLLMObsAnnotationQueuesWit new GenericType() {}); } + /** + * List custom evaluator configurations. + * + *

See {@link #listLLMObsCustomEvalConfigsWithHttpInfo}. + * + * @return LLMObsCustomEvalConfigListResponse + * @throws ApiException if fails to make API call + */ + public LLMObsCustomEvalConfigListResponse listLLMObsCustomEvalConfigs() throws ApiException { + return listLLMObsCustomEvalConfigsWithHttpInfo().getData(); + } + + /** + * List custom evaluator configurations. + * + *

See {@link #listLLMObsCustomEvalConfigsWithHttpInfoAsync}. + * + * @return CompletableFuture<LLMObsCustomEvalConfigListResponse> + */ + public CompletableFuture listLLMObsCustomEvalConfigsAsync() { + return listLLMObsCustomEvalConfigsWithHttpInfoAsync() + .thenApply( + response -> { + return response.getData(); + }); + } + + /** + * List all custom LLM Observability evaluator configurations for the organization. + * + * @return ApiResponse<LLMObsCustomEvalConfigListResponse> + * @throws ApiException if fails to make API call + * @http.response.details + * + * + * + * + * + * + * + *
Response details
Status Code Description Response Headers
200 OK -
401 Unauthorized -
403 Forbidden -
429 Too many requests -
+ */ + public ApiResponse listLLMObsCustomEvalConfigsWithHttpInfo() + throws ApiException { + // Check if unstable operation is enabled + String operationId = "listLLMObsCustomEvalConfigs"; + if (apiClient.isUnstableOperationEnabled("v2." + operationId)) { + apiClient.getLogger().warning(String.format("Using unstable operation '%s'", operationId)); + } else { + throw new ApiException(0, String.format("Unstable operation '%s' is disabled", operationId)); + } + Object localVarPostBody = null; + // create path and map variables + String localVarPath = "/api/unstable/llm-obs/config/evaluators/custom"; + + Map localVarHeaderParams = new HashMap(); + + Invocation.Builder builder = + apiClient.createBuilder( + "v2.LlmObservabilityApi.listLLMObsCustomEvalConfigs", + localVarPath, + new ArrayList(), + localVarHeaderParams, + new HashMap(), + new String[] {"application/json"}, + new String[] {"apiKeyAuth", "appKeyAuth"}); + return apiClient.invokeAPI( + "GET", + builder, + localVarHeaderParams, + new String[] {}, + localVarPostBody, + new HashMap(), + false, + new GenericType() {}); + } + + /** + * List custom evaluator configurations. + * + *

See {@link #listLLMObsCustomEvalConfigsWithHttpInfo}. + * + * @return CompletableFuture<ApiResponse<LLMObsCustomEvalConfigListResponse>> + */ + public CompletableFuture> + listLLMObsCustomEvalConfigsWithHttpInfoAsync() { + // Check if unstable operation is enabled + String operationId = "listLLMObsCustomEvalConfigs"; + if (apiClient.isUnstableOperationEnabled("v2." + operationId)) { + apiClient.getLogger().warning(String.format("Using unstable operation '%s'", operationId)); + } else { + CompletableFuture> result = + new CompletableFuture<>(); + result.completeExceptionally( + new ApiException(0, String.format("Unstable operation '%s' is disabled", operationId))); + return result; + } + Object localVarPostBody = null; + // create path and map variables + String localVarPath = "/api/unstable/llm-obs/config/evaluators/custom"; + + Map localVarHeaderParams = new HashMap(); + + Invocation.Builder builder; + try { + builder = + apiClient.createBuilder( + "v2.LlmObservabilityApi.listLLMObsCustomEvalConfigs", + localVarPath, + new ArrayList(), + localVarHeaderParams, + new HashMap(), + new String[] {"application/json"}, + new String[] {"apiKeyAuth", "appKeyAuth"}); + } catch (ApiException ex) { + CompletableFuture> result = + new CompletableFuture<>(); + result.completeExceptionally(ex); + return result; + } + return apiClient.invokeAPIAsync( + "GET", + builder, + localVarHeaderParams, + new String[] {}, + localVarPostBody, + new HashMap(), + false, + new GenericType() {}); + } + /** Manage optional parameters to listLLMObsDatasetRecords. */ public static class ListLLMObsDatasetRecordsOptionalParameters { private Long filterVersion; diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigBedrockOptions.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigBedrockOptions.java index 1e191a6e1e4..a8803e6c9be 100644 --- a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigBedrockOptions.java +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigBedrockOptions.java @@ -17,14 +17,41 @@ import java.util.Objects; /** AWS Bedrock-specific options for LLM provider configuration. */ -@JsonPropertyOrder({LLMObsCustomEvalConfigBedrockOptions.JSON_PROPERTY_REGION}) +@JsonPropertyOrder({ + LLMObsCustomEvalConfigBedrockOptions.JSON_PROPERTY_INFERENCE_PROFILE, + LLMObsCustomEvalConfigBedrockOptions.JSON_PROPERTY_REGION +}) @jakarta.annotation.Generated( value = "https://github.com/DataDog/datadog-api-client-java/blob/master/.generator") public class LLMObsCustomEvalConfigBedrockOptions { @JsonIgnore public boolean unparsed = false; + public static final String JSON_PROPERTY_INFERENCE_PROFILE = "inference_profile"; + private String inferenceProfile; + public static final String JSON_PROPERTY_REGION = "region"; private String region; + public LLMObsCustomEvalConfigBedrockOptions inferenceProfile(String inferenceProfile) { + this.inferenceProfile = inferenceProfile; + return this; + } + + /** + * Bedrock inference profile identifier, such as an application inference profile ARN. + * + * @return inferenceProfile + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_INFERENCE_PROFILE) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public String getInferenceProfile() { + return inferenceProfile; + } + + public void setInferenceProfile(String inferenceProfile) { + this.inferenceProfile = inferenceProfile; + } + public LLMObsCustomEvalConfigBedrockOptions region(String region) { this.region = region; return this; @@ -103,20 +130,23 @@ public boolean equals(Object o) { } LLMObsCustomEvalConfigBedrockOptions llmObsCustomEvalConfigBedrockOptions = (LLMObsCustomEvalConfigBedrockOptions) o; - return Objects.equals(this.region, llmObsCustomEvalConfigBedrockOptions.region) + return Objects.equals( + this.inferenceProfile, llmObsCustomEvalConfigBedrockOptions.inferenceProfile) + && Objects.equals(this.region, llmObsCustomEvalConfigBedrockOptions.region) && Objects.equals( this.additionalProperties, llmObsCustomEvalConfigBedrockOptions.additionalProperties); } @Override public int hashCode() { - return Objects.hash(region, additionalProperties); + return Objects.hash(inferenceProfile, region, additionalProperties); } @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("class LLMObsCustomEvalConfigBedrockOptions {\n"); + sb.append(" inferenceProfile: ").append(toIndentedString(inferenceProfile)).append("\n"); sb.append(" region: ").append(toIndentedString(region)).append("\n"); sb.append(" additionalProperties: ") .append(toIndentedString(additionalProperties)) diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMJudgeConfig.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMJudgeConfig.java index 1370c30e8a3..1462aa0afcf 100644 --- a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMJudgeConfig.java +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMJudgeConfig.java @@ -23,12 +23,15 @@ /** LLM judge configuration for a custom evaluator. */ @JsonPropertyOrder({ LLMObsCustomEvalConfigLLMJudgeConfig.JSON_PROPERTY_ASSESSMENT_CRITERIA, + LLMObsCustomEvalConfigLLMJudgeConfig.JSON_PROPERTY_CONTEXT_QUERY, LLMObsCustomEvalConfigLLMJudgeConfig.JSON_PROPERTY_INFERENCE_PARAMS, LLMObsCustomEvalConfigLLMJudgeConfig.JSON_PROPERTY_LAST_USED_LIBRARY_PROMPT_TEMPLATE_NAME, LLMObsCustomEvalConfigLLMJudgeConfig.JSON_PROPERTY_MODIFIED_LIBRARY_PROMPT_TEMPLATE, LLMObsCustomEvalConfigLLMJudgeConfig.JSON_PROPERTY_OUTPUT_SCHEMA, LLMObsCustomEvalConfigLLMJudgeConfig.JSON_PROPERTY_PARSING_TYPE, - LLMObsCustomEvalConfigLLMJudgeConfig.JSON_PROPERTY_PROMPT_TEMPLATE + LLMObsCustomEvalConfigLLMJudgeConfig.JSON_PROPERTY_PROMPT_TEMPLATE, + LLMObsCustomEvalConfigLLMJudgeConfig.JSON_PROPERTY_TARGET_QUERY, + LLMObsCustomEvalConfigLLMJudgeConfig.JSON_PROPERTY_USER_SPECIFIED_JSON_POST_PROCESSING_FUNCTION }) @jakarta.annotation.Generated( value = "https://github.com/DataDog/datadog-api-client-java/blob/master/.generator") @@ -37,6 +40,9 @@ public class LLMObsCustomEvalConfigLLMJudgeConfig { public static final String JSON_PROPERTY_ASSESSMENT_CRITERIA = "assessment_criteria"; private LLMObsCustomEvalConfigAssessmentCriteria assessmentCriteria; + public static final String JSON_PROPERTY_CONTEXT_QUERY = "context_query"; + private JsonNullable contextQuery = JsonNullable.undefined(); + public static final String JSON_PROPERTY_INFERENCE_PARAMS = "inference_params"; private LLMObsCustomEvalConfigInferenceParams inferenceParams; @@ -58,6 +64,14 @@ public class LLMObsCustomEvalConfigLLMJudgeConfig { public static final String JSON_PROPERTY_PROMPT_TEMPLATE = "prompt_template"; private List promptTemplate = null; + public static final String JSON_PROPERTY_TARGET_QUERY = "target_query"; + private JsonNullable targetQuery = JsonNullable.undefined(); + + public static final String JSON_PROPERTY_USER_SPECIFIED_JSON_POST_PROCESSING_FUNCTION = + "user_specified_json_post_processing_function"; + private JsonNullable userSpecifiedJsonPostProcessingFunction = + JsonNullable.undefined(); + public LLMObsCustomEvalConfigLLMJudgeConfig() {} @JsonCreator @@ -94,6 +108,37 @@ public void setAssessmentCriteria(LLMObsCustomEvalConfigAssessmentCriteria asses } } + public LLMObsCustomEvalConfigLLMJudgeConfig contextQuery(String contextQuery) { + this.contextQuery = JsonNullable.of(contextQuery); + return this; + } + + /** + * Query used to extract additional context for the evaluation. + * + * @return contextQuery + */ + @jakarta.annotation.Nullable + @JsonIgnore + public String getContextQuery() { + return contextQuery.orElse(null); + } + + @JsonProperty(JSON_PROPERTY_CONTEXT_QUERY) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public JsonNullable getContextQuery_JsonNullable() { + return contextQuery; + } + + @JsonProperty(JSON_PROPERTY_CONTEXT_QUERY) + public void setContextQuery_JsonNullable(JsonNullable contextQuery) { + this.contextQuery = contextQuery; + } + + public void setContextQuery(String contextQuery) { + this.contextQuery = JsonNullable.of(contextQuery); + } + public LLMObsCustomEvalConfigLLMJudgeConfig inferenceParams( LLMObsCustomEvalConfigInferenceParams inferenceParams) { this.inferenceParams = inferenceParams; @@ -299,6 +344,73 @@ public void setPromptTemplate(List promptTe } } + public LLMObsCustomEvalConfigLLMJudgeConfig targetQuery(String targetQuery) { + this.targetQuery = JsonNullable.of(targetQuery); + return this; + } + + /** + * Query used to extract the target value to evaluate. + * + * @return targetQuery + */ + @jakarta.annotation.Nullable + @JsonIgnore + public String getTargetQuery() { + return targetQuery.orElse(null); + } + + @JsonProperty(JSON_PROPERTY_TARGET_QUERY) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public JsonNullable getTargetQuery_JsonNullable() { + return targetQuery; + } + + @JsonProperty(JSON_PROPERTY_TARGET_QUERY) + public void setTargetQuery_JsonNullable(JsonNullable targetQuery) { + this.targetQuery = targetQuery; + } + + public void setTargetQuery(String targetQuery) { + this.targetQuery = JsonNullable.of(targetQuery); + } + + public LLMObsCustomEvalConfigLLMJudgeConfig userSpecifiedJsonPostProcessingFunction( + String userSpecifiedJsonPostProcessingFunction) { + this.userSpecifiedJsonPostProcessingFunction = + JsonNullable.of(userSpecifiedJsonPostProcessingFunction); + return this; + } + + /** + * User-provided function applied to post-process the JSON output of the LLM judge. + * + * @return userSpecifiedJsonPostProcessingFunction + */ + @jakarta.annotation.Nullable + @JsonIgnore + public String getUserSpecifiedJsonPostProcessingFunction() { + return userSpecifiedJsonPostProcessingFunction.orElse(null); + } + + @JsonProperty(JSON_PROPERTY_USER_SPECIFIED_JSON_POST_PROCESSING_FUNCTION) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public JsonNullable getUserSpecifiedJsonPostProcessingFunction_JsonNullable() { + return userSpecifiedJsonPostProcessingFunction; + } + + @JsonProperty(JSON_PROPERTY_USER_SPECIFIED_JSON_POST_PROCESSING_FUNCTION) + public void setUserSpecifiedJsonPostProcessingFunction_JsonNullable( + JsonNullable userSpecifiedJsonPostProcessingFunction) { + this.userSpecifiedJsonPostProcessingFunction = userSpecifiedJsonPostProcessingFunction; + } + + public void setUserSpecifiedJsonPostProcessingFunction( + String userSpecifiedJsonPostProcessingFunction) { + this.userSpecifiedJsonPostProcessingFunction = + JsonNullable.of(userSpecifiedJsonPostProcessingFunction); + } + /** * A container for additional, undeclared properties. This is a holder for any undeclared * properties as specified with the 'additionalProperties' keyword in the OAS document. @@ -358,6 +470,7 @@ public boolean equals(Object o) { (LLMObsCustomEvalConfigLLMJudgeConfig) o; return Objects.equals( this.assessmentCriteria, llmObsCustomEvalConfigLlmJudgeConfig.assessmentCriteria) + && Objects.equals(this.contextQuery, llmObsCustomEvalConfigLlmJudgeConfig.contextQuery) && Objects.equals( this.inferenceParams, llmObsCustomEvalConfigLlmJudgeConfig.inferenceParams) && Objects.equals( @@ -369,6 +482,10 @@ public boolean equals(Object o) { && Objects.equals(this.outputSchema, llmObsCustomEvalConfigLlmJudgeConfig.outputSchema) && Objects.equals(this.parsingType, llmObsCustomEvalConfigLlmJudgeConfig.parsingType) && Objects.equals(this.promptTemplate, llmObsCustomEvalConfigLlmJudgeConfig.promptTemplate) + && Objects.equals(this.targetQuery, llmObsCustomEvalConfigLlmJudgeConfig.targetQuery) + && Objects.equals( + this.userSpecifiedJsonPostProcessingFunction, + llmObsCustomEvalConfigLlmJudgeConfig.userSpecifiedJsonPostProcessingFunction) && Objects.equals( this.additionalProperties, llmObsCustomEvalConfigLlmJudgeConfig.additionalProperties); } @@ -377,12 +494,15 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash( assessmentCriteria, + contextQuery, inferenceParams, lastUsedLibraryPromptTemplateName, modifiedLibraryPromptTemplate, outputSchema, parsingType, promptTemplate, + targetQuery, + userSpecifiedJsonPostProcessingFunction, additionalProperties); } @@ -391,6 +511,7 @@ public String toString() { StringBuilder sb = new StringBuilder(); sb.append("class LLMObsCustomEvalConfigLLMJudgeConfig {\n"); sb.append(" assessmentCriteria: ").append(toIndentedString(assessmentCriteria)).append("\n"); + sb.append(" contextQuery: ").append(toIndentedString(contextQuery)).append("\n"); sb.append(" inferenceParams: ").append(toIndentedString(inferenceParams)).append("\n"); sb.append(" lastUsedLibraryPromptTemplateName: ") .append(toIndentedString(lastUsedLibraryPromptTemplateName)) @@ -401,6 +522,10 @@ public String toString() { sb.append(" outputSchema: ").append(toIndentedString(outputSchema)).append("\n"); sb.append(" parsingType: ").append(toIndentedString(parsingType)).append("\n"); sb.append(" promptTemplate: ").append(toIndentedString(promptTemplate)).append("\n"); + sb.append(" targetQuery: ").append(toIndentedString(targetQuery)).append("\n"); + sb.append(" userSpecifiedJsonPostProcessingFunction: ") + .append(toIndentedString(userSpecifiedJsonPostProcessingFunction)) + .append("\n"); sb.append(" additionalProperties: ") .append(toIndentedString(additionalProperties)) .append("\n"); diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigListResponse.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigListResponse.java new file mode 100644 index 00000000000..5d7969cd981 --- /dev/null +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigListResponse.java @@ -0,0 +1,165 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +package com.datadog.api.client.v2.model; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** Response containing a list of custom LLM Observability evaluator configurations. */ +@JsonPropertyOrder({LLMObsCustomEvalConfigListResponse.JSON_PROPERTY_DATA}) +@jakarta.annotation.Generated( + value = "https://github.com/DataDog/datadog-api-client-java/blob/master/.generator") +public class LLMObsCustomEvalConfigListResponse { + @JsonIgnore public boolean unparsed = false; + public static final String JSON_PROPERTY_DATA = "data"; + private List data = new ArrayList<>(); + + public LLMObsCustomEvalConfigListResponse() {} + + @JsonCreator + public LLMObsCustomEvalConfigListResponse( + @JsonProperty(required = true, value = JSON_PROPERTY_DATA) + List data) { + this.data = data; + for (LLMObsCustomEvalConfigData item : data) { + this.unparsed |= item.unparsed; + } + } + + public LLMObsCustomEvalConfigListResponse data(List data) { + this.data = data; + for (LLMObsCustomEvalConfigData item : data) { + this.unparsed |= item.unparsed; + } + return this; + } + + public LLMObsCustomEvalConfigListResponse addDataItem(LLMObsCustomEvalConfigData dataItem) { + this.data.add(dataItem); + this.unparsed |= dataItem.unparsed; + return this; + } + + /** + * List of custom evaluator configuration data objects. + * + * @return data + */ + @JsonProperty(JSON_PROPERTY_DATA) + @JsonInclude(value = JsonInclude.Include.ALWAYS) + public List getData() { + return data; + } + + public void setData(List data) { + this.data = data; + if (data != null) { + for (LLMObsCustomEvalConfigData item : data) { + this.unparsed |= item.unparsed; + } + } + } + + /** + * A container for additional, undeclared properties. This is a holder for any undeclared + * properties as specified with the 'additionalProperties' keyword in the OAS document. + */ + private Map additionalProperties; + + /** + * Set the additional (undeclared) property with the specified name and value. If the property + * does not already exist, create it otherwise replace it. + * + * @param key The arbitrary key to set + * @param value The associated value + * @return LLMObsCustomEvalConfigListResponse + */ + @JsonAnySetter + public LLMObsCustomEvalConfigListResponse putAdditionalProperty(String key, Object value) { + if (this.additionalProperties == null) { + this.additionalProperties = new HashMap(); + } + this.additionalProperties.put(key, value); + return this; + } + + /** + * Return the additional (undeclared) property. + * + * @return The additional properties + */ + @JsonAnyGetter + public Map getAdditionalProperties() { + return additionalProperties; + } + + /** + * Return the additional (undeclared) property with the specified name. + * + * @param key The arbitrary key to get + * @return The specific additional property for the given key + */ + public Object getAdditionalProperty(String key) { + if (this.additionalProperties == null) { + return null; + } + return this.additionalProperties.get(key); + } + + /** Return true if this LLMObsCustomEvalConfigListResponse object is equal to o. */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LLMObsCustomEvalConfigListResponse llmObsCustomEvalConfigListResponse = + (LLMObsCustomEvalConfigListResponse) o; + return Objects.equals(this.data, llmObsCustomEvalConfigListResponse.data) + && Objects.equals( + this.additionalProperties, llmObsCustomEvalConfigListResponse.additionalProperties); + } + + @Override + public int hashCode() { + return Objects.hash(data, additionalProperties); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class LLMObsCustomEvalConfigListResponse {\n"); + sb.append(" data: ").append(toIndentedString(data)).append("\n"); + sb.append(" additionalProperties: ") + .append(toIndentedString(additionalProperties)) + .append("\n"); + sb.append('}'); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces (except the first line). + */ + private String toIndentedString(Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigParsingType.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigParsingType.java index 1308b7476cf..02480aeccf3 100644 --- a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigParsingType.java +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigParsingType.java @@ -24,12 +24,14 @@ public class LLMObsCustomEvalConfigParsingType extends ModelEnum { private static final Set allowedValues = - new HashSet(Arrays.asList("structured_output", "json")); + new HashSet(Arrays.asList("structured_output", "json", "keyword_search")); public static final LLMObsCustomEvalConfigParsingType STRUCTURED_OUTPUT = new LLMObsCustomEvalConfigParsingType("structured_output"); public static final LLMObsCustomEvalConfigParsingType JSON = new LLMObsCustomEvalConfigParsingType("json"); + public static final LLMObsCustomEvalConfigParsingType KEYWORD_SEARCH = + new LLMObsCustomEvalConfigParsingType("keyword_search"); LLMObsCustomEvalConfigParsingType(String value) { super(value, allowedValues); diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigTarget.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigTarget.java index 1515448c611..879f3c68dbe 100644 --- a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigTarget.java +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigTarget.java @@ -13,9 +13,12 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.UUID; import org.openapitools.jackson.nullable.JsonNullable; /** Target application configuration for a custom evaluator. */ @@ -23,6 +26,7 @@ LLMObsCustomEvalConfigTarget.JSON_PROPERTY_APPLICATION_NAME, LLMObsCustomEvalConfigTarget.JSON_PROPERTY_ENABLED, LLMObsCustomEvalConfigTarget.JSON_PROPERTY_EVAL_SCOPE, + LLMObsCustomEvalConfigTarget.JSON_PROPERTY_EXPERIMENT_PROJECT_IDS, LLMObsCustomEvalConfigTarget.JSON_PROPERTY_FILTER, LLMObsCustomEvalConfigTarget.JSON_PROPERTY_ROOT_SPANS_ONLY, LLMObsCustomEvalConfigTarget.JSON_PROPERTY_SAMPLING_PERCENTAGE @@ -40,6 +44,9 @@ public class LLMObsCustomEvalConfigTarget { public static final String JSON_PROPERTY_EVAL_SCOPE = "eval_scope"; private LLMObsCustomEvalConfigEvalScope evalScope; + public static final String JSON_PROPERTY_EXPERIMENT_PROJECT_IDS = "experiment_project_ids"; + private List experimentProjectIds = null; + public static final String JSON_PROPERTY_FILTER = "filter"; private JsonNullable filter = JsonNullable.undefined(); @@ -124,6 +131,35 @@ public void setEvalScope(LLMObsCustomEvalConfigEvalScope evalScope) { this.evalScope = evalScope; } + public LLMObsCustomEvalConfigTarget experimentProjectIds(List experimentProjectIds) { + this.experimentProjectIds = experimentProjectIds; + return this; + } + + public LLMObsCustomEvalConfigTarget addExperimentProjectIdsItem(UUID experimentProjectIdsItem) { + if (this.experimentProjectIds == null) { + this.experimentProjectIds = new ArrayList<>(); + } + this.experimentProjectIds.add(experimentProjectIdsItem); + return this; + } + + /** + * Experiment project IDs this evaluator is scoped to. + * + * @return experimentProjectIds + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_EXPERIMENT_PROJECT_IDS) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public List getExperimentProjectIds() { + return experimentProjectIds; + } + + public void setExperimentProjectIds(List experimentProjectIds) { + this.experimentProjectIds = experimentProjectIds; + } + public LLMObsCustomEvalConfigTarget filter(String filter) { this.filter = JsonNullable.of(filter); return this; @@ -276,6 +312,8 @@ public boolean equals(Object o) { return Objects.equals(this.applicationName, llmObsCustomEvalConfigTarget.applicationName) && Objects.equals(this.enabled, llmObsCustomEvalConfigTarget.enabled) && Objects.equals(this.evalScope, llmObsCustomEvalConfigTarget.evalScope) + && Objects.equals( + this.experimentProjectIds, llmObsCustomEvalConfigTarget.experimentProjectIds) && Objects.equals(this.filter, llmObsCustomEvalConfigTarget.filter) && Objects.equals(this.rootSpansOnly, llmObsCustomEvalConfigTarget.rootSpansOnly) && Objects.equals(this.samplingPercentage, llmObsCustomEvalConfigTarget.samplingPercentage) @@ -289,6 +327,7 @@ public int hashCode() { applicationName, enabled, evalScope, + experimentProjectIds, filter, rootSpansOnly, samplingPercentage, @@ -302,6 +341,9 @@ public String toString() { sb.append(" applicationName: ").append(toIndentedString(applicationName)).append("\n"); sb.append(" enabled: ").append(toIndentedString(enabled)).append("\n"); sb.append(" evalScope: ").append(toIndentedString(evalScope)).append("\n"); + sb.append(" experimentProjectIds: ") + .append(toIndentedString(experimentProjectIds)) + .append("\n"); sb.append(" filter: ").append(toIndentedString(filter)).append("\n"); sb.append(" rootSpansOnly: ").append(toIndentedString(rootSpansOnly)).append("\n"); sb.append(" samplingPercentage: ").append(toIndentedString(samplingPercentage)).append("\n"); diff --git a/src/test/resources/com/datadog/api/client/v2/api/llm_observability.feature b/src/test/resources/com/datadog/api/client/v2/api/llm_observability.feature index 2c23f9f6d7b..b1897f532b9 100644 --- a/src/test/resources/com/datadog/api/client/v2/api/llm_observability.feature +++ b/src/test/resources/com/datadog/api/client/v2/api/llm_observability.feature @@ -338,7 +338,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsCustomEvalConfig" enabled And new "UpdateLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "context_query": "@input.context", "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}], "target_query": "@output.value", "user_specified_json_post_processing_function": null}, "llm_provider": {"bedrock": {"inference_profile": "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", "region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "experiment_project_ids": [], "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} When the request is sent Then the response status is 400 Bad Request @@ -347,7 +347,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsCustomEvalConfig" enabled And new "UpdateLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "context_query": "@input.context", "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}], "target_query": "@output.value", "user_specified_json_post_processing_function": null}, "llm_provider": {"bedrock": {"inference_profile": "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", "region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "experiment_project_ids": [], "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} When the request is sent Then the response status is 404 Not Found @@ -356,7 +356,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsCustomEvalConfig" enabled And new "UpdateLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "context_query": "@input.context", "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}], "target_query": "@output.value", "user_specified_json_post_processing_function": null}, "llm_provider": {"bedrock": {"inference_profile": "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", "region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "experiment_project_ids": [], "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} When the request is sent Then the response status is 200 OK @@ -365,7 +365,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsCustomEvalConfig" enabled And new "UpdateLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "context_query": "@input.context", "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}], "target_query": "@output.value", "user_specified_json_post_processing_function": null}, "llm_provider": {"bedrock": {"inference_profile": "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", "region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "experiment_project_ids": [], "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} When the request is sent Then the response status is 422 Unprocessable Entity @@ -1110,6 +1110,13 @@ Feature: LLM Observability When the request is sent Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability + Scenario: List custom evaluator configurations returns "OK" response + Given operation "ListLLMObsCustomEvalConfigs" enabled + And new "ListLLMObsCustomEvalConfigs" request + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: List events for an LLM Observability experiment returns "Bad Request" response Given operation "ListLLMObsExperimentEvents" enabled diff --git a/src/test/resources/com/datadog/api/client/v2/api/undo.json b/src/test/resources/com/datadog/api/client/v2/api/undo.json index d93991ac7d7..2a7fb64a0a9 100644 --- a/src/test/resources/com/datadog/api/client/v2/api/undo.json +++ b/src/test/resources/com/datadog/api/client/v2/api/undo.json @@ -123,6 +123,12 @@ "type": "safe" } }, + "ListLLMObsCustomEvalConfigs": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, "DeleteLLMObsCustomEvalConfig": { "tag": "LLM Observability", "undo": {