diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index 9d6b430aeeca..b556f5b82634 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -52338,6 +52338,10 @@ components: LLMObsCustomEvalConfigBedrockOptions: description: AWS Bedrock-specific options for LLM provider configuration. properties: + inference_profile: + description: Bedrock inference profile identifier, such as an application inference profile ARN. + example: "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123" + type: string region: description: AWS region for Bedrock. example: "us-east-1" @@ -52428,6 +52432,11 @@ components: properties: assessment_criteria: $ref: "#/components/schemas/LLMObsCustomEvalConfigAssessmentCriteria" + context_query: + description: Query used to extract additional context for the evaluation. + example: "@input.context" + nullable: true + type: string inference_params: $ref: "#/components/schemas/LLMObsCustomEvalConfigInferenceParams" last_used_library_prompt_template_name: @@ -52452,6 +52461,15 @@ components: items: $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptMessage" type: array + target_query: + description: Query used to extract the target value to evaluate. + example: "@output.value" + nullable: true + type: string + user_specified_json_post_processing_function: + description: User-provided function applied to post-process the JSON output of the LLM judge. + nullable: true + type: string required: - inference_params type: object @@ -52473,16 +52491,29 @@ components: vertex_ai: $ref: "#/components/schemas/LLMObsCustomEvalConfigVertexAIOptions" type: object + LLMObsCustomEvalConfigListResponse: + description: Response containing a list of custom LLM Observability evaluator configurations. + properties: + data: + description: List of custom evaluator configuration data objects. + items: + $ref: "#/components/schemas/LLMObsCustomEvalConfigData" + type: array + required: + - data + type: object LLMObsCustomEvalConfigParsingType: description: Output parsing type for a custom LLM judge evaluator. enum: - structured_output - json + - keyword_search example: "structured_output" type: string x-enum-varnames: - STRUCTURED_OUTPUT - JSON + - KEYWORD_SEARCH LLMObsCustomEvalConfigPromptContent: description: A content block within a prompt message. properties: @@ -52589,6 +52620,13 @@ components: eval_scope: $ref: "#/components/schemas/LLMObsCustomEvalConfigEvalScope" nullable: true + experiment_project_ids: + description: Experiment project IDs this evaluator is scoped to. + items: + description: An experiment project ID. + format: uuid + type: string + type: array filter: description: Filter expression to select which spans to evaluate. example: "@service:my-service" @@ -116425,6 +116463,67 @@ paths: x-unstable: |- This endpoint is in Preview and may introduce breaking changes. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/unstable/llm-obs/config/evaluators/custom: + get: + description: List all custom LLM Observability evaluator configurations for the organization. + operationId: ListLLMObsCustomEvalConfigs + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + - attributes: + category: "Custom" + created_at: "2024-01-15T10:30:00Z" + created_by: + email: "user@example.com" + eval_name: "my-custom-evaluator" + last_updated_by: + email: "user@example.com" + llm_judge_config: + inference_params: + max_tokens: 1024 + temperature: 0.7 + parsing_type: "structured_output" + llm_provider: + integration_provider: "openai" + model_name: "gpt-4o" + target: + application_name: "my-llm-app" + enabled: true + sampling_percentage: 50.0 + updated_at: "2024-01-15T10:30:00Z" + id: "my-custom-evaluator" + type: "evaluator_config" + schema: + $ref: "#/components/schemas/LLMObsCustomEvalConfigListResponse" + description: OK + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: List custom evaluator configurations + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). /api/unstable/llm-obs/config/evaluators/custom/{eval_name}: delete: description: Delete a custom LLM Observability evaluator configuration by its name. diff --git a/examples/v2/llm-observability/ListLLMObsCustomEvalConfigs.rb b/examples/v2/llm-observability/ListLLMObsCustomEvalConfigs.rb new file mode 100644 index 000000000000..8d603011f886 --- /dev/null +++ b/examples/v2/llm-observability/ListLLMObsCustomEvalConfigs.rb @@ -0,0 +1,8 @@ +# List custom evaluator configurations returns "OK" response + +require "datadog_api_client" +DatadogAPIClient.configure do |config| + config.unstable_operations["v2.list_llm_obs_custom_eval_configs".to_sym] = true +end +api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new +p api_instance.list_llm_obs_custom_eval_configs() diff --git a/examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.rb b/examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.rb index 432cc216e9e1..2a23fbad0870 100644 --- a/examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.rb +++ b/examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.rb @@ -21,6 +21,7 @@ ], pass_when: true, }), + context_query: "@input.context", inference_params: DatadogAPIClient::V2::LLMObsCustomEvalConfigInferenceParams.new({ frequency_penalty: 0.0, max_tokens: 1024, @@ -59,9 +60,12 @@ role: "user", }), ], + target_query: "@output.value", + user_specified_json_post_processing_function: nil, }), llm_provider: DatadogAPIClient::V2::LLMObsCustomEvalConfigLLMProvider.new({ bedrock: DatadogAPIClient::V2::LLMObsCustomEvalConfigBedrockOptions.new({ + inference_profile: "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", region: "us-east-1", }), integration_account_id: "my-account-id", @@ -76,6 +80,7 @@ application_name: "my-llm-app", enabled: true, eval_scope: DatadogAPIClient::V2::LLMObsCustomEvalConfigEvalScope::SPAN, + experiment_project_ids: [], filter: "@service:my-service", root_spans_only: true, sampling_percentage: 50.0, diff --git a/features/v2/llm_observability.feature b/features/v2/llm_observability.feature index 2c23f9f6d7b4..b1897f532b9f 100644 --- a/features/v2/llm_observability.feature +++ b/features/v2/llm_observability.feature @@ -338,7 +338,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsCustomEvalConfig" enabled And new "UpdateLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "context_query": "@input.context", "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}], "target_query": "@output.value", "user_specified_json_post_processing_function": null}, "llm_provider": {"bedrock": {"inference_profile": "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", "region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "experiment_project_ids": [], "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} When the request is sent Then the response status is 400 Bad Request @@ -347,7 +347,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsCustomEvalConfig" enabled And new "UpdateLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "context_query": "@input.context", "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}], "target_query": "@output.value", "user_specified_json_post_processing_function": null}, "llm_provider": {"bedrock": {"inference_profile": "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", "region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "experiment_project_ids": [], "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} When the request is sent Then the response status is 404 Not Found @@ -356,7 +356,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsCustomEvalConfig" enabled And new "UpdateLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "context_query": "@input.context", "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}], "target_query": "@output.value", "user_specified_json_post_processing_function": null}, "llm_provider": {"bedrock": {"inference_profile": "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", "region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "experiment_project_ids": [], "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} When the request is sent Then the response status is 200 OK @@ -365,7 +365,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsCustomEvalConfig" enabled And new "UpdateLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "context_query": "@input.context", "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}], "target_query": "@output.value", "user_specified_json_post_processing_function": null}, "llm_provider": {"bedrock": {"inference_profile": "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", "region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "experiment_project_ids": [], "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} When the request is sent Then the response status is 422 Unprocessable Entity @@ -1110,6 +1110,13 @@ Feature: LLM Observability When the request is sent Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability + Scenario: List custom evaluator configurations returns "OK" response + Given operation "ListLLMObsCustomEvalConfigs" enabled + And new "ListLLMObsCustomEvalConfigs" request + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: List events for an LLM Observability experiment returns "Bad Request" response Given operation "ListLLMObsExperimentEvents" enabled diff --git a/features/v2/undo.json b/features/v2/undo.json index d93991ac7d73..2a7fb64a0a90 100644 --- a/features/v2/undo.json +++ b/features/v2/undo.json @@ -123,6 +123,12 @@ "type": "safe" } }, + "ListLLMObsCustomEvalConfigs": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, "DeleteLLMObsCustomEvalConfig": { "tag": "LLM Observability", "undo": { diff --git a/lib/datadog_api_client/configuration.rb b/lib/datadog_api_client/configuration.rb index c2ed6cc892b6..9d1245a1250d 100644 --- a/lib/datadog_api_client/configuration.rb +++ b/lib/datadog_api_client/configuration.rb @@ -240,6 +240,7 @@ def initialize "v2.get_llm_obs_prompt": false, "v2.get_llm_obs_prompt_version": false, "v2.list_llm_obs_annotation_queues": false, + "v2.list_llm_obs_custom_eval_configs": false, "v2.list_llm_obs_dataset_records": false, "v2.list_llm_obs_datasets": false, "v2.list_llm_obs_dataset_versions": false, diff --git a/lib/datadog_api_client/inflector.rb b/lib/datadog_api_client/inflector.rb index 2323ebd9b927..376f05d1142a 100644 --- a/lib/datadog_api_client/inflector.rb +++ b/lib/datadog_api_client/inflector.rb @@ -4315,6 +4315,7 @@ def overrides "v2.llm_obs_custom_eval_config_eval_scope" => "LLMObsCustomEvalConfigEvalScope", "v2.llm_obs_custom_eval_config_inference_params" => "LLMObsCustomEvalConfigInferenceParams", "v2.llm_obs_custom_eval_config_integration_provider" => "LLMObsCustomEvalConfigIntegrationProvider", + "v2.llm_obs_custom_eval_config_list_response" => "LLMObsCustomEvalConfigListResponse", "v2.llm_obs_custom_eval_config_llm_judge_config" => "LLMObsCustomEvalConfigLLMJudgeConfig", "v2.llm_obs_custom_eval_config_llm_provider" => "LLMObsCustomEvalConfigLLMProvider", "v2.llm_obs_custom_eval_config_parsing_type" => "LLMObsCustomEvalConfigParsingType", diff --git a/lib/datadog_api_client/v2/api/llm_observability_api.rb b/lib/datadog_api_client/v2/api/llm_observability_api.rb index a0991588041a..c419cc5cb7b5 100644 --- a/lib/datadog_api_client/v2/api/llm_observability_api.rb +++ b/lib/datadog_api_client/v2/api/llm_observability_api.rb @@ -2701,6 +2701,72 @@ def list_llm_obs_annotation_queues_with_http_info(opts = {}) return data, status_code, headers end + # List custom evaluator configurations. + # + # @see #list_llm_obs_custom_eval_configs_with_http_info + def list_llm_obs_custom_eval_configs(opts = {}) + data, _status_code, _headers = list_llm_obs_custom_eval_configs_with_http_info(opts) + data + end + + # List custom evaluator configurations. + # + # List all custom LLM Observability evaluator configurations for the organization. + # + # @param opts [Hash] the optional parameters + # @return [Array<(LLMObsCustomEvalConfigListResponse, Integer, Hash)>] LLMObsCustomEvalConfigListResponse data, response status code and response headers + def list_llm_obs_custom_eval_configs_with_http_info(opts = {}) + unstable_enabled = @api_client.config.unstable_operations["v2.list_llm_obs_custom_eval_configs".to_sym] + if unstable_enabled + @api_client.config.logger.warn format("Using unstable operation '%s'", "v2.list_llm_obs_custom_eval_configs") + else + raise DatadogAPIClient::APIError.new(message: format("Unstable operation '%s' is disabled", "v2.list_llm_obs_custom_eval_configs")) + end + + if @api_client.config.debugging + @api_client.config.logger.debug 'Calling API: LLMObservabilityAPI.list_llm_obs_custom_eval_configs ...' + end + # resource path + local_var_path = '/api/unstable/llm-obs/config/evaluators/custom' + + # query parameters + query_params = opts[:query_params] || {} + + # header parameters + header_params = opts[:header_params] || {} + # HTTP header 'Accept' (if needed) + header_params['Accept'] = @api_client.select_header_accept(['application/json']) + + # form parameters + form_params = opts[:form_params] || {} + + # http body (model) + post_body = opts[:debug_body] + + # return_type + return_type = opts[:debug_return_type] || 'LLMObsCustomEvalConfigListResponse' + + # auth_names + auth_names = opts[:debug_auth_names] || [:apiKeyAuth, :appKeyAuth] + + new_options = opts.merge( + :operation => :list_llm_obs_custom_eval_configs, + :header_params => header_params, + :query_params => query_params, + :form_params => form_params, + :body => post_body, + :auth_names => auth_names, + :return_type => return_type, + :api_version => "V2" + ) + + data, status_code, headers = @api_client.call_api(Net::HTTP::Get, local_var_path, new_options) + if @api_client.config.debugging + @api_client.config.logger.debug "API called: LLMObservabilityAPI#list_llm_obs_custom_eval_configs\nData: #{data.inspect}\nStatus code: #{status_code}\nHeaders: #{headers}" + end + return data, status_code, headers + end + # List LLM Observability dataset records. # # @see #list_llm_obs_dataset_records_with_http_info diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_bedrock_options.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_bedrock_options.rb index 261bed2ec872..558cc15c4485 100644 --- a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_bedrock_options.rb +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_bedrock_options.rb @@ -21,6 +21,9 @@ module DatadogAPIClient::V2 class LLMObsCustomEvalConfigBedrockOptions include BaseGenericModel + # Bedrock inference profile identifier, such as an application inference profile ARN. + attr_accessor :inference_profile + # AWS region for Bedrock. attr_accessor :region @@ -30,6 +33,7 @@ class LLMObsCustomEvalConfigBedrockOptions # @!visibility private def self.attribute_map { + :'inference_profile' => :'inference_profile', :'region' => :'region' } end @@ -38,6 +42,7 @@ def self.attribute_map # @!visibility private def self.openapi_types { + :'inference_profile' => :'String', :'region' => :'String' } end @@ -60,6 +65,10 @@ def initialize(attributes = {}) end } + if attributes.key?(:'inference_profile') + self.inference_profile = attributes[:'inference_profile'] + end + if attributes.key?(:'region') self.region = attributes[:'region'] end @@ -91,6 +100,7 @@ def to_hash def ==(o) return true if self.equal?(o) self.class == o.class && + inference_profile == o.inference_profile && region == o.region && additional_properties == o.additional_properties end @@ -99,7 +109,7 @@ def ==(o) # @return [Integer] Hash code # @!visibility private def hash - [region, additional_properties].hash + [inference_profile, region, additional_properties].hash end end end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_list_response.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_list_response.rb new file mode 100644 index 000000000000..aea95f4d1a79 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_list_response.rb @@ -0,0 +1,125 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # Response containing a list of custom LLM Observability evaluator configurations. + class LLMObsCustomEvalConfigListResponse + include BaseGenericModel + + # List of custom evaluator configuration data objects. + attr_reader :data + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'data' => :'data' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'data' => :'Array' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsCustomEvalConfigListResponse` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'data') + if (value = attributes[:'data']).is_a?(Array) + self.data = value + end + end + end + + # Check to see if the all the properties in the model are valid + # @return true if the model is valid + # @!visibility private + def valid? + return false if @data.nil? + true + end + + # Custom attribute writer method with validation + # @param data [Object] Object to be assigned + # @!visibility private + def data=(data) + if data.nil? + fail ArgumentError, 'invalid value for "data", data cannot be nil.' + end + @data = data + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + data == o.data && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [data, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_llm_judge_config.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_llm_judge_config.rb index 3db3100c6789..c0bac64f8d5c 100644 --- a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_llm_judge_config.rb +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_llm_judge_config.rb @@ -24,6 +24,9 @@ class LLMObsCustomEvalConfigLLMJudgeConfig # Criteria used to assess the pass/fail result of a custom evaluator. attr_accessor :assessment_criteria + # Query used to extract additional context for the evaluation. + attr_accessor :context_query + # LLM inference parameters for a custom evaluator. attr_reader :inference_params @@ -42,6 +45,12 @@ class LLMObsCustomEvalConfigLLMJudgeConfig # List of messages forming the LLM judge prompt template. attr_accessor :prompt_template + # Query used to extract the target value to evaluate. + attr_accessor :target_query + + # User-provided function applied to post-process the JSON output of the LLM judge. + attr_accessor :user_specified_json_post_processing_function + attr_accessor :additional_properties # Attribute mapping from ruby-style variable name to JSON key. @@ -49,12 +58,15 @@ class LLMObsCustomEvalConfigLLMJudgeConfig def self.attribute_map { :'assessment_criteria' => :'assessment_criteria', + :'context_query' => :'context_query', :'inference_params' => :'inference_params', :'last_used_library_prompt_template_name' => :'last_used_library_prompt_template_name', :'modified_library_prompt_template' => :'modified_library_prompt_template', :'output_schema' => :'output_schema', :'parsing_type' => :'parsing_type', - :'prompt_template' => :'prompt_template' + :'prompt_template' => :'prompt_template', + :'target_query' => :'target_query', + :'user_specified_json_post_processing_function' => :'user_specified_json_post_processing_function' } end @@ -63,12 +75,15 @@ def self.attribute_map def self.openapi_types { :'assessment_criteria' => :'LLMObsCustomEvalConfigAssessmentCriteria', + :'context_query' => :'String', :'inference_params' => :'LLMObsCustomEvalConfigInferenceParams', :'last_used_library_prompt_template_name' => :'String', :'modified_library_prompt_template' => :'Boolean', :'output_schema' => :'Hash', :'parsing_type' => :'LLMObsCustomEvalConfigParsingType', - :'prompt_template' => :'Array' + :'prompt_template' => :'Array', + :'target_query' => :'String', + :'user_specified_json_post_processing_function' => :'String' } end @@ -76,9 +91,12 @@ def self.openapi_types # @!visibility private def self.openapi_nullable Set.new([ + :'context_query', :'last_used_library_prompt_template_name', :'modified_library_prompt_template', :'output_schema', + :'target_query', + :'user_specified_json_post_processing_function', ]) end @@ -104,6 +122,10 @@ def initialize(attributes = {}) self.assessment_criteria = attributes[:'assessment_criteria'] end + if attributes.key?(:'context_query') + self.context_query = attributes[:'context_query'] + end + if attributes.key?(:'inference_params') self.inference_params = attributes[:'inference_params'] end @@ -129,6 +151,14 @@ def initialize(attributes = {}) self.prompt_template = value end end + + if attributes.key?(:'target_query') + self.target_query = attributes[:'target_query'] + end + + if attributes.key?(:'user_specified_json_post_processing_function') + self.user_specified_json_post_processing_function = attributes[:'user_specified_json_post_processing_function'] + end end # Check to see if the all the properties in the model are valid @@ -176,12 +206,15 @@ def ==(o) return true if self.equal?(o) self.class == o.class && assessment_criteria == o.assessment_criteria && + context_query == o.context_query && inference_params == o.inference_params && last_used_library_prompt_template_name == o.last_used_library_prompt_template_name && modified_library_prompt_template == o.modified_library_prompt_template && output_schema == o.output_schema && parsing_type == o.parsing_type && prompt_template == o.prompt_template && + target_query == o.target_query && + user_specified_json_post_processing_function == o.user_specified_json_post_processing_function && additional_properties == o.additional_properties end @@ -189,7 +222,7 @@ def ==(o) # @return [Integer] Hash code # @!visibility private def hash - [assessment_criteria, inference_params, last_used_library_prompt_template_name, modified_library_prompt_template, output_schema, parsing_type, prompt_template, additional_properties].hash + [assessment_criteria, context_query, inference_params, last_used_library_prompt_template_name, modified_library_prompt_template, output_schema, parsing_type, prompt_template, target_query, user_specified_json_post_processing_function, additional_properties].hash end end end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_parsing_type.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_parsing_type.rb index fd202fb5ffd4..733e51567317 100644 --- a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_parsing_type.rb +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_parsing_type.rb @@ -23,5 +23,6 @@ class LLMObsCustomEvalConfigParsingType STRUCTURED_OUTPUT = "structured_output".freeze JSON = "json".freeze + KEYWORD_SEARCH = "keyword_search".freeze end end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_target.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_target.rb index ca61ee2c392b..10792ac1de14 100644 --- a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_target.rb +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_target.rb @@ -30,6 +30,9 @@ class LLMObsCustomEvalConfigTarget # Scope at which to evaluate spans. attr_accessor :eval_scope + # Experiment project IDs this evaluator is scoped to. + attr_accessor :experiment_project_ids + # Filter expression to select which spans to evaluate. attr_accessor :filter @@ -48,6 +51,7 @@ def self.attribute_map :'application_name' => :'application_name', :'enabled' => :'enabled', :'eval_scope' => :'eval_scope', + :'experiment_project_ids' => :'experiment_project_ids', :'filter' => :'filter', :'root_spans_only' => :'root_spans_only', :'sampling_percentage' => :'sampling_percentage' @@ -61,6 +65,7 @@ def self.openapi_types :'application_name' => :'String', :'enabled' => :'Boolean', :'eval_scope' => :'LLMObsCustomEvalConfigEvalScope', + :'experiment_project_ids' => :'Array', :'filter' => :'String', :'root_spans_only' => :'Boolean', :'sampling_percentage' => :'Float' @@ -107,6 +112,12 @@ def initialize(attributes = {}) self.eval_scope = attributes[:'eval_scope'] end + if attributes.key?(:'experiment_project_ids') + if (value = attributes[:'experiment_project_ids']).is_a?(Array) + self.experiment_project_ids = value + end + end + if attributes.key?(:'filter') self.filter = attributes[:'filter'] end @@ -178,6 +189,7 @@ def ==(o) application_name == o.application_name && enabled == o.enabled && eval_scope == o.eval_scope && + experiment_project_ids == o.experiment_project_ids && filter == o.filter && root_spans_only == o.root_spans_only && sampling_percentage == o.sampling_percentage && @@ -188,7 +200,7 @@ def ==(o) # @return [Integer] Hash code # @!visibility private def hash - [application_name, enabled, eval_scope, filter, root_spans_only, sampling_percentage, additional_properties].hash + [application_name, enabled, eval_scope, experiment_project_ids, filter, root_spans_only, sampling_percentage, additional_properties].hash end end end