Add list custom LLM Obs evaluator configs endpoint

.generator/schemas/v2/openapi.yaml

@@ -52338,6 +52338,10 @@ components:
     LLMObsCustomEvalConfigBedrockOptions:
       description: AWS Bedrock-specific options for LLM provider configuration.
       properties:
+        inference_profile:
+          description: Bedrock inference profile identifier, such as an application inference profile ARN.
+          example: "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123"
+          type: string
         region:
           description: AWS region for Bedrock.
           example: "us-east-1"
@@ -52428,6 +52432,11 @@ components:
       properties:
         assessment_criteria:
           $ref: "#/components/schemas/LLMObsCustomEvalConfigAssessmentCriteria"
+        context_query:
+          description: Query used to extract additional context for the evaluation.
+          example: "@input.context"
+          nullable: true
+          type: string
         inference_params:
           $ref: "#/components/schemas/LLMObsCustomEvalConfigInferenceParams"
         last_used_library_prompt_template_name:
@@ -52452,6 +52461,15 @@ components:
           items:
             $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptMessage"
           type: array
+        target_query:
+          description: Query used to extract the target value to evaluate.
+          example: "@output.value"
+          nullable: true
+          type: string
+        user_specified_json_post_processing_function:
+          description: User-provided function applied to post-process the JSON output of the LLM judge.
+          nullable: true
+          type: string
       required:
         - inference_params
       type: object
@@ -52473,16 +52491,29 @@ components:
         vertex_ai:
           $ref: "#/components/schemas/LLMObsCustomEvalConfigVertexAIOptions"
       type: object
+    LLMObsCustomEvalConfigListResponse:
+      description: Response containing a list of custom LLM Observability evaluator configurations.
+      properties:
+        data:
+          description: List of custom evaluator configuration data objects.
+          items:
+            $ref: "#/components/schemas/LLMObsCustomEvalConfigData"
+          type: array
+      required:
+        - data
+      type: object
     LLMObsCustomEvalConfigParsingType:
       description: Output parsing type for a custom LLM judge evaluator.
       enum:
         - structured_output
         - json
+        - keyword_search
       example: "structured_output"
       type: string
       x-enum-varnames:
         - STRUCTURED_OUTPUT
         - JSON
+        - KEYWORD_SEARCH
     LLMObsCustomEvalConfigPromptContent:
       description: A content block within a prompt message.
       properties:
@@ -52589,6 +52620,13 @@ components:
         eval_scope:
           $ref: "#/components/schemas/LLMObsCustomEvalConfigEvalScope"
           nullable: true
+        experiment_project_ids:
+          description: Experiment project IDs this evaluator is scoped to.
+          items:
+            description: An experiment project ID.
+            format: uuid
+            type: string
+          type: array
         filter:
           description: Filter expression to select which spans to evaluate.
           example: "@service:my-service"
@@ -116425,6 +116463,67 @@ paths:
       x-unstable: |-
         This endpoint is in Preview and may introduce breaking changes.
         If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/).
+  /api/unstable/llm-obs/config/evaluators/custom:
+    get:
+      description: List all custom LLM Observability evaluator configurations for the organization.
+      operationId: ListLLMObsCustomEvalConfigs
+      responses:
+        "200":
+          content:
+            application/json:
+              examples:
+                default:
+                  value:
+                    data:
+                      - attributes:
+                          category: "Custom"
+                          created_at: "2024-01-15T10:30:00Z"
+                          created_by:
+                            email: "user@example.com"
+                          eval_name: "my-custom-evaluator"
+                          last_updated_by:
+                            email: "user@example.com"
+                          llm_judge_config:
+                            inference_params:
+                              max_tokens: 1024
+                              temperature: 0.7
+                            parsing_type: "structured_output"
+                          llm_provider:
+                            integration_provider: "openai"
+                            model_name: "gpt-4o"
+                          target:
+                            application_name: "my-llm-app"
+                            enabled: true
+                            sampling_percentage: 50.0
+                          updated_at: "2024-01-15T10:30:00Z"
+                        id: "my-custom-evaluator"
+                        type: "evaluator_config"
+              schema:
+                $ref: "#/components/schemas/LLMObsCustomEvalConfigListResponse"
+          description: OK
+        "401":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/JSONAPIErrorResponse"
+          description: Unauthorized
+        "403":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/JSONAPIErrorResponse"
+          description: Forbidden
+        "429":
+          $ref: "#/components/responses/TooManyRequestsResponse"
+      security:
+        - apiKeyAuth: []
+          appKeyAuth: []
+      summary: List custom evaluator configurations
+      tags:
+        - LLM Observability
+      x-unstable: |-
+        **Note**: This endpoint is in preview and is subject to change.
+        If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/).
   /api/unstable/llm-obs/config/evaluators/custom/{eval_name}:
     delete:
       description: Delete a custom LLM Observability evaluator configuration by its name.

examples/v2/llm-observability/ListLLMObsCustomEvalConfigs.rb

@@ -0,0 +1,8 @@
+# List custom evaluator configurations returns "OK" response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.unstable_operations["v2.list_llm_obs_custom_eval_configs".to_sym] = true
+end
+api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new
+p api_instance.list_llm_obs_custom_eval_configs()

examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.rb

@@ -21,6 +21,7 @@
           ],
           pass_when: true,
         }),
+        context_query: "@input.context",
         inference_params: DatadogAPIClient::V2::LLMObsCustomEvalConfigInferenceParams.new({
           frequency_penalty: 0.0,
           max_tokens: 1024,
@@ -59,9 +60,12 @@
             role: "user",
           }),
         ],
+        target_query: "@output.value",
+        user_specified_json_post_processing_function: nil,
       }),
       llm_provider: DatadogAPIClient::V2::LLMObsCustomEvalConfigLLMProvider.new({
         bedrock: DatadogAPIClient::V2::LLMObsCustomEvalConfigBedrockOptions.new({
+          inference_profile: "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123",
           region: "us-east-1",
         }),
         integration_account_id: "my-account-id",
@@ -76,6 +80,7 @@
         application_name: "my-llm-app",
         enabled: true,
         eval_scope: DatadogAPIClient::V2::LLMObsCustomEvalConfigEvalScope::SPAN,
+        experiment_project_ids: [],
         filter: "@service:my-service",
         root_spans_only: true,
         sampling_percentage: 50.0,