Sign in Dashboard Get started
  1. Packages
  2. Packages
  3. Grafana Cloud
  4. API Docs
  5. agento11y
  6. Evaluator
Viewing docs for Grafana v2.38.0
published on Friday, Aug 7, 2026 by pulumiverse
Schema (JSON)
pulumiverse/pulumi-grafana

grafana.agento11y.Evaluator

grafana logo
Viewing docs for Grafana v2.38.0
published on Friday, Aug 7, 2026 by pulumiverse
Schema (JSON)
pulumiverse/pulumi-grafana

    Manages a Grafana Agent Observability evaluator definition. Evaluators score agent generations or conversations (LLM judge, JSON schema, regex, or heuristic).

    Requires a Grafana instance with the grafana-agento11y-app plugin installed. Writes require a user or service account with the grafana-agento11y-app.eval:write permission, which only the Admin basic role grants by default.

    Example Usage

    import * as pulumi from "@pulumi/pulumi";
import * as grafana from "@pulumiverse/grafana";

const example = new grafana.agento11y.Evaluator("example", {
    evaluatorId: "no_secrets",
    version: "1",
    kind: "regex",
    description: "Flags assistant responses that leak secrets.",
    config: JSON.stringify({
        pattern: "(?i)(api[_-]?key|password|secret)",
    }),
    outputKeys: JSON.stringify([{
        key: "clean",
        type: "bool",
    }]),
});

    import pulumi
import json
import pulumiverse_grafana as grafana

example = grafana.agento11y.Evaluator("example",
    evaluator_id="no_secrets",
    version="1",
    kind="regex",
    description="Flags assistant responses that leak secrets.",
    config=json.dumps({
        "pattern": "(?i)(api[_-]?key|password|secret)",
    }),
    output_keys=json.dumps([{
        "key": "clean",
        "type": "bool",
    }]))

    package main

import (
	"encoding/json"

	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
	"github.com/pulumiverse/pulumi-grafana/sdk/v2/go/grafana/agento11y"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		tmpJSON0, err := json.Marshal(map[string]interface{}{
			"pattern": "(?i)(api[_-]?key|password|secret)",
		})
		if err != nil {
			return err
		}
		json0 := string(tmpJSON0)
		tmpJSON1, err := json.Marshal([]map[string]interface{}{
			map[string]interface{}{
				"key":  "clean",
				"type": "bool",
			},
		})
		if err != nil {
			return err
		}
		json1 := string(tmpJSON1)
		_, err = agento11y.NewEvaluator(ctx, "example", &agento11y.EvaluatorArgs{
			EvaluatorId: pulumi.String("no_secrets"),
			Version:     pulumi.String("1"),
			Kind:        pulumi.String("regex"),
			Description: pulumi.String("Flags assistant responses that leak secrets."),
			Config:      pulumi.String(pulumi.String(json0)),
			OutputKeys:  pulumi.String(pulumi.String(json1)),
		})
		if err != nil {
			return err
		}
		return nil
	})
}

    using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using Pulumi;
using Grafana = Pulumiverse.Grafana;

return await Deployment.RunAsync(() => 
{
    var example = new Grafana.Agento11y.Evaluator("example", new()
    {
        EvaluatorId = "no_secrets",
        Version = "1",
        Kind = "regex",
        Description = "Flags assistant responses that leak secrets.",
        Config = JsonSerializer.Serialize(new Dictionary<string, object?>
        {
            ["pattern"] = "(?i)(api[_-]?key|password|secret)",
        }),
        OutputKeys = JsonSerializer.Serialize(new[]
        {
            new Dictionary<string, object?>
            {
                ["key"] = "clean",
                ["type"] = "bool",
            },
        }),
    });

});

    package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.grafana.agento11y.Evaluator;
import com.pulumi.grafana.agento11y.EvaluatorArgs;
import static com.pulumi.codegen.internal.Serialization.*;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var example = new Evaluator("example", EvaluatorArgs.builder()
            .evaluatorId("no_secrets")
            .version("1")
            .kind("regex")
            .description("Flags assistant responses that leak secrets.")
            .config(serializeJson(
                jsonObject(
                    jsonProperty("pattern", "(?i)(api[_-]?key|password|secret)")
                )))
            .outputKeys(serializeJson(
                jsonArray(jsonObject(
                    jsonProperty("key", "clean"),
                    jsonProperty("type", "bool")
                ))))
            .build());

    }
}

    resources:
  example:
    type: grafana:agento11y:Evaluator
    properties:
      evaluatorId: no_secrets
      version: '1'
      kind: regex
      description: Flags assistant responses that leak secrets.
      config:
        fn::toJSON:
          pattern: (?i)(api[_-]?key|password|secret)
      outputKeys:
        fn::toJSON:
          - key: clean
            type: bool

    Example coming soon!

    Create Evaluator Resource

    Resources are created with functions called constructors. To learn more about declaring and configuring resources, see Resources.

    Constructor syntax

    new Evaluator(name: string, args: EvaluatorArgs, opts?: CustomResourceOptions);
    @overload
def Evaluator(resource_name: str,
              args: EvaluatorArgs,
              opts: Optional[ResourceOptions] = None)

@overload
def Evaluator(resource_name: str,
              opts: Optional[ResourceOptions] = None,
              config: Optional[str] = None,
              evaluator_id: Optional[str] = None,
              kind: Optional[str] = None,
              output_keys: Optional[str] = None,
              version: Optional[str] = None,
              description: Optional[str] = None)
    func NewEvaluator(ctx *Context, name string, args EvaluatorArgs, opts ...ResourceOption) (*Evaluator, error)
    public Evaluator(string name, EvaluatorArgs args, CustomResourceOptions? opts = null)
    public Evaluator(String name, EvaluatorArgs args)
public Evaluator(String name, EvaluatorArgs args, CustomResourceOptions options)

    type: grafana:agento11y:Evaluator
properties: # The arguments to resource properties.
options: # Bag of options to control resource's behavior.


    resource "grafana_agento11y_evaluator" "name" {
    # resource properties
}

    Parameters

    name string
    The unique name of the resource.
    args EvaluatorArgs
    The arguments to resource properties.
    opts CustomResourceOptions
    Bag of options to control resource's behavior.
    resource_name str
    The unique name of the resource.
    args EvaluatorArgs
    The arguments to resource properties.
    opts ResourceOptions
    Bag of options to control resource's behavior.
    ctx Context
    Context object for the current deployment.
    name string
    The unique name of the resource.
    args EvaluatorArgs
    The arguments to resource properties.
    opts ResourceOption
    Bag of options to control resource's behavior.
    name string
    The unique name of the resource.
    args EvaluatorArgs
    The arguments to resource properties.
    opts CustomResourceOptions
    Bag of options to control resource's behavior.
    name String
    The unique name of the resource.
    args EvaluatorArgs
    The arguments to resource properties.
    options CustomResourceOptions
    Bag of options to control resource's behavior.

    Constructor example

    The following reference example uses placeholder values for all input properties.

    var evaluatorResource = new Grafana.Agento11y.Evaluator("evaluatorResource", new()
{
    Config = "string",
    EvaluatorId = "string",
    Kind = "string",
    OutputKeys = "string",
    Version = "string",
    Description = "string",
});

    example, err := agento11y.NewEvaluator(ctx, "evaluatorResource", &agento11y.EvaluatorArgs{
	Config:      pulumi.String("string"),
	EvaluatorId: pulumi.String("string"),
	Kind:        pulumi.String("string"),
	OutputKeys:  pulumi.String("string"),
	Version:     pulumi.String("string"),
	Description: pulumi.String("string"),
})

    resource "grafana_agento11y_evaluator" "evaluatorResource" {
  lifecycle {
    create_before_destroy = true
  }
  config       = "string"
  evaluator_id = "string"
  kind         = "string"
  output_keys  = "string"
  version      = "string"
  description  = "string"
}

    var evaluatorResource = new Evaluator("evaluatorResource", EvaluatorArgs.builder()
    .config("string")
    .evaluatorId("string")
    .kind("string")
    .outputKeys("string")
    .version("string")
    .description("string")
    .build());

    evaluator_resource = grafana.agento11y.Evaluator("evaluatorResource",
    config="string",
    evaluator_id="string",
    kind="string",
    output_keys="string",
    version="string",
    description="string")

    const evaluatorResource = new grafana.agento11y.Evaluator("evaluatorResource", {
    config: "string",
    evaluatorId: "string",
    kind: "string",
    outputKeys: "string",
    version: "string",
    description: "string",
});

    type: grafana:agento11y:Evaluator
properties:
    config: string
    description: string
    evaluatorId: string
    kind: string
    outputKeys: string
    version: string

    Evaluator Resource Properties

    To learn more about resource properties and how to use them, see Inputs and Outputs in the Architecture and Concepts docs.

    Inputs

    In Python, inputs that are objects can be passed either as argument classes or as dictionary literals.

    The Evaluator resource accepts the following input properties:

    Config string
    Kind-specific evaluator configuration, encoded as a JSON object string. The server normalizes this payload, so it is managed from configuration and not refreshed from the API.
    EvaluatorId string
    Tenant-unique identifier of the evaluator. Changing this forces a new resource.
    Kind string
    The evaluator kind. One of llmJudge, jsonSchema, regex, heuristic.
    OutputKeys string
    JSON array of output key definitions produced by the evaluator (for example [{"key":"score","type":"number","passThreshold":0.5}]). Managed from configuration and not refreshed from the API.
    Version string
    Version label of the evaluator definition.
    Description string
    Optional human-readable description of the evaluator.
    Config string
    Kind-specific evaluator configuration, encoded as a JSON object string. The server normalizes this payload, so it is managed from configuration and not refreshed from the API.
    EvaluatorId string
    Tenant-unique identifier of the evaluator. Changing this forces a new resource.
    Kind string
    The evaluator kind. One of llmJudge, jsonSchema, regex, heuristic.
    OutputKeys string
    JSON array of output key definitions produced by the evaluator (for example [{"key":"score","type":"number","passThreshold":0.5}]). Managed from configuration and not refreshed from the API.
    Version string
    Version label of the evaluator definition.
    Description string
    Optional human-readable description of the evaluator.
    config string
    Kind-specific evaluator configuration, encoded as a JSON object string. The server normalizes this payload, so it is managed from configuration and not refreshed from the API.
    evaluator_id string
    Tenant-unique identifier of the evaluator. Changing this forces a new resource.
    kind string
    The evaluator kind. One of llmJudge, jsonSchema, regex, heuristic.
    output_keys string
    JSON array of output key definitions produced by the evaluator (for example [{"key":"score","type":"number","passThreshold":0.5}]). Managed from configuration and not refreshed from the API.
    version string
    Version label of the evaluator definition.
    description string
    Optional human-readable description of the evaluator.
    config String
    Kind-specific evaluator configuration, encoded as a JSON object string. The server normalizes this payload, so it is managed from configuration and not refreshed from the API.
    evaluatorId String
    Tenant-unique identifier of the evaluator. Changing this forces a new resource.
    kind String
    The evaluator kind. One of llmJudge, jsonSchema, regex, heuristic.
    outputKeys String
    JSON array of output key definitions produced by the evaluator (for example [{"key":"score","type":"number","passThreshold":0.5}]). Managed from configuration and not refreshed from the API.
    version String
    Version label of the evaluator definition.
    description String
    Optional human-readable description of the evaluator.
    config string
    Kind-specific evaluator configuration, encoded as a JSON object string. The server normalizes this payload, so it is managed from configuration and not refreshed from the API.
    evaluatorId string
    Tenant-unique identifier of the evaluator. Changing this forces a new resource.
    kind string
    The evaluator kind. One of llmJudge, jsonSchema, regex, heuristic.
    outputKeys string
    JSON array of output key definitions produced by the evaluator (for example [{"key":"score","type":"number","passThreshold":0.5}]). Managed from configuration and not refreshed from the API.
    version string
    Version label of the evaluator definition.
    description string
    Optional human-readable description of the evaluator.
    config str
    Kind-specific evaluator configuration, encoded as a JSON object string. The server normalizes this payload, so it is managed from configuration and not refreshed from the API.
    evaluator_id str
    Tenant-unique identifier of the evaluator. Changing this forces a new resource.
    kind str
    The evaluator kind. One of llmJudge, jsonSchema, regex, heuristic.
    output_keys str
    JSON array of output key definitions produced by the evaluator (for example [{"key":"score","type":"number","passThreshold":0.5}]). Managed from configuration and not refreshed from the API.
    version str
    Version label of the evaluator definition.
    description str
    Optional human-readable description of the evaluator.
    config String
    Kind-specific evaluator configuration, encoded as a JSON object string. The server normalizes this payload, so it is managed from configuration and not refreshed from the API.
    evaluatorId String
    Tenant-unique identifier of the evaluator. Changing this forces a new resource.
    kind String
    The evaluator kind. One of llmJudge, jsonSchema, regex, heuristic.
    outputKeys String
    JSON array of output key definitions produced by the evaluator (for example [{"key":"score","type":"number","passThreshold":0.5}]). Managed from configuration and not refreshed from the API.
    version String
    Version label of the evaluator definition.
    description String
    Optional human-readable description of the evaluator.

    Outputs

    All input properties are implicitly available as output properties. Additionally, the Evaluator resource produces the following output properties:

    Id string
    The provider-assigned unique ID for this managed resource.
    Id string
    The provider-assigned unique ID for this managed resource.
    id string
    The provider-assigned unique ID for this managed resource.
    id String
    The provider-assigned unique ID for this managed resource.
    id string
    The provider-assigned unique ID for this managed resource.
    id str
    The provider-assigned unique ID for this managed resource.
    id String
    The provider-assigned unique ID for this managed resource.

    Look up Existing Evaluator Resource

    Get an existing Evaluator resource’s state with the given name, ID, and optional extra properties used to qualify the lookup.

    public static get(name: string, id: Input<ID>, state?: EvaluatorState, opts?: CustomResourceOptions): Evaluator
    @staticmethod
def get(resource_name: str,
        id: str,
        opts: Optional[ResourceOptions] = None,
        config: Optional[str] = None,
        description: Optional[str] = None,
        evaluator_id: Optional[str] = None,
        kind: Optional[str] = None,
        output_keys: Optional[str] = None,
        version: Optional[str] = None) -> Evaluator
    func GetEvaluator(ctx *Context, name string, id IDInput, state *EvaluatorState, opts ...ResourceOption) (*Evaluator, error)
    public static Evaluator Get(string name, Input<string> id, EvaluatorState? state, CustomResourceOptions? opts = null)
    public static Evaluator get(String name, Output<String> id, EvaluatorState state, CustomResourceOptions options)
    resources:  _:    type: grafana:agento11y:Evaluator    get:      id: ${id}
    import {
  to = grafana_agento11y_evaluator.example
  id = "${id}"
}
    name
    The unique name of the resulting resource.
    id
    The unique provider ID of the resource to lookup.
    state
    Any extra arguments used during the lookup.
    opts
    A bag of options that control this resource's behavior.
    resource_name
    The unique name of the resulting resource.
    id
    The unique provider ID of the resource to lookup.
    name
    The unique name of the resulting resource.
    id
    The unique provider ID of the resource to lookup.
    state
    Any extra arguments used during the lookup.
    opts
    A bag of options that control this resource's behavior.
    name
    The unique name of the resulting resource.
    id
    The unique provider ID of the resource to lookup.
    state
    Any extra arguments used during the lookup.
    opts
    A bag of options that control this resource's behavior.
    name
    The unique name of the resulting resource.
    id
    The unique provider ID of the resource to lookup.
    state
    Any extra arguments used during the lookup.
    opts
    A bag of options that control this resource's behavior.
    The following state arguments are supported:
    Config string
    Kind-specific evaluator configuration, encoded as a JSON object string. The server normalizes this payload, so it is managed from configuration and not refreshed from the API.
    Description string
    Optional human-readable description of the evaluator.
    EvaluatorId string
    Tenant-unique identifier of the evaluator. Changing this forces a new resource.
    Kind string
    The evaluator kind. One of llmJudge, jsonSchema, regex, heuristic.
    OutputKeys string
    JSON array of output key definitions produced by the evaluator (for example [{"key":"score","type":"number","passThreshold":0.5}]). Managed from configuration and not refreshed from the API.
    Version string
    Version label of the evaluator definition.
    Config string
    Kind-specific evaluator configuration, encoded as a JSON object string. The server normalizes this payload, so it is managed from configuration and not refreshed from the API.
    Description string
    Optional human-readable description of the evaluator.
    EvaluatorId string
    Tenant-unique identifier of the evaluator. Changing this forces a new resource.
    Kind string
    The evaluator kind. One of llmJudge, jsonSchema, regex, heuristic.
    OutputKeys string
    JSON array of output key definitions produced by the evaluator (for example [{"key":"score","type":"number","passThreshold":0.5}]). Managed from configuration and not refreshed from the API.
    Version string
    Version label of the evaluator definition.
    config string
    Kind-specific evaluator configuration, encoded as a JSON object string. The server normalizes this payload, so it is managed from configuration and not refreshed from the API.
    description string
    Optional human-readable description of the evaluator.
    evaluator_id string
    Tenant-unique identifier of the evaluator. Changing this forces a new resource.
    kind string
    The evaluator kind. One of llmJudge, jsonSchema, regex, heuristic.
    output_keys string
    JSON array of output key definitions produced by the evaluator (for example [{"key":"score","type":"number","passThreshold":0.5}]). Managed from configuration and not refreshed from the API.
    version string
    Version label of the evaluator definition.
    config String
    Kind-specific evaluator configuration, encoded as a JSON object string. The server normalizes this payload, so it is managed from configuration and not refreshed from the API.
    description String
    Optional human-readable description of the evaluator.
    evaluatorId String
    Tenant-unique identifier of the evaluator. Changing this forces a new resource.
    kind String
    The evaluator kind. One of llmJudge, jsonSchema, regex, heuristic.
    outputKeys String
    JSON array of output key definitions produced by the evaluator (for example [{"key":"score","type":"number","passThreshold":0.5}]). Managed from configuration and not refreshed from the API.
    version String
    Version label of the evaluator definition.
    config string
    Kind-specific evaluator configuration, encoded as a JSON object string. The server normalizes this payload, so it is managed from configuration and not refreshed from the API.
    description string
    Optional human-readable description of the evaluator.
    evaluatorId string
    Tenant-unique identifier of the evaluator. Changing this forces a new resource.
    kind string
    The evaluator kind. One of llmJudge, jsonSchema, regex, heuristic.
    outputKeys string
    JSON array of output key definitions produced by the evaluator (for example [{"key":"score","type":"number","passThreshold":0.5}]). Managed from configuration and not refreshed from the API.
    version string
    Version label of the evaluator definition.
    config str
    Kind-specific evaluator configuration, encoded as a JSON object string. The server normalizes this payload, so it is managed from configuration and not refreshed from the API.
    description str
    Optional human-readable description of the evaluator.
    evaluator_id str
    Tenant-unique identifier of the evaluator. Changing this forces a new resource.
    kind str
    The evaluator kind. One of llmJudge, jsonSchema, regex, heuristic.
    output_keys str
    JSON array of output key definitions produced by the evaluator (for example [{"key":"score","type":"number","passThreshold":0.5}]). Managed from configuration and not refreshed from the API.
    version str
    Version label of the evaluator definition.
    config String
    Kind-specific evaluator configuration, encoded as a JSON object string. The server normalizes this payload, so it is managed from configuration and not refreshed from the API.
    description String
    Optional human-readable description of the evaluator.
    evaluatorId String
    Tenant-unique identifier of the evaluator. Changing this forces a new resource.
    kind String
    The evaluator kind. One of llmJudge, jsonSchema, regex, heuristic.
    outputKeys String
    JSON array of output key definitions produced by the evaluator (for example [{"key":"score","type":"number","passThreshold":0.5}]). Managed from configuration and not refreshed from the API.
    version String
    Version label of the evaluator definition.

    Import

    terraform import grafana_agento11y_evaluator.name "{{ evaluator_id }}"

    To learn more about importing existing cloud resources, see Importing resources.

    Package Details

    Repository
    grafana pulumiverse/pulumi-grafana
    License
    Apache-2.0
    Notes
    This Pulumi package is based on the grafana Terraform Provider.
    grafana logo
    Viewing docs for Grafana v2.38.0
    published on Friday, Aug 7, 2026 by pulumiverse
    Schema (JSON)
    pulumiverse/pulumi-grafana

      Try Pulumi Cloud free.
      Your team will thank you.

      Start free trial