published on Wednesday, Nov 18, 2026 by Pulumi
published on Wednesday, Nov 18, 2026 by Pulumi
An evaluation item represents a single prompt and optional model/candidate responses, ground truth, and rubrics for evaluation in Vertex AI.
To get more information about EvaluationItem, see:
- API documentation
- How-to Guides
Example Usage
Vertex Ai Evaluation Item
import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";
const evaluationItem = new gcp.vertex.AiEvaluationItem("evaluation_item", {
region: "us-central1",
displayName: "eval-item",
evaluationItemType: "REQUEST",
labels: {
env: "test",
team: "evaluation",
},
metadata: JSON.stringify({
author: "terraform-test",
version: "1.0",
}),
evaluationRequest: {
prompt: {
text: "What is the capital of France?",
},
goldenResponse: {
candidate: "ground-truth",
text: "Paris",
},
candidateResponses: [
{
candidate: "model-a",
text: "Paris",
},
{
candidate: "model-b",
value: JSON.stringify({
response: "Paris",
}),
},
{
candidate: "model-c",
agentData: JSON.stringify({
agents: {
agent_1: {
agentId: "agent_1",
agentType: "LlmAgent",
description: "Assistant",
instruction: "Be helpful",
},
},
}),
},
],
rubrics: [{
name: "test_rubric_group",
groupId: "test_rubric_group_id",
displayName: "Test Rubric Group",
rubrics: [{
rubricId: "test_rubric_1",
type: "SUMMARIZATION_QUALITY",
importance: "HIGH",
content: {
property: {
description: "The model response is concise and accurate.",
},
},
}],
}],
},
});
import pulumi
import json
import pulumi_gcp as gcp
evaluation_item = gcp.vertex.AiEvaluationItem("evaluation_item",
region="us-central1",
display_name="eval-item",
evaluation_item_type="REQUEST",
labels={
"env": "test",
"team": "evaluation",
},
metadata=json.dumps({
"author": "terraform-test",
"version": "1.0",
}),
evaluation_request={
"prompt": {
"text": "What is the capital of France?",
},
"golden_response": {
"candidate": "ground-truth",
"text": "Paris",
},
"candidate_responses": [
{
"candidate": "model-a",
"text": "Paris",
},
{
"candidate": "model-b",
"value": json.dumps({
"response": "Paris",
}),
},
{
"candidate": "model-c",
"agent_data": json.dumps({
"agents": {
"agent_1": {
"agentId": "agent_1",
"agentType": "LlmAgent",
"description": "Assistant",
"instruction": "Be helpful",
},
},
}),
},
],
"rubrics": [{
"name": "test_rubric_group",
"group_id": "test_rubric_group_id",
"display_name": "Test Rubric Group",
"rubrics": [{
"rubric_id": "test_rubric_1",
"type": "SUMMARIZATION_QUALITY",
"importance": "HIGH",
"content": {
"property": {
"description": "The model response is concise and accurate.",
},
},
}],
}],
})
package main
import (
"encoding/json"
"github.com/pulumi/pulumi-gcp/sdk/v10/go/gcp/vertex"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
tmpJSON0, err := json.Marshal(map[string]string{
"author": "terraform-test",
"version": "1.0",
})
if err != nil {
return err
}
json0 := string(tmpJSON0)
tmpJSON1, err := json.Marshal(map[string]string{
"response": "Paris",
})
if err != nil {
return err
}
json1 := string(tmpJSON1)
tmpJSON2, err := json.Marshal(map[string]map[string]map[string]string{
"agents": map[string]map[string]string{
"agent_1": map[string]string{
"agentId": "agent_1",
"agentType": "LlmAgent",
"description": "Assistant",
"instruction": "Be helpful",
},
},
})
if err != nil {
return err
}
json2 := string(tmpJSON2)
_, err = vertex.NewAiEvaluationItem(ctx, "evaluation_item", &vertex.AiEvaluationItemArgs{
Region: pulumi.String("us-central1"),
DisplayName: pulumi.String("eval-item"),
EvaluationItemType: pulumi.String("REQUEST"),
Labels: pulumi.StringMap{
"env": pulumi.String("test"),
"team": pulumi.String("evaluation"),
},
Metadata: pulumi.String(json0),
EvaluationRequest: &vertex.AiEvaluationItemEvaluationRequestArgs{
Prompt: &vertex.AiEvaluationItemEvaluationRequestPromptArgs{
Text: pulumi.String("What is the capital of France?"),
},
GoldenResponse: &vertex.AiEvaluationItemEvaluationRequestGoldenResponseArgs{
Candidate: pulumi.String("ground-truth"),
Text: pulumi.String("Paris"),
},
CandidateResponses: vertex.AiEvaluationItemEvaluationRequestCandidateResponseArray{
&vertex.AiEvaluationItemEvaluationRequestCandidateResponseArgs{
Candidate: pulumi.String("model-a"),
Text: pulumi.String("Paris"),
},
&vertex.AiEvaluationItemEvaluationRequestCandidateResponseArgs{
Candidate: pulumi.String("model-b"),
Value: pulumi.String(json1),
},
&vertex.AiEvaluationItemEvaluationRequestCandidateResponseArgs{
Candidate: pulumi.String("model-c"),
AgentData: pulumi.String(json2),
},
},
Rubrics: vertex.AiEvaluationItemEvaluationRequestRubricArray{
&vertex.AiEvaluationItemEvaluationRequestRubricArgs{
Name: pulumi.String("test_rubric_group"),
GroupId: pulumi.String("test_rubric_group_id"),
DisplayName: pulumi.String("Test Rubric Group"),
Rubrics: vertex.AiEvaluationItemEvaluationRequestRubricRubricArray{
&vertex.AiEvaluationItemEvaluationRequestRubricRubricArgs{
RubricId: pulumi.String("test_rubric_1"),
Type: pulumi.String("SUMMARIZATION_QUALITY"),
Importance: pulumi.String("HIGH"),
Content: &vertex.AiEvaluationItemEvaluationRequestRubricRubricContentArgs{
Property: &vertex.AiEvaluationItemEvaluationRequestRubricRubricContentPropertyArgs{
Description: pulumi.String("The model response is concise and accurate."),
},
},
},
},
},
},
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using Pulumi;
using Gcp = Pulumi.Gcp;
return await Deployment.RunAsync(() =>
{
var evaluationItem = new Gcp.Vertex.AiEvaluationItem("evaluation_item", new()
{
Region = "us-central1",
DisplayName = "eval-item",
EvaluationItemType = "REQUEST",
Labels =
{
{ "env", "test" },
{ "team", "evaluation" },
},
Metadata = JsonSerializer.Serialize(new Dictionary<string, object?>
{
["author"] = "terraform-test",
["version"] = "1.0",
}),
EvaluationRequest = new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestArgs
{
Prompt = new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestPromptArgs
{
Text = "What is the capital of France?",
},
GoldenResponse = new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestGoldenResponseArgs
{
Candidate = "ground-truth",
Text = "Paris",
},
CandidateResponses = new[]
{
new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestCandidateResponseArgs
{
Candidate = "model-a",
Text = "Paris",
},
new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestCandidateResponseArgs
{
Candidate = "model-b",
Value = JsonSerializer.Serialize(new Dictionary<string, object?>
{
["response"] = "Paris",
}),
},
new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestCandidateResponseArgs
{
Candidate = "model-c",
AgentData = JsonSerializer.Serialize(new Dictionary<string, object?>
{
["agents"] = new Dictionary<string, object?>
{
["agent_1"] = new Dictionary<string, object?>
{
["agentId"] = "agent_1",
["agentType"] = "LlmAgent",
["description"] = "Assistant",
["instruction"] = "Be helpful",
},
},
}),
},
},
Rubrics = new[]
{
new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestRubricArgs
{
Name = "test_rubric_group",
GroupId = "test_rubric_group_id",
DisplayName = "Test Rubric Group",
Rubrics = new[]
{
new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestRubricRubricArgs
{
RubricId = "test_rubric_1",
Type = "SUMMARIZATION_QUALITY",
Importance = "HIGH",
Content = new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestRubricRubricContentArgs
{
Property = new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestRubricRubricContentPropertyArgs
{
Description = "The model response is concise and accurate.",
},
},
},
},
},
},
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.vertex.AiEvaluationItem;
import com.pulumi.gcp.vertex.AiEvaluationItemArgs;
import com.pulumi.gcp.vertex.inputs.AiEvaluationItemEvaluationRequestArgs;
import com.pulumi.gcp.vertex.inputs.AiEvaluationItemEvaluationRequestPromptArgs;
import com.pulumi.gcp.vertex.inputs.AiEvaluationItemEvaluationRequestGoldenResponseArgs;
import com.pulumi.gcp.vertex.inputs.AiEvaluationItemEvaluationRequestCandidateResponseArgs;
import com.pulumi.gcp.vertex.inputs.AiEvaluationItemEvaluationRequestRubricArgs;
import com.pulumi.gcp.vertex.inputs.AiEvaluationItemEvaluationRequestRubricRubricArgs;
import com.pulumi.gcp.vertex.inputs.AiEvaluationItemEvaluationRequestRubricRubricContentArgs;
import com.pulumi.gcp.vertex.inputs.AiEvaluationItemEvaluationRequestRubricRubricContentPropertyArgs;
import static com.pulumi.codegen.internal.Serialization.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var evaluationItem = new AiEvaluationItem("evaluationItem", AiEvaluationItemArgs.builder()
.region("us-central1")
.displayName("eval-item")
.evaluationItemType("REQUEST")
.labels(Map.ofEntries(
Map.entry("env", "test"),
Map.entry("team", "evaluation")
))
.metadata(serializeJson(
jsonObject(
jsonProperty("author", "terraform-test"),
jsonProperty("version", "1.0")
)))
.evaluationRequest(AiEvaluationItemEvaluationRequestArgs.builder()
.prompt(AiEvaluationItemEvaluationRequestPromptArgs.builder()
.text("What is the capital of France?")
.build())
.goldenResponse(AiEvaluationItemEvaluationRequestGoldenResponseArgs.builder()
.candidate("ground-truth")
.text("Paris")
.build())
.candidateResponses(
AiEvaluationItemEvaluationRequestCandidateResponseArgs.builder()
.candidate("model-a")
.text("Paris")
.build(),
AiEvaluationItemEvaluationRequestCandidateResponseArgs.builder()
.candidate("model-b")
.value(serializeJson(
jsonObject(
jsonProperty("response", "Paris")
)))
.build(),
AiEvaluationItemEvaluationRequestCandidateResponseArgs.builder()
.candidate("model-c")
.agentData(serializeJson(
jsonObject(
jsonProperty("agents", jsonObject(
jsonProperty("agent_1", jsonObject(
jsonProperty("agentId", "agent_1"),
jsonProperty("agentType", "LlmAgent"),
jsonProperty("description", "Assistant"),
jsonProperty("instruction", "Be helpful")
))
))
)))
.build())
.rubrics(AiEvaluationItemEvaluationRequestRubricArgs.builder()
.name("test_rubric_group")
.groupId("test_rubric_group_id")
.displayName("Test Rubric Group")
.rubrics(AiEvaluationItemEvaluationRequestRubricRubricArgs.builder()
.rubricId("test_rubric_1")
.type("SUMMARIZATION_QUALITY")
.importance("HIGH")
.content(AiEvaluationItemEvaluationRequestRubricRubricContentArgs.builder()
.property(AiEvaluationItemEvaluationRequestRubricRubricContentPropertyArgs.builder()
.description("The model response is concise and accurate.")
.build())
.build())
.build())
.build())
.build())
.build());
}
}
resources:
evaluationItem:
type: gcp:vertex:AiEvaluationItem
name: evaluation_item
properties:
region: us-central1
displayName: eval-item
evaluationItemType: REQUEST
labels:
env: test
team: evaluation
metadata:
fn::toJSON:
author: terraform-test
version: '1.0'
evaluationRequest:
prompt:
text: What is the capital of France?
goldenResponse:
candidate: ground-truth
text: Paris
candidateResponses:
- candidate: model-a
text: Paris
- candidate: model-b
value:
fn::toJSON:
response: Paris
- candidate: model-c
agentData:
fn::toJSON:
agents:
agent_1:
agentId: agent_1
agentType: LlmAgent
description: Assistant
instruction: Be helpful
rubrics:
- name: test_rubric_group
groupId: test_rubric_group_id
displayName: Test Rubric Group
rubrics:
- rubricId: test_rubric_1
type: SUMMARIZATION_QUALITY
importance: HIGH
content:
property:
description: The model response is concise and accurate.
terraform {
required_providers {
gcp = {
source = "pulumi/gcp"
}
}
}
resource "gcp_vertex_ai_evaluation_item" "evaluation_item" {
lifecycle {
create_before_destroy = true
}
region = "us-central1"
display_name = "eval-item"
evaluation_item_type = "REQUEST"
labels = {
"env" = "test"
"team" = "evaluation"
}
metadata = jsonencode({
"author" = "terraform-test"
"version" = "1.0"
})
evaluation_request = {
prompt = {
text = "What is the capital of France?"
}
golden_response = {
candidate = "ground-truth"
text = "Paris"
}
candidate_responses = [{
candidate = "model-a"
text = "Paris"
}, {
candidate = "model-b"
value = jsonencode({
"response" = "Paris"
})
}, {
candidate = "model-c"
agent_data = jsonencode({
"agents" = {
"agent_1" = {
"agentId" = "agent_1"
"agentType" = "LlmAgent"
"description" = "Assistant"
"instruction" = "Be helpful"
}
}
})
}]
rubrics = [{
name = "test_rubric_group"
group_id = "test_rubric_group_id"
display_name = "Test Rubric Group"
rubrics = [{
rubric_id = "test_rubric_1"
type = "SUMMARIZATION_QUALITY"
importance = "HIGH"
content = {
property = {
description = "The model response is concise and accurate."
}
}
}]
}]
}
}
Vertex Ai Evaluation Item Gcs
import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";
const project = gcp.organizations.getProject({});
const bucket = new gcp.storage.Bucket("bucket", {
name: "eval-item-bucket",
location: "us-central1",
uniformBucketLevelAccess: true,
forceDestroy: true,
});
const viewer = new gcp.storage.BucketIAMMember("viewer", {
bucket: bucket.name,
role: "roles/storage.objectViewer",
member: project.then(project => `serviceAccount:service-${project.number}@gcp-sa-aiplatform.iam.gserviceaccount.com`),
});
const object = new gcp.storage.BucketObject("object", {
name: "eval_request.json",
bucket: bucket.name,
content: JSON.stringify({
prompt: {
text: "What is the capital of France?",
},
}),
});
const evaluationItemGcs = new gcp.vertex.AiEvaluationItem("evaluation_item_gcs", {
region: "us-central1",
displayName: "eval-item",
evaluationItemType: "REQUEST",
gcsUri: pulumi.interpolate`gs://${bucket.name}/${object.name}`,
}, {
dependsOn: [
viewer,
object,
],
});
import pulumi
import json
import pulumi_gcp as gcp
project = gcp.organizations.get_project()
bucket = gcp.storage.Bucket("bucket",
name="eval-item-bucket",
location="us-central1",
uniform_bucket_level_access=True,
force_destroy=True)
viewer = gcp.storage.BucketIAMMember("viewer",
bucket=bucket.name,
role="roles/storage.objectViewer",
member=f"serviceAccount:service-{project.number}@gcp-sa-aiplatform.iam.gserviceaccount.com")
object = gcp.storage.BucketObject("object",
name="eval_request.json",
bucket=bucket.name,
content=json.dumps({
"prompt": {
"text": "What is the capital of France?",
},
}))
evaluation_item_gcs = gcp.vertex.AiEvaluationItem("evaluation_item_gcs",
region="us-central1",
display_name="eval-item",
evaluation_item_type="REQUEST",
gcs_uri=pulumi.Output.all(
bucketName=bucket.name,
objectName=object.name
).apply(lambda resolved_outputs: f"gs://{resolved_outputs['bucketName']}/{resolved_outputs['objectName']}")
,
opts = pulumi.ResourceOptions(depends_on=[
viewer,
object,
]))
package main
import (
"encoding/json"
"fmt"
"github.com/pulumi/pulumi-gcp/sdk/v10/go/gcp/organizations"
"github.com/pulumi/pulumi-gcp/sdk/v10/go/gcp/storage"
"github.com/pulumi/pulumi-gcp/sdk/v10/go/gcp/vertex"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
project, err := organizations.LookupProject(ctx, &organizations.LookupProjectArgs{}, nil)
if err != nil {
return err
}
bucket, err := storage.NewBucket(ctx, "bucket", &storage.BucketArgs{
Name: pulumi.String("eval-item-bucket"),
Location: pulumi.String("us-central1"),
UniformBucketLevelAccess: pulumi.Bool(true),
ForceDestroy: pulumi.Bool(true),
})
if err != nil {
return err
}
viewer, err := storage.NewBucketIAMMember(ctx, "viewer", &storage.BucketIAMMemberArgs{
Bucket: bucket.Name,
Role: pulumi.String("roles/storage.objectViewer"),
Member: pulumi.Sprintf("serviceAccount:service-%v@gcp-sa-aiplatform.iam.gserviceaccount.com", project.Number),
})
if err != nil {
return err
}
tmpJSON0, err := json.Marshal(map[string]map[string]string{
"prompt": map[string]string{
"text": "What is the capital of France?",
},
})
if err != nil {
return err
}
json0 := string(tmpJSON0)
object, err := storage.NewBucketObject(ctx, "object", &storage.BucketObjectArgs{
Name: pulumi.String("eval_request.json"),
Bucket: bucket.Name,
Content: pulumi.String(json0),
})
if err != nil {
return err
}
_, err = vertex.NewAiEvaluationItem(ctx, "evaluation_item_gcs", &vertex.AiEvaluationItemArgs{
Region: pulumi.String("us-central1"),
DisplayName: pulumi.String("eval-item"),
EvaluationItemType: pulumi.String("REQUEST"),
GcsUri: pulumi.All(bucket.Name, object.Name).ApplyT(func(_args []interface{}) (string, error) {
bucketName := _args[0].(string)
objectName := _args[1].(string)
return fmt.Sprintf("gs://%v/%v", bucketName, objectName), nil
}).(pulumi.StringOutput),
}, pulumi.DependsOn([]pulumi.Resource{
viewer,
object,
}))
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using Pulumi;
using Gcp = Pulumi.Gcp;
return await Deployment.RunAsync(() =>
{
var project = Gcp.Organizations.GetProject.Invoke();
var bucket = new Gcp.Storage.Bucket("bucket", new()
{
Name = "eval-item-bucket",
Location = "us-central1",
UniformBucketLevelAccess = true,
ForceDestroy = true,
});
var viewer = new Gcp.Storage.BucketIAMMember("viewer", new()
{
Bucket = bucket.Name,
Role = "roles/storage.objectViewer",
Member = $"serviceAccount:service-{project.Apply(getProjectResult => getProjectResult.Number)}@gcp-sa-aiplatform.iam.gserviceaccount.com",
});
var @object = new Gcp.Storage.BucketObject("object", new()
{
Name = "eval_request.json",
Bucket = bucket.Name,
Content = JsonSerializer.Serialize(new Dictionary<string, object?>
{
["prompt"] = new Dictionary<string, object?>
{
["text"] = "What is the capital of France?",
},
}),
});
var evaluationItemGcs = new Gcp.Vertex.AiEvaluationItem("evaluation_item_gcs", new()
{
Region = "us-central1",
DisplayName = "eval-item",
EvaluationItemType = "REQUEST",
GcsUri = Output.Tuple(bucket.Name, @object.Name).Apply(values =>
{
var bucketName = values.Item1;
var objectName = values.Item2;
return $"gs://{bucketName}/{objectName}";
}),
}, new CustomResourceOptions
{
DependsOn =
{
viewer,
@object,
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.organizations.OrganizationsFunctions;
import com.pulumi.gcp.organizations.inputs.GetProjectArgs;
import com.pulumi.gcp.storage.Bucket;
import com.pulumi.gcp.storage.BucketArgs;
import com.pulumi.gcp.storage.BucketIAMMember;
import com.pulumi.gcp.storage.BucketIAMMemberArgs;
import com.pulumi.gcp.storage.BucketObject;
import com.pulumi.gcp.storage.BucketObjectArgs;
import com.pulumi.gcp.vertex.AiEvaluationItem;
import com.pulumi.gcp.vertex.AiEvaluationItemArgs;
import static com.pulumi.codegen.internal.Serialization.*;
import com.pulumi.resources.CustomResourceOptions;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
final var project = OrganizationsFunctions.getProject(GetProjectArgs.builder()
.build());
var bucket = new Bucket("bucket", BucketArgs.builder()
.name("eval-item-bucket")
.location("us-central1")
.uniformBucketLevelAccess(true)
.forceDestroy(true)
.build());
var viewer = new BucketIAMMember("viewer", BucketIAMMemberArgs.builder()
.bucket(bucket.name())
.role("roles/storage.objectViewer")
.member(String.format("serviceAccount:service-%s@gcp-sa-aiplatform.iam.gserviceaccount.com", project.number()))
.build());
var object = new BucketObject("object", BucketObjectArgs.builder()
.name("eval_request.json")
.bucket(bucket.name())
.content(serializeJson(
jsonObject(
jsonProperty("prompt", jsonObject(
jsonProperty("text", "What is the capital of France?")
))
)))
.build());
var evaluationItemGcs = new AiEvaluationItem("evaluationItemGcs", AiEvaluationItemArgs.builder()
.region("us-central1")
.displayName("eval-item")
.evaluationItemType("REQUEST")
.gcsUri(Output.tuple(bucket.name(), object.name()).applyValue(values -> {
var bucketName = values.t1;
var objectName = values.t2;
return String.format("gs://%s/%s", bucketName,objectName);
}))
.build(), CustomResourceOptions.builder()
.dependsOn(
viewer,
object)
.build());
}
}
resources:
bucket:
type: gcp:storage:Bucket
properties:
name: eval-item-bucket
location: us-central1
uniformBucketLevelAccess: true
forceDestroy: true
viewer:
type: gcp:storage:BucketIAMMember
properties:
bucket: ${bucket.name}
role: roles/storage.objectViewer
member: serviceAccount:service-${project.number}@gcp-sa-aiplatform.iam.gserviceaccount.com
object:
type: gcp:storage:BucketObject
properties:
name: eval_request.json
bucket: ${bucket.name}
content:
fn::toJSON:
prompt:
text: What is the capital of France?
evaluationItemGcs:
type: gcp:vertex:AiEvaluationItem
name: evaluation_item_gcs
properties:
region: us-central1
displayName: eval-item
evaluationItemType: REQUEST
gcsUri: gs://${bucket.name}/${object.name}
options:
dependsOn:
- ${viewer}
- ${object}
variables:
project:
fn::invoke:
function: gcp:organizations:getProject
arguments: {}
terraform {
required_providers {
gcp = {
source = "pulumi/gcp"
}
}
}
data "gcp_organizations_project" "project" {
}
resource "gcp_storage_bucket" "bucket" {
lifecycle {
create_before_destroy = true
}
name = "eval-item-bucket"
location = "us-central1"
uniform_bucket_level_access = true
force_destroy = true
}
resource "gcp_storage_bucket_i_a_m_member" "viewer" {
lifecycle {
create_before_destroy = true
}
bucket = gcp_storage_bucket.bucket.name
role = "roles/storage.objectViewer"
member ="serviceAccount:service-${data.gcp_organizations_project.project.number}@gcp-sa-aiplatform.iam.gserviceaccount.com"
}
resource "gcp_storage_bucket_object" "object" {
lifecycle {
create_before_destroy = true
}
name = "eval_request.json"
bucket = gcp_storage_bucket.bucket.name
content = jsonencode({
"prompt" = {
"text" = "What is the capital of France?"
}
})
}
resource "gcp_vertex_ai_evaluation_item" "evaluation_item_gcs" {
depends_on = [gcp_storage_bucket_i_a_m_member.viewer, gcp_storage_bucket_object.object]
lifecycle {
create_before_destroy = true
}
region = "us-central1"
display_name = "eval-item"
evaluation_item_type = "REQUEST"
gcs_uri ="gs://${gcp_storage_bucket.bucket.name}/${gcp_storage_bucket_object.object.name}"
}
Create AiEvaluationItem Resource
Resources are created with functions called constructors. To learn more about declaring and configuring resources, see Resources.
Constructor syntax
new AiEvaluationItem(name: string, args: AiEvaluationItemArgs, opts?: CustomResourceOptions);@overload
def AiEvaluationItem(resource_name: str,
args: AiEvaluationItemArgs,
opts: Optional[ResourceOptions] = None)
@overload
def AiEvaluationItem(resource_name: str,
opts: Optional[ResourceOptions] = None,
display_name: Optional[str] = None,
evaluation_item_type: Optional[str] = None,
region: Optional[str] = None,
deletion_policy: Optional[str] = None,
evaluation_request: Optional[AiEvaluationItemEvaluationRequestArgs] = None,
gcs_uri: Optional[str] = None,
labels: Optional[Mapping[str, str]] = None,
metadata: Optional[str] = None,
project: Optional[str] = None)func NewAiEvaluationItem(ctx *Context, name string, args AiEvaluationItemArgs, opts ...ResourceOption) (*AiEvaluationItem, error)public AiEvaluationItem(string name, AiEvaluationItemArgs args, CustomResourceOptions? opts = null)
public AiEvaluationItem(String name, AiEvaluationItemArgs args)
public AiEvaluationItem(String name, AiEvaluationItemArgs args, CustomResourceOptions options)
type: gcp:vertex:AiEvaluationItem
properties: # The arguments to resource properties.
options: # Bag of options to control resource's behavior.
resource "gcp_vertex_ai_evaluation_item" "name" {
# resource properties
}Parameters
- name string
- The unique name of the resource.
- args AiEvaluationItemArgs
- The arguments to resource properties.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- resource_name str
- The unique name of the resource.
- args AiEvaluationItemArgs
- The arguments to resource properties.
- opts ResourceOptions
- Bag of options to control resource's behavior.
- ctx Context
- Context object for the current deployment.
- name string
- The unique name of the resource.
- args AiEvaluationItemArgs
- The arguments to resource properties.
- opts ResourceOption
- Bag of options to control resource's behavior.
- name string
- The unique name of the resource.
- args AiEvaluationItemArgs
- The arguments to resource properties.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- name String
- The unique name of the resource.
- args AiEvaluationItemArgs
- The arguments to resource properties.
- options CustomResourceOptions
- Bag of options to control resource's behavior.
Constructor example
The following reference example uses placeholder values for all input properties.
var aiEvaluationItemResource = new Gcp.Vertex.AiEvaluationItem("aiEvaluationItemResource", new()
{
DisplayName = "string",
EvaluationItemType = "string",
Region = "string",
DeletionPolicy = "string",
EvaluationRequest = new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestArgs
{
CandidateResponses = new[]
{
new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestCandidateResponseArgs
{
Candidate = "string",
AgentData = "string",
Errors = new[]
{
new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestCandidateResponseErrorArgs
{
Code = 0,
Message = "string",
},
},
Text = "string",
Value = "string",
},
},
GoldenResponse = new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestGoldenResponseArgs
{
Candidate = "string",
AgentData = "string",
Errors = new[]
{
new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestGoldenResponseErrorArgs
{
Code = 0,
Message = "string",
},
},
Text = "string",
Value = "string",
},
Prompt = new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestPromptArgs
{
AgentData = "string",
PromptTemplateData = new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestPromptPromptTemplateDataArgs
{
Values = "string",
},
Text = "string",
UserScenario = new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestPromptUserScenarioArgs
{
ConversationPlan = "string",
StartingPrompt = "string",
},
Value = "string",
},
Rubrics = new[]
{
new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestRubricArgs
{
Name = "string",
DisplayName = "string",
GroupId = "string",
Rubrics = new[]
{
new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestRubricRubricArgs
{
Content = new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestRubricRubricContentArgs
{
Property = new Gcp.Vertex.Inputs.AiEvaluationItemEvaluationRequestRubricRubricContentPropertyArgs
{
Description = "string",
},
},
Importance = "string",
RubricId = "string",
Type = "string",
},
},
},
},
},
GcsUri = "string",
Labels =
{
{ "string", "string" },
},
Metadata = "string",
Project = "string",
});
example, err := vertex.NewAiEvaluationItem(ctx, "aiEvaluationItemResource", &vertex.AiEvaluationItemArgs{
DisplayName: pulumi.String("string"),
EvaluationItemType: pulumi.String("string"),
Region: pulumi.String("string"),
DeletionPolicy: pulumi.String("string"),
EvaluationRequest: &vertex.AiEvaluationItemEvaluationRequestArgs{
CandidateResponses: vertex.AiEvaluationItemEvaluationRequestCandidateResponseArray{
&vertex.AiEvaluationItemEvaluationRequestCandidateResponseArgs{
Candidate: pulumi.String("string"),
AgentData: pulumi.String("string"),
Errors: vertex.AiEvaluationItemEvaluationRequestCandidateResponseErrorArray{
&vertex.AiEvaluationItemEvaluationRequestCandidateResponseErrorArgs{
Code: pulumi.Int(0),
Message: pulumi.String("string"),
},
},
Text: pulumi.String("string"),
Value: pulumi.String("string"),
},
},
GoldenResponse: &vertex.AiEvaluationItemEvaluationRequestGoldenResponseArgs{
Candidate: pulumi.String("string"),
AgentData: pulumi.String("string"),
Errors: vertex.AiEvaluationItemEvaluationRequestGoldenResponseErrorArray{
&vertex.AiEvaluationItemEvaluationRequestGoldenResponseErrorArgs{
Code: pulumi.Int(0),
Message: pulumi.String("string"),
},
},
Text: pulumi.String("string"),
Value: pulumi.String("string"),
},
Prompt: &vertex.AiEvaluationItemEvaluationRequestPromptArgs{
AgentData: pulumi.String("string"),
PromptTemplateData: &vertex.AiEvaluationItemEvaluationRequestPromptPromptTemplateDataArgs{
Values: pulumi.String("string"),
},
Text: pulumi.String("string"),
UserScenario: &vertex.AiEvaluationItemEvaluationRequestPromptUserScenarioArgs{
ConversationPlan: pulumi.String("string"),
StartingPrompt: pulumi.String("string"),
},
Value: pulumi.String("string"),
},
Rubrics: vertex.AiEvaluationItemEvaluationRequestRubricArray{
&vertex.AiEvaluationItemEvaluationRequestRubricArgs{
Name: pulumi.String("string"),
DisplayName: pulumi.String("string"),
GroupId: pulumi.String("string"),
Rubrics: vertex.AiEvaluationItemEvaluationRequestRubricRubricArray{
&vertex.AiEvaluationItemEvaluationRequestRubricRubricArgs{
Content: &vertex.AiEvaluationItemEvaluationRequestRubricRubricContentArgs{
Property: &vertex.AiEvaluationItemEvaluationRequestRubricRubricContentPropertyArgs{
Description: pulumi.String("string"),
},
},
Importance: pulumi.String("string"),
RubricId: pulumi.String("string"),
Type: pulumi.String("string"),
},
},
},
},
},
GcsUri: pulumi.String("string"),
Labels: pulumi.StringMap{
"string": pulumi.String("string"),
},
Metadata: pulumi.String("string"),
Project: pulumi.String("string"),
})
resource "gcp_vertex_ai_evaluation_item" "aiEvaluationItemResource" {
lifecycle {
create_before_destroy = true
}
display_name = "string"
evaluation_item_type = "string"
region = "string"
deletion_policy = "string"
evaluation_request = {
candidate_responses = [{
candidate = "string"
agent_data = "string"
errors = [{
code = 0
message = "string"
}]
text = "string"
value = "string"
}]
golden_response = {
candidate = "string"
agent_data = "string"
errors = [{
code = 0
message = "string"
}]
text = "string"
value = "string"
}
prompt = {
agent_data = "string"
prompt_template_data = {
values = "string"
}
text = "string"
user_scenario = {
conversation_plan = "string"
starting_prompt = "string"
}
value = "string"
}
rubrics = [{
name = "string"
display_name = "string"
group_id = "string"
rubrics = [{
content = {
property = {
description = "string"
}
}
importance = "string"
rubric_id = "string"
type = "string"
}]
}]
}
gcs_uri = "string"
labels = {
"string" = "string"
}
metadata = "string"
project = "string"
}
var aiEvaluationItemResource = new AiEvaluationItem("aiEvaluationItemResource", AiEvaluationItemArgs.builder()
.displayName("string")
.evaluationItemType("string")
.region("string")
.deletionPolicy("string")
.evaluationRequest(AiEvaluationItemEvaluationRequestArgs.builder()
.candidateResponses(AiEvaluationItemEvaluationRequestCandidateResponseArgs.builder()
.candidate("string")
.agentData("string")
.errors(AiEvaluationItemEvaluationRequestCandidateResponseErrorArgs.builder()
.code(0)
.message("string")
.build())
.text("string")
.value("string")
.build())
.goldenResponse(AiEvaluationItemEvaluationRequestGoldenResponseArgs.builder()
.candidate("string")
.agentData("string")
.errors(AiEvaluationItemEvaluationRequestGoldenResponseErrorArgs.builder()
.code(0)
.message("string")
.build())
.text("string")
.value("string")
.build())
.prompt(AiEvaluationItemEvaluationRequestPromptArgs.builder()
.agentData("string")
.promptTemplateData(AiEvaluationItemEvaluationRequestPromptPromptTemplateDataArgs.builder()
.values("string")
.build())
.text("string")
.userScenario(AiEvaluationItemEvaluationRequestPromptUserScenarioArgs.builder()
.conversationPlan("string")
.startingPrompt("string")
.build())
.value("string")
.build())
.rubrics(AiEvaluationItemEvaluationRequestRubricArgs.builder()
.name("string")
.displayName("string")
.groupId("string")
.rubrics(AiEvaluationItemEvaluationRequestRubricRubricArgs.builder()
.content(AiEvaluationItemEvaluationRequestRubricRubricContentArgs.builder()
.property(AiEvaluationItemEvaluationRequestRubricRubricContentPropertyArgs.builder()
.description("string")
.build())
.build())
.importance("string")
.rubricId("string")
.type("string")
.build())
.build())
.build())
.gcsUri("string")
.labels(Map.of("string", "string"))
.metadata("string")
.project("string")
.build());
ai_evaluation_item_resource = gcp.vertex.AiEvaluationItem("aiEvaluationItemResource",
display_name="string",
evaluation_item_type="string",
region="string",
deletion_policy="string",
evaluation_request={
"candidate_responses": [{
"candidate": "string",
"agent_data": "string",
"errors": [{
"code": 0,
"message": "string",
}],
"text": "string",
"value": "string",
}],
"golden_response": {
"candidate": "string",
"agent_data": "string",
"errors": [{
"code": 0,
"message": "string",
}],
"text": "string",
"value": "string",
},
"prompt": {
"agent_data": "string",
"prompt_template_data": {
"values": "string",
},
"text": "string",
"user_scenario": {
"conversation_plan": "string",
"starting_prompt": "string",
},
"value": "string",
},
"rubrics": [{
"name": "string",
"display_name": "string",
"group_id": "string",
"rubrics": [{
"content": {
"property": {
"description": "string",
},
},
"importance": "string",
"rubric_id": "string",
"type": "string",
}],
}],
},
gcs_uri="string",
labels={
"string": "string",
},
metadata="string",
project="string")
const aiEvaluationItemResource = new gcp.vertex.AiEvaluationItem("aiEvaluationItemResource", {
displayName: "string",
evaluationItemType: "string",
region: "string",
deletionPolicy: "string",
evaluationRequest: {
candidateResponses: [{
candidate: "string",
agentData: "string",
errors: [{
code: 0,
message: "string",
}],
text: "string",
value: "string",
}],
goldenResponse: {
candidate: "string",
agentData: "string",
errors: [{
code: 0,
message: "string",
}],
text: "string",
value: "string",
},
prompt: {
agentData: "string",
promptTemplateData: {
values: "string",
},
text: "string",
userScenario: {
conversationPlan: "string",
startingPrompt: "string",
},
value: "string",
},
rubrics: [{
name: "string",
displayName: "string",
groupId: "string",
rubrics: [{
content: {
property: {
description: "string",
},
},
importance: "string",
rubricId: "string",
type: "string",
}],
}],
},
gcsUri: "string",
labels: {
string: "string",
},
metadata: "string",
project: "string",
});
type: gcp:vertex:AiEvaluationItem
properties:
deletionPolicy: string
displayName: string
evaluationItemType: string
evaluationRequest:
candidateResponses:
- agentData: string
candidate: string
errors:
- code: 0
message: string
text: string
value: string
goldenResponse:
agentData: string
candidate: string
errors:
- code: 0
message: string
text: string
value: string
prompt:
agentData: string
promptTemplateData:
values: string
text: string
userScenario:
conversationPlan: string
startingPrompt: string
value: string
rubrics:
- displayName: string
groupId: string
name: string
rubrics:
- content:
property:
description: string
importance: string
rubricId: string
type: string
gcsUri: string
labels:
string: string
metadata: string
project: string
region: string
AiEvaluationItem Resource Properties
To learn more about resource properties and how to use them, see Inputs and Outputs in the Architecture and Concepts docs.
Inputs
In Python, inputs that are objects can be passed either as argument classes or as dictionary literals.
The AiEvaluationItem resource accepts the following input properties:
- Display
Name string - The user-friendly display name for the EvaluationItem.
- Evaluation
Item stringType - The type of the EvaluationItem.
Possible values are:
REQUEST,RESULT. - Region string
- The region of the EvaluationItem. eg us-central1
- Deletion
Policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- Evaluation
Request AiEvaluation Item Evaluation Request - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - Gcs
Uri string - The Cloud Storage object where the request or response is stored.
Exactly one of
evaluationRequestorgcsUrican be set. - Labels Dictionary<string, string>
Labels for the EvaluationItem.
Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field
effectiveLabelsfor all of the labels present on the resource.- Metadata string
- User-defined metadata for this evaluation item as a JSON string.
- Project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- Display
Name string - The user-friendly display name for the EvaluationItem.
- Evaluation
Item stringType - The type of the EvaluationItem.
Possible values are:
REQUEST,RESULT. - Region string
- The region of the EvaluationItem. eg us-central1
- Deletion
Policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- Evaluation
Request AiEvaluation Item Evaluation Request Args - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - Gcs
Uri string - The Cloud Storage object where the request or response is stored.
Exactly one of
evaluationRequestorgcsUrican be set. - Labels map[string]string
Labels for the EvaluationItem.
Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field
effectiveLabelsfor all of the labels present on the resource.- Metadata string
- User-defined metadata for this evaluation item as a JSON string.
- Project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- display_
name string - The user-friendly display name for the EvaluationItem.
- evaluation_
item_ stringtype - The type of the EvaluationItem.
Possible values are:
REQUEST,RESULT. - region string
- The region of the EvaluationItem. eg us-central1
- deletion_
policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- evaluation_
request object - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - gcs_
uri string - The Cloud Storage object where the request or response is stored.
Exactly one of
evaluationRequestorgcsUrican be set. - labels map(string)
Labels for the EvaluationItem.
Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field
effectiveLabelsfor all of the labels present on the resource.- metadata string
- User-defined metadata for this evaluation item as a JSON string.
- project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- display
Name String - The user-friendly display name for the EvaluationItem.
- evaluation
Item StringType - The type of the EvaluationItem.
Possible values are:
REQUEST,RESULT. - region String
- The region of the EvaluationItem. eg us-central1
- deletion
Policy String - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- evaluation
Request AiEvaluation Item Evaluation Request - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - gcs
Uri String - The Cloud Storage object where the request or response is stored.
Exactly one of
evaluationRequestorgcsUrican be set. - labels Map<String,String>
Labels for the EvaluationItem.
Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field
effectiveLabelsfor all of the labels present on the resource.- metadata String
- User-defined metadata for this evaluation item as a JSON string.
- project String
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- display
Name string - The user-friendly display name for the EvaluationItem.
- evaluation
Item stringType - The type of the EvaluationItem.
Possible values are:
REQUEST,RESULT. - region string
- The region of the EvaluationItem. eg us-central1
- deletion
Policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- evaluation
Request AiEvaluation Item Evaluation Request - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - gcs
Uri string - The Cloud Storage object where the request or response is stored.
Exactly one of
evaluationRequestorgcsUrican be set. - labels {[key: string]: string}
Labels for the EvaluationItem.
Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field
effectiveLabelsfor all of the labels present on the resource.- metadata string
- User-defined metadata for this evaluation item as a JSON string.
- project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- display_
name str - The user-friendly display name for the EvaluationItem.
- evaluation_
item_ strtype - The type of the EvaluationItem.
Possible values are:
REQUEST,RESULT. - region str
- The region of the EvaluationItem. eg us-central1
- deletion_
policy str - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- evaluation_
request AiEvaluation Item Evaluation Request Args - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - gcs_
uri str - The Cloud Storage object where the request or response is stored.
Exactly one of
evaluationRequestorgcsUrican be set. - labels Mapping[str, str]
Labels for the EvaluationItem.
Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field
effectiveLabelsfor all of the labels present on the resource.- metadata str
- User-defined metadata for this evaluation item as a JSON string.
- project str
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- display
Name String - The user-friendly display name for the EvaluationItem.
- evaluation
Item StringType - The type of the EvaluationItem.
Possible values are:
REQUEST,RESULT. - region String
- The region of the EvaluationItem. eg us-central1
- deletion
Policy String - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- evaluation
Request Property Map - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - gcs
Uri String - The Cloud Storage object where the request or response is stored.
Exactly one of
evaluationRequestorgcsUrican be set. - labels Map<String>
Labels for the EvaluationItem.
Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field
effectiveLabelsfor all of the labels present on the resource.- metadata String
- User-defined metadata for this evaluation item as a JSON string.
- project String
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
Outputs
All input properties are implicitly available as output properties. Additionally, the AiEvaluationItem resource produces the following output properties:
- Create
Time string - Timestamp when this EvaluationItem was created.
- Effective
Labels Dictionary<string, string> - All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
- Errors
List<Ai
Evaluation Item Error> - (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- Evaluation
Responses List<AiEvaluation Item Evaluation Response> - The response from evaluation. Structure is documented below.
- Id string
- The provider-assigned unique ID for this managed resource.
- Name string
- The short name of the EvaluationItem (the final component of the resource name).
- Pulumi
Labels Dictionary<string, string> - The combination of labels configured directly on the resource and default labels configured on the provider.
- Create
Time string - Timestamp when this EvaluationItem was created.
- Effective
Labels map[string]string - All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
- Errors
[]Ai
Evaluation Item Error - (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- Evaluation
Responses []AiEvaluation Item Evaluation Response - The response from evaluation. Structure is documented below.
- Id string
- The provider-assigned unique ID for this managed resource.
- Name string
- The short name of the EvaluationItem (the final component of the resource name).
- Pulumi
Labels map[string]string - The combination of labels configured directly on the resource and default labels configured on the provider.
- create_
time string - Timestamp when this EvaluationItem was created.
- effective_
labels map(string) - All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
- errors list(object)
- (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- evaluation_
responses list(object) - The response from evaluation. Structure is documented below.
- id string
- The provider-assigned unique ID for this managed resource.
- name string
- The short name of the EvaluationItem (the final component of the resource name).
- pulumi_
labels map(string) - The combination of labels configured directly on the resource and default labels configured on the provider.
- create
Time String - Timestamp when this EvaluationItem was created.
- effective
Labels Map<String,String> - All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
- errors
List<Ai
Evaluation Item Error> - (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- evaluation
Responses List<AiEvaluation Item Evaluation Response> - The response from evaluation. Structure is documented below.
- id String
- The provider-assigned unique ID for this managed resource.
- name String
- The short name of the EvaluationItem (the final component of the resource name).
- pulumi
Labels Map<String,String> - The combination of labels configured directly on the resource and default labels configured on the provider.
- create
Time string - Timestamp when this EvaluationItem was created.
- effective
Labels {[key: string]: string} - All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
- errors
Ai
Evaluation Item Error[] - (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- evaluation
Responses AiEvaluation Item Evaluation Response[] - The response from evaluation. Structure is documented below.
- id string
- The provider-assigned unique ID for this managed resource.
- name string
- The short name of the EvaluationItem (the final component of the resource name).
- pulumi
Labels {[key: string]: string} - The combination of labels configured directly on the resource and default labels configured on the provider.
- create_
time str - Timestamp when this EvaluationItem was created.
- effective_
labels Mapping[str, str] - All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
- errors
Sequence[Ai
Evaluation Item Error] - (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- evaluation_
responses Sequence[AiEvaluation Item Evaluation Response] - The response from evaluation. Structure is documented below.
- id str
- The provider-assigned unique ID for this managed resource.
- name str
- The short name of the EvaluationItem (the final component of the resource name).
- pulumi_
labels Mapping[str, str] - The combination of labels configured directly on the resource and default labels configured on the provider.
- create
Time String - Timestamp when this EvaluationItem was created.
- effective
Labels Map<String> - All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
- errors List<Property Map>
- (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- evaluation
Responses List<Property Map> - The response from evaluation. Structure is documented below.
- id String
- The provider-assigned unique ID for this managed resource.
- name String
- The short name of the EvaluationItem (the final component of the resource name).
- pulumi
Labels Map<String> - The combination of labels configured directly on the resource and default labels configured on the provider.
Look up Existing AiEvaluationItem Resource
Get an existing AiEvaluationItem resource’s state with the given name, ID, and optional extra properties used to qualify the lookup.
public static get(name: string, id: Input<ID>, state?: AiEvaluationItemState, opts?: CustomResourceOptions): AiEvaluationItem@staticmethod
def get(resource_name: str,
id: str,
opts: Optional[ResourceOptions] = None,
create_time: Optional[str] = None,
deletion_policy: Optional[str] = None,
display_name: Optional[str] = None,
effective_labels: Optional[Mapping[str, str]] = None,
errors: Optional[Sequence[AiEvaluationItemErrorArgs]] = None,
evaluation_item_type: Optional[str] = None,
evaluation_request: Optional[AiEvaluationItemEvaluationRequestArgs] = None,
evaluation_responses: Optional[Sequence[AiEvaluationItemEvaluationResponseArgs]] = None,
gcs_uri: Optional[str] = None,
labels: Optional[Mapping[str, str]] = None,
metadata: Optional[str] = None,
name: Optional[str] = None,
project: Optional[str] = None,
pulumi_labels: Optional[Mapping[str, str]] = None,
region: Optional[str] = None) -> AiEvaluationItemfunc GetAiEvaluationItem(ctx *Context, name string, id IDInput, state *AiEvaluationItemState, opts ...ResourceOption) (*AiEvaluationItem, error)public static AiEvaluationItem Get(string name, Input<string> id, AiEvaluationItemState? state, CustomResourceOptions? opts = null)public static AiEvaluationItem get(String name, Output<String> id, AiEvaluationItemState state, CustomResourceOptions options)resources: existingAiEvaluationItem: type: gcp:vertex:AiEvaluationItem get: id: ${id}import {
to = gcp_vertex_ai_evaluation_item.example
id = "${id}"
}
- name string
- The unique name of the resulting resource.
- id Input<ID>
- The unique provider ID of the resource to look up.
- state AiEvaluationItemState
- Known property values the provider needs, along with the ID, to find the resource. Usually not required.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- resource_name str
- The unique name of the resulting resource.
- id str
- The unique provider ID of the resource to look up.
- opts ResourceOptions
- Bag of options to control resource's behavior.
- ctx Context
- Context object for the current deployment.
- name string
- The unique name of the resulting resource.
- id IDInput
- The unique provider ID of the resource to look up.
- state AiEvaluationItemState
- Known property values the provider needs, along with the ID, to find the resource. Usually not required.
- opts ResourceOption
- Bag of options to control resource's behavior.
- name string
- The unique name of the resulting resource.
- id Input<string>
- The unique provider ID of the resource to look up.
- state AiEvaluationItemState
- Known property values the provider needs, along with the ID, to find the resource. Usually not required.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- name String
- The unique name of the resulting resource.
- id Output<String>
- The unique provider ID of the resource to look up.
- state AiEvaluationItemState
- Known property values the provider needs, along with the ID, to find the resource. Usually not required.
- options CustomResourceOptions
- Bag of options to control resource's behavior.
- Create
Time string - Timestamp when this EvaluationItem was created.
- Deletion
Policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- Display
Name string - The user-friendly display name for the EvaluationItem.
- Effective
Labels Dictionary<string, string> - All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
- Errors
List<Ai
Evaluation Item Error> - (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- Evaluation
Item stringType - The type of the EvaluationItem.
Possible values are:
REQUEST,RESULT. - Evaluation
Request AiEvaluation Item Evaluation Request - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - Evaluation
Responses List<AiEvaluation Item Evaluation Response> - The response from evaluation. Structure is documented below.
- Gcs
Uri string - The Cloud Storage object where the request or response is stored.
Exactly one of
evaluationRequestorgcsUrican be set. - Labels Dictionary<string, string>
Labels for the EvaluationItem.
Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field
effectiveLabelsfor all of the labels present on the resource.- Metadata string
- User-defined metadata for this evaluation item as a JSON string.
- Name string
- The short name of the EvaluationItem (the final component of the resource name).
- Project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- Pulumi
Labels Dictionary<string, string> - The combination of labels configured directly on the resource and default labels configured on the provider.
- Region string
- The region of the EvaluationItem. eg us-central1
- Create
Time string - Timestamp when this EvaluationItem was created.
- Deletion
Policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- Display
Name string - The user-friendly display name for the EvaluationItem.
- Effective
Labels map[string]string - All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
- Errors
[]Ai
Evaluation Item Error Args - (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- Evaluation
Item stringType - The type of the EvaluationItem.
Possible values are:
REQUEST,RESULT. - Evaluation
Request AiEvaluation Item Evaluation Request Args - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - Evaluation
Responses []AiEvaluation Item Evaluation Response Args - The response from evaluation. Structure is documented below.
- Gcs
Uri string - The Cloud Storage object where the request or response is stored.
Exactly one of
evaluationRequestorgcsUrican be set. - Labels map[string]string
Labels for the EvaluationItem.
Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field
effectiveLabelsfor all of the labels present on the resource.- Metadata string
- User-defined metadata for this evaluation item as a JSON string.
- Name string
- The short name of the EvaluationItem (the final component of the resource name).
- Project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- Pulumi
Labels map[string]string - The combination of labels configured directly on the resource and default labels configured on the provider.
- Region string
- The region of the EvaluationItem. eg us-central1
- create_
time string - Timestamp when this EvaluationItem was created.
- deletion_
policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- display_
name string - The user-friendly display name for the EvaluationItem.
- effective_
labels map(string) - All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
- errors list(object)
- (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- evaluation_
item_ stringtype - The type of the EvaluationItem.
Possible values are:
REQUEST,RESULT. - evaluation_
request object - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - evaluation_
responses list(object) - The response from evaluation. Structure is documented below.
- gcs_
uri string - The Cloud Storage object where the request or response is stored.
Exactly one of
evaluationRequestorgcsUrican be set. - labels map(string)
Labels for the EvaluationItem.
Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field
effectiveLabelsfor all of the labels present on the resource.- metadata string
- User-defined metadata for this evaluation item as a JSON string.
- name string
- The short name of the EvaluationItem (the final component of the resource name).
- project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- pulumi_
labels map(string) - The combination of labels configured directly on the resource and default labels configured on the provider.
- region string
- The region of the EvaluationItem. eg us-central1
- create
Time String - Timestamp when this EvaluationItem was created.
- deletion
Policy String - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- display
Name String - The user-friendly display name for the EvaluationItem.
- effective
Labels Map<String,String> - All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
- errors
List<Ai
Evaluation Item Error> - (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- evaluation
Item StringType - The type of the EvaluationItem.
Possible values are:
REQUEST,RESULT. - evaluation
Request AiEvaluation Item Evaluation Request - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - evaluation
Responses List<AiEvaluation Item Evaluation Response> - The response from evaluation. Structure is documented below.
- gcs
Uri String - The Cloud Storage object where the request or response is stored.
Exactly one of
evaluationRequestorgcsUrican be set. - labels Map<String,String>
Labels for the EvaluationItem.
Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field
effectiveLabelsfor all of the labels present on the resource.- metadata String
- User-defined metadata for this evaluation item as a JSON string.
- name String
- The short name of the EvaluationItem (the final component of the resource name).
- project String
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- pulumi
Labels Map<String,String> - The combination of labels configured directly on the resource and default labels configured on the provider.
- region String
- The region of the EvaluationItem. eg us-central1
- create
Time string - Timestamp when this EvaluationItem was created.
- deletion
Policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- display
Name string - The user-friendly display name for the EvaluationItem.
- effective
Labels {[key: string]: string} - All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
- errors
Ai
Evaluation Item Error[] - (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- evaluation
Item stringType - The type of the EvaluationItem.
Possible values are:
REQUEST,RESULT. - evaluation
Request AiEvaluation Item Evaluation Request - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - evaluation
Responses AiEvaluation Item Evaluation Response[] - The response from evaluation. Structure is documented below.
- gcs
Uri string - The Cloud Storage object where the request or response is stored.
Exactly one of
evaluationRequestorgcsUrican be set. - labels {[key: string]: string}
Labels for the EvaluationItem.
Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field
effectiveLabelsfor all of the labels present on the resource.- metadata string
- User-defined metadata for this evaluation item as a JSON string.
- name string
- The short name of the EvaluationItem (the final component of the resource name).
- project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- pulumi
Labels {[key: string]: string} - The combination of labels configured directly on the resource and default labels configured on the provider.
- region string
- The region of the EvaluationItem. eg us-central1
- create_
time str - Timestamp when this EvaluationItem was created.
- deletion_
policy str - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- display_
name str - The user-friendly display name for the EvaluationItem.
- effective_
labels Mapping[str, str] - All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
- errors
Sequence[Ai
Evaluation Item Error Args] - (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- evaluation_
item_ strtype - The type of the EvaluationItem.
Possible values are:
REQUEST,RESULT. - evaluation_
request AiEvaluation Item Evaluation Request Args - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - evaluation_
responses Sequence[AiEvaluation Item Evaluation Response Args] - The response from evaluation. Structure is documented below.
- gcs_
uri str - The Cloud Storage object where the request or response is stored.
Exactly one of
evaluationRequestorgcsUrican be set. - labels Mapping[str, str]
Labels for the EvaluationItem.
Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field
effectiveLabelsfor all of the labels present on the resource.- metadata str
- User-defined metadata for this evaluation item as a JSON string.
- name str
- The short name of the EvaluationItem (the final component of the resource name).
- project str
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- pulumi_
labels Mapping[str, str] - The combination of labels configured directly on the resource and default labels configured on the provider.
- region str
- The region of the EvaluationItem. eg us-central1
- create
Time String - Timestamp when this EvaluationItem was created.
- deletion
Policy String - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- display
Name String - The user-friendly display name for the EvaluationItem.
- effective
Labels Map<String> - All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
- errors List<Property Map>
- (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- evaluation
Item StringType - The type of the EvaluationItem.
Possible values are:
REQUEST,RESULT. - evaluation
Request Property Map - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - evaluation
Responses List<Property Map> - The response from evaluation. Structure is documented below.
- gcs
Uri String - The Cloud Storage object where the request or response is stored.
Exactly one of
evaluationRequestorgcsUrican be set. - labels Map<String>
Labels for the EvaluationItem.
Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field
effectiveLabelsfor all of the labels present on the resource.- metadata String
- User-defined metadata for this evaluation item as a JSON string.
- name String
- The short name of the EvaluationItem (the final component of the resource name).
- project String
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- pulumi
Labels Map<String> - The combination of labels configured directly on the resource and default labels configured on the provider.
- region String
- The region of the EvaluationItem. eg us-central1
Supporting Types
AiEvaluationItemError, AiEvaluationItemErrorArgs
AiEvaluationItemEvaluationRequest, AiEvaluationItemEvaluationRequestArgs
- Candidate
Responses List<AiEvaluation Item Evaluation Request Candidate Response> - Responses from model under test and other baseline models for comparison. Structure is documented below.
- Golden
Response AiEvaluation Item Evaluation Request Golden Response - The ideal response or ground truth. Structure is documented below.
- Prompt
Ai
Evaluation Item Evaluation Request Prompt - The request/prompt to evaluate. Structure is documented below.
- Rubrics
List<Ai
Evaluation Item Evaluation Request Rubric> - Named groups of rubrics associated with this prompt. The key is a user-defined name for the rubric group. Structure is documented below.
- Candidate
Responses []AiEvaluation Item Evaluation Request Candidate Response - Responses from model under test and other baseline models for comparison. Structure is documented below.
- Golden
Response AiEvaluation Item Evaluation Request Golden Response - The ideal response or ground truth. Structure is documented below.
- Prompt
Ai
Evaluation Item Evaluation Request Prompt - The request/prompt to evaluate. Structure is documented below.
- Rubrics
[]Ai
Evaluation Item Evaluation Request Rubric - Named groups of rubrics associated with this prompt. The key is a user-defined name for the rubric group. Structure is documented below.
- candidate_
responses list(object) - Responses from model under test and other baseline models for comparison. Structure is documented below.
- golden_
response object - The ideal response or ground truth. Structure is documented below.
- prompt object
- The request/prompt to evaluate. Structure is documented below.
- rubrics list(object)
- Named groups of rubrics associated with this prompt. The key is a user-defined name for the rubric group. Structure is documented below.
- candidate
Responses List<AiEvaluation Item Evaluation Request Candidate Response> - Responses from model under test and other baseline models for comparison. Structure is documented below.
- golden
Response AiEvaluation Item Evaluation Request Golden Response - The ideal response or ground truth. Structure is documented below.
- prompt
Ai
Evaluation Item Evaluation Request Prompt - The request/prompt to evaluate. Structure is documented below.
- rubrics
List<Ai
Evaluation Item Evaluation Request Rubric> - Named groups of rubrics associated with this prompt. The key is a user-defined name for the rubric group. Structure is documented below.
- candidate
Responses AiEvaluation Item Evaluation Request Candidate Response[] - Responses from model under test and other baseline models for comparison. Structure is documented below.
- golden
Response AiEvaluation Item Evaluation Request Golden Response - The ideal response or ground truth. Structure is documented below.
- prompt
Ai
Evaluation Item Evaluation Request Prompt - The request/prompt to evaluate. Structure is documented below.
- rubrics
Ai
Evaluation Item Evaluation Request Rubric[] - Named groups of rubrics associated with this prompt. The key is a user-defined name for the rubric group. Structure is documented below.
- candidate_
responses Sequence[AiEvaluation Item Evaluation Request Candidate Response] - Responses from model under test and other baseline models for comparison. Structure is documented below.
- golden_
response AiEvaluation Item Evaluation Request Golden Response - The ideal response or ground truth. Structure is documented below.
- prompt
Ai
Evaluation Item Evaluation Request Prompt - The request/prompt to evaluate. Structure is documented below.
- rubrics
Sequence[Ai
Evaluation Item Evaluation Request Rubric] - Named groups of rubrics associated with this prompt. The key is a user-defined name for the rubric group. Structure is documented below.
- candidate
Responses List<Property Map> - Responses from model under test and other baseline models for comparison. Structure is documented below.
- golden
Response Property Map - The ideal response or ground truth. Structure is documented below.
- prompt Property Map
- The request/prompt to evaluate. Structure is documented below.
- rubrics List<Property Map>
- Named groups of rubrics associated with this prompt. The key is a user-defined name for the rubric group. Structure is documented below.
AiEvaluationItemEvaluationRequestCandidateResponse, AiEvaluationItemEvaluationRequestCandidateResponseArgs
- Candidate string
- The name of the candidate that produced the response.
- Agent
Data string - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
field is used to provide the full output of an agent's run, including
all turns and events, for direct evaluation.
At most one of
text,value, oragentDatacan be set. - Errors
List<Ai
Evaluation Item Evaluation Request Candidate Response Error> (Output) Error while scraping model or agent. Structure is documented below.
The
errorblock contains:- Text string
- Text response.
At most one of
text,value, oragentDatacan be set. - Value string
- Fields and values that can be used to populate the response template, as a JSON string.
At most one of
text,value, oragentDatacan be set.
- Candidate string
- The name of the candidate that produced the response.
- Agent
Data string - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
field is used to provide the full output of an agent's run, including
all turns and events, for direct evaluation.
At most one of
text,value, oragentDatacan be set. - Errors
[]Ai
Evaluation Item Evaluation Request Candidate Response Error (Output) Error while scraping model or agent. Structure is documented below.
The
errorblock contains:- Text string
- Text response.
At most one of
text,value, oragentDatacan be set. - Value string
- Fields and values that can be used to populate the response template, as a JSON string.
At most one of
text,value, oragentDatacan be set.
- candidate string
- The name of the candidate that produced the response.
- agent_
data string - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
field is used to provide the full output of an agent's run, including
all turns and events, for direct evaluation.
At most one of
text,value, oragentDatacan be set. - errors list(object)
(Output) Error while scraping model or agent. Structure is documented below.
The
errorblock contains:- text string
- Text response.
At most one of
text,value, oragentDatacan be set. - value string
- Fields and values that can be used to populate the response template, as a JSON string.
At most one of
text,value, oragentDatacan be set.
- candidate String
- The name of the candidate that produced the response.
- agent
Data String - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
field is used to provide the full output of an agent's run, including
all turns and events, for direct evaluation.
At most one of
text,value, oragentDatacan be set. - errors
List<Ai
Evaluation Item Evaluation Request Candidate Response Error> (Output) Error while scraping model or agent. Structure is documented below.
The
errorblock contains:- text String
- Text response.
At most one of
text,value, oragentDatacan be set. - value String
- Fields and values that can be used to populate the response template, as a JSON string.
At most one of
text,value, oragentDatacan be set.
- candidate string
- The name of the candidate that produced the response.
- agent
Data string - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
field is used to provide the full output of an agent's run, including
all turns and events, for direct evaluation.
At most one of
text,value, oragentDatacan be set. - errors
Ai
Evaluation Item Evaluation Request Candidate Response Error[] (Output) Error while scraping model or agent. Structure is documented below.
The
errorblock contains:- text string
- Text response.
At most one of
text,value, oragentDatacan be set. - value string
- Fields and values that can be used to populate the response template, as a JSON string.
At most one of
text,value, oragentDatacan be set.
- candidate str
- The name of the candidate that produced the response.
- agent_
data str - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
field is used to provide the full output of an agent's run, including
all turns and events, for direct evaluation.
At most one of
text,value, oragentDatacan be set. - errors
Sequence[Ai
Evaluation Item Evaluation Request Candidate Response Error] (Output) Error while scraping model or agent. Structure is documented below.
The
errorblock contains:- text str
- Text response.
At most one of
text,value, oragentDatacan be set. - value str
- Fields and values that can be used to populate the response template, as a JSON string.
At most one of
text,value, oragentDatacan be set.
- candidate String
- The name of the candidate that produced the response.
- agent
Data String - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
field is used to provide the full output of an agent's run, including
all turns and events, for direct evaluation.
At most one of
text,value, oragentDatacan be set. - errors List<Property Map>
(Output) Error while scraping model or agent. Structure is documented below.
The
errorblock contains:- text String
- Text response.
At most one of
text,value, oragentDatacan be set. - value String
- Fields and values that can be used to populate the response template, as a JSON string.
At most one of
text,value, oragentDatacan be set.
AiEvaluationItemEvaluationRequestCandidateResponseError, AiEvaluationItemEvaluationRequestCandidateResponseErrorArgs
AiEvaluationItemEvaluationRequestGoldenResponse, AiEvaluationItemEvaluationRequestGoldenResponseArgs
- Candidate string
- The name of the candidate that produced the response.
- Agent
Data string - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
field is used to provide the full output of an agent's run, including
all turns and events, for direct evaluation.
At most one of
text,value, oragentDatacan be set. - Errors
List<Ai
Evaluation Item Evaluation Request Golden Response Error> (Output) Error while scraping model or agent. Structure is documented below.
The
errorblock contains:- Text string
- Text response.
At most one of
text,value, oragentDatacan be set. - Value string
- Fields and values that can be used to populate the response template, as a JSON string.
At most one of
text,value, oragentDatacan be set.
- Candidate string
- The name of the candidate that produced the response.
- Agent
Data string - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
field is used to provide the full output of an agent's run, including
all turns and events, for direct evaluation.
At most one of
text,value, oragentDatacan be set. - Errors
[]Ai
Evaluation Item Evaluation Request Golden Response Error (Output) Error while scraping model or agent. Structure is documented below.
The
errorblock contains:- Text string
- Text response.
At most one of
text,value, oragentDatacan be set. - Value string
- Fields and values that can be used to populate the response template, as a JSON string.
At most one of
text,value, oragentDatacan be set.
- candidate string
- The name of the candidate that produced the response.
- agent_
data string - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
field is used to provide the full output of an agent's run, including
all turns and events, for direct evaluation.
At most one of
text,value, oragentDatacan be set. - errors list(object)
(Output) Error while scraping model or agent. Structure is documented below.
The
errorblock contains:- text string
- Text response.
At most one of
text,value, oragentDatacan be set. - value string
- Fields and values that can be used to populate the response template, as a JSON string.
At most one of
text,value, oragentDatacan be set.
- candidate String
- The name of the candidate that produced the response.
- agent
Data String - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
field is used to provide the full output of an agent's run, including
all turns and events, for direct evaluation.
At most one of
text,value, oragentDatacan be set. - errors
List<Ai
Evaluation Item Evaluation Request Golden Response Error> (Output) Error while scraping model or agent. Structure is documented below.
The
errorblock contains:- text String
- Text response.
At most one of
text,value, oragentDatacan be set. - value String
- Fields and values that can be used to populate the response template, as a JSON string.
At most one of
text,value, oragentDatacan be set.
- candidate string
- The name of the candidate that produced the response.
- agent
Data string - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
field is used to provide the full output of an agent's run, including
all turns and events, for direct evaluation.
At most one of
text,value, oragentDatacan be set. - errors
Ai
Evaluation Item Evaluation Request Golden Response Error[] (Output) Error while scraping model or agent. Structure is documented below.
The
errorblock contains:- text string
- Text response.
At most one of
text,value, oragentDatacan be set. - value string
- Fields and values that can be used to populate the response template, as a JSON string.
At most one of
text,value, oragentDatacan be set.
- candidate str
- The name of the candidate that produced the response.
- agent_
data str - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
field is used to provide the full output of an agent's run, including
all turns and events, for direct evaluation.
At most one of
text,value, oragentDatacan be set. - errors
Sequence[Ai
Evaluation Item Evaluation Request Golden Response Error] (Output) Error while scraping model or agent. Structure is documented below.
The
errorblock contains:- text str
- Text response.
At most one of
text,value, oragentDatacan be set. - value str
- Fields and values that can be used to populate the response template, as a JSON string.
At most one of
text,value, oragentDatacan be set.
- candidate String
- The name of the candidate that produced the response.
- agent
Data String - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
field is used to provide the full output of an agent's run, including
all turns and events, for direct evaluation.
At most one of
text,value, oragentDatacan be set. - errors List<Property Map>
(Output) Error while scraping model or agent. Structure is documented below.
The
errorblock contains:- text String
- Text response.
At most one of
text,value, oragentDatacan be set. - value String
- Fields and values that can be used to populate the response template, as a JSON string.
At most one of
text,value, oragentDatacan be set.
AiEvaluationItemEvaluationRequestGoldenResponseError, AiEvaluationItemEvaluationRequestGoldenResponseErrorArgs
AiEvaluationItemEvaluationRequestPrompt, AiEvaluationItemEvaluationRequestPromptArgs
- Agent
Data string - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
serves as the input context for agent scraping.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. - Prompt
Template AiData Evaluation Item Evaluation Request Prompt Prompt Template Data - Prompt template data.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. Structure is documented below. - Text string
- Text prompt.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. - User
Scenario AiEvaluation Item Evaluation Request Prompt User Scenario - The generated user scenario used to drive multi-turn agent running results.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. Structure is documented below. - Value string
- Fields and values that can be used to populate the prompt template, as a JSON string.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set.
- Agent
Data string - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
serves as the input context for agent scraping.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. - Prompt
Template AiData Evaluation Item Evaluation Request Prompt Prompt Template Data - Prompt template data.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. Structure is documented below. - Text string
- Text prompt.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. - User
Scenario AiEvaluation Item Evaluation Request Prompt User Scenario - The generated user scenario used to drive multi-turn agent running results.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. Structure is documented below. - Value string
- Fields and values that can be used to populate the prompt template, as a JSON string.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set.
- agent_
data string - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
serves as the input context for agent scraping.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. - prompt_
template_ objectdata - Prompt template data.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. Structure is documented below. - text string
- Text prompt.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. - user_
scenario object - The generated user scenario used to drive multi-turn agent running results.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. Structure is documented below. - value string
- Fields and values that can be used to populate the prompt template, as a JSON string.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set.
- agent
Data String - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
serves as the input context for agent scraping.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. - prompt
Template AiData Evaluation Item Evaluation Request Prompt Prompt Template Data - Prompt template data.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. Structure is documented below. - text String
- Text prompt.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. - user
Scenario AiEvaluation Item Evaluation Request Prompt User Scenario - The generated user scenario used to drive multi-turn agent running results.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. Structure is documented below. - value String
- Fields and values that can be used to populate the prompt template, as a JSON string.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set.
- agent
Data string - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
serves as the input context for agent scraping.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. - prompt
Template AiData Evaluation Item Evaluation Request Prompt Prompt Template Data - Prompt template data.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. Structure is documented below. - text string
- Text prompt.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. - user
Scenario AiEvaluation Item Evaluation Request Prompt User Scenario - The generated user scenario used to drive multi-turn agent running results.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. Structure is documented below. - value string
- Fields and values that can be used to populate the prompt template, as a JSON string.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set.
- agent_
data str - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
serves as the input context for agent scraping.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. - prompt_
template_ Aidata Evaluation Item Evaluation Request Prompt Prompt Template Data - Prompt template data.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. Structure is documented below. - text str
- Text prompt.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. - user_
scenario AiEvaluation Item Evaluation Request Prompt User Scenario - The generated user scenario used to drive multi-turn agent running results.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. Structure is documented below. - value str
- Fields and values that can be used to populate the prompt template, as a JSON string.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set.
- agent
Data String - Represents the complete execution trace of a multi-turn conversation,
which can involve single or multiple agents, as a JSON string. This
serves as the input context for agent scraping.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. - prompt
Template Property MapData - Prompt template data.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. Structure is documented below. - text String
- Text prompt.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. - user
Scenario Property Map - The generated user scenario used to drive multi-turn agent running results.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set. Structure is documented below. - value String
- Fields and values that can be used to populate the prompt template, as a JSON string.
At most one of
text,value,promptTemplateData,agentData, oruserScenariocan be set.
AiEvaluationItemEvaluationRequestPromptPromptTemplateData, AiEvaluationItemEvaluationRequestPromptPromptTemplateDataArgs
- Values string
- The values for fields in the prompt template, as a JSON string mapping field names to Content objects.
- Values string
- The values for fields in the prompt template, as a JSON string mapping field names to Content objects.
- values string
- The values for fields in the prompt template, as a JSON string mapping field names to Content objects.
- values String
- The values for fields in the prompt template, as a JSON string mapping field names to Content objects.
- values string
- The values for fields in the prompt template, as a JSON string mapping field names to Content objects.
- values str
- The values for fields in the prompt template, as a JSON string mapping field names to Content objects.
- values String
- The values for fields in the prompt template, as a JSON string mapping field names to Content objects.
AiEvaluationItemEvaluationRequestPromptUserScenario, AiEvaluationItemEvaluationRequestPromptUserScenarioArgs
- Conversation
Plan string - The plan for the conversation, used to drive the multi-turn agent run and generate the simulated agent evaluation dataset.
- Starting
Prompt string - The prompt that starts the conversation between the simulated user and the agent under test.
- Conversation
Plan string - The plan for the conversation, used to drive the multi-turn agent run and generate the simulated agent evaluation dataset.
- Starting
Prompt string - The prompt that starts the conversation between the simulated user and the agent under test.
- conversation_
plan string - The plan for the conversation, used to drive the multi-turn agent run and generate the simulated agent evaluation dataset.
- starting_
prompt string - The prompt that starts the conversation between the simulated user and the agent under test.
- conversation
Plan String - The plan for the conversation, used to drive the multi-turn agent run and generate the simulated agent evaluation dataset.
- starting
Prompt String - The prompt that starts the conversation between the simulated user and the agent under test.
- conversation
Plan string - The plan for the conversation, used to drive the multi-turn agent run and generate the simulated agent evaluation dataset.
- starting
Prompt string - The prompt that starts the conversation between the simulated user and the agent under test.
- conversation_
plan str - The plan for the conversation, used to drive the multi-turn agent run and generate the simulated agent evaluation dataset.
- starting_
prompt str - The prompt that starts the conversation between the simulated user and the agent under test.
- conversation
Plan String - The plan for the conversation, used to drive the multi-turn agent run and generate the simulated agent evaluation dataset.
- starting
Prompt String - The prompt that starts the conversation between the simulated user and the agent under test.
AiEvaluationItemEvaluationRequestRubric, AiEvaluationItemEvaluationRequestRubricArgs
- Name string
- The identifier for this object. Format specified above.
- Display
Name string - Human-readable name for the group.
- Group
Id string - Unique identifier for the group.
- Rubrics
List<Ai
Evaluation Item Evaluation Request Rubric Rubric> - Rubrics that are part of this group. Structure is documented below.
- Name string
- The identifier for this object. Format specified above.
- Display
Name string - Human-readable name for the group.
- Group
Id string - Unique identifier for the group.
- Rubrics
[]Ai
Evaluation Item Evaluation Request Rubric Rubric - Rubrics that are part of this group. Structure is documented below.
- name string
- The identifier for this object. Format specified above.
- display_
name string - Human-readable name for the group.
- group_
id string - Unique identifier for the group.
- rubrics list(object)
- Rubrics that are part of this group. Structure is documented below.
- name String
- The identifier for this object. Format specified above.
- display
Name String - Human-readable name for the group.
- group
Id String - Unique identifier for the group.
- rubrics
List<Ai
Evaluation Item Evaluation Request Rubric Rubric> - Rubrics that are part of this group. Structure is documented below.
- name string
- The identifier for this object. Format specified above.
- display
Name string - Human-readable name for the group.
- group
Id string - Unique identifier for the group.
- rubrics
Ai
Evaluation Item Evaluation Request Rubric Rubric[] - Rubrics that are part of this group. Structure is documented below.
- name str
- The identifier for this object. Format specified above.
- display_
name str - Human-readable name for the group.
- group_
id str - Unique identifier for the group.
- rubrics
Sequence[Ai
Evaluation Item Evaluation Request Rubric Rubric] - Rubrics that are part of this group. Structure is documented below.
- name String
- The identifier for this object. Format specified above.
- display
Name String - Human-readable name for the group.
- group
Id String - Unique identifier for the group.
- rubrics List<Property Map>
- Rubrics that are part of this group. Structure is documented below.
AiEvaluationItemEvaluationRequestRubricRubric, AiEvaluationItemEvaluationRequestRubricRubricArgs
- Content
Ai
Evaluation Item Evaluation Request Rubric Rubric Content - The actual testable criteria for the rubric. Structure is documented below.
- Importance string
- The relative importance of this rubric.
Possible values are:
HIGH,MEDIUM,LOW. - Rubric
Id string - Unique identifier for the rubric.
- Type string
- A type designator for the rubric, e.g.
SUMMARIZATION_QUALITY.
- Content
Ai
Evaluation Item Evaluation Request Rubric Rubric Content - The actual testable criteria for the rubric. Structure is documented below.
- Importance string
- The relative importance of this rubric.
Possible values are:
HIGH,MEDIUM,LOW. - Rubric
Id string - Unique identifier for the rubric.
- Type string
- A type designator for the rubric, e.g.
SUMMARIZATION_QUALITY.
- content object
- The actual testable criteria for the rubric. Structure is documented below.
- importance string
- The relative importance of this rubric.
Possible values are:
HIGH,MEDIUM,LOW. - rubric_
id string - Unique identifier for the rubric.
- type string
- A type designator for the rubric, e.g.
SUMMARIZATION_QUALITY.
- content
Ai
Evaluation Item Evaluation Request Rubric Rubric Content - The actual testable criteria for the rubric. Structure is documented below.
- importance String
- The relative importance of this rubric.
Possible values are:
HIGH,MEDIUM,LOW. - rubric
Id String - Unique identifier for the rubric.
- type String
- A type designator for the rubric, e.g.
SUMMARIZATION_QUALITY.
- content
Ai
Evaluation Item Evaluation Request Rubric Rubric Content - The actual testable criteria for the rubric. Structure is documented below.
- importance string
- The relative importance of this rubric.
Possible values are:
HIGH,MEDIUM,LOW. - rubric
Id string - Unique identifier for the rubric.
- type string
- A type designator for the rubric, e.g.
SUMMARIZATION_QUALITY.
- content
Ai
Evaluation Item Evaluation Request Rubric Rubric Content - The actual testable criteria for the rubric. Structure is documented below.
- importance str
- The relative importance of this rubric.
Possible values are:
HIGH,MEDIUM,LOW. - rubric_
id str - Unique identifier for the rubric.
- type str
- A type designator for the rubric, e.g.
SUMMARIZATION_QUALITY.
- content Property Map
- The actual testable criteria for the rubric. Structure is documented below.
- importance String
- The relative importance of this rubric.
Possible values are:
HIGH,MEDIUM,LOW. - rubric
Id String - Unique identifier for the rubric.
- type String
- A type designator for the rubric, e.g.
SUMMARIZATION_QUALITY.
AiEvaluationItemEvaluationRequestRubricRubricContent, AiEvaluationItemEvaluationRequestRubricRubricContentArgs
- Property
Ai
Evaluation Item Evaluation Request Rubric Rubric Content Property - Evaluation criteria based on a specific property. Structure is documented below.
- Property
Ai
Evaluation Item Evaluation Request Rubric Rubric Content Property - Evaluation criteria based on a specific property. Structure is documented below.
- property
Ai
Evaluation Item Evaluation Request Rubric Rubric Content Property - Evaluation criteria based on a specific property. Structure is documented below.
- property
Ai
Evaluation Item Evaluation Request Rubric Rubric Content Property - Evaluation criteria based on a specific property. Structure is documented below.
- property
Ai
Evaluation Item Evaluation Request Rubric Rubric Content Property - Evaluation criteria based on a specific property. Structure is documented below.
- property Property Map
- Evaluation criteria based on a specific property. Structure is documented below.
AiEvaluationItemEvaluationRequestRubricRubricContentProperty, AiEvaluationItemEvaluationRequestRubricRubricContentPropertyArgs
- Description string
- Description of the property being evaluated.
- Description string
- Description of the property being evaluated.
- description string
- Description of the property being evaluated.
- description String
- Description of the property being evaluated.
- description string
- Description of the property being evaluated.
- description str
- Description of the property being evaluated.
- description String
- Description of the property being evaluated.
AiEvaluationItemEvaluationResponse, AiEvaluationItemEvaluationResponseArgs
- Candidate
Results List<AiEvaluation Item Evaluation Response Candidate Result> - (Output) The results for the metric. Structure is documented below.
- Evaluation
Request string - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - Evaluation
Run string - (Output)
The evaluation run that was used to generate the result. Format:
projects/{project}/locations/{location}/evaluationRuns/{evaluation_run} - Metadata string
- User-defined metadata for this evaluation item as a JSON string.
- Metric string
- (Output) The metric that was evaluated.
- Request string
- (Output) The request that was evaluated, as a JSON string.
- Candidate
Results []AiEvaluation Item Evaluation Response Candidate Result - (Output) The results for the metric. Structure is documented below.
- Evaluation
Request string - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - Evaluation
Run string - (Output)
The evaluation run that was used to generate the result. Format:
projects/{project}/locations/{location}/evaluationRuns/{evaluation_run} - Metadata string
- User-defined metadata for this evaluation item as a JSON string.
- Metric string
- (Output) The metric that was evaluated.
- Request string
- (Output) The request that was evaluated, as a JSON string.
- candidate_
results list(object) - (Output) The results for the metric. Structure is documented below.
- evaluation_
request string - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - evaluation_
run string - (Output)
The evaluation run that was used to generate the result. Format:
projects/{project}/locations/{location}/evaluationRuns/{evaluation_run} - metadata string
- User-defined metadata for this evaluation item as a JSON string.
- metric string
- (Output) The metric that was evaluated.
- request string
- (Output) The request that was evaluated, as a JSON string.
- candidate
Results List<AiEvaluation Item Evaluation Response Candidate Result> - (Output) The results for the metric. Structure is documented below.
- evaluation
Request String - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - evaluation
Run String - (Output)
The evaluation run that was used to generate the result. Format:
projects/{project}/locations/{location}/evaluationRuns/{evaluation_run} - metadata String
- User-defined metadata for this evaluation item as a JSON string.
- metric String
- (Output) The metric that was evaluated.
- request String
- (Output) The request that was evaluated, as a JSON string.
- candidate
Results AiEvaluation Item Evaluation Response Candidate Result[] - (Output) The results for the metric. Structure is documented below.
- evaluation
Request string - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - evaluation
Run string - (Output)
The evaluation run that was used to generate the result. Format:
projects/{project}/locations/{location}/evaluationRuns/{evaluation_run} - metadata string
- User-defined metadata for this evaluation item as a JSON string.
- metric string
- (Output) The metric that was evaluated.
- request string
- (Output) The request that was evaluated, as a JSON string.
- candidate_
results Sequence[AiEvaluation Item Evaluation Response Candidate Result] - (Output) The results for the metric. Structure is documented below.
- evaluation_
request str - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - evaluation_
run str - (Output)
The evaluation run that was used to generate the result. Format:
projects/{project}/locations/{location}/evaluationRuns/{evaluation_run} - metadata str
- User-defined metadata for this evaluation item as a JSON string.
- metric str
- (Output) The metric that was evaluated.
- request str
- (Output) The request that was evaluated, as a JSON string.
- candidate
Results List<Property Map> - (Output) The results for the metric. Structure is documented below.
- evaluation
Request String - The request to evaluate.
Exactly one of
evaluationRequestorgcsUrican be set. Structure is documented below. - evaluation
Run String - (Output)
The evaluation run that was used to generate the result. Format:
projects/{project}/locations/{location}/evaluationRuns/{evaluation_run} - metadata String
- User-defined metadata for this evaluation item as a JSON string.
- metric String
- (Output) The metric that was evaluated.
- request String
- (Output) The request that was evaluated, as a JSON string.
AiEvaluationItemEvaluationResponseCandidateResult, AiEvaluationItemEvaluationResponseCandidateResultArgs
- Additional
Results string - (Output) Additional results for the metric, as a JSON string.
- Candidate string
- (Output) The candidate that is being evaluated.
- Errors
List<Ai
Evaluation Item Evaluation Response Candidate Result Error> - (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- Explanation string
- (Output) The explanation for the metric.
- Metric string
- (Output) The metric that was evaluated.
- Rubric
Verdicts List<AiEvaluation Item Evaluation Response Candidate Result Rubric Verdict> - (Output) The rubric verdicts for the metric. Structure is documented below.
- Score double
- (Output) The score for the metric.
- Additional
Results string - (Output) Additional results for the metric, as a JSON string.
- Candidate string
- (Output) The candidate that is being evaluated.
- Errors
[]Ai
Evaluation Item Evaluation Response Candidate Result Error - (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- Explanation string
- (Output) The explanation for the metric.
- Metric string
- (Output) The metric that was evaluated.
- Rubric
Verdicts []AiEvaluation Item Evaluation Response Candidate Result Rubric Verdict - (Output) The rubric verdicts for the metric. Structure is documented below.
- Score float64
- (Output) The score for the metric.
- additional_
results string - (Output) Additional results for the metric, as a JSON string.
- candidate string
- (Output) The candidate that is being evaluated.
- errors list(object)
- (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- explanation string
- (Output) The explanation for the metric.
- metric string
- (Output) The metric that was evaluated.
- rubric_
verdicts list(object) - (Output) The rubric verdicts for the metric. Structure is documented below.
- score number
- (Output) The score for the metric.
- additional
Results String - (Output) Additional results for the metric, as a JSON string.
- candidate String
- (Output) The candidate that is being evaluated.
- errors
List<Ai
Evaluation Item Evaluation Response Candidate Result Error> - (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- explanation String
- (Output) The explanation for the metric.
- metric String
- (Output) The metric that was evaluated.
- rubric
Verdicts List<AiEvaluation Item Evaluation Response Candidate Result Rubric Verdict> - (Output) The rubric verdicts for the metric. Structure is documented below.
- score Double
- (Output) The score for the metric.
- additional
Results string - (Output) Additional results for the metric, as a JSON string.
- candidate string
- (Output) The candidate that is being evaluated.
- errors
Ai
Evaluation Item Evaluation Response Candidate Result Error[] - (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- explanation string
- (Output) The explanation for the metric.
- metric string
- (Output) The metric that was evaluated.
- rubric
Verdicts AiEvaluation Item Evaluation Response Candidate Result Rubric Verdict[] - (Output) The rubric verdicts for the metric. Structure is documented below.
- score number
- (Output) The score for the metric.
- additional_
results str - (Output) Additional results for the metric, as a JSON string.
- candidate str
- (Output) The candidate that is being evaluated.
- errors
Sequence[Ai
Evaluation Item Evaluation Response Candidate Result Error] - (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- explanation str
- (Output) The explanation for the metric.
- metric str
- (Output) The metric that was evaluated.
- rubric_
verdicts Sequence[AiEvaluation Item Evaluation Response Candidate Result Rubric Verdict] - (Output) The rubric verdicts for the metric. Structure is documented below.
- score float
- (Output) The score for the metric.
- additional
Results String - (Output) Additional results for the metric, as a JSON string.
- candidate String
- (Output) The candidate that is being evaluated.
- errors List<Property Map>
- (Output) Error while evaluating the candidate for the metric. Structure is documented below.
- explanation String
- (Output) The explanation for the metric.
- metric String
- (Output) The metric that was evaluated.
- rubric
Verdicts List<Property Map> - (Output) The rubric verdicts for the metric. Structure is documented below.
- score Number
- (Output) The score for the metric.
AiEvaluationItemEvaluationResponseCandidateResultError, AiEvaluationItemEvaluationResponseCandidateResultErrorArgs
AiEvaluationItemEvaluationResponseCandidateResultRubricVerdict, AiEvaluationItemEvaluationResponseCandidateResultRubricVerdictArgs
- Evaluated
Rubric string - (Output) The full rubric definition that was evaluated, as a JSON string.
- Reasoning string
- (Output) Human-readable reasoning for the verdict.
- Verdict bool
- (Output) Outcome of the evaluation against the rubric.
- Evaluated
Rubric string - (Output) The full rubric definition that was evaluated, as a JSON string.
- Reasoning string
- (Output) Human-readable reasoning for the verdict.
- Verdict bool
- (Output) Outcome of the evaluation against the rubric.
- evaluated_
rubric string - (Output) The full rubric definition that was evaluated, as a JSON string.
- reasoning string
- (Output) Human-readable reasoning for the verdict.
- verdict bool
- (Output) Outcome of the evaluation against the rubric.
- evaluated
Rubric String - (Output) The full rubric definition that was evaluated, as a JSON string.
- reasoning String
- (Output) Human-readable reasoning for the verdict.
- verdict Boolean
- (Output) Outcome of the evaluation against the rubric.
- evaluated
Rubric string - (Output) The full rubric definition that was evaluated, as a JSON string.
- reasoning string
- (Output) Human-readable reasoning for the verdict.
- verdict boolean
- (Output) Outcome of the evaluation against the rubric.
- evaluated_
rubric str - (Output) The full rubric definition that was evaluated, as a JSON string.
- reasoning str
- (Output) Human-readable reasoning for the verdict.
- verdict bool
- (Output) Outcome of the evaluation against the rubric.
- evaluated
Rubric String - (Output) The full rubric definition that was evaluated, as a JSON string.
- reasoning String
- (Output) Human-readable reasoning for the verdict.
- verdict Boolean
- (Output) Outcome of the evaluation against the rubric.
Import
EvaluationItem can be imported using any of these accepted formats:
projects/{{project}}/locations/{{region}}/evaluationItems/{{name}}{{project}}/{{region}}/{{name}}{{region}}/{{name}}{{name}}
When using the pulumi import command, EvaluationItem can be imported using one of the formats above. For example:
$ pulumi import gcp:vertex/aiEvaluationItem:AiEvaluationItem default projects/{{project}}/locations/{{region}}/evaluationItems/{{name}}
$ pulumi import gcp:vertex/aiEvaluationItem:AiEvaluationItem default {{project}}/{{region}}/{{name}}
$ pulumi import gcp:vertex/aiEvaluationItem:AiEvaluationItem default {{region}}/{{name}}
$ pulumi import gcp:vertex/aiEvaluationItem:AiEvaluationItem default {{name}}
To learn more about importing existing cloud resources, see Importing resources.
Package Details
- Repository
- Google Cloud (GCP) pulumi/pulumi-gcp
- License
- Apache-2.0
- Notes
- This Pulumi package is based on the
google-betaTerraform Provider.
published on Wednesday, Nov 18, 2026 by Pulumi