Viewing docs for Google Cloud v9.23.0
published on Thursday, May 7, 2026 by Pulumi
published on Thursday, May 7, 2026 by Pulumi
Viewing docs for Google Cloud v9.23.0
published on Thursday, May 7, 2026 by Pulumi
published on Thursday, May 7, 2026 by Pulumi
Customer Engagement Suite Evaluation
Warning: This resource is in beta, and should be used with the terraform-provider-google-beta provider. See Provider Versions for more details on beta resources.
To get more information about Evaluation, see:
Example Usage
Ces Evaluation Basic
import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";
const app = new gcp.ces.App("app", {
appId: "app-id",
location: "us",
displayName: "my-app",
languageSettings: {
defaultLanguageCode: "en-US",
},
timeZoneSettings: {
timeZone: "America/Los_Angeles",
},
});
const cesEvaluationBasic = new gcp.ces.Evaluation("ces_evaluation_basic", {
evaluationId: "eval-basic",
displayName: "my-evaluation-basic",
location: "us",
app: app.appId,
});
import pulumi
import pulumi_gcp as gcp
app = gcp.ces.App("app",
app_id="app-id",
location="us",
display_name="my-app",
language_settings={
"default_language_code": "en-US",
},
time_zone_settings={
"time_zone": "America/Los_Angeles",
})
ces_evaluation_basic = gcp.ces.Evaluation("ces_evaluation_basic",
evaluation_id="eval-basic",
display_name="my-evaluation-basic",
location="us",
app=app.app_id)
package main
import (
"github.com/pulumi/pulumi-gcp/sdk/v9/go/gcp/ces"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
app, err := ces.NewApp(ctx, "app", &ces.AppArgs{
AppId: pulumi.String("app-id"),
Location: pulumi.String("us"),
DisplayName: pulumi.String("my-app"),
LanguageSettings: &ces.AppLanguageSettingsArgs{
DefaultLanguageCode: pulumi.String("en-US"),
},
TimeZoneSettings: &ces.AppTimeZoneSettingsArgs{
TimeZone: pulumi.String("America/Los_Angeles"),
},
})
if err != nil {
return err
}
_, err = ces.NewEvaluation(ctx, "ces_evaluation_basic", &ces.EvaluationArgs{
EvaluationId: pulumi.String("eval-basic"),
DisplayName: pulumi.String("my-evaluation-basic"),
Location: pulumi.String("us"),
App: app.AppId,
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;
return await Deployment.RunAsync(() =>
{
var app = new Gcp.Ces.App("app", new()
{
AppId = "app-id",
Location = "us",
DisplayName = "my-app",
LanguageSettings = new Gcp.Ces.Inputs.AppLanguageSettingsArgs
{
DefaultLanguageCode = "en-US",
},
TimeZoneSettings = new Gcp.Ces.Inputs.AppTimeZoneSettingsArgs
{
TimeZone = "America/Los_Angeles",
},
});
var cesEvaluationBasic = new Gcp.Ces.Evaluation("ces_evaluation_basic", new()
{
EvaluationId = "eval-basic",
DisplayName = "my-evaluation-basic",
Location = "us",
App = app.AppId,
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.ces.App;
import com.pulumi.gcp.ces.AppArgs;
import com.pulumi.gcp.ces.inputs.AppLanguageSettingsArgs;
import com.pulumi.gcp.ces.inputs.AppTimeZoneSettingsArgs;
import com.pulumi.gcp.ces.Evaluation;
import com.pulumi.gcp.ces.EvaluationArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var app = new App("app", AppArgs.builder()
.appId("app-id")
.location("us")
.displayName("my-app")
.languageSettings(AppLanguageSettingsArgs.builder()
.defaultLanguageCode("en-US")
.build())
.timeZoneSettings(AppTimeZoneSettingsArgs.builder()
.timeZone("America/Los_Angeles")
.build())
.build());
var cesEvaluationBasic = new Evaluation("cesEvaluationBasic", EvaluationArgs.builder()
.evaluationId("eval-basic")
.displayName("my-evaluation-basic")
.location("us")
.app(app.appId())
.build());
}
}
resources:
app:
type: gcp:ces:App
properties:
appId: app-id
location: us
displayName: my-app
languageSettings:
defaultLanguageCode: en-US
timeZoneSettings:
timeZone: America/Los_Angeles
cesEvaluationBasic:
type: gcp:ces:Evaluation
name: ces_evaluation_basic
properties:
evaluationId: eval-basic
displayName: my-evaluation-basic
location: us
app: ${app.appId}
Example coming soon!
Ces Evaluation Full
import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";
const app = new gcp.ces.App("app", {
appId: "app-id-full",
location: "us",
displayName: "my-app-full",
languageSettings: {
defaultLanguageCode: "en-US",
},
timeZoneSettings: {
timeZone: "America/Los_Angeles",
},
});
const tool = new gcp.ces.Tool("tool", {
location: "us",
app: app.appId,
toolId: "tool-id-full",
executionType: "SYNCHRONOUS",
pythonFunction: {
name: "example_function",
pythonCode: "def example_function() -> int: return 0",
},
});
const cesEvaluationFull = new gcp.ces.Evaluation("ces_evaluation_full", {
evaluationId: "evaluation-id-full",
displayName: "my-evaluation-full",
location: "us",
app: app.appId,
description: "Full evaluation for testing",
tags: [
"test",
"full",
],
golden: {
evaluationExpectations: [pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/evaluationExpectations/dummy-exp`],
turns: [{
steps: [
{
userInput: {
text: "Hello",
willContinue: true,
},
},
{
userInput: {
variables: {
key: "value",
},
},
},
{
expectation: {
note: "Expect tool call",
toolCall: {
id: "tool-call-id",
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
args: {
param: "value",
},
},
},
},
{
expectation: {
note: "Expect agent response",
agentResponse: {
role: "agent",
chunks: [
{
updatedVariables: {
key: "value",
},
},
{
agentTransfer: {
targetAgent: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/agents/dummy-agent`,
},
},
{
toolCall: {
id: "tool-call-id-3",
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
},
},
{
toolResponse: {
id: "tool-call-id-3",
response: {
result: "success",
},
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
},
},
],
},
},
},
{
expectation: {
note: "Expect toolset tool call",
toolCall: {
id: "tool-call-id-2",
toolsetTool: {
toolset: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset`,
toolId: "dummy-tool",
},
},
},
},
{
agentTransfer: {
targetAgent: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/agents/dummy-agent`,
},
},
{
expectation: {
agentTransfer: {
targetAgent: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/agents/dummy-agent`,
displayName: "dummy-agent",
},
},
},
{
expectation: {
note: "Expect mock tool response",
mockToolResponse: {
id: "tool-call-id-4",
response: {
result: "mocked",
},
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
},
},
},
{
expectation: {
note: "Expect tool response",
toolResponse: {
toolsetTool: {
toolset: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset`,
toolId: "dummy-tool",
},
},
},
},
{
expectation: {
note: "Expect updated variables",
updatedVariables: {
notes: "Some notes",
},
},
},
{
userInput: {
toolResponses: {
toolResponses: [{
id: "tool-call-id-5",
response: {
result: "user-provided",
},
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
}],
},
},
},
{
userInput: {
event: {
event: "my-event",
},
},
},
{
userInput: {
dtmf: "1234",
},
},
{
userInput: {
audio: "c29tZSBhdWRpbyBkYXRh",
},
},
{
userInput: {
blob: {
mimeType: "text/plain",
data: "c29tZSBibG9iIGRhdGE=",
},
},
},
{
userInput: {
image: {
mimeType: "image/png",
data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
},
},
},
{
userInput: {
toolResponses: {
toolResponses: [{
id: "tool-call-id",
response: {
result: "success",
},
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
}],
},
},
},
{
expectation: {
note: "Expect tool response",
toolResponse: {
id: "tool-call-id",
response: {
result: "success",
},
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
},
},
},
{
expectation: {
note: "Expect agent response with chunks",
agentResponse: {
role: "agent",
chunks: [
{
text: "Hello again",
},
{
text: "Hello again transcript",
},
{
blob: {
mimeType: "text/plain",
data: "c29tZSBibG9iIGRhdGE=",
},
},
{
image: {
mimeType: "image/png",
data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
},
},
{
toolCall: {
id: "tool-call-id-3",
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
args: {
param: "value",
},
},
},
],
},
},
},
],
}],
},
});
import pulumi
import pulumi_gcp as gcp
app = gcp.ces.App("app",
app_id="app-id-full",
location="us",
display_name="my-app-full",
language_settings={
"default_language_code": "en-US",
},
time_zone_settings={
"time_zone": "America/Los_Angeles",
})
tool = gcp.ces.Tool("tool",
location="us",
app=app.app_id,
tool_id="tool-id-full",
execution_type="SYNCHRONOUS",
python_function={
"name": "example_function",
"python_code": "def example_function() -> int: return 0",
})
ces_evaluation_full = gcp.ces.Evaluation("ces_evaluation_full",
evaluation_id="evaluation-id-full",
display_name="my-evaluation-full",
location="us",
app=app.app_id,
description="Full evaluation for testing",
tags=[
"test",
"full",
],
golden={
"evaluation_expectations": [pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/evaluationExpectations/dummy-exp")
],
"turns": [{
"steps": [
{
"user_input": {
"text": "Hello",
"will_continue": True,
},
},
{
"user_input": {
"variables": {
"key": "value",
},
},
},
{
"expectation": {
"note": "Expect tool call",
"tool_call": {
"id": "tool-call-id",
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
"args": {
"param": "value",
},
},
},
},
{
"expectation": {
"note": "Expect agent response",
"agent_response": {
"role": "agent",
"chunks": [
{
"updated_variables": {
"key": "value",
},
},
{
"agent_transfer": {
"target_agent": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/agents/dummy-agent")
,
},
},
{
"tool_call": {
"id": "tool-call-id-3",
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
},
},
{
"tool_response": {
"id": "tool-call-id-3",
"response": {
"result": "success",
},
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
},
},
],
},
},
},
{
"expectation": {
"note": "Expect toolset tool call",
"tool_call": {
"id": "tool-call-id-2",
"toolset_tool": {
"toolset": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/toolsets/dummy-toolset")
,
"tool_id": "dummy-tool",
},
},
},
},
{
"agent_transfer": {
"target_agent": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/agents/dummy-agent")
,
},
},
{
"expectation": {
"agent_transfer": {
"target_agent": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/agents/dummy-agent")
,
"display_name": "dummy-agent",
},
},
},
{
"expectation": {
"note": "Expect mock tool response",
"mock_tool_response": {
"id": "tool-call-id-4",
"response": {
"result": "mocked",
},
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
},
},
},
{
"expectation": {
"note": "Expect tool response",
"tool_response": {
"toolset_tool": {
"toolset": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/toolsets/dummy-toolset")
,
"tool_id": "dummy-tool",
},
},
},
},
{
"expectation": {
"note": "Expect updated variables",
"updated_variables": {
"notes": "Some notes",
},
},
},
{
"user_input": {
"tool_responses": {
"tool_responses": [{
"id": "tool-call-id-5",
"response": {
"result": "user-provided",
},
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
}],
},
},
},
{
"user_input": {
"event": {
"event": "my-event",
},
},
},
{
"user_input": {
"dtmf": "1234",
},
},
{
"user_input": {
"audio": "c29tZSBhdWRpbyBkYXRh",
},
},
{
"user_input": {
"blob": {
"mime_type": "text/plain",
"data": "c29tZSBibG9iIGRhdGE=",
},
},
},
{
"user_input": {
"image": {
"mime_type": "image/png",
"data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
},
},
},
{
"user_input": {
"tool_responses": {
"tool_responses": [{
"id": "tool-call-id",
"response": {
"result": "success",
},
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
}],
},
},
},
{
"expectation": {
"note": "Expect tool response",
"tool_response": {
"id": "tool-call-id",
"response": {
"result": "success",
},
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
},
},
},
{
"expectation": {
"note": "Expect agent response with chunks",
"agent_response": {
"role": "agent",
"chunks": [
{
"text": "Hello again",
},
{
"text": "Hello again transcript",
},
{
"blob": {
"mime_type": "text/plain",
"data": "c29tZSBibG9iIGRhdGE=",
},
},
{
"image": {
"mime_type": "image/png",
"data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
},
},
{
"tool_call": {
"id": "tool-call-id-3",
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
"args": {
"param": "value",
},
},
},
],
},
},
},
],
}],
})
package main
import (
"fmt"
"github.com/pulumi/pulumi-gcp/sdk/v9/go/gcp/ces"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
app, err := ces.NewApp(ctx, "app", &ces.AppArgs{
AppId: pulumi.String("app-id-full"),
Location: pulumi.String("us"),
DisplayName: pulumi.String("my-app-full"),
LanguageSettings: &ces.AppLanguageSettingsArgs{
DefaultLanguageCode: pulumi.String("en-US"),
},
TimeZoneSettings: &ces.AppTimeZoneSettingsArgs{
TimeZone: pulumi.String("America/Los_Angeles"),
},
})
if err != nil {
return err
}
tool, err := ces.NewTool(ctx, "tool", &ces.ToolArgs{
Location: pulumi.String("us"),
App: app.AppId,
ToolId: pulumi.String("tool-id-full"),
ExecutionType: pulumi.String("SYNCHRONOUS"),
PythonFunction: &ces.ToolPythonFunctionArgs{
Name: pulumi.String("example_function"),
PythonCode: pulumi.String("def example_function() -> int: return 0"),
},
})
if err != nil {
return err
}
_, err = ces.NewEvaluation(ctx, "ces_evaluation_full", &ces.EvaluationArgs{
EvaluationId: pulumi.String("evaluation-id-full"),
DisplayName: pulumi.String("my-evaluation-full"),
Location: pulumi.String("us"),
App: app.AppId,
Description: pulumi.String("Full evaluation for testing"),
Tags: pulumi.StringArray{
pulumi.String("test"),
pulumi.String("full"),
},
Golden: &ces.EvaluationGoldenArgs{
EvaluationExpectations: pulumi.StringArray{
pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/evaluationExpectations/dummy-exp", project, appId), nil
}).(pulumi.StringOutput),
},
Turns: ces.EvaluationGoldenTurnArray{
&ces.EvaluationGoldenTurnArgs{
Steps: ces.EvaluationGoldenTurnStepArray{
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
Text: pulumi.String("Hello"),
WillContinue: pulumi.Bool(true),
},
},
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
Variables: pulumi.StringMap{
"key": pulumi.String("value"),
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect tool call"),
ToolCall: &ces.EvaluationGoldenTurnStepExpectationToolCallArgs{
Id: pulumi.String("tool-call-id"),
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
Args: pulumi.StringMap{
"param": pulumi.String("value"),
},
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect agent response"),
AgentResponse: &ces.EvaluationGoldenTurnStepExpectationAgentResponseArgs{
Role: pulumi.String("agent"),
Chunks: ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArray{
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
UpdatedVariables: pulumi.StringMap{
"key": pulumi.String("value"),
},
},
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
AgentTransfer: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkAgentTransferArgs{
TargetAgent: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/agents/dummy-agent", project, appId), nil
}).(pulumi.StringOutput),
},
},
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
ToolCall: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs{
Id: pulumi.String("tool-call-id-3"),
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
},
},
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
ToolResponse: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs{
Id: pulumi.String("tool-call-id-3"),
Response: pulumi.StringMap{
"result": pulumi.String("success"),
},
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
},
},
},
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect toolset tool call"),
ToolCall: &ces.EvaluationGoldenTurnStepExpectationToolCallArgs{
Id: pulumi.String("tool-call-id-2"),
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationToolCallToolsetToolArgs{
Toolset: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/toolsets/dummy-toolset", project, appId), nil
}).(pulumi.StringOutput),
ToolId: pulumi.String("dummy-tool"),
},
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
AgentTransfer: &ces.EvaluationGoldenTurnStepAgentTransferArgs{
TargetAgent: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/agents/dummy-agent", project, appId), nil
}).(pulumi.StringOutput),
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
AgentTransfer: &ces.EvaluationGoldenTurnStepExpectationAgentTransferArgs{
TargetAgent: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/agents/dummy-agent", project, appId), nil
}).(pulumi.StringOutput),
DisplayName: pulumi.String("dummy-agent"),
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect mock tool response"),
MockToolResponse: &ces.EvaluationGoldenTurnStepExpectationMockToolResponseArgs{
Id: pulumi.String("tool-call-id-4"),
Response: pulumi.StringMap{
"result": pulumi.String("mocked"),
},
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect tool response"),
ToolResponse: &ces.EvaluationGoldenTurnStepExpectationToolResponseArgs{
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationToolResponseToolsetToolArgs{
Toolset: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/toolsets/dummy-toolset", project, appId), nil
}).(pulumi.StringOutput),
ToolId: pulumi.String("dummy-tool"),
},
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect updated variables"),
UpdatedVariables: &ces.EvaluationGoldenTurnStepExpectationUpdatedVariablesArgs{
Notes: pulumi.String("Some notes"),
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
ToolResponses: &ces.EvaluationGoldenTurnStepUserInputToolResponsesArgs{
ToolResponses: ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArray{
&ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs{
Id: pulumi.String("tool-call-id-5"),
Response: pulumi.StringMap{
"result": pulumi.String("user-provided"),
},
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
},
},
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
Event: &ces.EvaluationGoldenTurnStepUserInputEventArgs{
Event: pulumi.String("my-event"),
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
Dtmf: pulumi.String("1234"),
},
},
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
Audio: pulumi.String("c29tZSBhdWRpbyBkYXRh"),
},
},
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
Blob: &ces.EvaluationGoldenTurnStepUserInputBlobArgs{
MimeType: pulumi.String("text/plain"),
Data: pulumi.String("c29tZSBibG9iIGRhdGE="),
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
Image: &ces.EvaluationGoldenTurnStepUserInputImageArgs{
MimeType: pulumi.String("image/png"),
Data: pulumi.String("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="),
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
ToolResponses: &ces.EvaluationGoldenTurnStepUserInputToolResponsesArgs{
ToolResponses: ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArray{
&ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs{
Id: pulumi.String("tool-call-id"),
Response: pulumi.StringMap{
"result": pulumi.String("success"),
},
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
},
},
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect tool response"),
ToolResponse: &ces.EvaluationGoldenTurnStepExpectationToolResponseArgs{
Id: pulumi.String("tool-call-id"),
Response: pulumi.StringMap{
"result": pulumi.String("success"),
},
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect agent response with chunks"),
AgentResponse: &ces.EvaluationGoldenTurnStepExpectationAgentResponseArgs{
Role: pulumi.String("agent"),
Chunks: ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArray{
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
Text: pulumi.String("Hello again"),
},
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
Text: pulumi.String("Hello again transcript"),
},
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
Blob: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkBlobArgs{
MimeType: pulumi.String("text/plain"),
Data: pulumi.String("c29tZSBibG9iIGRhdGE="),
},
},
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
Image: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkImageArgs{
MimeType: pulumi.String("image/png"),
Data: pulumi.String("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="),
},
},
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
ToolCall: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs{
Id: pulumi.String("tool-call-id-3"),
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
Args: pulumi.StringMap{
"param": pulumi.String("value"),
},
},
},
},
},
},
},
},
},
},
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;
return await Deployment.RunAsync(() =>
{
var app = new Gcp.Ces.App("app", new()
{
AppId = "app-id-full",
Location = "us",
DisplayName = "my-app-full",
LanguageSettings = new Gcp.Ces.Inputs.AppLanguageSettingsArgs
{
DefaultLanguageCode = "en-US",
},
TimeZoneSettings = new Gcp.Ces.Inputs.AppTimeZoneSettingsArgs
{
TimeZone = "America/Los_Angeles",
},
});
var tool = new Gcp.Ces.Tool("tool", new()
{
Location = "us",
App = app.AppId,
ToolId = "tool-id-full",
ExecutionType = "SYNCHRONOUS",
PythonFunction = new Gcp.Ces.Inputs.ToolPythonFunctionArgs
{
Name = "example_function",
PythonCode = "def example_function() -> int: return 0",
},
});
var cesEvaluationFull = new Gcp.Ces.Evaluation("ces_evaluation_full", new()
{
EvaluationId = "evaluation-id-full",
DisplayName = "my-evaluation-full",
Location = "us",
App = app.AppId,
Description = "Full evaluation for testing",
Tags = new[]
{
"test",
"full",
},
Golden = new Gcp.Ces.Inputs.EvaluationGoldenArgs
{
EvaluationExpectations = new[]
{
Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/evaluationExpectations/dummy-exp";
}),
},
Turns = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnArgs
{
Steps = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
Text = "Hello",
WillContinue = true,
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
Variables =
{
{ "key", "value" },
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect tool call",
ToolCall = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolCallArgs
{
Id = "tool-call-id",
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
Args =
{
{ "param", "value" },
},
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect agent response",
AgentResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseArgs
{
Role = "agent",
Chunks = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
UpdatedVariables =
{
{ "key", "value" },
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
AgentTransfer = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkAgentTransferArgs
{
TargetAgent = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/agents/dummy-agent";
}),
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
ToolCall = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs
{
Id = "tool-call-id-3",
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
ToolResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs
{
Id = "tool-call-id-3",
Response =
{
{ "result", "success" },
},
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
},
},
},
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect toolset tool call",
ToolCall = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolCallArgs
{
Id = "tool-call-id-2",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolCallToolsetToolArgs
{
Toolset = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/toolsets/dummy-toolset";
}),
ToolId = "dummy-tool",
},
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
AgentTransfer = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepAgentTransferArgs
{
TargetAgent = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/agents/dummy-agent";
}),
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
AgentTransfer = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentTransferArgs
{
TargetAgent = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/agents/dummy-agent";
}),
DisplayName = "dummy-agent",
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect mock tool response",
MockToolResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationMockToolResponseArgs
{
Id = "tool-call-id-4",
Response =
{
{ "result", "mocked" },
},
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect tool response",
ToolResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolResponseArgs
{
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolResponseToolsetToolArgs
{
Toolset = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/toolsets/dummy-toolset";
}),
ToolId = "dummy-tool",
},
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect updated variables",
UpdatedVariables = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationUpdatedVariablesArgs
{
Notes = "Some notes",
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
ToolResponses = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesArgs
{
ToolResponses = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs
{
Id = "tool-call-id-5",
Response =
{
{ "result", "user-provided" },
},
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
},
},
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
Event = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputEventArgs
{
Event = "my-event",
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
Dtmf = "1234",
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
Audio = "c29tZSBhdWRpbyBkYXRh",
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
Blob = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputBlobArgs
{
MimeType = "text/plain",
Data = "c29tZSBibG9iIGRhdGE=",
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
Image = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputImageArgs
{
MimeType = "image/png",
Data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
ToolResponses = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesArgs
{
ToolResponses = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs
{
Id = "tool-call-id",
Response =
{
{ "result", "success" },
},
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
},
},
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect tool response",
ToolResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolResponseArgs
{
Id = "tool-call-id",
Response =
{
{ "result", "success" },
},
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect agent response with chunks",
AgentResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseArgs
{
Role = "agent",
Chunks = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
Text = "Hello again",
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
Text = "Hello again transcript",
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
Blob = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkBlobArgs
{
MimeType = "text/plain",
Data = "c29tZSBibG9iIGRhdGE=",
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
Image = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkImageArgs
{
MimeType = "image/png",
Data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
ToolCall = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs
{
Id = "tool-call-id-3",
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
Args =
{
{ "param", "value" },
},
},
},
},
},
},
},
},
},
},
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.ces.App;
import com.pulumi.gcp.ces.AppArgs;
import com.pulumi.gcp.ces.inputs.AppLanguageSettingsArgs;
import com.pulumi.gcp.ces.inputs.AppTimeZoneSettingsArgs;
import com.pulumi.gcp.ces.Tool;
import com.pulumi.gcp.ces.ToolArgs;
import com.pulumi.gcp.ces.inputs.ToolPythonFunctionArgs;
import com.pulumi.gcp.ces.Evaluation;
import com.pulumi.gcp.ces.EvaluationArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var app = new App("app", AppArgs.builder()
.appId("app-id-full")
.location("us")
.displayName("my-app-full")
.languageSettings(AppLanguageSettingsArgs.builder()
.defaultLanguageCode("en-US")
.build())
.timeZoneSettings(AppTimeZoneSettingsArgs.builder()
.timeZone("America/Los_Angeles")
.build())
.build());
var tool = new Tool("tool", ToolArgs.builder()
.location("us")
.app(app.appId())
.toolId("tool-id-full")
.executionType("SYNCHRONOUS")
.pythonFunction(ToolPythonFunctionArgs.builder()
.name("example_function")
.pythonCode("def example_function() -> int: return 0")
.build())
.build());
var cesEvaluationFull = new Evaluation("cesEvaluationFull", EvaluationArgs.builder()
.evaluationId("evaluation-id-full")
.displayName("my-evaluation-full")
.location("us")
.app(app.appId())
.description("Full evaluation for testing")
.tags(
"test",
"full")
.golden(EvaluationGoldenArgs.builder()
.evaluationExpectations(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/evaluationExpectations/dummy-exp", project,appId);
}))
.turns(EvaluationGoldenTurnArgs.builder()
.steps(
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.text("Hello")
.willContinue(true)
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.variables(Map.of("key", "value"))
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect tool call")
.toolCall(EvaluationGoldenTurnStepExpectationToolCallArgs.builder()
.id("tool-call-id")
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.args(Map.of("param", "value"))
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect agent response")
.agentResponse(EvaluationGoldenTurnStepExpectationAgentResponseArgs.builder()
.role("agent")
.chunks(
EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.updatedVariables(Map.of("key", "value"))
.build(),
EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.agentTransfer(EvaluationGoldenTurnStepExpectationAgentResponseChunkAgentTransferArgs.builder()
.targetAgent(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/agents/dummy-agent", project,appId);
}))
.build())
.build(),
EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.toolCall(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs.builder()
.id("tool-call-id-3")
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.build())
.build(),
EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.toolResponse(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs.builder()
.id("tool-call-id-3")
.response(Map.of("result", "success"))
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.build())
.build())
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect toolset tool call")
.toolCall(EvaluationGoldenTurnStepExpectationToolCallArgs.builder()
.id("tool-call-id-2")
.toolsetTool(EvaluationGoldenTurnStepExpectationToolCallToolsetToolArgs.builder()
.toolset(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/toolsets/dummy-toolset", project,appId);
}))
.toolId("dummy-tool")
.build())
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.agentTransfer(EvaluationGoldenTurnStepAgentTransferArgs.builder()
.targetAgent(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/agents/dummy-agent", project,appId);
}))
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.agentTransfer(EvaluationGoldenTurnStepExpectationAgentTransferArgs.builder()
.targetAgent(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/agents/dummy-agent", project,appId);
}))
.displayName("dummy-agent")
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect mock tool response")
.mockToolResponse(EvaluationGoldenTurnStepExpectationMockToolResponseArgs.builder()
.id("tool-call-id-4")
.response(Map.of("result", "mocked"))
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect tool response")
.toolResponse(EvaluationGoldenTurnStepExpectationToolResponseArgs.builder()
.toolsetTool(EvaluationGoldenTurnStepExpectationToolResponseToolsetToolArgs.builder()
.toolset(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/toolsets/dummy-toolset", project,appId);
}))
.toolId("dummy-tool")
.build())
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect updated variables")
.updatedVariables(EvaluationGoldenTurnStepExpectationUpdatedVariablesArgs.builder()
.notes("Some notes")
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.toolResponses(EvaluationGoldenTurnStepUserInputToolResponsesArgs.builder()
.toolResponses(EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs.builder()
.id("tool-call-id-5")
.response(Map.of("result", "user-provided"))
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.build())
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.event(EvaluationGoldenTurnStepUserInputEventArgs.builder()
.event("my-event")
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.dtmf("1234")
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.audio("c29tZSBhdWRpbyBkYXRh")
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.blob(EvaluationGoldenTurnStepUserInputBlobArgs.builder()
.mimeType("text/plain")
.data("c29tZSBibG9iIGRhdGE=")
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.image(EvaluationGoldenTurnStepUserInputImageArgs.builder()
.mimeType("image/png")
.data("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=")
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.toolResponses(EvaluationGoldenTurnStepUserInputToolResponsesArgs.builder()
.toolResponses(EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs.builder()
.id("tool-call-id")
.response(Map.of("result", "success"))
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.build())
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect tool response")
.toolResponse(EvaluationGoldenTurnStepExpectationToolResponseArgs.builder()
.id("tool-call-id")
.response(Map.of("result", "success"))
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect agent response with chunks")
.agentResponse(EvaluationGoldenTurnStepExpectationAgentResponseArgs.builder()
.role("agent")
.chunks(
EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.text("Hello again")
.build(),
EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.text("Hello again transcript")
.build(),
EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.blob(EvaluationGoldenTurnStepExpectationAgentResponseChunkBlobArgs.builder()
.mimeType("text/plain")
.data("c29tZSBibG9iIGRhdGE=")
.build())
.build(),
EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.image(EvaluationGoldenTurnStepExpectationAgentResponseChunkImageArgs.builder()
.mimeType("image/png")
.data("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=")
.build())
.build(),
EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.toolCall(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs.builder()
.id("tool-call-id-3")
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.args(Map.of("param", "value"))
.build())
.build())
.build())
.build())
.build())
.build())
.build())
.build());
}
}
resources:
app:
type: gcp:ces:App
properties:
appId: app-id-full
location: us
displayName: my-app-full
languageSettings:
defaultLanguageCode: en-US
timeZoneSettings:
timeZone: America/Los_Angeles
tool:
type: gcp:ces:Tool
properties:
location: us
app: ${app.appId}
toolId: tool-id-full
executionType: SYNCHRONOUS
pythonFunction:
name: example_function
pythonCode: 'def example_function() -> int: return 0'
cesEvaluationFull:
type: gcp:ces:Evaluation
name: ces_evaluation_full
properties:
evaluationId: evaluation-id-full
displayName: my-evaluation-full
location: us
app: ${app.appId}
description: Full evaluation for testing
tags:
- test
- full
golden:
evaluationExpectations:
- projects/${app.project}/locations/us/apps/${app.appId}/evaluationExpectations/dummy-exp
turns:
- steps:
- userInput:
text: Hello
willContinue: true
- userInput:
variables:
key: value
- expectation:
note: Expect tool call
toolCall:
id: tool-call-id
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
args:
param: value
- expectation:
note: Expect agent response
agentResponse:
role: agent
chunks:
- updatedVariables:
key: value
- agentTransfer:
targetAgent: projects/${app.project}/locations/us/apps/${app.appId}/agents/dummy-agent
- toolCall:
id: tool-call-id-3
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
- toolResponse:
id: tool-call-id-3
response:
result: success
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
- expectation:
note: Expect toolset tool call
toolCall:
id: tool-call-id-2
toolsetTool:
toolset: projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset
toolId: dummy-tool
- agentTransfer:
targetAgent: projects/${app.project}/locations/us/apps/${app.appId}/agents/dummy-agent
- expectation:
agentTransfer:
targetAgent: projects/${app.project}/locations/us/apps/${app.appId}/agents/dummy-agent
displayName: dummy-agent
- expectation:
note: Expect mock tool response
mockToolResponse:
id: tool-call-id-4
response:
result: mocked
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
- expectation:
note: Expect tool response
toolResponse:
toolsetTool:
toolset: projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset
toolId: dummy-tool
- expectation:
note: Expect updated variables
updatedVariables:
notes: Some notes
- userInput:
toolResponses:
toolResponses:
- id: tool-call-id-5
response:
result: user-provided
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
- userInput:
event:
event: my-event
- userInput:
dtmf: '1234'
- userInput:
audio: c29tZSBhdWRpbyBkYXRh
- userInput:
blob:
mimeType: text/plain
data: c29tZSBibG9iIGRhdGE=
- userInput:
image:
mimeType: image/png
data: iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=
- userInput:
toolResponses:
toolResponses:
- id: tool-call-id
response:
result: success
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
- expectation:
note: Expect tool response
toolResponse:
id: tool-call-id
response:
result: success
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
- expectation:
note: Expect agent response with chunks
agentResponse:
role: agent
chunks:
- text: Hello again
- text: Hello again transcript
- blob:
mimeType: text/plain
data: c29tZSBibG9iIGRhdGE=
- image:
mimeType: image/png
data: iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=
- toolCall:
id: tool-call-id-3
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
args:
param: value
Example coming soon!
Ces Evaluation Toolset
import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";
const app = new gcp.ces.App("app", {
appId: "app-id-toolset",
location: "us",
displayName: "my-app-toolset",
languageSettings: {
defaultLanguageCode: "en-US",
},
timeZoneSettings: {
timeZone: "America/Los_Angeles",
},
});
const cesEvaluationToolset = new gcp.ces.Evaluation("ces_evaluation_toolset", {
evaluationId: "eval-toolset",
displayName: "my-evaluation-toolset",
location: "us",
app: app.appId,
golden: {
turns: [{
steps: [
{
expectation: {
note: "Expect agent response with toolset tool call",
agentResponse: {
role: "agent",
chunks: [{
toolCall: {
id: "tool-call-id",
toolsetTool: {
toolset: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset`,
toolId: "dummy-tool",
},
},
}],
},
},
},
{
expectation: {
note: "Expect agent response with toolset tool response",
agentResponse: {
role: "agent",
chunks: [{
toolResponse: {
id: "tool-call-id",
response: {
result: "success",
},
toolsetTool: {
toolset: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset`,
toolId: "dummy-tool",
},
},
}],
},
},
},
{
expectation: {
note: "Expect mock tool response with toolset tool",
mockToolResponse: {
id: "tool-call-id-2",
response: {
result: "mocked",
},
toolsetTool: {
toolset: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset`,
toolId: "dummy-tool",
},
},
},
},
{
userInput: {
toolResponses: {
toolResponses: [{
id: "tool-call-id-3",
response: {
result: "user-provided",
},
toolsetTool: {
toolset: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset`,
toolId: "dummy-tool",
},
}],
},
},
},
],
}],
},
});
import pulumi
import pulumi_gcp as gcp
app = gcp.ces.App("app",
app_id="app-id-toolset",
location="us",
display_name="my-app-toolset",
language_settings={
"default_language_code": "en-US",
},
time_zone_settings={
"time_zone": "America/Los_Angeles",
})
ces_evaluation_toolset = gcp.ces.Evaluation("ces_evaluation_toolset",
evaluation_id="eval-toolset",
display_name="my-evaluation-toolset",
location="us",
app=app.app_id,
golden={
"turns": [{
"steps": [
{
"expectation": {
"note": "Expect agent response with toolset tool call",
"agent_response": {
"role": "agent",
"chunks": [{
"tool_call": {
"id": "tool-call-id",
"toolset_tool": {
"toolset": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/toolsets/dummy-toolset")
,
"tool_id": "dummy-tool",
},
},
}],
},
},
},
{
"expectation": {
"note": "Expect agent response with toolset tool response",
"agent_response": {
"role": "agent",
"chunks": [{
"tool_response": {
"id": "tool-call-id",
"response": {
"result": "success",
},
"toolset_tool": {
"toolset": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/toolsets/dummy-toolset")
,
"tool_id": "dummy-tool",
},
},
}],
},
},
},
{
"expectation": {
"note": "Expect mock tool response with toolset tool",
"mock_tool_response": {
"id": "tool-call-id-2",
"response": {
"result": "mocked",
},
"toolset_tool": {
"toolset": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/toolsets/dummy-toolset")
,
"tool_id": "dummy-tool",
},
},
},
},
{
"user_input": {
"tool_responses": {
"tool_responses": [{
"id": "tool-call-id-3",
"response": {
"result": "user-provided",
},
"toolset_tool": {
"toolset": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/toolsets/dummy-toolset")
,
"tool_id": "dummy-tool",
},
}],
},
},
},
],
}],
})
package main
import (
"fmt"
"github.com/pulumi/pulumi-gcp/sdk/v9/go/gcp/ces"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
app, err := ces.NewApp(ctx, "app", &ces.AppArgs{
AppId: pulumi.String("app-id-toolset"),
Location: pulumi.String("us"),
DisplayName: pulumi.String("my-app-toolset"),
LanguageSettings: &ces.AppLanguageSettingsArgs{
DefaultLanguageCode: pulumi.String("en-US"),
},
TimeZoneSettings: &ces.AppTimeZoneSettingsArgs{
TimeZone: pulumi.String("America/Los_Angeles"),
},
})
if err != nil {
return err
}
_, err = ces.NewEvaluation(ctx, "ces_evaluation_toolset", &ces.EvaluationArgs{
EvaluationId: pulumi.String("eval-toolset"),
DisplayName: pulumi.String("my-evaluation-toolset"),
Location: pulumi.String("us"),
App: app.AppId,
Golden: &ces.EvaluationGoldenArgs{
Turns: ces.EvaluationGoldenTurnArray{
&ces.EvaluationGoldenTurnArgs{
Steps: ces.EvaluationGoldenTurnStepArray{
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect agent response with toolset tool call"),
AgentResponse: &ces.EvaluationGoldenTurnStepExpectationAgentResponseArgs{
Role: pulumi.String("agent"),
Chunks: ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArray{
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
ToolCall: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs{
Id: pulumi.String("tool-call-id"),
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallToolsetToolArgs{
Toolset: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/toolsets/dummy-toolset", project, appId), nil
}).(pulumi.StringOutput),
ToolId: pulumi.String("dummy-tool"),
},
},
},
},
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect agent response with toolset tool response"),
AgentResponse: &ces.EvaluationGoldenTurnStepExpectationAgentResponseArgs{
Role: pulumi.String("agent"),
Chunks: ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArray{
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
ToolResponse: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs{
Id: pulumi.String("tool-call-id"),
Response: pulumi.StringMap{
"result": pulumi.String("success"),
},
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseToolsetToolArgs{
Toolset: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/toolsets/dummy-toolset", project, appId), nil
}).(pulumi.StringOutput),
ToolId: pulumi.String("dummy-tool"),
},
},
},
},
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect mock tool response with toolset tool"),
MockToolResponse: &ces.EvaluationGoldenTurnStepExpectationMockToolResponseArgs{
Id: pulumi.String("tool-call-id-2"),
Response: pulumi.StringMap{
"result": pulumi.String("mocked"),
},
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationMockToolResponseToolsetToolArgs{
Toolset: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/toolsets/dummy-toolset", project, appId), nil
}).(pulumi.StringOutput),
ToolId: pulumi.String("dummy-tool"),
},
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
ToolResponses: &ces.EvaluationGoldenTurnStepUserInputToolResponsesArgs{
ToolResponses: ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArray{
&ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs{
Id: pulumi.String("tool-call-id-3"),
Response: pulumi.StringMap{
"result": pulumi.String("user-provided"),
},
ToolsetTool: &ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseToolsetToolArgs{
Toolset: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/toolsets/dummy-toolset", project, appId), nil
}).(pulumi.StringOutput),
ToolId: pulumi.String("dummy-tool"),
},
},
},
},
},
},
},
},
},
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;
return await Deployment.RunAsync(() =>
{
var app = new Gcp.Ces.App("app", new()
{
AppId = "app-id-toolset",
Location = "us",
DisplayName = "my-app-toolset",
LanguageSettings = new Gcp.Ces.Inputs.AppLanguageSettingsArgs
{
DefaultLanguageCode = "en-US",
},
TimeZoneSettings = new Gcp.Ces.Inputs.AppTimeZoneSettingsArgs
{
TimeZone = "America/Los_Angeles",
},
});
var cesEvaluationToolset = new Gcp.Ces.Evaluation("ces_evaluation_toolset", new()
{
EvaluationId = "eval-toolset",
DisplayName = "my-evaluation-toolset",
Location = "us",
App = app.AppId,
Golden = new Gcp.Ces.Inputs.EvaluationGoldenArgs
{
Turns = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnArgs
{
Steps = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect agent response with toolset tool call",
AgentResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseArgs
{
Role = "agent",
Chunks = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
ToolCall = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs
{
Id = "tool-call-id",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallToolsetToolArgs
{
Toolset = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/toolsets/dummy-toolset";
}),
ToolId = "dummy-tool",
},
},
},
},
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect agent response with toolset tool response",
AgentResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseArgs
{
Role = "agent",
Chunks = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
ToolResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs
{
Id = "tool-call-id",
Response =
{
{ "result", "success" },
},
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseToolsetToolArgs
{
Toolset = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/toolsets/dummy-toolset";
}),
ToolId = "dummy-tool",
},
},
},
},
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect mock tool response with toolset tool",
MockToolResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationMockToolResponseArgs
{
Id = "tool-call-id-2",
Response =
{
{ "result", "mocked" },
},
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationMockToolResponseToolsetToolArgs
{
Toolset = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/toolsets/dummy-toolset";
}),
ToolId = "dummy-tool",
},
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
ToolResponses = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesArgs
{
ToolResponses = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs
{
Id = "tool-call-id-3",
Response =
{
{ "result", "user-provided" },
},
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseToolsetToolArgs
{
Toolset = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/toolsets/dummy-toolset";
}),
ToolId = "dummy-tool",
},
},
},
},
},
},
},
},
},
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.ces.App;
import com.pulumi.gcp.ces.AppArgs;
import com.pulumi.gcp.ces.inputs.AppLanguageSettingsArgs;
import com.pulumi.gcp.ces.inputs.AppTimeZoneSettingsArgs;
import com.pulumi.gcp.ces.Evaluation;
import com.pulumi.gcp.ces.EvaluationArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var app = new App("app", AppArgs.builder()
.appId("app-id-toolset")
.location("us")
.displayName("my-app-toolset")
.languageSettings(AppLanguageSettingsArgs.builder()
.defaultLanguageCode("en-US")
.build())
.timeZoneSettings(AppTimeZoneSettingsArgs.builder()
.timeZone("America/Los_Angeles")
.build())
.build());
var cesEvaluationToolset = new Evaluation("cesEvaluationToolset", EvaluationArgs.builder()
.evaluationId("eval-toolset")
.displayName("my-evaluation-toolset")
.location("us")
.app(app.appId())
.golden(EvaluationGoldenArgs.builder()
.turns(EvaluationGoldenTurnArgs.builder()
.steps(
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect agent response with toolset tool call")
.agentResponse(EvaluationGoldenTurnStepExpectationAgentResponseArgs.builder()
.role("agent")
.chunks(EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.toolCall(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs.builder()
.id("tool-call-id")
.toolsetTool(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallToolsetToolArgs.builder()
.toolset(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/toolsets/dummy-toolset", project,appId);
}))
.toolId("dummy-tool")
.build())
.build())
.build())
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect agent response with toolset tool response")
.agentResponse(EvaluationGoldenTurnStepExpectationAgentResponseArgs.builder()
.role("agent")
.chunks(EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.toolResponse(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs.builder()
.id("tool-call-id")
.response(Map.of("result", "success"))
.toolsetTool(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseToolsetToolArgs.builder()
.toolset(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/toolsets/dummy-toolset", project,appId);
}))
.toolId("dummy-tool")
.build())
.build())
.build())
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect mock tool response with toolset tool")
.mockToolResponse(EvaluationGoldenTurnStepExpectationMockToolResponseArgs.builder()
.id("tool-call-id-2")
.response(Map.of("result", "mocked"))
.toolsetTool(EvaluationGoldenTurnStepExpectationMockToolResponseToolsetToolArgs.builder()
.toolset(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/toolsets/dummy-toolset", project,appId);
}))
.toolId("dummy-tool")
.build())
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.toolResponses(EvaluationGoldenTurnStepUserInputToolResponsesArgs.builder()
.toolResponses(EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs.builder()
.id("tool-call-id-3")
.response(Map.of("result", "user-provided"))
.toolsetTool(EvaluationGoldenTurnStepUserInputToolResponsesToolResponseToolsetToolArgs.builder()
.toolset(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/toolsets/dummy-toolset", project,appId);
}))
.toolId("dummy-tool")
.build())
.build())
.build())
.build())
.build())
.build())
.build())
.build());
}
}
resources:
app:
type: gcp:ces:App
properties:
appId: app-id-toolset
location: us
displayName: my-app-toolset
languageSettings:
defaultLanguageCode: en-US
timeZoneSettings:
timeZone: America/Los_Angeles
cesEvaluationToolset:
type: gcp:ces:Evaluation
name: ces_evaluation_toolset
properties:
evaluationId: eval-toolset
displayName: my-evaluation-toolset
location: us
app: ${app.appId}
golden:
turns:
- steps:
- expectation:
note: Expect agent response with toolset tool call
agentResponse:
role: agent
chunks:
- toolCall:
id: tool-call-id
toolsetTool:
toolset: projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset
toolId: dummy-tool
- expectation:
note: Expect agent response with toolset tool response
agentResponse:
role: agent
chunks:
- toolResponse:
id: tool-call-id
response:
result: success
toolsetTool:
toolset: projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset
toolId: dummy-tool
- expectation:
note: Expect mock tool response with toolset tool
mockToolResponse:
id: tool-call-id-2
response:
result: mocked
toolsetTool:
toolset: projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset
toolId: dummy-tool
- userInput:
toolResponses:
toolResponses:
- id: tool-call-id-3
response:
result: user-provided
toolsetTool:
toolset: projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset
toolId: dummy-tool
Example coming soon!
Create Evaluation Resource
Resources are created with functions called constructors. To learn more about declaring and configuring resources, see Resources.
Constructor syntax
new Evaluation(name: string, args: EvaluationArgs, opts?: CustomResourceOptions);@overload
def Evaluation(resource_name: str,
args: EvaluationArgs,
opts: Optional[ResourceOptions] = None)
@overload
def Evaluation(resource_name: str,
opts: Optional[ResourceOptions] = None,
app: Optional[str] = None,
display_name: Optional[str] = None,
evaluation_id: Optional[str] = None,
location: Optional[str] = None,
description: Optional[str] = None,
golden: Optional[EvaluationGoldenArgs] = None,
project: Optional[str] = None,
tags: Optional[Sequence[str]] = None)func NewEvaluation(ctx *Context, name string, args EvaluationArgs, opts ...ResourceOption) (*Evaluation, error)public Evaluation(string name, EvaluationArgs args, CustomResourceOptions? opts = null)
public Evaluation(String name, EvaluationArgs args)
public Evaluation(String name, EvaluationArgs args, CustomResourceOptions options)
type: gcp:ces:Evaluation
properties: # The arguments to resource properties.
options: # Bag of options to control resource's behavior.
resource "gcp_ces_evaluation" "name" {
# resource properties
}Parameters
- name string
- The unique name of the resource.
- args EvaluationArgs
- The arguments to resource properties.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- resource_name str
- The unique name of the resource.
- args EvaluationArgs
- The arguments to resource properties.
- opts ResourceOptions
- Bag of options to control resource's behavior.
- ctx Context
- Context object for the current deployment.
- name string
- The unique name of the resource.
- args EvaluationArgs
- The arguments to resource properties.
- opts ResourceOption
- Bag of options to control resource's behavior.
- name string
- The unique name of the resource.
- args EvaluationArgs
- The arguments to resource properties.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- name String
- The unique name of the resource.
- args EvaluationArgs
- The arguments to resource properties.
- options CustomResourceOptions
- Bag of options to control resource's behavior.
Constructor example
The following reference example uses placeholder values for all input properties.
var evaluationResource = new Gcp.Ces.Evaluation("evaluationResource", new()
{
App = "string",
DisplayName = "string",
EvaluationId = "string",
Location = "string",
Description = "string",
Golden = new Gcp.Ces.Inputs.EvaluationGoldenArgs
{
Turns = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnArgs
{
Steps = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
AgentTransfer = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepAgentTransferArgs
{
TargetAgent = "string",
DisplayName = "string",
},
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
AgentResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseArgs
{
Chunks = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
AgentTransfer = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkAgentTransferArgs
{
TargetAgent = "string",
DisplayName = "string",
},
Blob = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkBlobArgs
{
Data = "string",
MimeType = "string",
},
Image = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkImageArgs
{
Data = "string",
MimeType = "string",
},
Text = "string",
ToolCall = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs
{
Args =
{
{ "string", "string" },
},
DisplayName = "string",
Id = "string",
Tool = "string",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallToolsetToolArgs
{
Toolset = "string",
ToolId = "string",
},
},
ToolResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs
{
DisplayName = "string",
Id = "string",
Response =
{
{ "string", "string" },
},
Tool = "string",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseToolsetToolArgs
{
Toolset = "string",
ToolId = "string",
},
},
UpdatedVariables =
{
{ "string", "string" },
},
},
},
Role = "string",
},
AgentTransfer = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentTransferArgs
{
DisplayName = "string",
TargetAgent = "string",
},
MockToolResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationMockToolResponseArgs
{
DisplayName = "string",
Id = "string",
Response =
{
{ "string", "string" },
},
Tool = "string",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationMockToolResponseToolsetToolArgs
{
Toolset = "string",
ToolId = "string",
},
},
Note = "string",
ToolCall = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolCallArgs
{
Args =
{
{ "string", "string" },
},
DisplayName = "string",
Id = "string",
Tool = "string",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolCallToolsetToolArgs
{
Toolset = "string",
ToolId = "string",
},
},
ToolResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolResponseArgs
{
DisplayName = "string",
Id = "string",
Response =
{
{ "string", "string" },
},
Tool = "string",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolResponseToolsetToolArgs
{
Toolset = "string",
ToolId = "string",
},
},
UpdatedVariables = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationUpdatedVariablesArgs
{
Notes = "string",
},
},
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
Audio = "string",
Blob = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputBlobArgs
{
Data = "string",
MimeType = "string",
},
Dtmf = "string",
Event = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputEventArgs
{
Event = "string",
},
Image = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputImageArgs
{
Data = "string",
MimeType = "string",
},
Text = "string",
ToolResponses = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesArgs
{
ToolResponses = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs
{
Response =
{
{ "string", "string" },
},
DisplayName = "string",
Id = "string",
Tool = "string",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseToolsetToolArgs
{
Toolset = "string",
ToolId = "string",
},
},
},
},
Variables =
{
{ "string", "string" },
},
WillContinue = false,
},
},
},
RootSpans = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnRootSpanArgs
{
Attributes = "string",
ChildSpans = "string",
Duration = "string",
EndTime = "string",
Name = "string",
StartTime = "string",
},
},
},
},
EvaluationExpectations = new[]
{
"string",
},
},
Project = "string",
Tags = new[]
{
"string",
},
});
example, err := ces.NewEvaluation(ctx, "evaluationResource", &ces.EvaluationArgs{
App: pulumi.String("string"),
DisplayName: pulumi.String("string"),
EvaluationId: pulumi.String("string"),
Location: pulumi.String("string"),
Description: pulumi.String("string"),
Golden: &ces.EvaluationGoldenArgs{
Turns: ces.EvaluationGoldenTurnArray{
&ces.EvaluationGoldenTurnArgs{
Steps: ces.EvaluationGoldenTurnStepArray{
&ces.EvaluationGoldenTurnStepArgs{
AgentTransfer: &ces.EvaluationGoldenTurnStepAgentTransferArgs{
TargetAgent: pulumi.String("string"),
DisplayName: pulumi.String("string"),
},
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
AgentResponse: &ces.EvaluationGoldenTurnStepExpectationAgentResponseArgs{
Chunks: ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArray{
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
AgentTransfer: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkAgentTransferArgs{
TargetAgent: pulumi.String("string"),
DisplayName: pulumi.String("string"),
},
Blob: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkBlobArgs{
Data: pulumi.String("string"),
MimeType: pulumi.String("string"),
},
Image: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkImageArgs{
Data: pulumi.String("string"),
MimeType: pulumi.String("string"),
},
Text: pulumi.String("string"),
ToolCall: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs{
Args: pulumi.StringMap{
"string": pulumi.String("string"),
},
DisplayName: pulumi.String("string"),
Id: pulumi.String("string"),
Tool: pulumi.String("string"),
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallToolsetToolArgs{
Toolset: pulumi.String("string"),
ToolId: pulumi.String("string"),
},
},
ToolResponse: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs{
DisplayName: pulumi.String("string"),
Id: pulumi.String("string"),
Response: pulumi.StringMap{
"string": pulumi.String("string"),
},
Tool: pulumi.String("string"),
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseToolsetToolArgs{
Toolset: pulumi.String("string"),
ToolId: pulumi.String("string"),
},
},
UpdatedVariables: pulumi.StringMap{
"string": pulumi.String("string"),
},
},
},
Role: pulumi.String("string"),
},
AgentTransfer: &ces.EvaluationGoldenTurnStepExpectationAgentTransferArgs{
DisplayName: pulumi.String("string"),
TargetAgent: pulumi.String("string"),
},
MockToolResponse: &ces.EvaluationGoldenTurnStepExpectationMockToolResponseArgs{
DisplayName: pulumi.String("string"),
Id: pulumi.String("string"),
Response: pulumi.StringMap{
"string": pulumi.String("string"),
},
Tool: pulumi.String("string"),
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationMockToolResponseToolsetToolArgs{
Toolset: pulumi.String("string"),
ToolId: pulumi.String("string"),
},
},
Note: pulumi.String("string"),
ToolCall: &ces.EvaluationGoldenTurnStepExpectationToolCallArgs{
Args: pulumi.StringMap{
"string": pulumi.String("string"),
},
DisplayName: pulumi.String("string"),
Id: pulumi.String("string"),
Tool: pulumi.String("string"),
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationToolCallToolsetToolArgs{
Toolset: pulumi.String("string"),
ToolId: pulumi.String("string"),
},
},
ToolResponse: &ces.EvaluationGoldenTurnStepExpectationToolResponseArgs{
DisplayName: pulumi.String("string"),
Id: pulumi.String("string"),
Response: pulumi.StringMap{
"string": pulumi.String("string"),
},
Tool: pulumi.String("string"),
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationToolResponseToolsetToolArgs{
Toolset: pulumi.String("string"),
ToolId: pulumi.String("string"),
},
},
UpdatedVariables: &ces.EvaluationGoldenTurnStepExpectationUpdatedVariablesArgs{
Notes: pulumi.String("string"),
},
},
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
Audio: pulumi.String("string"),
Blob: &ces.EvaluationGoldenTurnStepUserInputBlobArgs{
Data: pulumi.String("string"),
MimeType: pulumi.String("string"),
},
Dtmf: pulumi.String("string"),
Event: &ces.EvaluationGoldenTurnStepUserInputEventArgs{
Event: pulumi.String("string"),
},
Image: &ces.EvaluationGoldenTurnStepUserInputImageArgs{
Data: pulumi.String("string"),
MimeType: pulumi.String("string"),
},
Text: pulumi.String("string"),
ToolResponses: &ces.EvaluationGoldenTurnStepUserInputToolResponsesArgs{
ToolResponses: ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArray{
&ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs{
Response: pulumi.StringMap{
"string": pulumi.String("string"),
},
DisplayName: pulumi.String("string"),
Id: pulumi.String("string"),
Tool: pulumi.String("string"),
ToolsetTool: &ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseToolsetToolArgs{
Toolset: pulumi.String("string"),
ToolId: pulumi.String("string"),
},
},
},
},
Variables: pulumi.StringMap{
"string": pulumi.String("string"),
},
WillContinue: pulumi.Bool(false),
},
},
},
RootSpans: ces.EvaluationGoldenTurnRootSpanArray{
&ces.EvaluationGoldenTurnRootSpanArgs{
Attributes: pulumi.String("string"),
ChildSpans: pulumi.String("string"),
Duration: pulumi.String("string"),
EndTime: pulumi.String("string"),
Name: pulumi.String("string"),
StartTime: pulumi.String("string"),
},
},
},
},
EvaluationExpectations: pulumi.StringArray{
pulumi.String("string"),
},
},
Project: pulumi.String("string"),
Tags: pulumi.StringArray{
pulumi.String("string"),
},
})
resource "gcp_ces_evaluation" "evaluationResource" {
app = "string"
display_name = "string"
evaluation_id = "string"
location = "string"
description = "string"
golden = {
turns = [{
"steps" = [{
"agentTransfer" = {
"targetAgent" = "string"
"displayName" = "string"
}
"expectation" = {
"agentResponse" = {
"chunks" = [{
"agentTransfer" = {
"targetAgent" = "string"
"displayName" = "string"
}
"blob" = {
"data" = "string"
"mimeType" = "string"
}
"image" = {
"data" = "string"
"mimeType" = "string"
}
"text" = "string"
"toolCall" = {
"args" = {
"string" = "string"
}
"displayName" = "string"
"id" = "string"
"tool" = "string"
"toolsetTool" = {
"toolset" = "string"
"toolId" = "string"
}
}
"toolResponse" = {
"displayName" = "string"
"id" = "string"
"response" = {
"string" = "string"
}
"tool" = "string"
"toolsetTool" = {
"toolset" = "string"
"toolId" = "string"
}
}
"updatedVariables" = {
"string" = "string"
}
}]
"role" = "string"
}
"agentTransfer" = {
"displayName" = "string"
"targetAgent" = "string"
}
"mockToolResponse" = {
"displayName" = "string"
"id" = "string"
"response" = {
"string" = "string"
}
"tool" = "string"
"toolsetTool" = {
"toolset" = "string"
"toolId" = "string"
}
}
"note" = "string"
"toolCall" = {
"args" = {
"string" = "string"
}
"displayName" = "string"
"id" = "string"
"tool" = "string"
"toolsetTool" = {
"toolset" = "string"
"toolId" = "string"
}
}
"toolResponse" = {
"displayName" = "string"
"id" = "string"
"response" = {
"string" = "string"
}
"tool" = "string"
"toolsetTool" = {
"toolset" = "string"
"toolId" = "string"
}
}
"updatedVariables" = {
"notes" = "string"
}
}
"userInput" = {
"audio" = "string"
"blob" = {
"data" = "string"
"mimeType" = "string"
}
"dtmf" = "string"
"event" = {
"event" = "string"
}
"image" = {
"data" = "string"
"mimeType" = "string"
}
"text" = "string"
"toolResponses" = {
"toolResponses" = [{
"response" = {
"string" = "string"
}
"displayName" = "string"
"id" = "string"
"tool" = "string"
"toolsetTool" = {
"toolset" = "string"
"toolId" = "string"
}
}]
}
"variables" = {
"string" = "string"
}
"willContinue" = false
}
}]
"rootSpans" = [{
"attributes" = "string"
"childSpans" = "string"
"duration" = "string"
"endTime" = "string"
"name" = "string"
"startTime" = "string"
}]
}]
evaluation_expectations = ["string"]
}
project = "string"
tags = ["string"]
}
var evaluationResource = new Evaluation("evaluationResource", EvaluationArgs.builder()
.app("string")
.displayName("string")
.evaluationId("string")
.location("string")
.description("string")
.golden(EvaluationGoldenArgs.builder()
.turns(EvaluationGoldenTurnArgs.builder()
.steps(EvaluationGoldenTurnStepArgs.builder()
.agentTransfer(EvaluationGoldenTurnStepAgentTransferArgs.builder()
.targetAgent("string")
.displayName("string")
.build())
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.agentResponse(EvaluationGoldenTurnStepExpectationAgentResponseArgs.builder()
.chunks(EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.agentTransfer(EvaluationGoldenTurnStepExpectationAgentResponseChunkAgentTransferArgs.builder()
.targetAgent("string")
.displayName("string")
.build())
.blob(EvaluationGoldenTurnStepExpectationAgentResponseChunkBlobArgs.builder()
.data("string")
.mimeType("string")
.build())
.image(EvaluationGoldenTurnStepExpectationAgentResponseChunkImageArgs.builder()
.data("string")
.mimeType("string")
.build())
.text("string")
.toolCall(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs.builder()
.args(Map.of("string", "string"))
.displayName("string")
.id("string")
.tool("string")
.toolsetTool(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallToolsetToolArgs.builder()
.toolset("string")
.toolId("string")
.build())
.build())
.toolResponse(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs.builder()
.displayName("string")
.id("string")
.response(Map.of("string", "string"))
.tool("string")
.toolsetTool(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseToolsetToolArgs.builder()
.toolset("string")
.toolId("string")
.build())
.build())
.updatedVariables(Map.of("string", "string"))
.build())
.role("string")
.build())
.agentTransfer(EvaluationGoldenTurnStepExpectationAgentTransferArgs.builder()
.displayName("string")
.targetAgent("string")
.build())
.mockToolResponse(EvaluationGoldenTurnStepExpectationMockToolResponseArgs.builder()
.displayName("string")
.id("string")
.response(Map.of("string", "string"))
.tool("string")
.toolsetTool(EvaluationGoldenTurnStepExpectationMockToolResponseToolsetToolArgs.builder()
.toolset("string")
.toolId("string")
.build())
.build())
.note("string")
.toolCall(EvaluationGoldenTurnStepExpectationToolCallArgs.builder()
.args(Map.of("string", "string"))
.displayName("string")
.id("string")
.tool("string")
.toolsetTool(EvaluationGoldenTurnStepExpectationToolCallToolsetToolArgs.builder()
.toolset("string")
.toolId("string")
.build())
.build())
.toolResponse(EvaluationGoldenTurnStepExpectationToolResponseArgs.builder()
.displayName("string")
.id("string")
.response(Map.of("string", "string"))
.tool("string")
.toolsetTool(EvaluationGoldenTurnStepExpectationToolResponseToolsetToolArgs.builder()
.toolset("string")
.toolId("string")
.build())
.build())
.updatedVariables(EvaluationGoldenTurnStepExpectationUpdatedVariablesArgs.builder()
.notes("string")
.build())
.build())
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.audio("string")
.blob(EvaluationGoldenTurnStepUserInputBlobArgs.builder()
.data("string")
.mimeType("string")
.build())
.dtmf("string")
.event(EvaluationGoldenTurnStepUserInputEventArgs.builder()
.event("string")
.build())
.image(EvaluationGoldenTurnStepUserInputImageArgs.builder()
.data("string")
.mimeType("string")
.build())
.text("string")
.toolResponses(EvaluationGoldenTurnStepUserInputToolResponsesArgs.builder()
.toolResponses(EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs.builder()
.response(Map.of("string", "string"))
.displayName("string")
.id("string")
.tool("string")
.toolsetTool(EvaluationGoldenTurnStepUserInputToolResponsesToolResponseToolsetToolArgs.builder()
.toolset("string")
.toolId("string")
.build())
.build())
.build())
.variables(Map.of("string", "string"))
.willContinue(false)
.build())
.build())
.rootSpans(EvaluationGoldenTurnRootSpanArgs.builder()
.attributes("string")
.childSpans("string")
.duration("string")
.endTime("string")
.name("string")
.startTime("string")
.build())
.build())
.evaluationExpectations("string")
.build())
.project("string")
.tags("string")
.build());
evaluation_resource = gcp.ces.Evaluation("evaluationResource",
app="string",
display_name="string",
evaluation_id="string",
location="string",
description="string",
golden={
"turns": [{
"steps": [{
"agent_transfer": {
"target_agent": "string",
"display_name": "string",
},
"expectation": {
"agent_response": {
"chunks": [{
"agent_transfer": {
"target_agent": "string",
"display_name": "string",
},
"blob": {
"data": "string",
"mime_type": "string",
},
"image": {
"data": "string",
"mime_type": "string",
},
"text": "string",
"tool_call": {
"args": {
"string": "string",
},
"display_name": "string",
"id": "string",
"tool": "string",
"toolset_tool": {
"toolset": "string",
"tool_id": "string",
},
},
"tool_response": {
"display_name": "string",
"id": "string",
"response": {
"string": "string",
},
"tool": "string",
"toolset_tool": {
"toolset": "string",
"tool_id": "string",
},
},
"updated_variables": {
"string": "string",
},
}],
"role": "string",
},
"agent_transfer": {
"display_name": "string",
"target_agent": "string",
},
"mock_tool_response": {
"display_name": "string",
"id": "string",
"response": {
"string": "string",
},
"tool": "string",
"toolset_tool": {
"toolset": "string",
"tool_id": "string",
},
},
"note": "string",
"tool_call": {
"args": {
"string": "string",
},
"display_name": "string",
"id": "string",
"tool": "string",
"toolset_tool": {
"toolset": "string",
"tool_id": "string",
},
},
"tool_response": {
"display_name": "string",
"id": "string",
"response": {
"string": "string",
},
"tool": "string",
"toolset_tool": {
"toolset": "string",
"tool_id": "string",
},
},
"updated_variables": {
"notes": "string",
},
},
"user_input": {
"audio": "string",
"blob": {
"data": "string",
"mime_type": "string",
},
"dtmf": "string",
"event": {
"event": "string",
},
"image": {
"data": "string",
"mime_type": "string",
},
"text": "string",
"tool_responses": {
"tool_responses": [{
"response": {
"string": "string",
},
"display_name": "string",
"id": "string",
"tool": "string",
"toolset_tool": {
"toolset": "string",
"tool_id": "string",
},
}],
},
"variables": {
"string": "string",
},
"will_continue": False,
},
}],
"root_spans": [{
"attributes": "string",
"child_spans": "string",
"duration": "string",
"end_time": "string",
"name": "string",
"start_time": "string",
}],
}],
"evaluation_expectations": ["string"],
},
project="string",
tags=["string"])
const evaluationResource = new gcp.ces.Evaluation("evaluationResource", {
app: "string",
displayName: "string",
evaluationId: "string",
location: "string",
description: "string",
golden: {
turns: [{
steps: [{
agentTransfer: {
targetAgent: "string",
displayName: "string",
},
expectation: {
agentResponse: {
chunks: [{
agentTransfer: {
targetAgent: "string",
displayName: "string",
},
blob: {
data: "string",
mimeType: "string",
},
image: {
data: "string",
mimeType: "string",
},
text: "string",
toolCall: {
args: {
string: "string",
},
displayName: "string",
id: "string",
tool: "string",
toolsetTool: {
toolset: "string",
toolId: "string",
},
},
toolResponse: {
displayName: "string",
id: "string",
response: {
string: "string",
},
tool: "string",
toolsetTool: {
toolset: "string",
toolId: "string",
},
},
updatedVariables: {
string: "string",
},
}],
role: "string",
},
agentTransfer: {
displayName: "string",
targetAgent: "string",
},
mockToolResponse: {
displayName: "string",
id: "string",
response: {
string: "string",
},
tool: "string",
toolsetTool: {
toolset: "string",
toolId: "string",
},
},
note: "string",
toolCall: {
args: {
string: "string",
},
displayName: "string",
id: "string",
tool: "string",
toolsetTool: {
toolset: "string",
toolId: "string",
},
},
toolResponse: {
displayName: "string",
id: "string",
response: {
string: "string",
},
tool: "string",
toolsetTool: {
toolset: "string",
toolId: "string",
},
},
updatedVariables: {
notes: "string",
},
},
userInput: {
audio: "string",
blob: {
data: "string",
mimeType: "string",
},
dtmf: "string",
event: {
event: "string",
},
image: {
data: "string",
mimeType: "string",
},
text: "string",
toolResponses: {
toolResponses: [{
response: {
string: "string",
},
displayName: "string",
id: "string",
tool: "string",
toolsetTool: {
toolset: "string",
toolId: "string",
},
}],
},
variables: {
string: "string",
},
willContinue: false,
},
}],
rootSpans: [{
attributes: "string",
childSpans: "string",
duration: "string",
endTime: "string",
name: "string",
startTime: "string",
}],
}],
evaluationExpectations: ["string"],
},
project: "string",
tags: ["string"],
});
type: gcp:ces:Evaluation
properties:
app: string
description: string
displayName: string
evaluationId: string
golden:
evaluationExpectations:
- string
turns:
- rootSpans:
- attributes: string
childSpans: string
duration: string
endTime: string
name: string
startTime: string
steps:
- agentTransfer:
displayName: string
targetAgent: string
expectation:
agentResponse:
chunks:
- agentTransfer:
displayName: string
targetAgent: string
blob:
data: string
mimeType: string
image:
data: string
mimeType: string
text: string
toolCall:
args:
string: string
displayName: string
id: string
tool: string
toolsetTool:
toolId: string
toolset: string
toolResponse:
displayName: string
id: string
response:
string: string
tool: string
toolsetTool:
toolId: string
toolset: string
updatedVariables:
string: string
role: string
agentTransfer:
displayName: string
targetAgent: string
mockToolResponse:
displayName: string
id: string
response:
string: string
tool: string
toolsetTool:
toolId: string
toolset: string
note: string
toolCall:
args:
string: string
displayName: string
id: string
tool: string
toolsetTool:
toolId: string
toolset: string
toolResponse:
displayName: string
id: string
response:
string: string
tool: string
toolsetTool:
toolId: string
toolset: string
updatedVariables:
notes: string
userInput:
audio: string
blob:
data: string
mimeType: string
dtmf: string
event:
event: string
image:
data: string
mimeType: string
text: string
toolResponses:
toolResponses:
- displayName: string
id: string
response:
string: string
tool: string
toolsetTool:
toolId: string
toolset: string
variables:
string: string
willContinue: false
location: string
project: string
tags:
- string
Evaluation Resource Properties
To learn more about resource properties and how to use them, see Inputs and Outputs in the Architecture and Concepts docs.
Inputs
In Python, inputs that are objects can be passed either as argument classes or as dictionary literals.
The Evaluation resource accepts the following input properties:
- App string
- (Required)
- Display
Name string - User-defined display name of the evaluation.
- Evaluation
Id string - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- Location string
- (Required)
- Description string
- User-defined description of the evaluation.
- Golden
Evaluation
Golden - Golden input. Structure is documented below.
- Project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- List<string>
- User defined tags to categorize the evaluation.
- App string
- (Required)
- Display
Name string - User-defined display name of the evaluation.
- Evaluation
Id string - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- Location string
- (Required)
- Description string
- User-defined description of the evaluation.
- Golden
Evaluation
Golden Args - Golden input. Structure is documented below.
- Project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- []string
- User defined tags to categorize the evaluation.
- app string
- (Required)
- display_
name string - User-defined display name of the evaluation.
- evaluation_
id string - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- location string
- (Required)
- description string
- User-defined description of the evaluation.
- golden object
- Golden input. Structure is documented below.
- project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- list(string)
- User defined tags to categorize the evaluation.
- app String
- (Required)
- display
Name String - User-defined display name of the evaluation.
- evaluation
Id String - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- location String
- (Required)
- description String
- User-defined description of the evaluation.
- golden
Evaluation
Golden - Golden input. Structure is documented below.
- project String
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- List<String>
- User defined tags to categorize the evaluation.
- app string
- (Required)
- display
Name string - User-defined display name of the evaluation.
- evaluation
Id string - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- location string
- (Required)
- description string
- User-defined description of the evaluation.
- golden
Evaluation
Golden - Golden input. Structure is documented below.
- project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- string[]
- User defined tags to categorize the evaluation.
- app str
- (Required)
- display_
name str - User-defined display name of the evaluation.
- evaluation_
id str - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- location str
- (Required)
- description str
- User-defined description of the evaluation.
- golden
Evaluation
Golden Args - Golden input. Structure is documented below.
- project str
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- Sequence[str]
- User defined tags to categorize the evaluation.
- app String
- (Required)
- display
Name String - User-defined display name of the evaluation.
- evaluation
Id String - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- location String
- (Required)
- description String
- User-defined description of the evaluation.
- golden Property Map
- Golden input. Structure is documented below.
- project String
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- List<String>
- User defined tags to categorize the evaluation.
Outputs
All input properties are implicitly available as output properties. Additionally, the Evaluation resource produces the following output properties:
- Create
Time string - Timestamp when the evaluation was created.
- Created
By string - The user who created the evaluation.
- Etag string
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- Evaluation
Datasets List<string> - List of evaluation datasets the evaluation belongs to.
- Evaluation
Runs List<string> - The evaluation runs for this evaluation.
- Id string
- The provider-assigned unique ID for this managed resource.
- Invalid bool
- Whether the evaluation is invalid.
- Last
Updated stringBy - The user who last updated the evaluation.
- Name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - Update
Time string - Timestamp when the evaluation was last updated.
- Create
Time string - Timestamp when the evaluation was created.
- Created
By string - The user who created the evaluation.
- Etag string
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- Evaluation
Datasets []string - List of evaluation datasets the evaluation belongs to.
- Evaluation
Runs []string - The evaluation runs for this evaluation.
- Id string
- The provider-assigned unique ID for this managed resource.
- Invalid bool
- Whether the evaluation is invalid.
- Last
Updated stringBy - The user who last updated the evaluation.
- Name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - Update
Time string - Timestamp when the evaluation was last updated.
- create_
time string - Timestamp when the evaluation was created.
- created_
by string - The user who created the evaluation.
- etag string
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation_
datasets list(string) - List of evaluation datasets the evaluation belongs to.
- evaluation_
runs list(string) - The evaluation runs for this evaluation.
- id string
- The provider-assigned unique ID for this managed resource.
- invalid bool
- Whether the evaluation is invalid.
- last_
updated_ stringby - The user who last updated the evaluation.
- name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - update_
time string - Timestamp when the evaluation was last updated.
- create
Time String - Timestamp when the evaluation was created.
- created
By String - The user who created the evaluation.
- etag String
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation
Datasets List<String> - List of evaluation datasets the evaluation belongs to.
- evaluation
Runs List<String> - The evaluation runs for this evaluation.
- id String
- The provider-assigned unique ID for this managed resource.
- invalid Boolean
- Whether the evaluation is invalid.
- last
Updated StringBy - The user who last updated the evaluation.
- name String
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - update
Time String - Timestamp when the evaluation was last updated.
- create
Time string - Timestamp when the evaluation was created.
- created
By string - The user who created the evaluation.
- etag string
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation
Datasets string[] - List of evaluation datasets the evaluation belongs to.
- evaluation
Runs string[] - The evaluation runs for this evaluation.
- id string
- The provider-assigned unique ID for this managed resource.
- invalid boolean
- Whether the evaluation is invalid.
- last
Updated stringBy - The user who last updated the evaluation.
- name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - update
Time string - Timestamp when the evaluation was last updated.
- create_
time str - Timestamp when the evaluation was created.
- created_
by str - The user who created the evaluation.
- etag str
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation_
datasets Sequence[str] - List of evaluation datasets the evaluation belongs to.
- evaluation_
runs Sequence[str] - The evaluation runs for this evaluation.
- id str
- The provider-assigned unique ID for this managed resource.
- invalid bool
- Whether the evaluation is invalid.
- last_
updated_ strby - The user who last updated the evaluation.
- name str
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - update_
time str - Timestamp when the evaluation was last updated.
- create
Time String - Timestamp when the evaluation was created.
- created
By String - The user who created the evaluation.
- etag String
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation
Datasets List<String> - List of evaluation datasets the evaluation belongs to.
- evaluation
Runs List<String> - The evaluation runs for this evaluation.
- id String
- The provider-assigned unique ID for this managed resource.
- invalid Boolean
- Whether the evaluation is invalid.
- last
Updated StringBy - The user who last updated the evaluation.
- name String
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - update
Time String - Timestamp when the evaluation was last updated.
Look up Existing Evaluation Resource
Get an existing Evaluation resource’s state with the given name, ID, and optional extra properties used to qualify the lookup.
public static get(name: string, id: Input<ID>, state?: EvaluationState, opts?: CustomResourceOptions): Evaluation@staticmethod
def get(resource_name: str,
id: str,
opts: Optional[ResourceOptions] = None,
app: Optional[str] = None,
create_time: Optional[str] = None,
created_by: Optional[str] = None,
description: Optional[str] = None,
display_name: Optional[str] = None,
etag: Optional[str] = None,
evaluation_datasets: Optional[Sequence[str]] = None,
evaluation_id: Optional[str] = None,
evaluation_runs: Optional[Sequence[str]] = None,
golden: Optional[EvaluationGoldenArgs] = None,
invalid: Optional[bool] = None,
last_updated_by: Optional[str] = None,
location: Optional[str] = None,
name: Optional[str] = None,
project: Optional[str] = None,
tags: Optional[Sequence[str]] = None,
update_time: Optional[str] = None) -> Evaluationfunc GetEvaluation(ctx *Context, name string, id IDInput, state *EvaluationState, opts ...ResourceOption) (*Evaluation, error)public static Evaluation Get(string name, Input<string> id, EvaluationState? state, CustomResourceOptions? opts = null)public static Evaluation get(String name, Output<String> id, EvaluationState state, CustomResourceOptions options)resources: _: type: gcp:ces:Evaluation get: id: ${id}import {
to = gcp_ces_evaluation.example
id = "${id}"
}
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- resource_name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- App string
- (Required)
- Create
Time string - Timestamp when the evaluation was created.
- Created
By string - The user who created the evaluation.
- Description string
- User-defined description of the evaluation.
- Display
Name string - User-defined display name of the evaluation.
- Etag string
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- Evaluation
Datasets List<string> - List of evaluation datasets the evaluation belongs to.
- Evaluation
Id string - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- Evaluation
Runs List<string> - The evaluation runs for this evaluation.
- Golden
Evaluation
Golden - Golden input. Structure is documented below.
- Invalid bool
- Whether the evaluation is invalid.
- Last
Updated stringBy - The user who last updated the evaluation.
- Location string
- (Required)
- Name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - Project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- List<string>
- User defined tags to categorize the evaluation.
- Update
Time string - Timestamp when the evaluation was last updated.
- App string
- (Required)
- Create
Time string - Timestamp when the evaluation was created.
- Created
By string - The user who created the evaluation.
- Description string
- User-defined description of the evaluation.
- Display
Name string - User-defined display name of the evaluation.
- Etag string
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- Evaluation
Datasets []string - List of evaluation datasets the evaluation belongs to.
- Evaluation
Id string - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- Evaluation
Runs []string - The evaluation runs for this evaluation.
- Golden
Evaluation
Golden Args - Golden input. Structure is documented below.
- Invalid bool
- Whether the evaluation is invalid.
- Last
Updated stringBy - The user who last updated the evaluation.
- Location string
- (Required)
- Name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - Project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- []string
- User defined tags to categorize the evaluation.
- Update
Time string - Timestamp when the evaluation was last updated.
- app string
- (Required)
- create_
time string - Timestamp when the evaluation was created.
- created_
by string - The user who created the evaluation.
- description string
- User-defined description of the evaluation.
- display_
name string - User-defined display name of the evaluation.
- etag string
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation_
datasets list(string) - List of evaluation datasets the evaluation belongs to.
- evaluation_
id string - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- evaluation_
runs list(string) - The evaluation runs for this evaluation.
- golden object
- Golden input. Structure is documented below.
- invalid bool
- Whether the evaluation is invalid.
- last_
updated_ stringby - The user who last updated the evaluation.
- location string
- (Required)
- name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- list(string)
- User defined tags to categorize the evaluation.
- update_
time string - Timestamp when the evaluation was last updated.
- app String
- (Required)
- create
Time String - Timestamp when the evaluation was created.
- created
By String - The user who created the evaluation.
- description String
- User-defined description of the evaluation.
- display
Name String - User-defined display name of the evaluation.
- etag String
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation
Datasets List<String> - List of evaluation datasets the evaluation belongs to.
- evaluation
Id String - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- evaluation
Runs List<String> - The evaluation runs for this evaluation.
- golden
Evaluation
Golden - Golden input. Structure is documented below.
- invalid Boolean
- Whether the evaluation is invalid.
- last
Updated StringBy - The user who last updated the evaluation.
- location String
- (Required)
- name String
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - project String
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- List<String>
- User defined tags to categorize the evaluation.
- update
Time String - Timestamp when the evaluation was last updated.
- app string
- (Required)
- create
Time string - Timestamp when the evaluation was created.
- created
By string - The user who created the evaluation.
- description string
- User-defined description of the evaluation.
- display
Name string - User-defined display name of the evaluation.
- etag string
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation
Datasets string[] - List of evaluation datasets the evaluation belongs to.
- evaluation
Id string - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- evaluation
Runs string[] - The evaluation runs for this evaluation.
- golden
Evaluation
Golden - Golden input. Structure is documented below.
- invalid boolean
- Whether the evaluation is invalid.
- last
Updated stringBy - The user who last updated the evaluation.
- location string
- (Required)
- name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- string[]
- User defined tags to categorize the evaluation.
- update
Time string - Timestamp when the evaluation was last updated.
- app str
- (Required)
- create_
time str - Timestamp when the evaluation was created.
- created_
by str - The user who created the evaluation.
- description str
- User-defined description of the evaluation.
- display_
name str - User-defined display name of the evaluation.
- etag str
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation_
datasets Sequence[str] - List of evaluation datasets the evaluation belongs to.
- evaluation_
id str - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- evaluation_
runs Sequence[str] - The evaluation runs for this evaluation.
- golden
Evaluation
Golden Args - Golden input. Structure is documented below.
- invalid bool
- Whether the evaluation is invalid.
- last_
updated_ strby - The user who last updated the evaluation.
- location str
- (Required)
- name str
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - project str
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- Sequence[str]
- User defined tags to categorize the evaluation.
- update_
time str - Timestamp when the evaluation was last updated.
- app String
- (Required)
- create
Time String - Timestamp when the evaluation was created.
- created
By String - The user who created the evaluation.
- description String
- User-defined description of the evaluation.
- display
Name String - User-defined display name of the evaluation.
- etag String
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation
Datasets List<String> - List of evaluation datasets the evaluation belongs to.
- evaluation
Id String - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- evaluation
Runs List<String> - The evaluation runs for this evaluation.
- golden Property Map
- Golden input. Structure is documented below.
- invalid Boolean
- Whether the evaluation is invalid.
- last
Updated StringBy - The user who last updated the evaluation.
- location String
- (Required)
- name String
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - project String
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- List<String>
- User defined tags to categorize the evaluation.
- update
Time String - Timestamp when the evaluation was last updated.
Supporting Types
EvaluationGolden, EvaluationGoldenArgs
- Turns
List<Evaluation
Golden Turn> - The golden turns required to replay a golden conversation. Structure is documented below.
- Evaluation
Expectations List<string> - The evaluation expectations to evaluate the replayed conversation against. Format: projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluationExpectation}
- Turns
[]Evaluation
Golden Turn - The golden turns required to replay a golden conversation. Structure is documented below.
- Evaluation
Expectations []string - The evaluation expectations to evaluate the replayed conversation against. Format: projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluationExpectation}
- turns list(object)
- The golden turns required to replay a golden conversation. Structure is documented below.
- evaluation_
expectations list(string) - The evaluation expectations to evaluate the replayed conversation against. Format: projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluationExpectation}
- turns
List<Evaluation
Golden Turn> - The golden turns required to replay a golden conversation. Structure is documented below.
- evaluation
Expectations List<String> - The evaluation expectations to evaluate the replayed conversation against. Format: projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluationExpectation}
- turns
Evaluation
Golden Turn[] - The golden turns required to replay a golden conversation. Structure is documented below.
- evaluation
Expectations string[] - The evaluation expectations to evaluate the replayed conversation against. Format: projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluationExpectation}
- turns
Sequence[Evaluation
Golden Turn] - The golden turns required to replay a golden conversation. Structure is documented below.
- evaluation_
expectations Sequence[str] - The evaluation expectations to evaluate the replayed conversation against. Format: projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluationExpectation}
- turns List<Property Map>
- The golden turns required to replay a golden conversation. Structure is documented below.
- evaluation
Expectations List<String> - The evaluation expectations to evaluate the replayed conversation against. Format: projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluationExpectation}
EvaluationGoldenTurn, EvaluationGoldenTurnArgs
- Steps
List<Evaluation
Golden Turn Step> The sequence of steps required to replay a golden conversation turn. Structure is documented below.
The
rootSpanblock contains:- Root
Spans List<EvaluationGolden Turn Root Span> - (Output) The root span of the golden turn for processing and maintaining audio information. Structure is documented below.
- Steps
[]Evaluation
Golden Turn Step The sequence of steps required to replay a golden conversation turn. Structure is documented below.
The
rootSpanblock contains:- Root
Spans []EvaluationGolden Turn Root Span - (Output) The root span of the golden turn for processing and maintaining audio information. Structure is documented below.
- steps list(object)
The sequence of steps required to replay a golden conversation turn. Structure is documented below.
The
rootSpanblock contains:- root_
spans list(object) - (Output) The root span of the golden turn for processing and maintaining audio information. Structure is documented below.
- steps
List<Evaluation
Golden Turn Step> The sequence of steps required to replay a golden conversation turn. Structure is documented below.
The
rootSpanblock contains:- root
Spans List<EvaluationGolden Turn Root Span> - (Output) The root span of the golden turn for processing and maintaining audio information. Structure is documented below.
- steps
Evaluation
Golden Turn Step[] The sequence of steps required to replay a golden conversation turn. Structure is documented below.
The
rootSpanblock contains:- root
Spans EvaluationGolden Turn Root Span[] - (Output) The root span of the golden turn for processing and maintaining audio information. Structure is documented below.
- steps
Sequence[Evaluation
Golden Turn Step] The sequence of steps required to replay a golden conversation turn. Structure is documented below.
The
rootSpanblock contains:- root_
spans Sequence[EvaluationGolden Turn Root Span] - (Output) The root span of the golden turn for processing and maintaining audio information. Structure is documented below.
- steps List<Property Map>
The sequence of steps required to replay a golden conversation turn. Structure is documented below.
The
rootSpanblock contains:- root
Spans List<Property Map> - (Output) The root span of the golden turn for processing and maintaining audio information. Structure is documented below.
EvaluationGoldenTurnRootSpan, EvaluationGoldenTurnRootSpanArgs
- Attributes string
- Output only. Key-value attributes associated with the span.
- Child
Spans string - Output only. The child spans that are nested under this span.
- Duration string
- Output only. The duration of the span.
- End
Time string - Output only. The end time of the span.
- Name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - Start
Time string - Output only. The start time of the span.
- Attributes string
- Output only. Key-value attributes associated with the span.
- Child
Spans string - Output only. The child spans that are nested under this span.
- Duration string
- Output only. The duration of the span.
- End
Time string - Output only. The end time of the span.
- Name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - Start
Time string - Output only. The start time of the span.
- attributes string
- Output only. Key-value attributes associated with the span.
- child_
spans string - Output only. The child spans that are nested under this span.
- duration string
- Output only. The duration of the span.
- end_
time string - Output only. The end time of the span.
- name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - start_
time string - Output only. The start time of the span.
- attributes String
- Output only. Key-value attributes associated with the span.
- child
Spans String - Output only. The child spans that are nested under this span.
- duration String
- Output only. The duration of the span.
- end
Time String - Output only. The end time of the span.
- name String
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - start
Time String - Output only. The start time of the span.
- attributes string
- Output only. Key-value attributes associated with the span.
- child
Spans string - Output only. The child spans that are nested under this span.
- duration string
- Output only. The duration of the span.
- end
Time string - Output only. The end time of the span.
- name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - start
Time string - Output only. The start time of the span.
- attributes str
- Output only. Key-value attributes associated with the span.
- child_
spans str - Output only. The child spans that are nested under this span.
- duration str
- Output only. The duration of the span.
- end_
time str - Output only. The end time of the span.
- name str
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - start_
time str - Output only. The start time of the span.
- attributes String
- Output only. Key-value attributes associated with the span.
- child
Spans String - Output only. The child spans that are nested under this span.
- duration String
- Output only. The duration of the span.
- end
Time String - Output only. The end time of the span.
- name String
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - start
Time String - Output only. The start time of the span.
EvaluationGoldenTurnStep, EvaluationGoldenTurnStepArgs
- Agent
Transfer EvaluationGolden Turn Step Agent Transfer - Represents an event indicating the transfer of the conversation to a different agent. Structure is documented below.
- Expectation
Evaluation
Golden Turn Step Expectation - Executes an expectation on the current turn to verify the agent's behavior. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- User
Input EvaluationGolden Turn Step User Input - Represents user input for the conversation. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- Agent
Transfer EvaluationGolden Turn Step Agent Transfer - Represents an event indicating the transfer of the conversation to a different agent. Structure is documented below.
- Expectation
Evaluation
Golden Turn Step Expectation - Executes an expectation on the current turn to verify the agent's behavior. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- User
Input EvaluationGolden Turn Step User Input - Represents user input for the conversation. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- agent_
transfer object - Represents an event indicating the transfer of the conversation to a different agent. Structure is documented below.
- expectation object
- Executes an expectation on the current turn to verify the agent's behavior. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- user_
input object - Represents user input for the conversation. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- agent
Transfer EvaluationGolden Turn Step Agent Transfer - Represents an event indicating the transfer of the conversation to a different agent. Structure is documented below.
- expectation
Evaluation
Golden Turn Step Expectation - Executes an expectation on the current turn to verify the agent's behavior. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- user
Input EvaluationGolden Turn Step User Input - Represents user input for the conversation. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- agent
Transfer EvaluationGolden Turn Step Agent Transfer - Represents an event indicating the transfer of the conversation to a different agent. Structure is documented below.
- expectation
Evaluation
Golden Turn Step Expectation - Executes an expectation on the current turn to verify the agent's behavior. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- user
Input EvaluationGolden Turn Step User Input - Represents user input for the conversation. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- agent_
transfer EvaluationGolden Turn Step Agent Transfer - Represents an event indicating the transfer of the conversation to a different agent. Structure is documented below.
- expectation
Evaluation
Golden Turn Step Expectation - Executes an expectation on the current turn to verify the agent's behavior. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- user_
input EvaluationGolden Turn Step User Input - Represents user input for the conversation. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- agent
Transfer Property Map - Represents an event indicating the transfer of the conversation to a different agent. Structure is documented below.
- expectation Property Map
- Executes an expectation on the current turn to verify the agent's behavior. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- user
Input Property Map - Represents user input for the conversation. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
EvaluationGoldenTurnStepAgentTransfer, EvaluationGoldenTurnStepAgentTransferArgs
- Target
Agent string - The resource name of the target agent.
- Display
Name string - The display name of the target agent.
- Target
Agent string - The resource name of the target agent.
- Display
Name string - The display name of the target agent.
- target_
agent string - The resource name of the target agent.
- display_
name string - The display name of the target agent.
- target
Agent String - The resource name of the target agent.
- display
Name String - The display name of the target agent.
- target
Agent string - The resource name of the target agent.
- display
Name string - The display name of the target agent.
- target_
agent str - The resource name of the target agent.
- display_
name str - The display name of the target agent.
- target
Agent String - The resource name of the target agent.
- display
Name String - The display name of the target agent.
EvaluationGoldenTurnStepExpectation, EvaluationGoldenTurnStepExpectationArgs
- Agent
Response EvaluationGolden Turn Step Expectation Agent Response - Expectation on agent response. Structure is documented below.
- Agent
Transfer EvaluationGolden Turn Step Expectation Agent Transfer - Expectation on agent transfer. Structure is documented below.
- Mock
Tool EvaluationResponse Golden Turn Step Expectation Mock Tool Response - A mock response to provide if a tool is called. Structure is documented below.
- Note string
- A note describing the expectation.
- Tool
Call EvaluationGolden Turn Step Expectation Tool Call - Expectation on tool call. Structure is documented below.
- Tool
Response EvaluationGolden Turn Step Expectation Tool Response - Expectation on tool response. Structure is documented below.
- Updated
Variables EvaluationGolden Turn Step Expectation Updated Variables - Expectation on updated variables. Structure is documented below.
- Agent
Response EvaluationGolden Turn Step Expectation Agent Response - Expectation on agent response. Structure is documented below.
- Agent
Transfer EvaluationGolden Turn Step Expectation Agent Transfer - Expectation on agent transfer. Structure is documented below.
- Mock
Tool EvaluationResponse Golden Turn Step Expectation Mock Tool Response - A mock response to provide if a tool is called. Structure is documented below.
- Note string
- A note describing the expectation.
- Tool
Call EvaluationGolden Turn Step Expectation Tool Call - Expectation on tool call. Structure is documented below.
- Tool
Response EvaluationGolden Turn Step Expectation Tool Response - Expectation on tool response. Structure is documented below.
- Updated
Variables EvaluationGolden Turn Step Expectation Updated Variables - Expectation on updated variables. Structure is documented below.
- agent_
response object - Expectation on agent response. Structure is documented below.
- agent_
transfer object - Expectation on agent transfer. Structure is documented below.
- mock_
tool_ objectresponse - A mock response to provide if a tool is called. Structure is documented below.
- note string
- A note describing the expectation.
- tool_
call object - Expectation on tool call. Structure is documented below.
- tool_
response object - Expectation on tool response. Structure is documented below.
- updated_
variables object - Expectation on updated variables. Structure is documented below.
- agent
Response EvaluationGolden Turn Step Expectation Agent Response - Expectation on agent response. Structure is documented below.
- agent
Transfer EvaluationGolden Turn Step Expectation Agent Transfer - Expectation on agent transfer. Structure is documented below.
- mock
Tool EvaluationResponse Golden Turn Step Expectation Mock Tool Response - A mock response to provide if a tool is called. Structure is documented below.
- note String
- A note describing the expectation.
- tool
Call EvaluationGolden Turn Step Expectation Tool Call - Expectation on tool call. Structure is documented below.
- tool
Response EvaluationGolden Turn Step Expectation Tool Response - Expectation on tool response. Structure is documented below.
- updated
Variables EvaluationGolden Turn Step Expectation Updated Variables - Expectation on updated variables. Structure is documented below.
- agent
Response EvaluationGolden Turn Step Expectation Agent Response - Expectation on agent response. Structure is documented below.
- agent
Transfer EvaluationGolden Turn Step Expectation Agent Transfer - Expectation on agent transfer. Structure is documented below.
- mock
Tool EvaluationResponse Golden Turn Step Expectation Mock Tool Response - A mock response to provide if a tool is called. Structure is documented below.
- note string
- A note describing the expectation.
- tool
Call EvaluationGolden Turn Step Expectation Tool Call - Expectation on tool call. Structure is documented below.
- tool
Response EvaluationGolden Turn Step Expectation Tool Response - Expectation on tool response. Structure is documented below.
- updated
Variables EvaluationGolden Turn Step Expectation Updated Variables - Expectation on updated variables. Structure is documented below.
- agent_
response EvaluationGolden Turn Step Expectation Agent Response - Expectation on agent response. Structure is documented below.
- agent_
transfer EvaluationGolden Turn Step Expectation Agent Transfer - Expectation on agent transfer. Structure is documented below.
- mock_
tool_ Evaluationresponse Golden Turn Step Expectation Mock Tool Response - A mock response to provide if a tool is called. Structure is documented below.
- note str
- A note describing the expectation.
- tool_
call EvaluationGolden Turn Step Expectation Tool Call - Expectation on tool call. Structure is documented below.
- tool_
response EvaluationGolden Turn Step Expectation Tool Response - Expectation on tool response. Structure is documented below.
- updated_
variables EvaluationGolden Turn Step Expectation Updated Variables - Expectation on updated variables. Structure is documented below.
- agent
Response Property Map - Expectation on agent response. Structure is documented below.
- agent
Transfer Property Map - Expectation on agent transfer. Structure is documented below.
- mock
Tool Property MapResponse - A mock response to provide if a tool is called. Structure is documented below.
- note String
- A note describing the expectation.
- tool
Call Property Map - Expectation on tool call. Structure is documented below.
- tool
Response Property Map - Expectation on tool response. Structure is documented below.
- updated
Variables Property Map - Expectation on updated variables. Structure is documented below.
EvaluationGoldenTurnStepExpectationAgentResponse, EvaluationGoldenTurnStepExpectationAgentResponseArgs
- Chunks
List<Evaluation
Golden Turn Step Expectation Agent Response Chunk> - Content of the message as a series of chunks. Structure is documented below.
- Role string
- The role within the conversation, e.g., user, agent.
- Chunks
[]Evaluation
Golden Turn Step Expectation Agent Response Chunk - Content of the message as a series of chunks. Structure is documented below.
- Role string
- The role within the conversation, e.g., user, agent.
- chunks list(object)
- Content of the message as a series of chunks. Structure is documented below.
- role string
- The role within the conversation, e.g., user, agent.
- chunks
List<Evaluation
Golden Turn Step Expectation Agent Response Chunk> - Content of the message as a series of chunks. Structure is documented below.
- role String
- The role within the conversation, e.g., user, agent.
- chunks
Evaluation
Golden Turn Step Expectation Agent Response Chunk[] - Content of the message as a series of chunks. Structure is documented below.
- role string
- The role within the conversation, e.g., user, agent.
- chunks
Sequence[Evaluation
Golden Turn Step Expectation Agent Response Chunk] - Content of the message as a series of chunks. Structure is documented below.
- role str
- The role within the conversation, e.g., user, agent.
- chunks List<Property Map>
- Content of the message as a series of chunks. Structure is documented below.
- role String
- The role within the conversation, e.g., user, agent.
EvaluationGoldenTurnStepExpectationAgentResponseChunk, EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
- Agent
Transfer EvaluationGolden Turn Step Expectation Agent Response Chunk Agent Transfer - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- Blob
Evaluation
Golden Turn Step Expectation Agent Response Chunk Blob - Represents a blob input or output in the conversation. Structure is documented below.
- Image
Evaluation
Golden Turn Step Expectation Agent Response Chunk Image - Represents an image input or output in the conversation. Structure is documented below.
- Text string
- Text data.
- Tool
Call EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call - Request for the client or the agent to execute the specified tool. Structure is documented below.
- Tool
Response EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response - The execution result of a specific tool from the client or the agent. Structure is documented below.
- Updated
Variables Dictionary<string, string> - Updated variables in JSON object format.
- Agent
Transfer EvaluationGolden Turn Step Expectation Agent Response Chunk Agent Transfer - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- Blob
Evaluation
Golden Turn Step Expectation Agent Response Chunk Blob - Represents a blob input or output in the conversation. Structure is documented below.
- Image
Evaluation
Golden Turn Step Expectation Agent Response Chunk Image - Represents an image input or output in the conversation. Structure is documented below.
- Text string
- Text data.
- Tool
Call EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call - Request for the client or the agent to execute the specified tool. Structure is documented below.
- Tool
Response EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response - The execution result of a specific tool from the client or the agent. Structure is documented below.
- Updated
Variables map[string]string - Updated variables in JSON object format.
- agent_
transfer object - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- blob object
- Represents a blob input or output in the conversation. Structure is documented below.
- image object
- Represents an image input or output in the conversation. Structure is documented below.
- text string
- Text data.
- tool_
call object - Request for the client or the agent to execute the specified tool. Structure is documented below.
- tool_
response object - The execution result of a specific tool from the client or the agent. Structure is documented below.
- updated_
variables map(string) - Updated variables in JSON object format.
- agent
Transfer EvaluationGolden Turn Step Expectation Agent Response Chunk Agent Transfer - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- blob
Evaluation
Golden Turn Step Expectation Agent Response Chunk Blob - Represents a blob input or output in the conversation. Structure is documented below.
- image
Evaluation
Golden Turn Step Expectation Agent Response Chunk Image - Represents an image input or output in the conversation. Structure is documented below.
- text String
- Text data.
- tool
Call EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call - Request for the client or the agent to execute the specified tool. Structure is documented below.
- tool
Response EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response - The execution result of a specific tool from the client or the agent. Structure is documented below.
- updated
Variables Map<String,String> - Updated variables in JSON object format.
- agent
Transfer EvaluationGolden Turn Step Expectation Agent Response Chunk Agent Transfer - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- blob
Evaluation
Golden Turn Step Expectation Agent Response Chunk Blob - Represents a blob input or output in the conversation. Structure is documented below.
- image
Evaluation
Golden Turn Step Expectation Agent Response Chunk Image - Represents an image input or output in the conversation. Structure is documented below.
- text string
- Text data.
- tool
Call EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call - Request for the client or the agent to execute the specified tool. Structure is documented below.
- tool
Response EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response - The execution result of a specific tool from the client or the agent. Structure is documented below.
- updated
Variables {[key: string]: string} - Updated variables in JSON object format.
- agent_
transfer EvaluationGolden Turn Step Expectation Agent Response Chunk Agent Transfer - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- blob
Evaluation
Golden Turn Step Expectation Agent Response Chunk Blob - Represents a blob input or output in the conversation. Structure is documented below.
- image
Evaluation
Golden Turn Step Expectation Agent Response Chunk Image - Represents an image input or output in the conversation. Structure is documented below.
- text str
- Text data.
- tool_
call EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call - Request for the client or the agent to execute the specified tool. Structure is documented below.
- tool_
response EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response - The execution result of a specific tool from the client or the agent. Structure is documented below.
- updated_
variables Mapping[str, str] - Updated variables in JSON object format.
- agent
Transfer Property Map - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- blob Property Map
- Represents a blob input or output in the conversation. Structure is documented below.
- image Property Map
- Represents an image input or output in the conversation. Structure is documented below.
- text String
- Text data.
- tool
Call Property Map - Request for the client or the agent to execute the specified tool. Structure is documented below.
- tool
Response Property Map - The execution result of a specific tool from the client or the agent. Structure is documented below.
- updated
Variables Map<String> - Updated variables in JSON object format.
EvaluationGoldenTurnStepExpectationAgentResponseChunkAgentTransfer, EvaluationGoldenTurnStepExpectationAgentResponseChunkAgentTransferArgs
- Target
Agent string - The resource name of the target agent.
- Display
Name string - The display name of the target agent.
- Target
Agent string - The resource name of the target agent.
- Display
Name string - The display name of the target agent.
- target_
agent string - The resource name of the target agent.
- display_
name string - The display name of the target agent.
- target
Agent String - The resource name of the target agent.
- display
Name String - The display name of the target agent.
- target
Agent string - The resource name of the target agent.
- display
Name string - The display name of the target agent.
- target_
agent str - The resource name of the target agent.
- display_
name str - The display name of the target agent.
- target
Agent String - The resource name of the target agent.
- display
Name String - The display name of the target agent.
EvaluationGoldenTurnStepExpectationAgentResponseChunkBlob, EvaluationGoldenTurnStepExpectationAgentResponseChunkBlobArgs
EvaluationGoldenTurnStepExpectationAgentResponseChunkImage, EvaluationGoldenTurnStepExpectationAgentResponseChunkImageArgs
EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCall, EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs
- Args Dictionary<string, string>
- The input parameters and values for the tool in JSON object format.
- Display
Name string - (Output) Display name of the tool.
- Id string
- The unique identifier of the tool call.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- Args map[string]string
- The input parameters and values for the tool in JSON object format.
- Display
Name string - (Output) Display name of the tool.
- Id string
- The unique identifier of the tool call.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args map(string)
- The input parameters and values for the tool in JSON object format.
- display_
name string - (Output) Display name of the tool.
- id string
- The unique identifier of the tool call.
- tool string
- The resource name of the tool.
- toolset_
tool object - A tool that is created from a toolset. Structure is documented below.
- args Map<String,String>
- The input parameters and values for the tool in JSON object format.
- display
Name String - (Output) Display name of the tool.
- id String
- The unique identifier of the tool call.
- tool String
- The resource name of the tool.
- toolset
Tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args {[key: string]: string}
- The input parameters and values for the tool in JSON object format.
- display
Name string - (Output) Display name of the tool.
- id string
- The unique identifier of the tool call.
- tool string
- The resource name of the tool.
- toolset
Tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args Mapping[str, str]
- The input parameters and values for the tool in JSON object format.
- display_
name str - (Output) Display name of the tool.
- id str
- The unique identifier of the tool call.
- tool str
- The resource name of the tool.
- toolset_
tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args Map<String>
- The input parameters and values for the tool in JSON object format.
- display
Name String - (Output) Display name of the tool.
- id String
- The unique identifier of the tool call.
- tool String
- The resource name of the tool.
- toolset
Tool Property Map - A tool that is created from a toolset. Structure is documented below.
EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallToolsetTool, EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallToolsetToolArgs
EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponse, EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs
- Display
Name string - (Output) Display name of the tool.
- Id string
- The matching ID of the tool call the response is for.
- Response Dictionary<string, string>
- The tool execution result in JSON object format.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- Display
Name string - (Output) Display name of the tool.
- Id string
- The matching ID of the tool call the response is for.
- Response map[string]string
- The tool execution result in JSON object format.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display_
name string - (Output) Display name of the tool.
- id string
- The matching ID of the tool call the response is for.
- response map(string)
- The tool execution result in JSON object format.
- tool string
- The resource name of the tool.
- toolset_
tool object - A tool that is created from a toolset. Structure is documented below.
- display
Name String - (Output) Display name of the tool.
- id String
- The matching ID of the tool call the response is for.
- response Map<String,String>
- The tool execution result in JSON object format.
- tool String
- The resource name of the tool.
- toolset
Tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display
Name string - (Output) Display name of the tool.
- id string
- The matching ID of the tool call the response is for.
- response {[key: string]: string}
- The tool execution result in JSON object format.
- tool string
- The resource name of the tool.
- toolset
Tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display_
name str - (Output) Display name of the tool.
- id str
- The matching ID of the tool call the response is for.
- response Mapping[str, str]
- The tool execution result in JSON object format.
- tool str
- The resource name of the tool.
- toolset_
tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display
Name String - (Output) Display name of the tool.
- id String
- The matching ID of the tool call the response is for.
- response Map<String>
- The tool execution result in JSON object format.
- tool String
- The resource name of the tool.
- toolset
Tool Property Map - A tool that is created from a toolset. Structure is documented below.
EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseToolsetTool, EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseToolsetToolArgs
EvaluationGoldenTurnStepExpectationAgentTransfer, EvaluationGoldenTurnStepExpectationAgentTransferArgs
- Display
Name string - The display name of the target agent.
- Target
Agent string - The resource name of the target agent.
- Display
Name string - The display name of the target agent.
- Target
Agent string - The resource name of the target agent.
- display_
name string - The display name of the target agent.
- target_
agent string - The resource name of the target agent.
- display
Name String - The display name of the target agent.
- target
Agent String - The resource name of the target agent.
- display
Name string - The display name of the target agent.
- target
Agent string - The resource name of the target agent.
- display_
name str - The display name of the target agent.
- target_
agent str - The resource name of the target agent.
- display
Name String - The display name of the target agent.
- target
Agent String - The resource name of the target agent.
EvaluationGoldenTurnStepExpectationMockToolResponse, EvaluationGoldenTurnStepExpectationMockToolResponseArgs
- Display
Name string - (Output) Display name of the tool.
- Id string
- The matching ID of the tool call the response is for.
- Response Dictionary<string, string>
- The tool execution result in JSON object format.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationGolden Turn Step Expectation Mock Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- Display
Name string - (Output) Display name of the tool.
- Id string
- The matching ID of the tool call the response is for.
- Response map[string]string
- The tool execution result in JSON object format.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationGolden Turn Step Expectation Mock Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display_
name string - (Output) Display name of the tool.
- id string
- The matching ID of the tool call the response is for.
- response map(string)
- The tool execution result in JSON object format.
- tool string
- The resource name of the tool.
- toolset_
tool object - A tool that is created from a toolset. Structure is documented below.
- display
Name String - (Output) Display name of the tool.
- id String
- The matching ID of the tool call the response is for.
- response Map<String,String>
- The tool execution result in JSON object format.
- tool String
- The resource name of the tool.
- toolset
Tool EvaluationGolden Turn Step Expectation Mock Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display
Name string - (Output) Display name of the tool.
- id string
- The matching ID of the tool call the response is for.
- response {[key: string]: string}
- The tool execution result in JSON object format.
- tool string
- The resource name of the tool.
- toolset
Tool EvaluationGolden Turn Step Expectation Mock Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display_
name str - (Output) Display name of the tool.
- id str
- The matching ID of the tool call the response is for.
- response Mapping[str, str]
- The tool execution result in JSON object format.
- tool str
- The resource name of the tool.
- toolset_
tool EvaluationGolden Turn Step Expectation Mock Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display
Name String - (Output) Display name of the tool.
- id String
- The matching ID of the tool call the response is for.
- response Map<String>
- The tool execution result in JSON object format.
- tool String
- The resource name of the tool.
- toolset
Tool Property Map - A tool that is created from a toolset. Structure is documented below.
EvaluationGoldenTurnStepExpectationMockToolResponseToolsetTool, EvaluationGoldenTurnStepExpectationMockToolResponseToolsetToolArgs
EvaluationGoldenTurnStepExpectationToolCall, EvaluationGoldenTurnStepExpectationToolCallArgs
- Args Dictionary<string, string>
- The input parameters and values for the tool in JSON object format.
- Display
Name string - (Output) Display name of the tool.
- Id string
- The unique identifier of the tool call.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationGolden Turn Step Expectation Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- Args map[string]string
- The input parameters and values for the tool in JSON object format.
- Display
Name string - (Output) Display name of the tool.
- Id string
- The unique identifier of the tool call.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationGolden Turn Step Expectation Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args map(string)
- The input parameters and values for the tool in JSON object format.
- display_
name string - (Output) Display name of the tool.
- id string
- The unique identifier of the tool call.
- tool string
- The resource name of the tool.
- toolset_
tool object - A tool that is created from a toolset. Structure is documented below.
- args Map<String,String>
- The input parameters and values for the tool in JSON object format.
- display
Name String - (Output) Display name of the tool.
- id String
- The unique identifier of the tool call.
- tool String
- The resource name of the tool.
- toolset
Tool EvaluationGolden Turn Step Expectation Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args {[key: string]: string}
- The input parameters and values for the tool in JSON object format.
- display
Name string - (Output) Display name of the tool.
- id string
- The unique identifier of the tool call.
- tool string
- The resource name of the tool.
- toolset
Tool EvaluationGolden Turn Step Expectation Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args Mapping[str, str]
- The input parameters and values for the tool in JSON object format.
- display_
name str - (Output) Display name of the tool.
- id str
- The unique identifier of the tool call.
- tool str
- The resource name of the tool.
- toolset_
tool EvaluationGolden Turn Step Expectation Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args Map<String>
- The input parameters and values for the tool in JSON object format.
- display
Name String - (Output) Display name of the tool.
- id String
- The unique identifier of the tool call.
- tool String
- The resource name of the tool.
- toolset
Tool Property Map - A tool that is created from a toolset. Structure is documented below.
EvaluationGoldenTurnStepExpectationToolCallToolsetTool, EvaluationGoldenTurnStepExpectationToolCallToolsetToolArgs
EvaluationGoldenTurnStepExpectationToolResponse, EvaluationGoldenTurnStepExpectationToolResponseArgs
- Display
Name string - (Output) Display name of the tool.
- Id string
- The matching ID of the tool call the response is for.
- Response Dictionary<string, string>
- The tool execution result in JSON object format.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationGolden Turn Step Expectation Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- Display
Name string - (Output) Display name of the tool.
- Id string
- The matching ID of the tool call the response is for.
- Response map[string]string
- The tool execution result in JSON object format.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationGolden Turn Step Expectation Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display_
name string - (Output) Display name of the tool.
- id string
- The matching ID of the tool call the response is for.
- response map(string)
- The tool execution result in JSON object format.
- tool string
- The resource name of the tool.
- toolset_
tool object - A tool that is created from a toolset. Structure is documented below.
- display
Name String - (Output) Display name of the tool.
- id String
- The matching ID of the tool call the response is for.
- response Map<String,String>
- The tool execution result in JSON object format.
- tool String
- The resource name of the tool.
- toolset
Tool EvaluationGolden Turn Step Expectation Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display
Name string - (Output) Display name of the tool.
- id string
- The matching ID of the tool call the response is for.
- response {[key: string]: string}
- The tool execution result in JSON object format.
- tool string
- The resource name of the tool.
- toolset
Tool EvaluationGolden Turn Step Expectation Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display_
name str - (Output) Display name of the tool.
- id str
- The matching ID of the tool call the response is for.
- response Mapping[str, str]
- The tool execution result in JSON object format.
- tool str
- The resource name of the tool.
- toolset_
tool EvaluationGolden Turn Step Expectation Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display
Name String - (Output) Display name of the tool.
- id String
- The matching ID of the tool call the response is for.
- response Map<String>
- The tool execution result in JSON object format.
- tool String
- The resource name of the tool.
- toolset
Tool Property Map - A tool that is created from a toolset. Structure is documented below.
EvaluationGoldenTurnStepExpectationToolResponseToolsetTool, EvaluationGoldenTurnStepExpectationToolResponseToolsetToolArgs
EvaluationGoldenTurnStepExpectationUpdatedVariables, EvaluationGoldenTurnStepExpectationUpdatedVariablesArgs
- Notes string
- Dummy property.
- Notes string
- Dummy property.
- notes string
- Dummy property.
- notes String
- Dummy property.
- notes string
- Dummy property.
- notes str
- Dummy property.
- notes String
- Dummy property.
EvaluationGoldenTurnStepUserInput, EvaluationGoldenTurnStepUserInputArgs
- Audio string
- Audio data from the end user.
- Blob
Evaluation
Golden Turn Step User Input Blob - Blob data from the end user. Structure is documented below.
- Dtmf string
- DTMF digits from the end user.
- Event
Evaluation
Golden Turn Step User Input Event - An event to be sent to the session. Structure is documented below.
- Image
Evaluation
Golden Turn Step User Input Image - Image data from the end user. Structure is documented below.
- Text string
- Natural language query.
- Tool
Responses EvaluationGolden Turn Step User Input Tool Responses - The list of tool execution results. Structure is documented below.
- Variables Dictionary<string, string>
- Map of variables to set.
- Will
Continue bool - Whether the session should continue.
- Audio string
- Audio data from the end user.
- Blob
Evaluation
Golden Turn Step User Input Blob - Blob data from the end user. Structure is documented below.
- Dtmf string
- DTMF digits from the end user.
- Event
Evaluation
Golden Turn Step User Input Event - An event to be sent to the session. Structure is documented below.
- Image
Evaluation
Golden Turn Step User Input Image - Image data from the end user. Structure is documented below.
- Text string
- Natural language query.
- Tool
Responses EvaluationGolden Turn Step User Input Tool Responses - The list of tool execution results. Structure is documented below.
- Variables map[string]string
- Map of variables to set.
- Will
Continue bool - Whether the session should continue.
- audio string
- Audio data from the end user.
- blob object
- Blob data from the end user. Structure is documented below.
- dtmf string
- DTMF digits from the end user.
- event object
- An event to be sent to the session. Structure is documented below.
- image object
- Image data from the end user. Structure is documented below.
- text string
- Natural language query.
- tool_
responses object - The list of tool execution results. Structure is documented below.
- variables map(string)
- Map of variables to set.
- will_
continue bool - Whether the session should continue.
- audio String
- Audio data from the end user.
- blob
Evaluation
Golden Turn Step User Input Blob - Blob data from the end user. Structure is documented below.
- dtmf String
- DTMF digits from the end user.
- event
Evaluation
Golden Turn Step User Input Event - An event to be sent to the session. Structure is documented below.
- image
Evaluation
Golden Turn Step User Input Image - Image data from the end user. Structure is documented below.
- text String
- Natural language query.
- tool
Responses EvaluationGolden Turn Step User Input Tool Responses - The list of tool execution results. Structure is documented below.
- variables Map<String,String>
- Map of variables to set.
- will
Continue Boolean - Whether the session should continue.
- audio string
- Audio data from the end user.
- blob
Evaluation
Golden Turn Step User Input Blob - Blob data from the end user. Structure is documented below.
- dtmf string
- DTMF digits from the end user.
- event
Evaluation
Golden Turn Step User Input Event - An event to be sent to the session. Structure is documented below.
- image
Evaluation
Golden Turn Step User Input Image - Image data from the end user. Structure is documented below.
- text string
- Natural language query.
- tool
Responses EvaluationGolden Turn Step User Input Tool Responses - The list of tool execution results. Structure is documented below.
- variables {[key: string]: string}
- Map of variables to set.
- will
Continue boolean - Whether the session should continue.
- audio str
- Audio data from the end user.
- blob
Evaluation
Golden Turn Step User Input Blob - Blob data from the end user. Structure is documented below.
- dtmf str
- DTMF digits from the end user.
- event
Evaluation
Golden Turn Step User Input Event - An event to be sent to the session. Structure is documented below.
- image
Evaluation
Golden Turn Step User Input Image - Image data from the end user. Structure is documented below.
- text str
- Natural language query.
- tool_
responses EvaluationGolden Turn Step User Input Tool Responses - The list of tool execution results. Structure is documented below.
- variables Mapping[str, str]
- Map of variables to set.
- will_
continue bool - Whether the session should continue.
- audio String
- Audio data from the end user.
- blob Property Map
- Blob data from the end user. Structure is documented below.
- dtmf String
- DTMF digits from the end user.
- event Property Map
- An event to be sent to the session. Structure is documented below.
- image Property Map
- Image data from the end user. Structure is documented below.
- text String
- Natural language query.
- tool
Responses Property Map - The list of tool execution results. Structure is documented below.
- variables Map<String>
- Map of variables to set.
- will
Continue Boolean - Whether the session should continue.
EvaluationGoldenTurnStepUserInputBlob, EvaluationGoldenTurnStepUserInputBlobArgs
EvaluationGoldenTurnStepUserInputEvent, EvaluationGoldenTurnStepUserInputEventArgs
- Event string
- Event name.
- Event string
- Event name.
- event string
- Event name.
- event String
- Event name.
- event string
- Event name.
- event str
- Event name.
- event String
- Event name.
EvaluationGoldenTurnStepUserInputImage, EvaluationGoldenTurnStepUserInputImageArgs
EvaluationGoldenTurnStepUserInputToolResponses, EvaluationGoldenTurnStepUserInputToolResponsesArgs
- Tool
Responses List<EvaluationGolden Turn Step User Input Tool Responses Tool Response> - The list of tool execution results. Structure is documented below.
- Tool
Responses []EvaluationGolden Turn Step User Input Tool Responses Tool Response - The list of tool execution results. Structure is documented below.
- tool_
responses list(object) - The list of tool execution results. Structure is documented below.
- tool
Responses List<EvaluationGolden Turn Step User Input Tool Responses Tool Response> - The list of tool execution results. Structure is documented below.
- tool
Responses EvaluationGolden Turn Step User Input Tool Responses Tool Response[] - The list of tool execution results. Structure is documented below.
- tool_
responses Sequence[EvaluationGolden Turn Step User Input Tool Responses Tool Response] - The list of tool execution results. Structure is documented below.
- tool
Responses List<Property Map> - The list of tool execution results. Structure is documented below.
EvaluationGoldenTurnStepUserInputToolResponsesToolResponse, EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs
- Response Dictionary<string, string>
- Required. The tool execution result in JSON.
- Display
Name string - (Output) Output only. Display name of the tool.
- Id string
- Optional. Matching ID of the tool call.
- Tool string
- Name of the tool to execute.
- Toolset
Tool EvaluationGolden Turn Step User Input Tool Responses Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- Response map[string]string
- Required. The tool execution result in JSON.
- Display
Name string - (Output) Output only. Display name of the tool.
- Id string
- Optional. Matching ID of the tool call.
- Tool string
- Name of the tool to execute.
- Toolset
Tool EvaluationGolden Turn Step User Input Tool Responses Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- response map(string)
- Required. The tool execution result in JSON.
- display_
name string - (Output) Output only. Display name of the tool.
- id string
- Optional. Matching ID of the tool call.
- tool string
- Name of the tool to execute.
- toolset_
tool object - The toolset tool that got executed. Structure is documented below.
- response Map<String,String>
- Required. The tool execution result in JSON.
- display
Name String - (Output) Output only. Display name of the tool.
- id String
- Optional. Matching ID of the tool call.
- tool String
- Name of the tool to execute.
- toolset
Tool EvaluationGolden Turn Step User Input Tool Responses Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- response {[key: string]: string}
- Required. The tool execution result in JSON.
- display
Name string - (Output) Output only. Display name of the tool.
- id string
- Optional. Matching ID of the tool call.
- tool string
- Name of the tool to execute.
- toolset
Tool EvaluationGolden Turn Step User Input Tool Responses Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- response Mapping[str, str]
- Required. The tool execution result in JSON.
- display_
name str - (Output) Output only. Display name of the tool.
- id str
- Optional. Matching ID of the tool call.
- tool str
- Name of the tool to execute.
- toolset_
tool EvaluationGolden Turn Step User Input Tool Responses Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- response Map<String>
- Required. The tool execution result in JSON.
- display
Name String - (Output) Output only. Display name of the tool.
- id String
- Optional. Matching ID of the tool call.
- tool String
- Name of the tool to execute.
- toolset
Tool Property Map - The toolset tool that got executed. Structure is documented below.
EvaluationGoldenTurnStepUserInputToolResponsesToolResponseToolsetTool, EvaluationGoldenTurnStepUserInputToolResponsesToolResponseToolsetToolArgs
Import
Evaluation can be imported using any of these accepted formats:
projects/{{project}}/locations/{{location}}/apps/{{app}}/evaluations/{{evaluation_id}}{{project}}/{{location}}/{{app}}/{{evaluation_id}}{{location}}/{{app}}/{{evaluation_id}}
When using the pulumi import command, Evaluation can be imported using one of the formats above. For example:
$ pulumi import gcp:ces/evaluation:Evaluation default projects/{{project}}/locations/{{location}}/apps/{{app}}/evaluations/{{evaluation_id}}
$ pulumi import gcp:ces/evaluation:Evaluation default {{project}}/{{location}}/{{app}}/{{evaluation_id}}
$ pulumi import gcp:ces/evaluation:Evaluation default {{location}}/{{app}}/{{evaluation_id}}
To learn more about importing existing cloud resources, see Importing resources.
Package Details
- Repository
- Google Cloud (GCP) Classic pulumi/pulumi-gcp
- License
- Apache-2.0
- Notes
- This Pulumi package is based on the
google-betaTerraform Provider.
Viewing docs for Google Cloud v9.23.0
published on Thursday, May 7, 2026 by Pulumi
published on Thursday, May 7, 2026 by Pulumi
