Skip to content

feat: GenAI Client(evals) - async auto-create EvaluationExperiment parity #7016

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
copybara-service merged 1 commit into from
Jul 29, 2026
+145 −23
Merged

feat: GenAI Client(evals) - async auto-create EvaluationExperiment parity #7016

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions agentplatform/_genai/evals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5504,6 +5504,7 @@ async def create_evaluation_run(
metrics: list[types.EvaluationRunMetricOrDict],
name: Optional[str] = None,
display_name: Optional[str] = None,
evaluation_experiment: Optional[str] = None,
agent_info: Optional[evals_types.AgentInfo] = None,
agent: Optional[str] = None,
user_simulator_config: Optional[evals_types.UserSimulatorConfigOrDict] = None,
Expand All @@ -5524,6 +5525,11 @@ async def create_evaluation_run(
metrics: The list of metrics to evaluate.
name: The name of the evaluation run.
display_name: The display name of the evaluation run.
evaluation_experiment: The resource name of an existing
EvaluationExperiment to group this run under. If omitted, a new
EvaluationExperiment is created automatically so the run is visible in
the Agent Platform UI. Pass an existing experiment name to group
multiple runs together.
agent_info: The agent info to evaluate. Mutually exclusive with
`inference_configs`.
agent: The agent resource name in str type. Accepts either an Agent
Expand Down Expand Up @@ -5682,11 +5688,23 @@ async def create_evaluation_run(
self._api_client, resolved_dataset, inference_configs, parsed_agent_info
)
resolved_labels = _evals_common._add_evaluation_run_labels(labels, agent)
resolved_name = name or f"evaluation_run_{uuid.uuid4()}"
resolved_display_name = display_name or name or f"evaluation_run_{uuid.uuid4()}"
resolved_experiment = evaluation_experiment
if resolved_experiment is None:
experiment_display_name = (
display_name or f"SDK Experiment {_evals_common._local_timestamp()}"
)
created_experiment = await self.create_evaluation_experiment(
display_name=experiment_display_name
)
resolved_experiment = created_experiment.name

# The backend rejects a caller-supplied `name` for runs that belong to an
# EvaluationExperiment and assigns the resource ID itself, so `name` is
# only used as a display_name fallback.
result = await self._create_evaluation_run(
name=resolved_name,
display_name=display_name or resolved_name,
display_name=resolved_display_name,
evaluation_experiment=resolved_experiment,
data_source=resolved_dataset,
evaluation_config=evaluation_config,
inference_configs=resolved_inference_configs,
Expand Down
125 changes: 108 additions & 17 deletions tests/unit/agentplatform/genai/test_evals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10635,25 +10635,33 @@ async def test_create_evaluation_run_async_passes_allow_cross_region_model(self)
return_value=self.mock_response
)
async_evals_module = evals.AsyncEvals(api_client_=self.mock_api_client)

await async_evals_module.create_evaluation_run(
dataset=agentplatform_genai_types.EvaluationRunDataSource(
evaluation_set="projects/123/locations/us-central1/evaluationSets/789"
),
metrics=[
agentplatform_genai_types.EvaluationRunMetric(
metric="general_quality_v1",
metric_config=agentplatform_genai_types.UnifiedMetric(
predefined_metric_spec=genai_types.PredefinedMetricSpec(
metric_spec_name="general_quality_v1",
)
),
)
],
dest="gs://test-bucket/output",
config={"allow_cross_region_model": True},
experiment = agentplatform_genai_types.EvaluationExperiment(
name="projects/123/locations/us-central1/evaluationExperiments/e1"
)

with mock.patch.object(
async_evals_module,
"create_evaluation_experiment",
new=mock.AsyncMock(return_value=experiment),
):
await async_evals_module.create_evaluation_run(
dataset=agentplatform_genai_types.EvaluationRunDataSource(
evaluation_set="projects/123/locations/us-central1/evaluationSets/789"
),
metrics=[
agentplatform_genai_types.EvaluationRunMetric(
metric="general_quality_v1",
metric_config=agentplatform_genai_types.UnifiedMetric(
predefined_metric_spec=genai_types.PredefinedMetricSpec(
metric_spec_name="general_quality_v1",
)
),
)
],
dest="gs://test-bucket/output",
config={"allow_cross_region_model": True},
)

self.mock_api_client.async_request.assert_called_once()
call_args = self.mock_api_client.async_request.call_args
request_body = call_args[0][2] # Third positional arg is the request dict
Expand Down Expand Up @@ -12414,3 +12422,86 @@ def test_uses_provided_experiment_without_creating(self):
request_body.get("evaluationExperiment")
== "projects/123/locations/us-central1/evaluationExperiments/existing"
)


class TestAsyncCreateEvaluationRunAutoExperiment:

def setup_method(self, method):
self.mock_api_client = mock.MagicMock()
self.mock_api_client.vertexai = True
self.mock_response = mock.MagicMock()
self.mock_response.body = json.dumps(
{
"name": "projects/123/locations/us-central1/evaluationRuns/456",
"displayName": "test_run",
"state": "PENDING",
"evaluationExperiment": (
"projects/123/locations/us-central1/evaluationExperiments/e1"
),
}
)
self.mock_api_client.async_request = mock.AsyncMock(
return_value=self.mock_response
)
self.dataset = agentplatform_genai_types.EvaluationRunDataSource(
evaluation_set="projects/123/locations/us-central1/evaluationSets/789"
)
self.metrics = [
agentplatform_genai_types.EvaluationRunMetric(
metric="general_quality_v1",
metric_config=agentplatform_genai_types.UnifiedMetric(
predefined_metric_spec=genai_types.PredefinedMetricSpec(
metric_spec_name="general_quality_v1",
)
),
)
]

@pytest.mark.asyncio
async def test_async_auto_creates_experiment_when_not_provided(self):
async_evals_module = evals.AsyncEvals(api_client_=self.mock_api_client)
experiment = agentplatform_genai_types.EvaluationExperiment(
name="projects/123/locations/us-central1/evaluationExperiments/e1"
)
with mock.patch.object(
async_evals_module,
"create_evaluation_experiment",
new=mock.AsyncMock(return_value=experiment),
) as mock_create_exp:
await async_evals_module.create_evaluation_run(
dataset=self.dataset,
metrics=self.metrics,
dest="gs://test-bucket/output",
display_name="my_run",
)

mock_create_exp.assert_awaited_once()
request_body = self.mock_api_client.async_request.call_args[0][2]
assert (
request_body.get("evaluationExperiment")
== "projects/123/locations/us-central1/evaluationExperiments/e1"
)

@pytest.mark.asyncio
async def test_async_uses_provided_experiment_without_creating(self):
async_evals_module = evals.AsyncEvals(api_client_=self.mock_api_client)
with mock.patch.object(
async_evals_module,
"create_evaluation_experiment",
new=mock.AsyncMock(),
) as mock_create_exp:
await async_evals_module.create_evaluation_run(
dataset=self.dataset,
metrics=self.metrics,
dest="gs://test-bucket/output",
evaluation_experiment=(
"projects/123/locations/us-central1/evaluationExperiments/existing"
),
)

mock_create_exp.assert_not_awaited()
request_body = self.mock_api_client.async_request.call_args[0][2]
assert (
request_body.get("evaluationExperiment")
== "projects/123/locations/us-central1/evaluationExperiments/existing"
)
19 changes: 16 additions & 3 deletions vertexai/_genai/evals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4949,6 +4949,7 @@ async def create_evaluation_run(
metrics: list[types.EvaluationRunMetricOrDict],
name: Optional[str] = None,
display_name: Optional[str] = None,
evaluation_experiment: Optional[str] = None,
agent_info: Optional[evals_types.AgentInfo] = None,
agent: Optional[str] = None,
user_simulator_config: Optional[evals_types.UserSimulatorConfigOrDict] = None,
Expand Down Expand Up @@ -5095,11 +5096,23 @@ async def create_evaluation_run(
self._api_client, resolved_dataset, inference_configs, parsed_agent_info
)
resolved_labels = _evals_common._add_evaluation_run_labels(labels, agent)
resolved_name = name or f"evaluation_run_{uuid.uuid4()}"
resolved_display_name = display_name or name or f"evaluation_run_{uuid.uuid4()}"
resolved_experiment = evaluation_experiment
if resolved_experiment is None:
experiment_display_name = (
display_name or f"SDK Experiment {_evals_common._local_timestamp()}"
)
created_experiment = await self.create_evaluation_experiment(
display_name=experiment_display_name
)
resolved_experiment = created_experiment.name

# The backend rejects a caller-supplied `name` for runs that belong to an
# EvaluationExperiment and assigns the resource ID itself, so `name` is
# only used as a display_name fallback.
result = await self._create_evaluation_run(
name=resolved_name,
display_name=display_name or resolved_name,
display_name=resolved_display_name,
evaluation_experiment=resolved_experiment,
data_source=resolved_dataset,
evaluation_config=evaluation_config,
inference_configs=resolved_inference_configs,
Expand Down
Loading