diff --git a/agentplatform/_genai/_transformers.py b/agentplatform/_genai/_transformers.py index fb6f477cf8..0de1aef4cf 100644 --- a/agentplatform/_genai/_transformers.py +++ b/agentplatform/_genai/_transformers.py @@ -72,9 +72,12 @@ def t_metrics( elif ( hasattr(metric, "remote_custom_function") and metric.remote_custom_function ): - metric_payload_item["custom_code_execution_spec"] = { + spec: dict[str, Any] = { "evaluation_function": metric.remote_custom_function } + if getattr(metric, "code_execution_region", None): + spec["code_execution_region"] = metric.code_execution_region + metric_payload_item["custom_code_execution_spec"] = spec elif ( isinstance(metric, types.CodeExecutionMetric) or ( @@ -82,9 +85,10 @@ def t_metrics( and isinstance(getattr(metric, "custom_function", None), str) ) ) and getattr(metric, "custom_function", None): - metric_payload_item["custom_code_execution_spec"] = { - "evaluation_function": metric.custom_function - } + spec = {"evaluation_function": metric.custom_function} + if getattr(metric, "code_execution_region", None): + spec["code_execution_region"] = metric.code_execution_region + metric_payload_item["custom_code_execution_spec"] = spec # LLM-based metrics elif hasattr(metric, "prompt_template") and metric.prompt_template: llm_based_spec: dict[str, Any] = { diff --git a/tests/unit/agentplatform/genai/replays/test_custom_code_execution_metric.py b/tests/unit/agentplatform/genai/replays/test_custom_code_execution_metric.py index 34aed0c68d..7195844187 100644 --- a/tests/unit/agentplatform/genai/replays/test_custom_code_execution_metric.py +++ b/tests/unit/agentplatform/genai/replays/test_custom_code_execution_metric.py @@ -43,9 +43,9 @@ def evaluate(instance): ], ) def test_custom_code_execution(client, custom_metric): - """Tests that custom code execution metric produces a correctly structured EvaluationResult.""" + """Tests that custom code execution metric produces a correctly structured EvaluationResult.""" - prompts_df = pd.DataFrame( + prompts_df = pd.DataFrame( { "prompt": ["What is 2+2?", "What is 3+3?"], "response": ["4", "5"], @@ -53,30 +53,71 @@ def test_custom_code_execution(client, custom_metric): } ) - eval_dataset = types.EvaluationDataset( + eval_dataset = types.EvaluationDataset( eval_dataset_df=prompts_df, candidate_name="test_model", ) - evaluation_result = client.evals.evaluate( + evaluation_result = client.evals.evaluate( dataset=eval_dataset, metrics=[custom_metric], ) - assert isinstance(evaluation_result, types.EvaluationResult) - - assert evaluation_result.summary_metrics is not None - assert evaluation_result.summary_metrics - for summary in evaluation_result.summary_metrics: - assert isinstance(summary, types.AggregatedMetricResult) - assert summary.metric_name == "my_custom_code_metric" - - assert evaluation_result.eval_case_results is not None - assert evaluation_result.eval_case_results - for case_result in evaluation_result.eval_case_results: - assert isinstance(case_result, types.EvalCaseResult) - assert case_result.eval_case_index is not None - assert case_result.response_candidate_results is not None + assert isinstance(evaluation_result, types.EvaluationResult) + + assert evaluation_result.summary_metrics is not None + assert evaluation_result.summary_metrics + for summary in evaluation_result.summary_metrics: + assert isinstance(summary, types.AggregatedMetricResult) + assert summary.metric_name == "my_custom_code_metric" + + assert evaluation_result.eval_case_results is not None + assert evaluation_result.eval_case_results + for case_result in evaluation_result.eval_case_results: + assert isinstance(case_result, types.EvalCaseResult) + assert case_result.eval_case_index is not None + assert case_result.response_candidate_results is not None + + +def test_custom_code_execution_with_region(client): + """Tests that code_execution_region is included in the custom code execution spec.""" + + prompts_df = pd.DataFrame({ + "prompt": ["What is 2+2?", "What is 3+3?"], + "response": ["4", "5"], + "reference": ["4", "6"], + }) + + eval_dataset = types.EvaluationDataset( + eval_dataset_df=prompts_df, + candidate_name="test_model", + ) + + metric = types.Metric( + name="my_custom_code_metric", + remote_custom_function=CODE_SNIPPET, + code_execution_region="europe-west3", + ) + + evaluation_result = client.evals.evaluate( + dataset=eval_dataset, + metrics=[metric], + ) + + assert isinstance(evaluation_result, types.EvaluationResult) + + assert evaluation_result.summary_metrics is not None + assert evaluation_result.summary_metrics + for summary in evaluation_result.summary_metrics: + assert isinstance(summary, types.AggregatedMetricResult) + assert summary.metric_name == "my_custom_code_metric" + + assert evaluation_result.eval_case_results is not None + assert evaluation_result.eval_case_results + for case_result in evaluation_result.eval_case_results: + assert isinstance(case_result, types.EvalCaseResult) + assert case_result.eval_case_index is not None + assert case_result.response_candidate_results is not None @pytest.mark.parametrize( diff --git a/vertexai/_genai/_transformers.py b/vertexai/_genai/_transformers.py index fb6f477cf8..0de1aef4cf 100644 --- a/vertexai/_genai/_transformers.py +++ b/vertexai/_genai/_transformers.py @@ -72,9 +72,12 @@ def t_metrics( elif ( hasattr(metric, "remote_custom_function") and metric.remote_custom_function ): - metric_payload_item["custom_code_execution_spec"] = { + spec: dict[str, Any] = { "evaluation_function": metric.remote_custom_function } + if getattr(metric, "code_execution_region", None): + spec["code_execution_region"] = metric.code_execution_region + metric_payload_item["custom_code_execution_spec"] = spec elif ( isinstance(metric, types.CodeExecutionMetric) or ( @@ -82,9 +85,10 @@ def t_metrics( and isinstance(getattr(metric, "custom_function", None), str) ) ) and getattr(metric, "custom_function", None): - metric_payload_item["custom_code_execution_spec"] = { - "evaluation_function": metric.custom_function - } + spec = {"evaluation_function": metric.custom_function} + if getattr(metric, "code_execution_region", None): + spec["code_execution_region"] = metric.code_execution_region + metric_payload_item["custom_code_execution_spec"] = spec # LLM-based metrics elif hasattr(metric, "prompt_template") and metric.prompt_template: llm_based_spec: dict[str, Any] = { diff --git a/vertexai/_genai/evals.py b/vertexai/_genai/evals.py index 3715571d96..fe7b75a812 100644 --- a/vertexai/_genai/evals.py +++ b/vertexai/_genai/evals.py @@ -206,6 +206,13 @@ def _CustomCodeExecutionSpec_from_vertex( getv(from_object, ["evaluation_function"]), ) + if getv(from_object, ["codeExecutionRegion"]) is not None: + setv( + to_object, + ["code_execution_region"], + getv(from_object, ["codeExecutionRegion"]), + ) + return to_object @@ -228,6 +235,13 @@ def _CustomCodeExecutionSpec_to_vertex( getv(from_object, ["remote_custom_function"]), ) + if getv(from_object, ["code_execution_region"]) is not None: + setv( + to_object, + ["codeExecutionRegion"], + getv(from_object, ["code_execution_region"]), + ) + return to_object diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index 17d95148d1..2d4c6db193 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -1816,6 +1816,10 @@ class Metric(_common.BaseModel): default=None, description="""The evaluation function for the custom code execution metric. This custom code is run remotely in the evaluation service.""", ) + code_execution_region: Optional[str] = Field( + default=None, + description="""Optional. The region to use for code execution. If set, the Code Execution Sandbox will be invoked in the specified region regardless of the request's originating region. Supported regions: us-central1, us-east1, us-east4, us-west1, us-west4, southamerica-east1, europe-west2, europe-west3, asia-east1, asia-south1, asia-southeast1. If unset, the request's originating region is used.""", + ) judge_model: Optional[str] = Field( default=None, description="""The judge model for the metric.""" ) @@ -2164,6 +2168,10 @@ def evaluate(instance: dict[str, Any]) -> float: Instance is the evaluation instance, any fields populated in the instance are available to the function as instance[field_name].""", ) + code_execution_region: Optional[str] = Field( + default=None, + description="""Optional. The region to use for code execution. If set, the Code Execution Sandbox will be invoked in the specified region regardless of the request's originating region. Supported regions: us-central1, us-east1, us-east4, us-west1, us-west4, southamerica-east1, europe-west2, europe-west3, asia-east1, asia-south1, asia-southeast1. If unset, the request's originating region is used.""", + ) class CustomCodeExecutionSpecDict(TypedDict, total=False):