Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,8 @@
side-by-side comparison.
* ``VpcConfig`` is at the step ``Arguments`` level (not inside the
``JobConfigDocument``).
* Hyperparameters (``eval_group_size``, ``sampling_temperature``,
``top_p``, ``max_tokens``, ``pass_k_values``, ``success_threshold``)
are emitted as string values under ``EvaluationConfig.HyperParameters``.
* User-changed hyperparameters are emitted as strings under
``EvaluationConfig.HyperParameters``, keyed by recipe parameter name.
* Lineage DAG:
``CreateEvaluationAction → (Evaluate…) → AssociateLineage``;
``AssociateLineage`` reads ``ServiceOutput.MlflowDetails.*`` from the
Expand All @@ -33,9 +32,7 @@

pipeline_name, role_arn, base_model_arn, agent_arn, agent_qualifier,
dataset_uri, s3_output_path, mlflow_resource_arn,
mlflow_experiment_name, eval_group_size,
sampling_temperature, top_p, max_tokens, pass_k_values,
success_threshold, model_package_group_arn, source_model_package_arn,
mlflow_experiment_name, model_package_group_arn, source_model_package_arn,
action_arn_prefix, dataset_artifact_arn, kms_key_arn, vpc_config,
vpc_security_group_ids, vpc_subnets, tags
"""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
)
from sagemaker.core.telemetry.telemetry_logging import _telemetry_emitter, TelemetryParamType
from sagemaker.train.common_utils.telemetry_params import BASE_EVALUATOR_TELEMETRY_PARAMS
from sagemaker.train.recipe_resolver import flatten_resolved_recipe
from sagemaker.core.telemetry.constants import Feature

if TYPE_CHECKING:
Expand Down Expand Up @@ -160,6 +161,7 @@ class MultiTurnRLEvaluator(BaseEvaluator):
_agent_arn_resolved: Optional[str] = None
_agent_kind: Optional[str] = None # "bedrock" | "lambda"
_hyperparameters: Optional[Any] = None
_hp_defaults: Optional[Dict[str, str]] = None

# --- Validators ------------------------------------------------------

Expand Down Expand Up @@ -315,9 +317,14 @@ def hyperparameters(self):
``get_info()``, and attribute-style read/write access with
hub-sourced validation (type + range).

Supported parameters (sourced from the AgentRFT evaluation recipe):
``eval_group_size``, ``sampling_temperature``, ``top_p``,
``max_tokens``, ``pass_k_values``, ``success_threshold``.
Parameter names and ranges come from the hub recipe; call
``get_info()`` to list them.

Example:

.. code:: python

evaluator.hyperparameters.sampling_max_tokens = 8192

Raises:
ValueError: If the base model name is not available or the hub
Expand Down Expand Up @@ -365,8 +372,32 @@ def hyperparameters(self):
override_params, param_names=list(override_params.keys()), return_full_spec=True
)
self._hyperparameters = FineTuningOptions(spec)
self._hp_defaults = self._hyperparameters.to_dict().copy()
return self._hyperparameters

def _get_user_changed_hyperparameters(self) -> Dict[str, str]:
"""Return hyperparameters that differ from the hub recipe defaults.

Returns:
Dict[str, str]: Changed values, keyed by recipe parameter name.
"""
final = self.hyperparameters.to_dict()

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟠 Medium: List-valued hyperparameters (e.g. pass_k_values) set via recipe or overrides are silently dropped. _get_effective_hyperparameters() uses flatten_resolved_recipe, which explicitly skips list values (elif not isinstance(v, list) in recipe_resolver.flatten_resolved_recipe), so the loop here never sees them. The only list values that survive are those captured from self.hyperparameters.to_dict(), i.e. those set via direct attribute assignment. The PR body states "This covers evaluator.hyperparameters.<name>, recipe and overrides" and "List values such as pass_k_values are forwarded", but the recipe/overrides path for list values does not work. The new test_list_values_are_forwarded only exercises direct assignment and test_overrides_dict_is_forwarded only uses scalars, so this gap is untested. Consider merging list values from the resolved recipe before dropping them, or documenting the limitation.

try:
resolved = self.get_resolved_recipe()
except ValueError: # no recipe, overrides or direct assignments
resolved = {}
values = flatten_resolved_recipe(resolved)
# flatten_resolved_recipe skips lists, so take them from the inference section.
values.update(
{k: v for k, v in resolved.get("inference", {}).items() if isinstance(v, list)}
)
for k, v in values.items():
if v is not None:
final[k] = v if isinstance(v, str) else str(v)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Low: In _get_user_changed_hyperparameters, recipe/override-sourced values are stringified (final[k] = str(v)) before the final comparison v != defaults.get(k), but defaults holds native-typed values from to_dict() (e.g. int 4096). So a value supplied via recipe/overrides that is equal to the recipe default (e.g. sampling_max_tokens: 4096) still compares unequal ("4096" != 4096) and gets sent in HyperParameters, whereas the same value set via direct attribute assignment is correctly suppressed. The effect is harmless since the service applies the same default, and MultiTurnRLTrainer has the identical behavior, so this is only a minor inconsistency. The new test test_value_equal_to_default_is_not_sent only exercises the direct-assignment path and wouldn't catch this; consider a test for the recipe/overrides-equals-default case if you want the suppression to be exact.


defaults = self._hp_defaults or {}
return {k: v for k, v in final.items() if v != defaults.get(k)}

# --- Helpers ---------------------------------------------------------

def _resolve_agent_arn(self) -> None:
Expand Down Expand Up @@ -428,14 +459,14 @@ def _build_template_context(
"""Assemble the rendering context expected by the MTRL templates."""
import json as _json

hparams: Dict[str, Any] = {}
hparams: Dict[str, str] = {}
try:
hparams = self._get_effective_hyperparameters()
hparams = self._get_user_changed_hyperparameters()
except Exception as e: # hub fetch can fail in offline envs
_logger.info(f"Skipping hub-sourced hyperparameters: {e}")

def _str_or_none(v):
return str(v) if v is not None else None
_logger.warning(
f"Could not load MTRL evaluation hyperparameters ({e}); the "
f"evaluation job will use the service defaults."
)

action_arn_prefix = (
f"arn:aws:sagemaker:{aws_context['region']}:{aws_context['account_id']}:action"
Expand Down Expand Up @@ -492,20 +523,8 @@ def _build_job_config_doc(include_mpc: bool, mlflow_run_name: str) -> str:
"BaseModelArn": base_model_arn,
"AcceptEula": True,
}
hp: Dict[str, str] = {}
for k in (
"eval_group_size",
"sampling_temperature",
"top_p",
"max_tokens",
"pass_k_values",
"success_threshold",
):
v = hparams.get(k)
if v is not None:
hp[k] = str(v)
if hp:
eval_cfg["HyperParameters"] = hp
if hparams:
eval_cfg["HyperParameters"] = dict(hparams)

doc: Dict[str, Any] = {
"AgentConfig": agent_cfg,
Expand Down Expand Up @@ -549,12 +568,6 @@ def _build_job_config_doc(include_mpc: bool, mlflow_run_name: str) -> str:
"mlflow_resource_arn": self.mlflow_resource_arn,
"mlflow_experiment_name": getattr(self, "mlflow_experiment_name", None)
or aws_context.get("pipeline_name"),
"eval_group_size": _str_or_none(hparams.get("eval_group_size")),
"sampling_temperature": _str_or_none(hparams.get("sampling_temperature")),
"top_p": _str_or_none(hparams.get("top_p")),
"max_tokens": _str_or_none(hparams.get("max_tokens")),
"pass_k_values": _str_or_none(hparams.get("pass_k_values")),
"success_threshold": _str_or_none(hparams.get("success_threshold")),
"stopping_condition": self.stopping_condition,
"model_package_group_arn": model_package_group_arn,
"source_model_package_arn": self._source_model_package_arn_cache,
Expand Down
141 changes: 141 additions & 0 deletions sagemaker-train/tests/unit/train/evaluate/test_mtrl_evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -319,3 +319,144 @@ def test_resolve_base_model_no_latest_job_uses_training_job(self):
resolver.resolve_model_info(mock_trainer)

mock_resolve.assert_called_once_with(SOURCE_MP_ARN)


# Mirrors the MTRLEvaluation override spec the hub publishes for MTRL models.
_HUB_MTRL_EVAL_SPEC = {
"base_model_name": {"type": "string", "default": BASE_MODEL, "required": True},
"model_name": {"type": "string", "default": BASE_MODEL, "required": True},
"name": {"type": "string", "default": "mtrl-eval-abcde", "required": True},
"eval_group_size": {"type": "integer", "min": 1, "max": 128, "default": 1},
"pass_k_values": {"type": "array", "default": [1, 2, 4, 8, 16, 32]},
"success_threshold": {"type": "float", "default": 1.0},
"temperature": {"type": "float", "min": 0, "max": 2, "default": 0},
"sampling_top_p": {"type": "float", "min": 0, "max": 1, "default": 1},
"sampling_max_tokens": {"type": "integer", "min": 512, "max": 8192, "default": 4096},
"rollout_max_concurrency": {"type": "integer", "min": 32, "max": 96, "default": 96},
"rollout_timeout": {"type": "float", "min": 300, "max": 86400, "default": 600.0},
"rollout_max_retries": {"type": "integer", "min": 1, "max": 10, "default": 3},
}

_AWS_CONTEXT = {"region": REGION, "account_id": "123456789012", "role_arn": ROLE}


@patch(
"sagemaker.train.common_utils.recipe_utils._get_evaluation_override_params",
return_value=_HUB_MTRL_EVAL_SPEC,
)
@patch("sagemaker.train.common_utils.model_resolution._resolve_base_model")
@patch("sagemaker.core.resources.Artifact")
class TestHyperparameterForwarding:
"""The JobConfigDocument must carry user-changed values under the recipe's names."""

def _create(self, mock_artifact, mock_resolve, **kwargs):
from sagemaker.train.common_utils.model_resolution import _ModelInfo, _ModelType

mock_resolve.return_value = _ModelInfo(
base_model_name=BASE_MODEL,
base_model_arn=(
f"arn:aws:sagemaker:{REGION}:aws:hub-content/SageMakerPublicHub/Model/"
f"{BASE_MODEL}/1.0.0"
),
source_model_package_arn=None,
model_type=_ModelType.JUMPSTART,
hub_content_name=BASE_MODEL,
additional_metadata={},
)
mock_artifact.get_all.return_value = iter([])
session = MagicMock()
session.boto_region_name = REGION
session.boto_session.region_name = REGION
session.sagemaker_config = {}
evaluator = MultiTurnRLEvaluator(
model=BASE_MODEL,
agent_config=AGENT_ARN,
dataset=DATASET,
s3_output_path=OUTPUT,
sagemaker_session=session,
region=REGION,
**kwargs,
)
evaluator._resolve_agent_arn()
return evaluator

@staticmethod
def _sent_hyperparameters(evaluator):
import json

ctx = evaluator._build_template_context(_AWS_CONTEXT, {}, None)
for key in ("job_config_document_str", "job_config_document_ft_str"):
doc = json.loads(ctx[key])
yield doc["EvaluationConfig"].get("HyperParameters")

def test_defaults_send_no_hyperparameters(self, mock_artifact, mock_resolve, _):
evaluator = self._create(mock_artifact, mock_resolve)
assert list(self._sent_hyperparameters(evaluator)) == [None, None]

def test_direct_assignment_uses_recipe_names(self, mock_artifact, mock_resolve, _):
evaluator = self._create(mock_artifact, mock_resolve)
evaluator.hyperparameters.sampling_max_tokens = 8192
evaluator.hyperparameters.sampling_top_p = 0.9
evaluator.hyperparameters.temperature = 0.7
evaluator.hyperparameters.rollout_timeout = 1200.0
evaluator.hyperparameters.rollout_max_concurrency = 48
evaluator.hyperparameters.rollout_max_retries = 5

expected = {
"sampling_max_tokens": "8192",
"sampling_top_p": "0.9",
"temperature": "0.7",
"rollout_timeout": "1200.0",
"rollout_max_concurrency": "48",
"rollout_max_retries": "5",
}
for sent in self._sent_hyperparameters(evaluator):
assert sent == expected

def test_list_values_are_forwarded(self, mock_artifact, mock_resolve, _):
evaluator = self._create(mock_artifact, mock_resolve)
evaluator.hyperparameters.pass_k_values = [1, 4]
for sent in self._sent_hyperparameters(evaluator):
assert sent == {"pass_k_values": "[1, 4]"}

def test_overrides_dict_is_forwarded(self, mock_artifact, mock_resolve, _):
evaluator = self._create(
mock_artifact,
mock_resolve,
overrides={"inference": {"sampling_max_tokens": 6000, "eval_group_size": 4}},
)
for sent in self._sent_hyperparameters(evaluator):
assert sent == {"sampling_max_tokens": "6000", "eval_group_size": "4"}

def test_list_value_in_overrides_is_forwarded(self, mock_artifact, mock_resolve, _):
evaluator = self._create(
mock_artifact,
mock_resolve,
overrides={"inference": {"pass_k_values": [1, 4], "sampling_max_tokens": 6000}},
)
for sent in self._sent_hyperparameters(evaluator):
assert sent == {"pass_k_values": "[1, 4]", "sampling_max_tokens": "6000"}

def test_list_value_in_recipe_is_forwarded(self, mock_artifact, mock_resolve, _, tmp_path):
recipe = tmp_path / "recipe.yaml"
recipe.write_text("inference:\n pass_k_values: [1, 2]\n eval_group_size: 2\n")
evaluator = self._create(mock_artifact, mock_resolve, recipe=str(recipe))
for sent in self._sent_hyperparameters(evaluator):
assert sent == {"pass_k_values": "[1, 2]", "eval_group_size": "2"}

def test_value_equal_to_default_is_not_sent(self, mock_artifact, mock_resolve, _):
evaluator = self._create(mock_artifact, mock_resolve)
evaluator.hyperparameters.sampling_max_tokens = 4096
evaluator.hyperparameters.eval_group_size = 8
for sent in self._sent_hyperparameters(evaluator):
assert sent == {"eval_group_size": "8"}

def test_range_is_validated_against_recipe(self, mock_artifact, mock_resolve, _):
evaluator = self._create(mock_artifact, mock_resolve)
with pytest.raises(ValueError, match="sampling_max_tokens must be <= 8192"):
evaluator.hyperparameters.sampling_max_tokens = 16384

def test_hub_failure_sends_no_hyperparameters(self, mock_artifact, mock_resolve, mock_hub):
mock_hub.side_effect = RuntimeError("hub unavailable")
evaluator = self._create(mock_artifact, mock_resolve)
assert list(self._sent_hyperparameters(evaluator)) == [None, None]
Loading