Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion src/openjd/model/_format_strings/_expression.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,24 @@ def evaluate(

raise ExpressionError(f"Nonvalid result type: {result} of type {type(result)}")

def evaluate_value(self, *, symtab: SymbolTable, path_format: Optional[Any] = None) -> Any:
@property
def is_expr(self) -> bool:
"""Whether this expression uses the EXPR language (Expression Language),
rather than the base 2023-09 value-reference grammar."""
return getattr(self._expresion_tree, "allows_nonscalar", False)

def evaluate_value(
self,
*,
symtab: SymbolTable,
path_format: Optional[Any] = None,
target_type: Optional[Any] = None,
) -> Any:
"""Evaluate the expression, preferring the engine's typed value form.

``target_type`` (an ``openjd.expr.ExprType``), when given, applies to
EXPR-backed expressions: the result is coerced toward it.

For EXPR-backed expressions this returns the engine ``ExprValue``,
which keeps the result's EXPR type (path/int/float/list/...) and its
string-rendering fidelity when re-seeded into a symbol table — the
Expand All @@ -103,6 +118,10 @@ def evaluate_value(self, *, symtab: SymbolTable, path_format: Optional[Any] = No
if evaluate_value is None:
return self.evaluate(symtab=symtab, path_format=path_format)
try:
if target_type is not None:
return evaluate_value(
symtab=symtab, path_format=path_format, target_type=target_type
)
return evaluate_value(symtab=symtab, path_format=path_format)
except ValueError as exc:
raise ExpressionError(f"Expression failed validation: {str(exc)}")
Expand Down
72 changes: 72 additions & 0 deletions src/openjd/model/_format_strings/_format_string.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

import functools
from dataclasses import dataclass
from typing import Any, Optional, Union

Expand All @@ -10,6 +11,16 @@
from .._types import ModelParsingContextInterface, SpecificationRevision


@functools.lru_cache(maxsize=None)
def _expr_type(type_string: str) -> Any:
"""The ``openjd.expr.ExprType`` for ``type_string``. Imported here, on
first use, because importing openjd.model must not load the native
extension; only an EXPR template reaches the callers."""
from openjd.expr import ExprType

return ExprType(type_string)


class _ReconstructionContext(ModelParsingContextInterface):
"""Minimal parsing context used when a FormatString is reconstructed by
pickle or the copy module (see ``FormatString.__getnewargs_ex__``).
Expand Down Expand Up @@ -224,6 +235,67 @@ def resolve_value(self, *, symtab: SymbolTable, path_format: Optional[object] =
)
return self.resolve(symtab=symtab, path_format=path_format)

def resolve_list_item(
self,
*,
symtab: SymbolTable,
target_type: str,
room: Optional[int] = None,
path_format: Optional[Any] = None,
) -> Optional[list[Any]]:
"""Resolve this format string as a list item (Expression Language §1.3.2).

When the format string is exactly one EXPR ``{{ ... }}`` expression,
with no text around it, the expression is evaluated with
``target_type`` (an EXPR type string of the form ``T? | list[T]``,
such as ``"int | list[int] | nulltype"``): a ``null`` result contributes no values, a list
contributes one value per member, and any other result contributes
itself. The values are the engine's ``ExprValue``s. A result that does
not coerce to the target raises. A list longer than ``room`` (when
given) returns ``None`` before its members are copied.

Any other format string (text around its expressions, several
expressions, or a template without EXPR, whose expressions are bare
value references that cannot produce ``null`` or a list) contributes
its resolved string, as :meth:`resolve` returns it.

Returns:
The values the element contributes, or ``None`` when it is a list
longer than ``room``.

Raises:
FormatStringError: if the expression cannot be evaluated or its
result does not coerce to ``target_type``.
"""
expressions = self.expressions
if len(expressions) == 1:
info = expressions[0]
expression = info.expression
whole_field = info.start_pos == 0 and info.end_pos == len(self.original_value)
if whole_field and expression is not None and getattr(expression, "is_expr", False):
try:
value = expression.evaluate_value(
symtab=symtab,
path_format=path_format,
target_type=_expr_type(target_type),
)
except ExpressionError as exc:
raise FormatStringError(
string=self.original_value,
start=info.start_pos,
end=info.end_pos,
expr=expression.expr,
details=str(exc),
)
if getattr(value, "is_null", False):
return []
if str(value.type).startswith("list["):
if room is not None and len(value) > room:
return None
return list(value)
return [value]
return [self.resolve(symtab=symtab, path_format=path_format)]

def _preprocess(
self, *, context: ModelParsingContextInterface
) -> list[Union[str, ExpressionInfo]]:
Expand Down
19 changes: 14 additions & 5 deletions src/openjd/model/_format_strings/_nodes.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,12 +220,17 @@ def _check_comprehension_shadowing(self, symbols: set) -> None:
"a variable that already exists at this location."
)

def _evaluate_raw(self, *, symtab: SymbolTable, path_format: Any = None) -> Any:
def _evaluate_raw(
self, *, symtab: SymbolTable, path_format: Any = None, target_type: Any = None
) -> Any:
"""Evaluate the expression and return the engine's ``ExprValue``.

Shared by :meth:`evaluate` (which unwraps to the native Python value)
and :meth:`evaluate_to_str` (which uses the engine's spec-defined string
coercion). Re-raises engine errors as the model's ``ExpressionError``.
``target_type`` (an ``openjd.expr.ExprType``), when given, is the
result's target type (Expression Language §1.2.3): the result is
coerced toward it, and evaluation fails when it cannot be.
"""
from ._expr_support import (
map_eval_error,
Expand All @@ -246,16 +251,20 @@ def _evaluate_raw(self, *, symtab: SymbolTable, path_format: Any = None) -> Any:
# is not absolute, and that reaches the caller as an expression
# error like any other.
profile = profile_for_symtab(symtab)
return self._parsed.evaluate(values=values, profile=profile, path_format=path_format)
return self._parsed.evaluate(
values=values, profile=profile, path_format=path_format, target_type=target_type
)
except Exception as exc: # noqa: BLE001 - boundary; re-raise as model error
raise map_eval_error(exc)

def evaluate(self, *, symtab: SymbolTable, path_format: Any = None) -> Any:
return self._evaluate_raw(symtab=symtab, path_format=path_format).item()

def evaluate_value(self, *, symtab: SymbolTable, path_format: Any = None) -> Any:
def evaluate_value(
self, *, symtab: SymbolTable, path_format: Any = None, target_type: Any = None
) -> Any:
"""Evaluate and return the engine's ``ExprValue`` (not unwrapped to a
native Python value).
native Python value), coerced toward ``target_type`` when given.

Callers that re-seed the result into a symbol table (e.g. the session
runtime's ``let`` bindings, RFC 0007) should use this form: the typed
Expand All @@ -265,7 +274,7 @@ def evaluate_value(self, *, symtab: SymbolTable, path_format: Any = None) -> Any
are preserved), matching the Rust runtime's natively typed symbol
table.
"""
return self._evaluate_raw(symtab=symtab, path_format=path_format)
return self._evaluate_raw(symtab=symtab, path_format=path_format, target_type=target_type)

def evaluate_to_str(self, *, symtab: SymbolTable, path_format: Any = None) -> str:
# ``str(ExprValue)`` applies the engine's RFC 0005 format-string
Expand Down
65 changes: 64 additions & 1 deletion src/openjd/model/_internal/_create_job.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

from .._symbol_table import SymbolTable
from .._format_strings import FormatString
from .._types import ModelParsingContextInterface, OpenJDModel
from .._types import ListItemSpec, ModelParsingContextInterface, OpenJDModel

__all__ = ("instantiate_model", "resolve_whole_field_typed_list")

Expand Down Expand Up @@ -262,6 +262,15 @@ def _instantiate_model( # noqa: C901
instantiated = coerce(model, field_name, raw_typed)
else:
instantiated = unwrap_typed_resolution(raw_typed)
elif (
isinstance(field_value, list)
and field_name in model._job_creation_metadata.list_item_fields
):
instantiated = _instantiate_list_items(
field_value,
symtab,
model._job_creation_metadata.list_item_fields[field_name],
)
elif isinstance(field_value, list):
if field_name in model._job_creation_metadata.reshape_field_to_dict:
key_field = model._job_creation_metadata.reshape_field_to_dict[field_name]
Expand Down Expand Up @@ -366,6 +375,60 @@ def _instantiate_list_field_as_list( # noqa: C901
return result


def _instantiate_list_items(
value: list[Any],
symtab: SymbolTable,
spec: ListItemSpec,
) -> list[Any]:
"""Instantiate a list whose format string elements are list items
(Expression Language §1.3.2; see ``JobCreationMetadata.list_item_fields``).

Each format string element contributes the values
:meth:`FormatString.resolve_list_item` resolves it to, and every value,
and every element that is not a format string, goes through
``spec.convert``. Errors are reported at the element's index in the list
as written. Resolution stops at the first element that takes the list
past ``spec.max_len``, before a flattened element's members are copied.

Raises:
ValidationError: if an element fails to resolve or convert.
ValueError: if the list resolves to more than ``spec.max_len``
elements, or a list that was not empty resolves to none.
"""
errors: list[InitErrorDetails] = []
result: list[Any] = []
for idx, item in enumerate(value):
resolved: Optional[list[Any]] = []
with capture_validation_errors(output_errors=errors, loc=(idx,), input=value):
if isinstance(item, FormatString):
resolved = item.resolve_list_item(
symtab=symtab,
target_type=spec.target_type,
room=spec.max_len - len(result),
)
else:
resolved = [item]
if resolved is not None:
result.extend(spec.convert(v) for v in resolved)
# The length is a property of the whole list, so it is reported at the
# field rather than at the element that went over. An element error
# found before it is reported instead, as openjd-rs does.
if resolved is None or len(result) > spec.max_len:
if errors:
break
raise ValueError(spec.too_long_message)

if errors:
raise ValidationError.from_exception_data(
title="_instantiate_list_items", line_errors=errors
)
# The non-empty rule is checked on the list as written, at decode; dropped
# elements can still leave it empty.
if value and not result:
raise ValueError("has no elements after resolution.")
return result


def _instantiate_list_field_as_dict( # noqa: C901
value: list[Any],
symtab: SymbolTable,
Expand Down
37 changes: 37 additions & 0 deletions src/openjd/model/_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,30 @@ class JobCreateAsMetadata:
callable: Optional[Callable[["OpenJDModel", dict], Type["OpenJDModel"]]] = field(default=None)


@dataclass(frozen=True, eq=False, **dataclass_kwargs)
class ListItemSpec:
"""How the elements of a list field resolve as list items (see
``JobCreationMetadata.list_item_fields``).

Attributes:
target_type: The EXPR type string of a whole-field EXPR element's
target, of the form ``T? | list[T]``, such as
``"int | list[int] | nulltype"``.
convert: Turns one resolved value, an ``openjd.expr.ExprValue`` from a
typed element or the ``str`` of any other element, into the
instantiated element. Raises ``ValueError`` to reject it.
max_len: The most elements the list may resolve to. Resolution stops
at the element that would take it past this, before that
element's members are copied, and fails with ``too_long_message``.
too_long_message: The error for a list past ``max_len``.
"""

target_type: str
convert: Callable[[Any], Any]
max_len: int
too_long_message: str


@dataclass(frozen=True, eq=False, **dataclass_kwargs)
class JobCreationMetadata:
"""Metadata for creating a job out of a job template.
Expand Down Expand Up @@ -302,6 +326,19 @@ class JobCreationMetadata:
native Python list unchanged.
"""

list_item_fields: dict[str, "ListItemSpec"] = field(default_factory=dict)
"""List fields (a subset of ``resolve_fields``) whose format string
elements are list items (Expression Language §1.3.2), keyed by field name.
Each element resolves through :meth:`FormatString.resolve_list_item` with
the spec's target type, so with EXPR a whole-field ``null`` drops the
element and a list flattens into one element per member. The spec's
``convert`` turns each resolved value into the instantiated element, and
may raise ``ValueError`` to reject it; the error is reported at the
element's index in the list as written. A list that was not empty as
written but resolves to no elements is rejected. Used by task-parameter
``range`` lists.
"""

create_as: Optional[JobCreateAsMetadata] = field(default=None)
"""The class to create the model as during instantiation. Options:
1. None -- create as the model class itself.
Expand Down
Loading
Loading