Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion pyiceberg/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -1678,6 +1678,8 @@ def _(file_type: FloatType, read_type: IcebergType) -> IcebergType:
@promote.register(StringType)
def _(file_type: StringType, read_type: IcebergType) -> IcebergType:
if isinstance(read_type, BinaryType):
# Not a valid promotion in the spec, but allowed on purpose: strings and binary are both stored as bytes, so one
# can be read as the other. See https://github.com/apache/iceberg/pull/5151. Schema updates use this rule too.
return read_type
else:
raise ResolveError(f"Cannot promote an string to {read_type}")
Expand All @@ -1686,6 +1688,7 @@ def _(file_type: StringType, read_type: IcebergType) -> IcebergType:
@promote.register(BinaryType)
def _(file_type: BinaryType, read_type: IcebergType) -> IcebergType:
if isinstance(read_type, StringType):
# Not a valid promotion in the spec, but allowed on purpose, see string -> binary above
return read_type
else:
raise ResolveError(f"Cannot promote an binary to {read_type}")
Expand All @@ -1705,7 +1708,9 @@ def _(file_type: DecimalType, read_type: IcebergType) -> IcebergType:
@promote.register(FixedType)
def _(file_type: FixedType, read_type: IcebergType) -> IcebergType:
if isinstance(read_type, UUIDType) and len(file_type) == 16:
# Since pyarrow reads parquet UUID as fixed 16-byte binary, the promotion is needed to ensure read compatibility
# Not a valid promotion in the spec, but allowed on purpose: older PyArrow versions had no UUID type and read
# Parquet UUIDs as fixed 16-byte binary. This keeps such columns readable as UUID, and lets update_column()
# turn them into UUID.
return read_type
else:
raise ResolveError(f"Cannot promote {file_type} to {read_type}")
Expand Down
2 changes: 2 additions & 0 deletions pyiceberg/table/update/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,8 @@ def update_column(
raise ValidationError(f"Cannot change column type: {field.field_type} is not a primitive")

if not self._allow_incompatible_changes and field.field_type != field_type:
# promote() is the one place for the allowed type changes. It also allows string <-> binary and
# fixed(16) -> uuid, which are not in the spec, see the comments there.
try:
promote(field.field_type, field_type)
except ResolveError as e:
Expand Down
12 changes: 12 additions & 0 deletions tests/test_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -1328,6 +1328,18 @@ def test_detect_invalid_promotion_string_to_float(table_v2: Table) -> None:
_ = UpdateSchema(transaction=Transaction(table_v2), schema=current_schema).union_by_name(new_schema)._apply()


def test_update_column_allows_non_spec_promotions(table_v2: Table) -> None:
# Not valid promotions in the spec, but allowed on purpose, see promote()
for current_type, new_type in [(StringType(), BinaryType()), (BinaryType(), StringType()), (FixedType(16), UUIDType())]:
current_schema = Schema(NestedField(field_id=1, name="aCol", field_type=current_type, required=False))
update = UpdateSchema(transaction=Transaction(table_v2), schema=current_schema).update_column("aCol", new_type)
assert update._apply().find_field("aCol").field_type == new_type

current_schema = Schema(NestedField(field_id=1, name="aCol", field_type=FixedType(15), required=False))
with pytest.raises(ValidationError, match=r"^Cannot change column type: aCol: fixed\[15\] -> uuid$"):
UpdateSchema(transaction=Transaction(table_v2), schema=current_schema).update_column("aCol", UUIDType())


# decimal(P,S) Fixed-point decimal; precision P, scale S -> Scale is fixed [1],
# precision must be 38 or less
def test_type_promote_decimal_to_fixed_scale_with_wider_precision(table_v2: Table) -> None:
Expand Down
Loading