Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion sagemaker-mlops/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ dependencies = [
"botocore>=1.43.127,<2.0",
"pyiceberg[glue]>=0.8.0",
"pyarrow>=14.0.0",
"s3fs",
]

[project.optional-dependencies]
Expand Down
46 changes: 33 additions & 13 deletions sagemaker-mlops/src/sagemaker/mlops/feature_store/feature_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
# Licensed under the Apache License, Version 2.0
"""Utilities for working with FeatureGroups and FeatureStores."""

import io
import logging
import os
import time
Expand All @@ -14,6 +15,8 @@
import pandas
import pandas as pd
from pandas import DataFrame, Series, read_csv
from pandas.io.common import infer_compression
from sagemaker.core.common_utils import pop_out_unused_kwarg
from sagemaker.core.shapes import FeatureValue, TtlDuration
from sagemaker.core.utils.utils import Unassigned
from sagemaker.mlops.feature_store import FeatureGroup as CoreFeatureGroup, FeatureGroup
Expand Down Expand Up @@ -857,7 +860,8 @@ def prepare_fg_from_dataframe_or_file(

Args:
feature_group_name (str): feature group name
dataframe_or_path (str, Path, pandas.DataFrame) : pandas.DataFrame or path to the data
dataframe_or_path (str, Path, pandas.DataFrame) : pandas.DataFrame, or a local path
or ``s3://`` URI to the data
verbose (bool) : True for displaying messages, False for silent method.
record_id (str, 'record_id'): (Optional) Feature identifier of the rows. If specified each
value of that feature has to be unique. If not specified or
Expand All @@ -882,17 +886,42 @@ def prepare_fg_from_dataframe_or_file(
if verbose:
logger.setLevel(logging.INFO)

# Resolve the session before reading any data, so that an S3 read below uses the
# caller's credentials instead of falling back to the default credential chain.
if session is not None:
sagemaker_session = session
elif role is not None and region is not None:
sagemaker_session = get_session_from_role(region=region)
else:
exc = Exception("Argument Session or role and region must be specified.")
logger.exception(exc)
raise exc

if isinstance(dataframe_or_path, DataFrame):
data = dataframe_or_path
elif isinstance(dataframe_or_path, str):
elif isinstance(dataframe_or_path, (str, Path)):
kwargs.pop("filepath_or_buffer", None)
data = read_csv(filepath_or_buffer=dataframe_or_path, **kwargs)
path = str(dataframe_or_path)
if path.startswith("s3://"):
# storage_options configures the fsspec filesystem pandas would have opened
# the URI with. The object is fetched here instead, so pandas receives a
# buffer and would reject the argument outright; warn and drop it.
pop_out_unused_kwarg("storage_options", kwargs)
# pandas infers compression from the file extension only when it is handed a
# path; it cannot infer from an in-memory buffer. Infer from the S3 key so
# that gzipped/bz2/zip/xz/zstd objects keep working as they did via s3fs.
if "compression" not in kwargs:
kwargs["compression"] = infer_compression(path, "infer")
s3_bytes = S3Downloader.read_bytes(path, sagemaker_session=sagemaker_session)
data = read_csv(filepath_or_buffer=io.BytesIO(s3_bytes), **kwargs)
else:
data = read_csv(filepath_or_buffer=dataframe_or_path, **kwargs)
else:
exc = Exception(
str(
f"Invalid type {type(dataframe_or_path)} for "
"argument dataframe_or_path. \nParameter must be"
" of type pandas.DataFrame or string"
" of type pandas.DataFrame, str or pathlib.Path"
)
)
logger.exception(exc)
Expand Down Expand Up @@ -923,15 +952,6 @@ def prepare_fg_from_dataframe_or_file(
current_time_sec = int(time.time() + 0.5)
data[event_id] = Series([current_time_sec] * lg_id, dtype="float64")

if session is not None:
sagemaker_session = session
elif role is not None and region is not None:
sagemaker_session = get_session_from_role(region=region)
else:
exc = Exception("Argument Session or role and region must be specified.")
logger.exception(exc)
raise exc

feature_group = FeatureGroup(name=feature_group_name, sagemaker_session=sagemaker_session)

feature_group.load_feature_definitions(data_frame=data)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@
# Licensed under the Apache License, Version 2.0
"""Unit tests for feature_utils.py"""

import gzip
import logging
from pathlib import Path

import pytest
from unittest.mock import Mock, patch, MagicMock
import pandas as pd
Expand Down Expand Up @@ -656,6 +660,218 @@ def test_raises_when_no_session_or_region(self):
feature_group_name="test-fg",
)

@patch("sagemaker.mlops.feature_store.feature_utils.FeatureGroup")
@patch("sagemaker.mlops.feature_store.feature_utils.S3Downloader")
def test_s3_path_is_read_via_s3_downloader(self, mock_s3_downloader, mock_fg_class):
"""An s3:// path is fetched with S3Downloader rather than handed to read_csv.

This is the behaviour that replaced the implicit s3fs/fsspec backend, so the
CSV bytes must round-trip into the DataFrame that feature definitions are
loaded from.
"""
mock_fg = MagicMock()
mock_fg_class.return_value = mock_fg
mock_session = MagicMock()
mock_s3_downloader.read_bytes.return_value = b"id,value\n1,1.1\n2,2.2\n"

from sagemaker.mlops.feature_store.feature_utils import prepare_fg_from_dataframe_or_file

result = prepare_fg_from_dataframe_or_file(
dataframe_or_path="s3://my-bucket/prefix/data.csv",
feature_group_name="test-fg",
session=mock_session,
record_id="id",
)

assert result == mock_fg
mock_s3_downloader.read_bytes.assert_called_once()

# The caller's session must be threaded through, otherwise the read falls back
# to the default credential chain instead of the caller's credentials.
_, kwargs = mock_s3_downloader.read_bytes.call_args
assert kwargs["sagemaker_session"] is mock_session

# The downloaded bytes, not the URI, must be what got parsed.
loaded = mock_fg.load_feature_definitions.call_args.kwargs["data_frame"]
assert list(loaded["id"]) == [1, 2]
assert list(loaded["value"]) == [1.1, 2.2]

@patch("sagemaker.mlops.feature_store.feature_utils.FeatureGroup")
@patch("sagemaker.mlops.feature_store.feature_utils.S3Downloader")
def test_s3_gzipped_object_is_decompressed(self, mock_s3_downloader, mock_fg_class):
"""A gzipped S3 object still parses, as it did when s3fs served the path.

pandas infers compression from a file extension only when it receives a path.
Handing it a buffer loses that, so the extension must be inferred from the key
or ``s3://.../data.csv.gz`` silently breaks with a UnicodeDecodeError.
"""
mock_fg = MagicMock()
mock_fg_class.return_value = mock_fg
mock_s3_downloader.read_bytes.return_value = gzip.compress(b"id,value\n1,1.1\n2,2.2\n")

from sagemaker.mlops.feature_store.feature_utils import prepare_fg_from_dataframe_or_file

prepare_fg_from_dataframe_or_file(
dataframe_or_path="s3://my-bucket/prefix/data.csv.gz",
feature_group_name="test-fg",
session=MagicMock(),
record_id="id",
)

loaded = mock_fg.load_feature_definitions.call_args.kwargs["data_frame"]
assert list(loaded["id"]) == [1, 2]
assert list(loaded["value"]) == [1.1, 2.2]

@patch("sagemaker.mlops.feature_store.feature_utils.FeatureGroup")
@patch("sagemaker.mlops.feature_store.feature_utils.S3Downloader")
def test_storage_options_is_dropped_with_a_warning(
self, mock_s3_downloader, mock_fg_class, caplog
):
"""storage_options is an fsspec concept and no longer applies.

It configured the filesystem pandas would have opened the URI with. The object
is now fetched before parsing, so pandas receives a buffer and rejects the
argument with "storage_options passed with file object or non-fsspec file path".
Rather than surface that, drop it and warn, so the read still succeeds.
"""
mock_fg = MagicMock()
mock_fg_class.return_value = mock_fg
mock_s3_downloader.read_bytes.return_value = b"id,value\n1,1.1\n2,2.2\n"

from sagemaker.mlops.feature_store.feature_utils import prepare_fg_from_dataframe_or_file

with caplog.at_level(logging.WARNING):
prepare_fg_from_dataframe_or_file(
dataframe_or_path="s3://my-bucket/prefix/data.csv",
feature_group_name="test-fg",
session=MagicMock(),
record_id="id",
storage_options={"anon": True},
)

assert "storage_options" in caplog.text
assert "will be ignored" in caplog.text

# The read still succeeds rather than raising the pandas ValueError.
loaded = mock_fg.load_feature_definitions.call_args.kwargs["data_frame"]
assert list(loaded["id"]) == [1, 2]

@patch("sagemaker.mlops.feature_store.feature_utils.FeatureGroup")
@patch("sagemaker.mlops.feature_store.feature_utils.S3Downloader")
def test_storage_options_still_reaches_read_csv_for_local_paths(
self, mock_s3_downloader, mock_fg_class
):
"""Only the s3:// branch drops it; a local path is pandas' business, not ours."""
mock_fg_class.return_value = MagicMock()

from sagemaker.mlops.feature_store import feature_utils

with patch.object(feature_utils, "read_csv") as mock_read_csv:
mock_read_csv.return_value = pd.DataFrame({"id": [1, 2], "value": [1.1, 2.2]})
feature_utils.prepare_fg_from_dataframe_or_file(
dataframe_or_path="/path/to/file.csv",
feature_group_name="test-fg",
session=MagicMock(),
record_id="id",
storage_options={"anon": True},
)

assert mock_read_csv.call_args.kwargs["storage_options"] == {"anon": True}
mock_s3_downloader.read_bytes.assert_not_called()

@patch("sagemaker.mlops.feature_store.feature_utils.FeatureGroup")
@patch("sagemaker.mlops.feature_store.feature_utils.S3Downloader")
def test_explicit_compression_kwarg_is_not_overridden(self, mock_s3_downloader, mock_fg_class):
"""An explicit compression= from the caller wins over inference from the key."""
mock_fg = MagicMock()
mock_fg_class.return_value = mock_fg
mock_s3_downloader.read_bytes.return_value = gzip.compress(b"id,value\n1,1.1\n")

from sagemaker.mlops.feature_store.feature_utils import prepare_fg_from_dataframe_or_file

# Key has no .gz suffix, so inference would say "no compression"; the caller's
# explicit gzip must be honoured instead.
prepare_fg_from_dataframe_or_file(
dataframe_or_path="s3://my-bucket/prefix/data-without-suffix",
feature_group_name="test-fg",
session=MagicMock(),
record_id="id",
compression="gzip",
)

loaded = mock_fg.load_feature_definitions.call_args.kwargs["data_frame"]
assert list(loaded["id"]) == [1]

@patch("sagemaker.mlops.feature_store.feature_utils.FeatureGroup")
@patch("sagemaker.mlops.feature_store.feature_utils.S3Downloader")
def test_s3_path_threads_session_resolved_from_role_and_region(
self, mock_s3_downloader, mock_fg_class
):
"""A role+region caller gets that session on the S3 read, not the default chain."""
mock_fg_class.return_value = MagicMock()
mock_s3_downloader.read_bytes.return_value = b"id,value\n1,1.1\n"

from sagemaker.mlops.feature_store import feature_utils

role_session = MagicMock()
with patch.object(
feature_utils, "get_session_from_role", return_value=role_session
) as mock_get_session:
feature_utils.prepare_fg_from_dataframe_or_file(
dataframe_or_path="s3://my-bucket/prefix/data.csv",
feature_group_name="test-fg",
role="arn:aws:iam::123:role/MyRole",
region="us-east-1",
record_id="id",
)

mock_get_session.assert_called_once_with(region="us-east-1")
_, kwargs = mock_s3_downloader.read_bytes.call_args
assert kwargs["sagemaker_session"] is role_session

@patch("sagemaker.mlops.feature_store.feature_utils.FeatureGroup")
@patch("sagemaker.mlops.feature_store.feature_utils.S3Downloader")
@patch("sagemaker.mlops.feature_store.feature_utils.read_csv")
def test_local_path_does_not_touch_s3_downloader(
self, mock_read_csv, mock_s3_downloader, mock_fg_class
):
"""A local path still goes straight to read_csv."""
mock_fg_class.return_value = MagicMock()
mock_read_csv.return_value = pd.DataFrame({"id": [1, 2], "value": [1.1, 2.2]})

from sagemaker.mlops.feature_store.feature_utils import prepare_fg_from_dataframe_or_file

prepare_fg_from_dataframe_or_file(
dataframe_or_path="/path/to/file.csv",
feature_group_name="test-fg",
session=MagicMock(),
record_id="id",
)

mock_read_csv.assert_called_once_with(filepath_or_buffer="/path/to/file.csv")
mock_s3_downloader.read_bytes.assert_not_called()

@patch("sagemaker.mlops.feature_store.feature_utils.FeatureGroup")
@patch("sagemaker.mlops.feature_store.feature_utils.read_csv")
def test_accepts_pathlib_path(self, mock_read_csv, mock_fg_class):
"""pathlib.Path is in the signature's Union, so it must not be rejected."""
mock_fg = MagicMock()
mock_fg_class.return_value = mock_fg
mock_read_csv.return_value = pd.DataFrame({"id": [1, 2], "value": [1.1, 2.2]})

from sagemaker.mlops.feature_store.feature_utils import prepare_fg_from_dataframe_or_file

local_path = Path("/path/to/file.csv")
result = prepare_fg_from_dataframe_or_file(
dataframe_or_path=local_path,
feature_group_name="test-fg",
session=MagicMock(),
record_id="id",
)

assert result == mock_fg
mock_read_csv.assert_called_once_with(filepath_or_buffer=local_path)

@patch("sagemaker.mlops.feature_store.feature_utils.FeatureGroup")
def test_creates_record_id_from_index(self, mock_fg_class):
mock_session = MagicMock()
Expand Down
Loading