Fix for simulation-based shadow pricing segmentation

activitysim/abm/tables/shadow_pricing.py

@@ -6,11 +6,13 @@
 import logging
 import multiprocessing
 import time
+import warnings
 from collections import OrderedDict
 from typing import Any, Literal
 
 import numpy as np
 import pandas as pd
+from pydantic import model_validator
 
 from activitysim.abm.tables.size_terms import size_terms as get_size_terms
 from activitysim.abm.tables.size_terms import tour_destination_size_terms
@@ -62,12 +64,6 @@
 TALLY_CHECKOUT = (1, -1)
 TALLY_PENDING_PERSONS = (2, -1)
 
-default_segment_to_name_dict = {
-    # model_selector : persons_segment_name
-    "school": "school_segment",
-    "workplace": "income_segment",
-}
-
 
 def size_table_name(model_selector):
     """
@@ -134,11 +130,20 @@ class ShadowPriceSettings(PydanticReadable, extra="forbid"):
 
     WRITE_ITERATION_CHOICES: bool = False
 
-    SEGMENT_TO_NAME: dict[str, str] = {
-        "school": "school_segment",
-        "workplace": "income_segment",
-    }  # pydantic uses deep copy, so mutable default value is ok here
-    """Mapping from model_selector to persons_segment_name."""
+    SEGMENT_TO_NAME: dict[str, str] | None = None
+    """Deprecated compatibility setting; chooser segmentation comes from model settings."""
+
+    @model_validator(mode="before")
+    @classmethod
+    def warn_deprecated_segment_to_name(cls, data: Any) -> Any:
+        if isinstance(data, dict) and "SEGMENT_TO_NAME" in data:
+            warnings.warn(
+                "SEGMENT_TO_NAME is deprecated and ignored; use "
+                "CHOOSER_SEGMENT_COLUMN_NAME in location choice settings instead",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+        return data
 
 
 class ShadowPriceCalculator:
@@ -176,6 +181,7 @@ def __init__(
         )
 
         self.model_selector = model_settings.MODEL_SELECTOR
+        self.chooser_segment_column = model_settings.CHOOSER_SEGMENT_COLUMN_NAME
 
         if (self.num_processes > 1) and not state.settings.fail_fast:
             # if we are multiprocessing, then fail_fast should be true or we will wait forever for failed processes
@@ -869,13 +875,34 @@ def update_shadow_prices(self, state):
             sampled_persons = pd.DataFrame()
             persons_merged = state.get_dataframe("persons_merged")
 
-            # need to join the segment to the choices to sample correct persons
-            segment_to_name_dict = self.shadow_settings.SEGMENT_TO_NAME
-            segment_name = segment_to_name_dict[self.model_selector]
+            # Use the model chooser segmentation to keep shadow-pricing resampling
+            # consistent with segment_ids in location choice settings.
+            segment_name = self.chooser_segment_column
+            if segment_name not in persons_merged.columns:
+                raise SystemConfigurationError(
+                    f"Missing chooser segment column '{segment_name}' in persons_merged "
+                    f"for {self.model_selector} simulation shadow pricing"
+                )
 
             if type(self.choices_synced) != pd.DataFrame:
                 self.choices_synced = self.choices_synced.to_frame()
 
+            # Fail fast on obvious misconfiguration instead of silently sampling no one.
+            segment_values = set(self.segment_ids.values())
+            chooser_values = set(
+                persons_merged[segment_name]
+                .reindex(self.choices_synced.index)
+                .dropna()
+                .unique()
+            )
+            if not segment_values.intersection(chooser_values):
+                raise SystemConfigurationError(
+                    f"No overlap between SEGMENT_IDS values ({sorted(segment_values)}) and "
+                    f"modeled chooser values in persons_merged['{segment_name}'] "
+                    f"for {self.model_selector} "
+                    "simulation shadow pricing"
+                )
+
             choices_synced = self.choices_synced.merge(
                 persons_merged[segment_name],
                 how="left",

activitysim/abm/test/test_misc/test_shadow_pricing_simulate.py

@@ -12,6 +12,7 @@
 from activitysim.abm.tables import shadow_pricing
 from activitysim.core import los, workflow
 from activitysim.core.configuration.logit import TourLocationComponentSettings
+from activitysim.core.exceptions import SystemConfigurationError
 
 LAND_USE_FIELDS = [
     "e01_nrm",
@@ -333,6 +334,9 @@ def persons() -> pd.DataFrame:
         }
     )
 
+    persons["school_segment_string"] = persons["school_segment"].astype(str)
+    persons["school_segment_string_99"] = "99"
+
     return persons
 
 
@@ -572,6 +576,163 @@ def test_shadow_pricing_simulate(state, model_settings, network_los):
         )
 
 
+def test_shadow_pricing_simulate_custom_segment(state, model_settings, network_los):
+    """Run simulation shadow pricing with string-valued chooser segments."""
+    segment_ids = {
+        "university": "3",
+        "highschool": "2",
+        "gradeschool": "1",
+    }
+    custom_model_settings = model_settings.model_copy(
+        update={
+            "CHOOSER_SEGMENT_COLUMN_NAME": "school_segment_string",
+            "SEGMENT_IDS": segment_ids,
+        }
+    )
+    custom_model_settings.LOGSUM_SETTINGS = None
+
+    spc = shadow_pricing.load_shadow_price_calculator(state, custom_model_settings)
+
+    max_iterations = 5
+    chooser_segment_column = "school_segment_string"
+    save_sample_df = choices_df = None
+    persons_merged = state.get_dataframe("persons_merged")
+
+    for iteration in range(1, max_iterations + 1):
+        old_shadow_prices = spc.shadow_prices["highschool"].values
+        persons_merged_df_ = persons_merged.copy()
+
+        if spc.use_shadow_pricing and iteration > 1:
+            spc.update_shadow_prices(state)
+
+            if spc.shadow_settings.SHADOW_PRICE_METHOD == "simulation":
+                persons_merged_df_ = persons_merged_df_[
+                    persons_merged_df_.index.isin(spc.sampled_persons.index)
+                ].sort_index()
+
+        choices_df_, save_sample_df = run_location_choice(
+            state,
+            persons_merged_df_,
+            network_los,
+            shadow_price_calculator=spc,
+            want_logsums=False,
+            want_sample_table=False,
+            estimator=None,
+            model_settings=custom_model_settings,
+            chunk_size=0,
+            chunk_tag="school_location_string_segment",
+            trace_label=f"school_location_string_segment_{iteration}",
+        )
+
+        if spc.use_shadow_pricing:
+            if (
+                spc.shadow_settings.SHADOW_PRICE_METHOD == "simulation"
+                and iteration > 1
+            ):
+                if len(choices_df_) != 0:
+                    choices_df = pd.concat([choices_df, choices_df_], axis=0)
+                    choices_df_index = choices_df_.index.name
+                    choices_df = choices_df.reset_index()
+                    choices_df = choices_df.drop_duplicates(
+                        subset=[choices_df_index], keep="last"
+                    )
+                    choices_df = choices_df.set_index(choices_df_index).sort_index()
+            else:
+                choices_df = choices_df_.copy()
+
+        new_shadow_prices = spc.shadow_prices["highschool"].values
+        assert not any((old_shadow_prices == -999) & (new_shadow_prices != -999))
+        check_shadow_prices(spc, iteration)
+
+        spc.set_choices(
+            choices=choices_df["choice"],
+            segment_ids=persons_merged[chooser_segment_column].reindex(
+                choices_df.index
+            ),
+        )
+
+
+def test_shadow_pricing_simulate_segment_values_do_not_overlap(
+    state, model_settings, network_los
+):
+    custom_segment_column = "school_segment_string_99"
+    custom_model_settings = model_settings.model_copy(
+        update={
+            "CHOOSER_SEGMENT_COLUMN_NAME": custom_segment_column,
+            "SEGMENT_IDS": {
+                "university": "3",
+                "highschool": "2",
+                "gradeschool": "1",
+            },
+        }
+    )
+    persons_merged = state.get_dataframe("persons_merged")
+    state.settings.use_shadow_pricing = True
+
+    try:
+        spc = shadow_pricing.load_shadow_price_calculator(state, custom_model_settings)
+        choices = pd.Series(22660, index=persons_merged.index, name="choice", dtype=int)
+        spc.set_choices(choices, persons_merged[custom_segment_column])
+
+        with pytest.raises(SystemConfigurationError) as error:
+            spc.update_shadow_prices(state)
+
+        assert str(error.value) == (
+            "No overlap between SEGMENT_IDS values (['1', '2', '3']) and "
+            "modeled chooser values in persons_merged['school_segment_string_99'] for school "
+            "simulation shadow pricing"
+        )
+    finally:
+        persons_merged.pop(custom_segment_column)
+
+
+def test_shadow_pricing_simulate_segment_overlap_ignores_unmodeled_choosers(
+    state, model_settings, network_los
+):
+    custom_segment_column = "school_segment_string_99"
+    custom_model_settings = model_settings.model_copy(
+        update={
+            "CHOOSER_SEGMENT_COLUMN_NAME": custom_segment_column,
+            "SEGMENT_IDS": {
+                "university": "3",
+                "highschool": "2",
+                "gradeschool": "1",
+            },
+        }
+    )
+    persons_merged = state.get_dataframe("persons_merged")
+    persons_merged[custom_segment_column] = "99"
+    unmodeled_chooser = persons_merged.index[0]
+    persons_merged.loc[unmodeled_chooser, custom_segment_column] = "1"
+    modeled_choosers = persons_merged.drop(index=unmodeled_chooser)
+    state.settings.use_shadow_pricing = True
+
+    try:
+        spc = shadow_pricing.load_shadow_price_calculator(state, custom_model_settings)
+        choices = pd.Series(
+            22660, index=modeled_choosers.index, name="choice", dtype=int
+        )
+        spc.set_choices(choices, modeled_choosers[custom_segment_column])
+
+        with pytest.raises(
+            SystemConfigurationError, match="No overlap between SEGMENT_IDS"
+        ):
+            spc.update_shadow_prices(state)
+    finally:
+        persons_merged.pop(custom_segment_column)
+
+
+def test_shadow_price_settings_accept_deprecated_segment_to_name():
+    legacy_setting = {"school": "school_segment"}
+
+    with pytest.warns(DeprecationWarning, match="SEGMENT_TO_NAME is deprecated"):
+        settings = shadow_pricing.ShadowPriceSettings.model_validate(
+            {"SEGMENT_TO_NAME": legacy_setting}
+        )
+
+    assert settings.SEGMENT_TO_NAME == legacy_setting
+
+
 def test_shadow_pricing_dedicated_rng_channel_eet_only(
     state, model_settings, network_los
 ):

docs/dev-guide/changes.md

@@ -14,6 +14,18 @@ branch (i.e., the main branch on GitHub), but not yet released in a stable versi
 of ActivitySim.
 No unreleased changes are currently documented.
 
+### Shadow Pricing Re-Simulation Segmentation
+
+In version 1.6.1, the segmentation used for shadow pricing re-simulation now 
+inherits the segmentation used in the work and school location choice simulation, 
+e.g., defined as `CHOOSER_SEGMENT_COLUMN_NAME` in the `workplace_location.yaml`. 
+The legacy setting `SEGMENT_TO_NAME` in the `shadow_pricing.yaml` is now deprecated 
+and ignored. This ensures consistency between the segmentation used for shadow 
+pricing re-simulation and the segmentation used for work and school location 
+choice simulation. See
+[Issue #1082](https://github.com/ActivitySim/activitysim/issues/1082) and
+[PR #1083](https://github.com/ActivitySim/activitysim/pull/1083).
+
 
 ## v1.6
 

docs/dev-guide/components/shadow_pricing.md

@@ -50,10 +50,32 @@ zone on or off for selection in the subsequent iterations. For this reason, warm
 for this approach is not applicable.)  This process repeats until the overall convergence criteria
 is met or the maximum number of allowed iterations is reached.
 
+Note that the simulation approach compares the modeled employment/enrollment share against the target 
+employment/enrollment share for each zone, rather than the absolute number. Therefore, it does not require
+the absolute total workers/students to match the regional employment/enrollment totals.
+
 Because the simulation approach only re-simulates workers / students who were over-assigned in the
 previous iteration, run time is significantly less (~90%) than the CTRAMP or DaySim approaches which
 re-simulate all workers and students at each iteration.
 
+When choosing workers or students for re-simulation, the simulation approach selects individuals from 
+overassigned zones by shadow pricing segment. In version 1.6.0 and prior, the segmentation was defaulted
+to use income groups for workers and school segments for students, and changable via the `SEGMENT_TO_NAME`
+setting in the `shadow_pricing.yaml`. However, this setting was easily overlooked or misconfigured, 
+which could cause the shadow pricing to be accidentally skipped or cause a mismatch between the segmentation 
+used for shadow pricing re-simulation and the segmentation used for work and school location choice simulation, 
+if the latter uses a different segmentation scheme from the default income and school segments. 
+In version 1.6.1, the segmentation used for shadow pricing re-simulation now inherits the segmentation 
+used in the work and school location choice simulation, e.g., defined as `CHOOSER_SEGMENT_COLUMN_NAME` in 
+the `workplace_location.yaml`. The legacy setting `SEGMENT_TO_NAME` in the `shadow_pricing.yaml` is now 
+deprecated and ignored.
+
+The segmentation in shadow pricing re-simulation, e.g., occupation or income groups, accommodates but 
+does not require occupation- or income-specific land use employment controls. For example, 
+the example SANDAG model segments workplace location choice by occupation, but all occupation segments 
+share the same `total_emp_revised` target. Under the SANDAG configuration, ActivitySim combines the modeled 
+workers across segments sharing that target when evaluating whether a zone is over- or under-assigned.
+
 ## Configuration
 
 ```{eval-rst}