From 37de80bc2e45a792b42bd49c8708ec9d99b80375 Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Tue, 4 Aug 2026 13:31:28 -0400
Subject: [PATCH 01/27] Add in dummy validation summary data and fix
scatterplot axes
---
dashboard/calculation_notes.yaml | 7 +-
.../pages/validation/_traffic/composition.py | 20 ++
.../pages/validation/_traffic/features.py | 60 ++--
.../validation/_traffic/selector_domains.py | 12 +
.../pages/validation/_traffic/transforms.py | 104 +++++--
dashboard/pages/validation/regional.py | 16 +-
dashboard/pages/validation/transit.py | 29 +-
dashboard/rendering/figures.py | 32 +-
processor/summarize/summaries/validation.py | 83 +++--
scripts/generate_validation_demo_fixtures.py | 289 ++++++++++++++++++
tests/test_dashboard_live.py | 10 +-
tests/test_figure_builders.py | 10 +-
tests/test_summary_cache.py | 65 +++-
tests/test_summary_regressions.py | 33 ++
tests/test_validation_derived.py | 83 +++++
wiki/24-summary-catalog.md | 2 +-
16 files changed, 722 insertions(+), 133 deletions(-)
create mode 100644 scripts/generate_validation_demo_fixtures.py
diff --git a/dashboard/calculation_notes.yaml b/dashboard/calculation_notes.yaml
index da1c174..0d16aa2 100644
--- a/dashboard/calculation_notes.yaml
+++ b/dashboard/calculation_notes.yaml
@@ -698,16 +698,17 @@ notes:
traffic.screenlines:
method: aligned_comparison
- method_text: Records are first summed by screenline, direction, and count period; observed and modeled totals with the same three-part key are then paired for the scatter plot.
+ method_text: Records are first summed by screenline, direction, count period, and facility type; observed and modeled totals with matching keys are then paired for the scatter plot and fitted with an ordinary least-squares trendline.
sources: [screenline_flow_comparisons]
summary: The scatter plot compares observed and modeled screenline flows.
source_filters:
- Screenline records without an identifier, direction, count period, or observed volume are excluded before matching modeled flows.
details:
Aggregation:
- - Flow records are grouped by screenline, direction, and count period before observed and modeled volumes are paired.
+ - Flow records are grouped by screenline, direction, count period, and facility type before observed and modeled volumes are paired.
Display:
- - Each point represents one comparable screenline-direction-period record.
+ - The time-period and facility-type selectors filter the paired records before fitting.
+ - Each run displays its fitted equation, R-squared value, and number of comparable records alongside a one-to-one reference line.
vmt.overview:
method: vmt
diff --git a/dashboard/pages/validation/_traffic/composition.py b/dashboard/pages/validation/_traffic/composition.py
index 230e17f..338f8c1 100644
--- a/dashboard/pages/validation/_traffic/composition.py
+++ b/dashboard/pages/validation/_traffic/composition.py
@@ -12,6 +12,7 @@
class TrafficPageCompositionMixin:
def build_page(self) -> pn.viewable.Viewable:
self.demo_facility_raw_by_label = {"All": "All"}
+ self.screenline_facility_raw_by_label = {"All": "All"}
self.demo_period_sel = self.selector(
"demo_period",
widget=pn.widgets.Select(
@@ -44,6 +45,20 @@ def build_page(self) -> pn.viewable.Viewable:
),
label="Top N by Modeled Volume",
)
+ self.screenline_period_sel = self.selector(
+ "screenline_period",
+ widget=pn.widgets.Select(
+ name="Time Period",
+ options=list(DEMO_TRAFFIC_TIME_PERIODS),
+ value="Day",
+ ),
+ label="Time Period",
+ )
+ self.screenline_facility_sel = self.select(
+ "screenline_facility_type",
+ "Facility Type",
+ options=self._screenline_facility_options,
+ )
observed_fit = self.feature("observed_model_fit")
facility = self.feature("facility_summaries")
links = self.feature("link_tables")
@@ -76,6 +91,7 @@ def build_page(self) -> pn.viewable.Viewable:
)
self._screenline_body = screenlines.section(
"body",
+ selectors=("screenline_period", "screenline_facility_type"),
render=self.render_screenline_flow_section,
)
return self.new_section(
@@ -104,6 +120,10 @@ def build_page(self) -> pn.viewable.Viewable:
self._external_top_body,
),
pn.pane.Markdown("### Screenline Flow Summaries"),
+ selector_row(
+ self.screenline_period_sel,
+ self.screenline_facility_sel,
+ ),
self.noted_section("traffic.screenlines", self._screenline_body),
sizing_mode="stretch_width",
)
diff --git a/dashboard/pages/validation/_traffic/features.py b/dashboard/pages/validation/_traffic/features.py
index 916fc10..9ab85ff 100644
--- a/dashboard/pages/validation/_traffic/features.py
+++ b/dashboard/pages/validation/_traffic/features.py
@@ -13,39 +13,39 @@
class TrafficFeatureMixin:
- def render_validation_chart(
- self,
- data_list: list[tuple[str, pl.DataFrame]] | None,
- *,
- title: str,
- detail: str,
- missing_summary_id: str,
- ) -> pn.viewable.Viewable:
- if data_list is None:
- return self.data_not_available_card(
- detail=detail,
- missing_items=[missing_summary_id],
- )
- chart_data = self.query(lambda: validation_chart_data(data_list))
- return self.plot.scatter(
- chart_data,
- x="observed_volume",
- y="modeled_volume",
- title=title,
- x_title="Observed Traffic Volume",
- y_title="Modeled Traffic Volume",
- )
-
def render_screenline_flow_section(self):
if not self.state.run_labels:
return [self.no_runs_message()]
-
+ data = self.data.summary("screenline_flow_comparisons", self.weighting_key)
+ if data is None:
+ return [
+ self.data_not_available_card(
+ detail="Screenline flow comparisons are unavailable.",
+ missing_items=["screenline_flow_comparisons"],
+ )
+ ]
+ period = str(self.screenline_period_sel.value)
+ facility_type = self.selected_screenline_facility_type_raw()
+ scatter_data = self.query(
+ lambda: screenline_scatter_data(
+ data,
+ period=period,
+ facility_type=facility_type,
+ )
+ )
+ fit_data = self.query(lambda: screenline_fit_line_data(scatter_data))
return [
- self.render_validation_chart(
- self.data.summary("screenline_flow_comparisons", self.weighting_key),
- title="Screenline Flow Comparisons",
- detail="Screenline flow comparisons are unavailable.",
- missing_summary_id="screenline_flow_comparisons",
+ self.plot.scatter(
+ scatter_data,
+ x="observed_volume",
+ y="modeled_volume",
+ title=f"Screenline Observed vs Modeled - {period}",
+ x_title="Observed Screenline Flow",
+ y_title="Modeled Screenline Flow",
+ fit_overlays=fit_data,
+ one_to_one=True,
+ legend_on_right=True,
+ panel_aspect_ratio=1.0,
)
]
@@ -162,6 +162,7 @@ def render_demo_traffic_section(self) -> list[pn.viewable.Viewable]:
y_title="Modeled Volume",
fit_overlays=fit_data,
one_to_one=True,
+ legend_on_right=True,
panel_aspect_ratio=1.0,
)
)
@@ -183,6 +184,7 @@ def render_demo_traffic_section(self) -> list[pn.viewable.Viewable]:
x_title="Observed Count",
y_title="Modeled Volume",
one_to_one=True,
+ legend_on_right=True,
panel_aspect_ratio=1.0,
)
)
diff --git a/dashboard/pages/validation/_traffic/selector_domains.py b/dashboard/pages/validation/_traffic/selector_domains.py
index 112e4bd..b09adc0 100644
--- a/dashboard/pages/validation/_traffic/selector_domains.py
+++ b/dashboard/pages/validation/_traffic/selector_domains.py
@@ -37,3 +37,15 @@ def selected_facility_type_raw(self) -> str:
selected = str(self.demo_facility_sel.value)
raw_value = self.demo_facility_raw_by_label.get(selected, selected)
return "All" if raw_value is None else str(raw_value)
+
+ def _screenline_facility_options(self) -> list[str]:
+ options, self.screenline_facility_raw_by_label = demo_facility_options(
+ self.data.summary("screenline_flow_comparisons", self.weighting_key),
+ config=self.config,
+ )
+ return options
+
+ def selected_screenline_facility_type_raw(self) -> str:
+ selected = str(self.screenline_facility_sel.value)
+ raw_value = self.screenline_facility_raw_by_label.get(selected, selected)
+ return "All" if raw_value is None else str(raw_value)
diff --git a/dashboard/pages/validation/_traffic/transforms.py b/dashboard/pages/validation/_traffic/transforms.py
index 5c5e6c4..7d292a0 100644
--- a/dashboard/pages/validation/_traffic/transforms.py
+++ b/dashboard/pages/validation/_traffic/transforms.py
@@ -16,28 +16,45 @@
from .contracts import *
-def validation_chart_data(
+def screenline_scatter_data(
data_list: list[tuple[str, pl.DataFrame]],
+ *,
+ period: str,
+ facility_type: str,
) -> list[tuple[str, pl.DataFrame]]:
- """Aggregate one validation summary list to one observed/modeled point per id."""
- out = []
+ """Filter screenline observed/modeled points for one period and facility."""
+ out: list[tuple[str, pl.DataFrame]] = []
+ required = {
+ "screenline_id",
+ "count_period",
+ "observed_volume",
+ "modeled_volume",
+ }
for label, df in nonempty(data_list):
- filtered = df
- id_col = None
- if "count_location_id" in filtered.columns:
- id_col = "count_location_id"
- elif "screenline_id" in filtered.columns:
- id_col = "screenline_id"
- if id_col is not None:
- filtered = (
- filtered.group_by(id_col)
- .agg(
- observed_volume=pl.col("observed_volume").sum(),
- modeled_volume=pl.col("modeled_volume").sum(),
- )
- .sort(id_col)
+ if not required.issubset(df.columns):
+ continue
+ filtered = df.with_columns(
+ pl.col("count_period").cast(pl.Utf8),
+ (
+ pl.col("facility_type").cast(pl.Utf8)
+ if "facility_type" in df.columns
+ else pl.lit("All")
+ ).alias("facility_type"),
+ ).filter(pl.col("count_period") == period)
+ if facility_type != "All":
+ filtered = filtered.filter(pl.col("facility_type") == facility_type)
+ out.append(
+ (
+ label,
+ filtered.select(
+ "screenline_id",
+ "facility_type",
+ "count_period",
+ "observed_volume",
+ "modeled_volume",
+ ).sort("screenline_id"),
)
- out.append((label, filtered))
+ )
return out
@@ -212,7 +229,9 @@ def demo_count_fit_line_data(
return out
-def _r_squared_from_points(points: pl.DataFrame) -> float | None:
+def _linear_fit_from_points(
+ points: pl.DataFrame,
+) -> tuple[float, float, float] | None:
if points.height < 2:
return None
x = [float(value) for value in points["observed_volume"].to_list()]
@@ -229,8 +248,51 @@ def _r_squared_from_points(points: pl.DataFrame) -> float | None:
sse = sum((yi - yhat) ** 2 for yi, yhat in zip(y, fitted))
ss_yy = sum((yi - y_mean) ** 2 for yi in y)
if math.isclose(ss_yy, 0.0):
- return 1.0 if math.isclose(sse, 0.0) else 0.0
- return max(0.0, min(1.0, 1.0 - sse / ss_yy))
+ r_squared = 1.0 if math.isclose(sse, 0.0) else 0.0
+ else:
+ r_squared = max(0.0, min(1.0, 1.0 - sse / ss_yy))
+ return slope, intercept, r_squared
+
+
+def _r_squared_from_points(points: pl.DataFrame) -> float | None:
+ fit = _linear_fit_from_points(points)
+ return None if fit is None else fit[2]
+
+
+def screenline_fit_line_data(
+ scatter_data: list[tuple[str, pl.DataFrame]],
+) -> list[tuple[str, pl.DataFrame]]:
+ """Build regression lines and annotations from filtered screenline points."""
+ out: list[tuple[str, pl.DataFrame]] = []
+ for label, df in nonempty(scatter_data):
+ points = df.select("observed_volume", "modeled_volume").drop_nulls()
+ fit = _linear_fit_from_points(points)
+ if fit is None:
+ continue
+ slope, intercept, r_squared = fit
+ observed_min = float(points["observed_volume"].min())
+ observed_max = float(points["observed_volume"].max())
+ sign = "+" if intercept >= 0 else "-"
+ annotation = (
+ f"{label}
y = {slope:.2f}x {sign} {abs(intercept):.2f}"
+ f"
R^2 = {r_squared:.2f}
n = {points.height}"
+ )
+ out.append(
+ (
+ label,
+ pl.DataFrame(
+ {
+ "observed_volume": [observed_min, observed_max],
+ "modeled_volume": [
+ slope * observed_min + intercept,
+ slope * observed_max + intercept,
+ ],
+ "annotation": [annotation, annotation],
+ }
+ ),
+ )
+ )
+ return out
def _fit_r_squared_lookup(
diff --git a/dashboard/pages/validation/regional.py b/dashboard/pages/validation/regional.py
index f6e7e14..a1a2983 100644
--- a/dashboard/pages/validation/regional.py
+++ b/dashboard/pages/validation/regional.py
@@ -14,18 +14,18 @@
TOTAL_FLOW_LABELS = {"total", "all", "all_geographies"}
FLOW_COMPARISON_OPTIONS = [
+ "Modeled",
"Observed",
"Difference",
- "Percent Difference",
- "Absolute Percent Difference",
- "Modeled",
+ "% Difference",
+ "Absolute % Difference",
]
FLOW_VALUE_COLUMNS = {
"Modeled": "modeled",
"Observed": "observed",
"Difference": "difference",
- "Percent Difference": "percent_difference",
- "Absolute Percent Difference": "absolute_percent_difference",
+ "% Difference": "percent_difference",
+ "Absolute % Difference": "absolute_percent_difference",
}
@@ -381,7 +381,7 @@ def flow_comparison_heatmap(
[lookup.get((origin, destination)) for destination in destinations]
for origin in origins
]
- if metric in {"Percent Difference", "Absolute Percent Difference"}:
+ if metric in {"% Difference", "Absolute % Difference"}:
text = [
["" if value is None else f"{float(value):,.1f}%" for value in row]
for row in z
@@ -392,7 +392,7 @@ def flow_comparison_heatmap(
for row in z
]
colorscale = (
- "RdBu_r" if metric in {"Difference", "Percent Difference"} else "Blues"
+ "RdBu_r" if metric in {"Difference", "% Difference"} else "Blues"
)
z_values = [
abs(float(value)) for row in z for value in row if value is not None
@@ -411,7 +411,7 @@ def flow_comparison_heatmap(
f"{metric}: %{{text}}"
),
}
- if metric in {"Difference", "Percent Difference"} and zmax is not None:
+ if metric in {"Difference", "% Difference"} and zmax is not None:
heatmap_kwargs.update(zmid=0, zmin=-zmax, zmax=zmax)
fig = go.Figure(data=go.Heatmap(**heatmap_kwargs))
fig.update_layout(
diff --git a/dashboard/pages/validation/transit.py b/dashboard/pages/validation/transit.py
index 0dcaf3d..85d36f8 100644
--- a/dashboard/pages/validation/transit.py
+++ b/dashboard/pages/validation/transit.py
@@ -8,7 +8,6 @@
from dashboard.rendering import selector_row
from dashboard.data_access import RunTables
from dashboard.helpers.category_helpers import (
- common_column_options,
column_options,
nonempty,
)
@@ -84,18 +83,19 @@ def build_page(self) -> pn.viewable.Viewable:
)
self._transfer_body = self.section(
"transit_transfer_body",
- selectors=("technology", "access_mode"),
+ selectors=("access_mode",),
render=self.render_transfer_section,
)
return self.new_section(
pn.pane.Markdown("## Transit Validation"),
pn.pane.Markdown("### Transit Boardings"),
- selector_row(self.technology_sel, self.access_mode_sel),
+ selector_row(self.technology_sel),
self._boardings_body,
self.section_note(
"transit_validation.boardings", self._boardings_body
),
pn.pane.Markdown("### Transfer Rate"),
+ selector_row(self.access_mode_sel),
self.noted_section(
"transit_validation.transfer_rate", self._transfer_body
),
@@ -103,11 +103,11 @@ def build_page(self) -> pn.viewable.Viewable:
)
def _technology_options(self) -> list[str]:
- options, _ = common_column_options(
+ options, _ = column_options(
self.data.summary(
"transit_boardings_by_operator_and_technology", self.weighting_key
- ),
- self.data.summary("transit_transfer_rate", self.weighting_key),
+ )
+ or [],
column="technology",
total_raw="All",
total_label="All",
@@ -176,16 +176,15 @@ def render_transfer_chart(self, operator_values: list[str]) -> pn.viewable.Viewa
detail="Transit transfer summaries are unavailable.",
missing_items=["transit_transfer_rate"],
)
- technology = self.technology_sel.value
access_mode = self.access_mode_sel.value
transfer_data = self.query(
- lambda: filter_transit_data(transfer_list, technology, access_mode)
+ lambda: filter_transit_data(transfer_list, "All", access_mode)
)
return self.plot.bar(
transfer_data,
x="operator",
y="transfer_rate",
- title=f"Transit Transfer Rate - {technology}, {access_mode}",
+ title=f"Transit Transfer Rate - {access_mode}",
x_title="Operator",
y_title="Boardings per Linked Trip",
value_mode="count",
@@ -199,23 +198,15 @@ def render_boardings_section(self):
"transit_boardings_by_operator_and_technology",
self.weighting_key,
)
- transfer_list = self.data.summary(
- "transit_transfer_rate",
- self.weighting_key,
- )
- operator_values = self._operator_values(boarding_list, transfer_list)
+ operator_values = self._operator_values(boarding_list)
return [self.render_boardings_chart(operator_values)]
def render_transfer_section(self):
if not self.state.run_labels:
return [self.no_runs_message()]
- boarding_list = self.data.summary(
- "transit_boardings_by_operator_and_technology",
- self.weighting_key,
- )
transfer_list = self.data.summary(
"transit_transfer_rate",
self.weighting_key,
)
- operator_values = self._operator_values(boarding_list, transfer_list)
+ operator_values = self._operator_values(transfer_list)
return [self.render_transfer_chart(operator_values)]
diff --git a/dashboard/rendering/figures.py b/dashboard/rendering/figures.py
index 250b9f8..579ad04 100644
--- a/dashboard/rendering/figures.py
+++ b/dashboard/rendering/figures.py
@@ -277,6 +277,7 @@ def scatter_figure(
fit_overlays: ChartTables | None = None,
fit_annotation: str = "annotation",
one_to_one: bool = False,
+ legend_on_right: bool = False,
) -> go.Figure:
_require_columns(data, "scatter", x, y)
figure = go.Figure()
@@ -310,11 +311,38 @@ def scatter_figure(
bordercolor=color, borderwidth=1,
)
if one_to_one:
- maximum = max([value for value in axis_values if value >= 0], default=1.0) or 1.0
+ if axis_values:
+ minimum = min(axis_values)
+ maximum = max(axis_values)
+ if minimum == maximum:
+ padding = max(abs(minimum) * 0.05, 1.0)
+ minimum -= padding
+ maximum += padding
+ else:
+ minimum, maximum = 0.0, 1.0
figure.add_trace(go.Scatter(
- name="1:1 line", x=[0.0, maximum], y=[0.0, maximum], mode="lines",
+ name="1:1 line", x=[minimum, maximum], y=[minimum, maximum], mode="lines",
line=dict(color="#BDBDBD", width=1.5, dash="dash"),
hoverinfo="skip", showlegend=False,
))
_layout(figure, title=title, x_title=x_title, y_title=y_title, height=height)
+ if one_to_one:
+ figure.update_xaxes(range=[minimum, maximum], constrain="domain")
+ figure.update_yaxes(
+ range=[minimum, maximum],
+ constrain="domain",
+ scaleanchor="x",
+ scaleratio=1.0,
+ )
+ if legend_on_right:
+ figure.update_layout(
+ legend=dict(
+ orientation="v",
+ x=1.02,
+ xanchor="left",
+ y=1.0,
+ yanchor="top",
+ ),
+ margin=dict(l=60, r=180, t=90, b=90),
+ )
return figure
diff --git a/processor/summarize/summaries/validation.py b/processor/summarize/summaries/validation.py
index a035ac6..39e2333 100644
--- a/processor/summarize/summaries/validation.py
+++ b/processor/summarize/summaries/validation.py
@@ -121,6 +121,7 @@ def traffic_count_comparisons(rd: RunData, config: Config) -> pl.DataFrame:
"screenline_id": pl.Utf8,
"direction": pl.Utf8,
"count_period": pl.Utf8,
+ "facility_type": pl.Utf8,
"observed_volume": pl.Float64,
"modeled_volume": pl.Float64,
},
@@ -130,6 +131,7 @@ def screenline_flow_comparisons(rd: RunData, config: Config) -> pl.DataFrame:
"screenline_id": pl.Utf8,
"direction": pl.Utf8,
"count_period": pl.Utf8,
+ "facility_type": pl.Utf8,
"observed_volume": pl.Float64,
"modeled_volume": pl.Float64,
}
@@ -146,39 +148,48 @@ def screenline_flow_comparisons(rd: RunData, config: Config) -> pl.DataFrame:
) or not required.issubset(set(rd.visum_screenline_flows.columns)):
return pl.DataFrame(schema=result_schema)
- observed = (
- rd.observed_screenline_flows.filter(
- pl.col("screenline_id").is_not_null()
- & pl.col("direction").is_not_null()
- & pl.col("count_period").is_not_null()
- & pl.col("volume").is_not_null()
+ def normalize(source: pl.DataFrame, value_column: str) -> pl.DataFrame:
+ facility_output = f"_{value_column}_facility_type"
+ facility_column = next(
+ (
+ column
+ for column in ("facility_type", "FACTYPE")
+ if column in source.columns
+ ),
+ None,
)
- .group_by(["screenline_id", "direction", "count_period"])
- .agg(observed_volume=pl.col("volume").sum())
- .with_columns(
- pl.col("screenline_id").cast(pl.Utf8),
- pl.col("direction").cast(pl.Utf8),
- pl.col("count_period").cast(pl.Utf8),
- pl.col("observed_volume").cast(pl.Float64),
+ return (
+ source.with_columns(
+ (
+ pl.col(facility_column).cast(pl.Utf8)
+ if facility_column is not None
+ else pl.lit(None, dtype=pl.Utf8)
+ ).alias("facility_type")
+ )
+ .filter(
+ pl.col("screenline_id").is_not_null()
+ & pl.col("direction").is_not_null()
+ & pl.col("count_period").is_not_null()
+ & pl.col("volume").is_not_null()
+ )
+ .group_by(["screenline_id", "direction", "count_period"])
+ .agg(
+ pl.col("volume").sum().cast(pl.Float64).alias(value_column),
+ pl.col("facility_type")
+ .drop_nulls()
+ .first()
+ .alias(facility_output),
+ )
+ .with_columns(
+ pl.col("screenline_id").cast(pl.Utf8),
+ pl.col("direction").cast(pl.Utf8),
+ pl.col("count_period").cast(pl.Utf8),
+ pl.col(facility_output).cast(pl.Utf8),
+ )
)
- )
- modeled = (
- rd.visum_screenline_flows.filter(
- pl.col("screenline_id").is_not_null()
- & pl.col("direction").is_not_null()
- & pl.col("count_period").is_not_null()
- & pl.col("volume").is_not_null()
- )
- .group_by(["screenline_id", "direction", "count_period"])
- .agg(modeled_volume=pl.col("volume").sum())
- .with_columns(
- pl.col("screenline_id").cast(pl.Utf8),
- pl.col("direction").cast(pl.Utf8),
- pl.col("count_period").cast(pl.Utf8),
- pl.col("modeled_volume").cast(pl.Float64),
- )
- )
+ observed = normalize(rd.observed_screenline_flows, "observed_volume")
+ modeled = normalize(rd.visum_screenline_flows, "modeled_volume")
return (
observed.join(
@@ -186,14 +197,24 @@ def screenline_flow_comparisons(rd: RunData, config: Config) -> pl.DataFrame:
on=["screenline_id", "direction", "count_period"],
how="inner",
)
+ .with_columns(
+ pl.coalesce(
+ [
+ pl.col("_modeled_volume_facility_type"),
+ pl.col("_observed_volume_facility_type"),
+ pl.lit("All"),
+ ]
+ ).alias("facility_type")
+ )
.select(
"screenline_id",
"direction",
"count_period",
+ "facility_type",
"observed_volume",
"modeled_volume",
)
- .sort(["screenline_id", "direction", "count_period"])
+ .sort(["screenline_id", "direction", "count_period", "facility_type"])
)
diff --git a/scripts/generate_validation_demo_fixtures.py b/scripts/generate_validation_demo_fixtures.py
new file mode 100644
index 0000000..de802b3
--- /dev/null
+++ b/scripts/generate_validation_demo_fixtures.py
@@ -0,0 +1,289 @@
+"""Generate deterministic estimated tables for validation-page demonstrations."""
+
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+SOURCE_DIR = PROJECT_ROOT / "outside_summary_tables"
+OUTPUT_DIR = SOURCE_DIR / "estimated_fixtures"
+RUNS = ("unfiltered", "filtered", "override", "estimation-output")
+RUN_BIASES = (-0.08, -0.03, 0.03, 0.08)
+PERIOD_COLUMNS = ("am_vol", "md_vol", "pm_vol", "day_vol")
+
+
+def _read_rows(path: Path) -> list[dict[str, str]]:
+ with path.open(newline="", encoding="utf-8-sig") as stream:
+ return list(csv.DictReader(stream))
+
+
+def _write_rows(path: Path, fieldnames: list[str], rows: list[dict[str, object]]) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ with path.open("w", newline="", encoding="utf-8") as stream:
+ writer = csv.DictWriter(stream, fieldnames=fieldnames)
+ writer.writeheader()
+ writer.writerows(rows)
+
+
+def _estimated_value(value: str | float, run_index: int, row_index: int, column_index: int) -> float:
+ variation = (((row_index + 3) * (column_index + 5) * (run_index + 2)) % 17 - 8) / 100
+ return max(0.0, float(value) * (1.0 + RUN_BIASES[run_index] + variation))
+
+
+def _write_count_locations(run: str, run_index: int) -> None:
+ source = _read_rows(SOURCE_DIR / "countLocCounts.csv")
+ rows: list[dict[str, object]] = []
+ for row_index, row in enumerate(source):
+ output: dict[str, object] = {"id": row["id"], "FACTYPE": row["FACTYPE"]}
+ for column_index, column in enumerate(PERIOD_COLUMNS):
+ output[column] = _estimated_value(
+ row[column], run_index, row_index, column_index
+ )
+ rows.append(output)
+ _write_rows(
+ OUTPUT_DIR / run / "count_location_volumes_validation_summary.csv",
+ ["id", "FACTYPE", *PERIOD_COLUMNS],
+ rows,
+ )
+
+
+def _write_links(run: str, run_index: int) -> None:
+ source = _read_rows(SOURCE_DIR / "allLinkSummary.csv")[:2000]
+ rows: list[dict[str, object]] = []
+ for row_index, row in enumerate(source):
+ output: dict[str, object] = {
+ "id": row["id"],
+ "From_Node": row["From_Node"],
+ "To_Node": row["To_Node"],
+ "FACTYPE": row["FACTYPE"],
+ }
+ for column_index, column in enumerate(PERIOD_COLUMNS):
+ output[column] = _estimated_value(
+ row[column], run_index, row_index, column_index
+ )
+ rows.append(output)
+ _write_rows(
+ OUTPUT_DIR / run / "link_validation_summary.csv",
+ ["id", "From_Node", "To_Node", "FACTYPE", *PERIOD_COLUMNS],
+ rows,
+ )
+
+
+def _write_screenlines(run: str, run_index: int) -> None:
+ counts = _read_rows(SOURCE_DIR / "countLocCounts.csv")[:16]
+ rows: list[dict[str, object]] = []
+ for row_index, row in enumerate(counts):
+ for column_index, (period, column) in enumerate(
+ zip(("AM", "MD", "PM", "Day"), PERIOD_COLUMNS)
+ ):
+ observed = float(row[column])
+ rows.append(
+ {
+ "screenline_id": f"SL-{row_index + 1:02d}",
+ "direction": "NB/EB" if row_index % 2 == 0 else "SB/WB",
+ "count_period": period,
+ "facility_type": row["FACTYPE"],
+ "observed_volume": observed,
+ "modeled_volume": _estimated_value(
+ observed, run_index, row_index, column_index
+ ),
+ }
+ )
+ _write_rows(
+ OUTPUT_DIR / run / "screenline_flow_comparisons.csv",
+ [
+ "screenline_id",
+ "direction",
+ "count_period",
+ "facility_type",
+ "observed_volume",
+ "modeled_volume",
+ ],
+ rows,
+ )
+
+
+def _write_commuting_flows(run: str, run_index: int) -> None:
+ rows: list[dict[str, object]] = []
+ for source_name, geography_type in (
+ ("countyFlows.csv", "district"),
+ ("countyFlows_JoJa.csv", "county"),
+ ):
+ for row_index, row in enumerate(_read_rows(SOURCE_DIR / source_name)):
+ origin = row.get("") or row.get("Origin")
+ if origin is None or origin.strip().lower() == "total":
+ continue
+ for column_index, (destination, value) in enumerate(row.items()):
+ if destination in {"", "Origin", "Total"}:
+ continue
+ rows.append(
+ {
+ "origin_geography_type": geography_type,
+ "origin_geography_id": origin,
+ "destination_geography_type": geography_type,
+ "destination_geography_id": destination,
+ "commuter_count": _estimated_value(
+ value, run_index, row_index, column_index
+ ),
+ }
+ )
+ _write_rows(
+ OUTPUT_DIR / run / "commuting_flows.csv",
+ [
+ "origin_geography_type",
+ "origin_geography_id",
+ "destination_geography_type",
+ "destination_geography_id",
+ "commuter_count",
+ ],
+ rows,
+ )
+
+
+def _write_wide_summary(
+ run: str,
+ run_index: int,
+ *,
+ source_name: str,
+ output_name: str,
+ category_column: str,
+ value_columns: list[str],
+) -> None:
+ source = _read_rows(SOURCE_DIR / source_name)
+ rows: list[dict[str, object]] = []
+ for row_index, row in enumerate(source):
+ output: dict[str, object] = {category_column: row[category_column]}
+ for column_index, column in enumerate(value_columns):
+ output[column] = _estimated_value(
+ row[column], run_index, row_index, column_index
+ )
+ output["Total"] = sum(float(output[column]) for column in value_columns)
+ rows.append(output)
+ _write_rows(
+ OUTPUT_DIR / run / output_name,
+ [category_column, *value_columns, "Total"],
+ rows,
+ )
+
+
+def _write_observed_series() -> None:
+ observed_dir = OUTPUT_DIR / "observed"
+ _write_rows(
+ observed_dir / "transit_boardings_by_operator_and_technology.csv",
+ ["operator", "technology", "boardings"],
+ [
+ {"operator": "City Transit", "technology": "Bus", "boardings": 18400.0},
+ {"operator": "Regional Transit", "technology": "Bus", "boardings": 9200.0},
+ {"operator": "Regional Transit", "technology": "Rail", "boardings": 6100.0},
+ ],
+ )
+ _write_rows(
+ observed_dir / "transit_transfer_rate.csv",
+ ["operator", "technology", "access_mode", "transfer_rate"],
+ [
+ {"operator": "City Transit", "technology": "Bus", "access_mode": "Walk", "transfer_rate": 1.21},
+ {"operator": "Regional Transit", "technology": "Bus", "access_mode": "Walk", "transfer_rate": 1.34},
+ {"operator": "Regional Transit", "technology": "Rail", "access_mode": "PNR", "transfer_rate": 1.47},
+ ],
+ )
+ _write_rows(
+ observed_dir / "bicycle_vmt_by_facility_type.csv",
+ ["facility_type", "bicycle_vmt"],
+ [
+ {"facility_type": "Protected Bike Lane", "bicycle_vmt": 12400.0},
+ {"facility_type": "Bike Lane", "bicycle_vmt": 18750.0},
+ {"facility_type": "Shared Roadway", "bicycle_vmt": 9300.0},
+ {"facility_type": "Multi-Use Path", "bicycle_vmt": 15600.0},
+ ],
+ )
+
+
+def _write_estimated_series(run: str, run_index: int) -> None:
+ observed_dir = OUTPUT_DIR / "observed"
+ table_specs = (
+ (
+ "transit_boardings_by_operator_and_technology.csv",
+ ["operator", "technology", "boardings"],
+ ["boardings"],
+ ),
+ (
+ "transit_transfer_rate.csv",
+ ["operator", "technology", "access_mode", "transfer_rate"],
+ ["transfer_rate"],
+ ),
+ (
+ "bicycle_vmt_by_facility_type.csv",
+ ["facility_type", "bicycle_vmt"],
+ ["bicycle_vmt"],
+ ),
+ )
+ for filename, fieldnames, value_columns in table_specs:
+ source = _read_rows(observed_dir / filename)
+ rows: list[dict[str, object]] = []
+ for row_index, row in enumerate(source):
+ output: dict[str, object] = dict(row)
+ for column_index, column in enumerate(value_columns):
+ output[column] = _estimated_value(
+ row[column], run_index, row_index, column_index
+ )
+ rows.append(output)
+ _write_rows(OUTPUT_DIR / run / filename, fieldnames, rows)
+
+
+def generate() -> None:
+ _write_observed_series()
+ for run_index, run in enumerate(RUNS):
+ _write_count_locations(run, run_index)
+ _write_links(run, run_index)
+ _write_screenlines(run, run_index)
+ _write_commuting_flows(run, run_index)
+ _write_estimated_series(run, run_index)
+ _write_wide_summary(
+ run,
+ run_index,
+ source_name="cvm_summary.csv",
+ output_name="commercial_vehicle_validation_summary.csv",
+ category_column="tod",
+ value_columns=["car", "mu", "su"],
+ )
+ _write_wide_summary(
+ run,
+ run_index,
+ source_name="cvm_vmt_summary.csv",
+ output_name="commercial_vehicle_vmt_validation_summary.csv",
+ category_column="tod",
+ value_columns=["car", "mu", "su"],
+ )
+ external_columns = [
+ "hbcoll",
+ "hbo",
+ "hbr",
+ "hbs",
+ "hbsch",
+ "hbw",
+ "nhbnw",
+ "nhbw",
+ "truck",
+ ]
+ _write_wide_summary(
+ run,
+ run_index,
+ source_name="ext_summary.csv",
+ output_name="external_trip_validation_summary.csv",
+ category_column="tod",
+ value_columns=external_columns,
+ )
+ _write_wide_summary(
+ run,
+ run_index,
+ source_name="ext_vmt_summary.csv",
+ output_name="external_vmt_validation_summary.csv",
+ category_column="tod",
+ value_columns=external_columns,
+ )
+
+
+if __name__ == "__main__":
+ generate()
diff --git a/tests/test_dashboard_live.py b/tests/test_dashboard_live.py
index 583d098..8b61943 100644
--- a/tests/test_dashboard_live.py
+++ b/tests/test_dashboard_live.py
@@ -2027,18 +2027,18 @@ def test_regional_validation_page_compares_county_flows_to_commuting_flows(
assert list(page.flow_matrix_sel.options) == ["County flows"]
assert list(page.comparison_metric_sel.options) == [
+ "Modeled",
"Observed",
"Difference",
- "Percent Difference",
- "Absolute Percent Difference",
- "Modeled",
+ "% Difference",
+ "Absolute % Difference",
]
chart = page.render_flow_section()
tabs = chart.objects[0]
plot = tabs.objects[0][0]
- assert plot.object.layout.title.text == "Observed County flows"
- assert plot.object.data[0].z == ([10.0, 5.0], [3.0, 20.0])
+ assert plot.object.layout.title.text == "Modeled County flows"
+ assert plot.object.data[0].z == ([12.0, 4.0], [3.0, 18.0])
page.comparison_metric_sel.value = "Difference"
chart = page.render_flow_section()
diff --git a/tests/test_figure_builders.py b/tests/test_figure_builders.py
index a447a5f..71dd44a 100644
--- a/tests/test_figure_builders.py
+++ b/tests/test_figure_builders.py
@@ -114,8 +114,14 @@ def test_scatter_chart_can_add_one_to_one_reference_line() -> None:
reference_line = chart.object.data[-1]
assert reference_line.name == "1:1 line"
- assert list(reference_line.x) == [0.0, 25.0]
- assert list(reference_line.y) == [0.0, 25.0]
+ assert list(reference_line.x) == [10.0, 25.0]
+ assert list(reference_line.y) == [10.0, 25.0]
assert reference_line.line.color == "#BDBDBD"
assert reference_line.line.dash == "dash"
assert reference_line.showlegend is False
+ assert list(chart.object.layout.xaxis.range) == [10.0, 25.0]
+ assert list(chart.object.layout.yaxis.range) == [10.0, 25.0]
+ assert chart.object.layout.xaxis.constrain == "domain"
+ assert chart.object.layout.yaxis.constrain == "domain"
+ assert chart.object.layout.yaxis.scaleanchor == "x"
+ assert chart.object.layout.yaxis.scaleratio == 1.0
diff --git a/tests/test_summary_cache.py b/tests/test_summary_cache.py
index 3cc3e2d..bc4ca6b 100644
--- a/tests/test_summary_cache.py
+++ b/tests/test_summary_cache.py
@@ -5225,6 +5225,7 @@ def test_traffic_validation_removes_direction_period_selectors_and_count_card(
"direction": ["outbound"],
"count_period": ["AM"],
"screenline_id": ["A"],
+ "facility_type": [3],
"observed_volume": [15.0],
"modeled_volume": [14.0],
}
@@ -5244,6 +5245,8 @@ def test_traffic_validation_removes_direction_period_selectors_and_count_card(
"demo_facility_type",
"demo_top_period",
"demo_top_n",
+ "screenline_period",
+ "screenline_facility_type",
]
assert page.demo_period_sel.name == "Period"
assert page.demo_top_period_sel.name == "Period"
@@ -5261,12 +5264,20 @@ def test_traffic_validation_removes_direction_period_selectors_and_count_card(
"demo_facility_type",
)
assert sections["link_tables.volume"].selector_ids == ("demo_period",)
- assert page.view.objects[-2].object == "### Screenline Flow Summaries"
+ assert sections["screenlines.body"].selector_ids == (
+ "screenline_period",
+ "screenline_facility_type",
+ )
+ assert page.view.objects[-3].object == "### Screenline Flow Summaries"
+ assert list(page.view.objects[-2].objects) == [
+ page.screenline_period_sel,
+ page.screenline_facility_sel,
+ ]
plot_titles = [
plot.object.layout.title.text
for plot in _collect_plotly_panes(page._screenline_body)
]
- assert plot_titles == ["Screenline Flow Comparisons"]
+ assert plot_titles == ["Screenline Observed vs Modeled - Day"]
def test_traffic_validation_external_volume_table_compares_observed_and_modeled(
@@ -5297,11 +5308,12 @@ def test_traffic_validation_external_volume_table_compares_observed_and_modeled(
),
"screenline_flow_comparisons": pl.DataFrame(
{
- "direction": ["outbound"],
- "count_period": ["AM"],
- "screenline_id": ["A"],
- "observed_volume": [15.0],
- "modeled_volume": [14.0],
+ "direction": ["outbound", "outbound", "inbound"],
+ "count_period": ["AM", "AM", "AM"],
+ "screenline_id": ["A", "B", "C"],
+ "facility_type": [3, 3, 4],
+ "observed_volume": [15.0, 25.0, 35.0],
+ "modeled_volume": [14.0, 27.0, 30.0],
}
),
"link_validation_summary": pl.DataFrame(
@@ -5353,6 +5365,8 @@ def test_traffic_validation_external_volume_table_compares_observed_and_modeled(
assert page.demo_top_n_sel.name == "Top N by Modeled Volume"
page.demo_period_sel.value = "AM"
page.demo_facility_sel.value = "Principal Arterial"
+ page.screenline_period_sel.value = "AM"
+ page.screenline_facility_sel.value = "Principal Arterial"
page.refresh(force=True)
tables = _collect_tabulators(page._external_top_body)
@@ -5459,11 +5473,18 @@ def test_traffic_validation_external_volume_table_compares_observed_and_modeled(
reference_line = count_plot.object.data[-1]
assert reference_line.name == "1:1 line"
- assert list(reference_line.x) == [0.0, 11.0]
- assert list(reference_line.y) == [0.0, 11.0]
+ assert list(reference_line.x) == [10.0, 11.0]
+ assert list(reference_line.y) == [10.0, 11.0]
assert reference_line.line.color == "#BDBDBD"
assert reference_line.line.dash == "dash"
assert reference_line.showlegend is False
+ assert list(count_plot.object.layout.xaxis.range) == [10.0, 11.0]
+ assert list(count_plot.object.layout.yaxis.range) == [10.0, 11.0]
+ assert count_plot.object.layout.xaxis.constrain == "domain"
+ assert count_plot.object.layout.yaxis.constrain == "domain"
+ assert count_plot.object.layout.yaxis.scaleanchor == "x"
+ assert count_plot.object.layout.legend.orientation == "v"
+ assert count_plot.object.layout.legend.x == 1.02
assert count_plot.sizing_mode == "scale_width"
assert count_plot.aspect_ratio == 1.0
assert list(bar_plot.object.data[0].x) == [
@@ -5472,13 +5493,22 @@ def test_traffic_validation_external_volume_table_compares_observed_and_modeled(
]
assert bar_plot.object.layout.showlegend is True
assert bar_plot.object.data[0].name == "Base"
- assert plot_titles[-1] == "Screenline Flow Comparisons"
+ assert plot_titles[-1] == "Screenline Observed vs Modeled - AM"
+ screenline_plot = _collect_plotly_panes(page._screenline_body)[0]
+ assert [trace.name for trace in screenline_plot.object.data] == [
+ "Base",
+ "Base fit",
+ "1:1 line",
+ ]
+ assert "R^2" in screenline_plot.object.layout.annotations[0].text
+ assert screenline_plot.object.layout.yaxis.scaleanchor == "x"
+ assert screenline_plot.object.layout.legend.x == 1.02
assert "Traffic Count Comparisons" not in plot_titles
assert "Demo Link Volume by Facility Type - Day" not in plot_titles
assert "Link Volume by Facility Type - AM" in plot_titles
-def test_transit_validation_technology_selector_uses_common_summary_options(
+def test_transit_validation_places_each_selector_with_its_plot(
tmp_path: Path,
) -> None:
config = _write_config(tmp_path)
@@ -5510,7 +5540,18 @@ def test_transit_validation_technology_selector_uses_common_summary_options(
page = TransitValidationPage(state, config)
page.refresh(force=True)
- assert list(page.technology_sel.options) == ["All", "bus"]
+ assert list(page.technology_sel.options) == ["All", "bus", "rail"]
+ assert list(page.view.objects[2].objects) == [page.technology_sel]
+ assert list(page.view.objects[6].objects) == [page.access_mode_sel]
+ sections = {section.section_id: section for section in page.registered_sections}
+ assert sections["transit_boardings_body"].selector_ids == ("technology",)
+ assert sections["transit_transfer_body"].selector_ids == ("access_mode",)
+
+ page.technology_sel.value = "rail"
+ page.access_mode_sel.value = "walk"
+ page.refresh(force=True)
+ transfer_plot = _collect_plotly_panes(page._transfer_body)[0]
+ assert transfer_plot.object.layout.title.text == "Transit Transfer Rate - walk"
def test_tour_distance_chart_casts_distance_bins_consistently_across_runs(
diff --git a/tests/test_summary_regressions.py b/tests/test_summary_regressions.py
index 9ff3f7b..4d0d571 100644
--- a/tests/test_summary_regressions.py
+++ b/tests/test_summary_regressions.py
@@ -7,6 +7,7 @@
from processor.models import RunData
from processor.summarize.summaries.demographics import hh_size, person_type
from processor.summarize.summaries.long_term_geography import free_parking
+from processor.summarize.summaries.validation import screenline_flow_comparisons
def _run(
@@ -39,6 +40,38 @@ def _config():
)
+def test_screenline_comparison_uses_available_facility_type() -> None:
+ run = _run()
+ run.observed_screenline_flows = pl.DataFrame(
+ {
+ "screenline_id": ["A"],
+ "direction": ["NB"],
+ "count_period": ["AM"],
+ "volume": [100.0],
+ }
+ )
+ run.visum_screenline_flows = pl.DataFrame(
+ {
+ "screenline_id": ["A"],
+ "direction": ["NB"],
+ "count_period": ["AM"],
+ "facility_type": [3],
+ "volume": [110.0],
+ }
+ )
+
+ assert screenline_flow_comparisons(run, None).to_dicts() == [
+ {
+ "screenline_id": "A",
+ "direction": "NB",
+ "count_period": "AM",
+ "facility_type": "3",
+ "observed_volume": 100.0,
+ "modeled_volume": 110.0,
+ }
+ ]
+
+
def test_household_size_summary_normalizes_integer_width_to_contract() -> None:
result = hh_size(
_run(
diff --git a/tests/test_validation_derived.py b/tests/test_validation_derived.py
index ace89f0..b67f7c5 100644
--- a/tests/test_validation_derived.py
+++ b/tests/test_validation_derived.py
@@ -18,6 +18,7 @@
)
from processor.summarize.external import load_summary_table_map, merge_summary_table_map_run
from runtime.config import Config
+from scripts import generate_validation_demo_fixtures as fixture_generator
def _write_config(tmp_path: Path) -> Config:
@@ -204,3 +205,85 @@ def test_summary_table_map_run_builds_and_caches_count_location_validation_deriv
loaded.summary_metadata_by_mode["weighted"][COUNT_LOCATION_FIT_ID]["state"]
== "available"
)
+
+
+def test_validation_demo_fixture_generator_writes_distinct_run_tables(
+ tmp_path: Path,
+ monkeypatch,
+) -> None:
+ source_dir = tmp_path / "source"
+ output_dir = source_dir / "estimated_fixtures"
+ source_dir.mkdir()
+ pl.DataFrame(
+ {
+ "id": [1],
+ "FACTYPE": [3],
+ "am_vol": [100.0],
+ "md_vol": [200.0],
+ "pm_vol": [150.0],
+ "day_vol": [500.0],
+ }
+ ).write_csv(source_dir / "countLocCounts.csv")
+ pl.DataFrame(
+ {
+ "id": [1],
+ "From_Node": [10],
+ "To_Node": [20],
+ "FACTYPE": [3],
+ "am_vol": [100.0],
+ "md_vol": [200.0],
+ "pm_vol": [150.0],
+ "day_vol": [500.0],
+ }
+ ).write_csv(source_dir / "allLinkSummary.csv")
+ for filename in ("cvm_summary.csv", "cvm_vmt_summary.csv"):
+ pl.DataFrame(
+ {"tod": ["AM"], "car": [10.0], "mu": [3.0], "su": [5.0], "Total": [18.0]}
+ ).write_csv(source_dir / filename)
+ external = {
+ "tod": ["AM"],
+ "hbcoll": [1.0],
+ "hbo": [2.0],
+ "hbr": [3.0],
+ "hbs": [4.0],
+ "hbsch": [5.0],
+ "hbw": [6.0],
+ "nhbnw": [7.0],
+ "nhbw": [8.0],
+ "truck": [9.0],
+ "Total": [45.0],
+ }
+ for filename in ("ext_summary.csv", "ext_vmt_summary.csv"):
+ pl.DataFrame(external).write_csv(source_dir / filename)
+ flow_matrix = pl.DataFrame(
+ {"": ["A", "Total"], "A": [10.0, 10.0], "Total": [10.0, 10.0]}
+ )
+ flow_matrix.write_csv(source_dir / "countyFlows.csv")
+ flow_matrix.write_csv(source_dir / "countyFlows_JoJa.csv")
+
+ monkeypatch.setattr(fixture_generator, "SOURCE_DIR", source_dir)
+ monkeypatch.setattr(fixture_generator, "OUTPUT_DIR", output_dir)
+ fixture_generator.generate()
+
+ day_values = []
+ for run in fixture_generator.RUNS:
+ run_dir = output_dir / run
+ assert len(list(run_dir.glob("*.csv"))) == 11
+ day_values.append(
+ pl.read_csv(run_dir / "count_location_volumes_validation_summary.csv")[
+ "day_vol"
+ ][0]
+ )
+ assert not pl.read_csv(run_dir / "screenline_flow_comparisons.csv").is_empty()
+ assert set(pl.read_csv(run_dir / "commuting_flows.csv")[
+ "origin_geography_type"
+ ]) == {"district", "county"}
+ commercial = pl.read_csv(
+ run_dir / "commercial_vehicle_validation_summary.csv"
+ )
+ assert commercial["Total"][0] == commercial.select(
+ pl.sum_horizontal("car", "mu", "su")
+ ).item()
+
+ assert len(set(day_values)) == 4
+ assert len(list((output_dir / "observed").glob("*.csv"))) == 3
diff --git a/wiki/24-summary-catalog.md b/wiki/24-summary-catalog.md
index a64deb1..e33ab4f 100644
--- a/wiki/24-summary-catalog.md
+++ b/wiki/24-summary-catalog.md
@@ -79,7 +79,7 @@ Total registered summaries: **100**
| `school_shadow_pricing_residual_histogram` | `school_shadow_pricing_residual_histogram.csv` | `processor.summarize.summaries.long_term_geography.school_shadow_pricing_residual_histogram` | `geography_type: String`
`student_type: String`
`bin_start: Float64`
`bin_end: Float64`
`geography_count: Float64` | land_use: `MAZ`, `enrollment_count`, `student_type`
per: `school_zone_id`, `is_student`, `finalweight`, `student_type` |
| `school_shadow_pricing_residuals` | `school_shadow_pricing_residuals.csv` | `processor.summarize.summaries.long_term_geography.school_shadow_pricing_residuals` | `geography_type: String`
`geography_id: String`
`student_type: String`
`target_count: Float64`
`modeled_count: Float64`
`residual_count: Float64`
`absolute_residual_count: Float64`
`percent_error: Float64` | land_use: `MAZ`, `enrollment_count`, `student_type`
per: `school_zone_id`, `is_student`, `finalweight`, `student_type` |
| `schoolkids_per_escorted_tour_by_student_count_and_direction` | `schoolkids_per_escorted_tour_by_student_count_and_direction.csv` | `processor.summarize.summaries.daily_travel_escort_counts.schoolkids_per_escorted_tour_by_student_count_and_direction` | `student_count: Int64`
`direction: String`
`avg_schoolkids_per_tour: Float64`
`tour_count: Float64` | hh: `household_id`, `finalweight`
per: `household_id`, `person_type`
tours: `school_esc_outbound`, `school_esc_inbound`, `num_escortees`, `finalweight` |
-| `screenline_flow_comparisons` | `screenline_flow_comparisons.csv` | `processor.summarize.summaries.validation.screenline_flow_comparisons` | `screenline_id: String`
`direction: String`
`count_period: String`
`observed_volume: Float64`
`modeled_volume: Float64` | - |
+| `screenline_flow_comparisons` | `screenline_flow_comparisons.csv` | `processor.summarize.summaries.validation.screenline_flow_comparisons` | `screenline_id: String`
`direction: String`
`count_period: String`
`facility_type: String`
`observed_volume: Float64`
`modeled_volume: Float64` | - |
| `skimjoin_tour_component_ecdf` | `skimjoin_tour_component_ecdf.csv` | `processor.summarize.summaries.skimjoin.tour_skim_component_ecdf` | `skim_scenario: String`
`tour_mode: String`
`component: String`
`percentile: Float64`
`value: Float64`
`n_valid: Float64` | tours: `tour_mode`, `finalweight` |
| `skimjoin_tour_component_stats` | `skimjoin_tour_component_stats.csv` | `processor.summarize.summaries.skimjoin.tour_skim_component_stats` | `skim_scenario: String`
`tour_mode: String`
`component: String`
`n_total: Float64`
`n_valid: Float64`
`mean: Float64`
`std: Float64`
`min: Float64`
`max: Float64`
`median: Float64`
`mode: Float64`
`zero_share: Float64`
`missing_share: Float64` | tours: `tour_mode`, `finalweight` |
| `skimjoin_trip_component_ecdf` | `skimjoin_trip_component_ecdf.csv` | `processor.summarize.summaries.skimjoin.trip_skim_component_ecdf` | `skim_scenario: String`
`trip_mode: String`
`component: String`
`percentile: Float64`
`value: Float64`
`n_valid: Float64` | trips: `trip_mode`, `finalweight` |
From d75eb9b22c4cc159736406cd3b72184d390e6ebb Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Tue, 4 Aug 2026 13:36:04 -0400
Subject: [PATCH 02/27] rename regional flows summaries to be more descriptive
---
dashboard/calculation_notes.yaml | 2 +-
dashboard/pages/validation/regional.py | 8 ++++----
.../summarize/summaries/validation_scaffolds.py | 16 ++++++++++------
tests/test_dashboard_live.py | 9 ++++++---
tests/test_runtime_workflows.py | 4 ++--
wiki/24-summary-catalog.md | 4 ++--
wiki/31-dashboard-pages.md | 2 +-
7 files changed, 26 insertions(+), 19 deletions(-)
diff --git a/dashboard/calculation_notes.yaml b/dashboard/calculation_notes.yaml
index 0d16aa2..a839ba5 100644
--- a/dashboard/calculation_notes.yaml
+++ b/dashboard/calculation_notes.yaml
@@ -607,7 +607,7 @@ notes:
regional_validation.flows:
method: aligned_comparison
method_text: Duplicate origin-destination pairs are summed within the observed and modeled sources, then the two matrices are joined by origin and destination before the selected cell-by-cell comparison is calculated.
- sources: [county_flows_validation_summary, county_flows_joja_validation_summary, commuting_flows]
+ sources: [district_commuting_flows_validation_summary, county_commuting_flows_validation_summary, commuting_flows]
summary: Observed and modeled origin-destination flow matrices are aligned by geography pair and displayed beside a comparison matrix.
source_filters:
- Modeled flows include only workers with known home and workplace zones.
diff --git a/dashboard/pages/validation/regional.py b/dashboard/pages/validation/regional.py
index a1a2983..c5fffba 100644
--- a/dashboard/pages/validation/regional.py
+++ b/dashboard/pages/validation/regional.py
@@ -37,11 +37,11 @@ class FlowOption:
FLOW_OPTIONS = {
"District flows": FlowOption(
- summary_id="county_flows_validation_summary",
+ summary_id="district_commuting_flows_validation_summary",
modeled_geography_types=("district", "home_district"),
),
"County flows": FlowOption(
- summary_id="county_flows_joja_validation_summary",
+ summary_id="county_commuting_flows_validation_summary",
modeled_geography_types=("county", "home_county"),
),
}
@@ -433,8 +433,8 @@ def flow_comparison_heatmap(
order=55,
default_enabled=False,
optional_summary_ids=(
- "county_flows_validation_summary",
- "county_flows_joja_validation_summary",
+ "district_commuting_flows_validation_summary",
+ "county_commuting_flows_validation_summary",
"commuting_flows",
),
)
diff --git a/processor/summarize/summaries/validation_scaffolds.py b/processor/summarize/summaries/validation_scaffolds.py
index 2015cc1..757399e 100644
--- a/processor/summarize/summaries/validation_scaffolds.py
+++ b/processor/summarize/summaries/validation_scaffolds.py
@@ -101,7 +101,7 @@ def count_location_fit_validation_summary(rd: RunData, config: Config) -> pl.Dat
@summary(
- id="county_flows_validation_summary",
+ id="district_commuting_flows_validation_summary",
build_by_default=False,
schema={
"": pl.Utf8,
@@ -112,12 +112,14 @@ def count_location_fit_validation_summary(rd: RunData, config: Config) -> pl.Dat
"Total": pl.Float64,
},
)
-def county_flows_validation_summary(rd: RunData, config: Config) -> pl.DataFrame:
- return county_flows_validation_summary.empty()
+def district_commuting_flows_validation_summary(
+ rd: RunData, config: Config
+) -> pl.DataFrame:
+ return district_commuting_flows_validation_summary.empty()
@summary(
- id="county_flows_joja_validation_summary",
+ id="county_commuting_flows_validation_summary",
build_by_default=False,
schema={
"": pl.Utf8,
@@ -127,8 +129,10 @@ def county_flows_validation_summary(rd: RunData, config: Config) -> pl.DataFrame
"Total": pl.Float64,
},
)
-def county_flows_joja_validation_summary(rd: RunData, config: Config) -> pl.DataFrame:
- return county_flows_joja_validation_summary.empty()
+def county_commuting_flows_validation_summary(
+ rd: RunData, config: Config
+) -> pl.DataFrame:
+ return county_commuting_flows_validation_summary.empty()
@summary(
diff --git a/tests/test_dashboard_live.py b/tests/test_dashboard_live.py
index 8b61943..564b8cc 100644
--- a/tests/test_dashboard_live.py
+++ b/tests/test_dashboard_live.py
@@ -2008,11 +2008,11 @@ def test_regional_validation_page_compares_county_flows_to_commuting_flows(
run_key="base",
summaries_by_mode={
"weighted": {
- "county_flows_joja_validation_summary": observed,
+ "county_commuting_flows_validation_summary": observed,
"commuting_flows": modeled,
},
"unweighted": {
- "county_flows_joja_validation_summary": observed,
+ "county_commuting_flows_validation_summary": observed,
"commuting_flows": modeled,
},
},
@@ -2154,7 +2154,10 @@ def test_data_requirements_for_pages_tracks_optional_summary_dependencies() -> N
assert "commercial_vmt_totals" not in requirements.required_summary_ids
assert "commercial_vmt_totals" not in requirements.optional_summary_ids
assert "auto_vmt_validation_summary" not in requirements.optional_summary_ids
- assert "county_flows_validation_summary" in requirements.optional_summary_ids
+ assert (
+ "district_commuting_flows_validation_summary"
+ in requirements.optional_summary_ids
+ )
assert "commuting_flows" in requirements.optional_summary_ids
assert "auto_vmt_validation_summary" not in requirements.summary_ids_for_pruning
diff --git a/tests/test_runtime_workflows.py b/tests/test_runtime_workflows.py
index 7d0903b..086e886 100644
--- a/tests/test_runtime_workflows.py
+++ b/tests/test_runtime_workflows.py
@@ -348,8 +348,8 @@ def test_validation_scaffold_summaries_are_registered_with_empty_contracts(
"count_location_volumes_validation_summary",
"count_location_scatter_validation_summary",
"count_location_fit_validation_summary",
- "county_flows_validation_summary",
- "county_flows_joja_validation_summary",
+ "district_commuting_flows_validation_summary",
+ "county_commuting_flows_validation_summary",
"commercial_vehicle_validation_summary",
"commercial_vehicle_vmt_validation_summary",
"external_trip_validation_summary",
diff --git a/wiki/24-summary-catalog.md b/wiki/24-summary-catalog.md
index e33ab4f..4b5423f 100644
--- a/wiki/24-summary-catalog.md
+++ b/wiki/24-summary-catalog.md
@@ -43,8 +43,8 @@ Total registered summaries: **100**
| `count_location_fit_validation_summary` | `count_location_fit_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.count_location_fit_validation_summary` | `facility_type: String`
`period: String`
`slope: Float64`
`intercept: Float64`
`r_squared: Float64`
`n_locations: Int64`
`observed_min: Float64`
`observed_max: Float64`
`equation_label: String`
`r_squared_label: String` | - |
| `count_location_scatter_validation_summary` | `count_location_scatter_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.count_location_scatter_validation_summary` | `id: Int64`
`facility_type: String`
`period: String`
`observed_volume: Float64`
`modeled_volume: Float64` | - |
| `count_location_volumes_validation_summary` | `count_location_volumes_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.count_location_volumes_validation_summary` | `id: Int64`
`FACTYPE: Int64`
`am_vol: Float64`
`md_vol: Float64`
`pm_vol: Float64`
`day_vol: Float64` | - |
-| `county_flows_joja_validation_summary` | `county_flows_joja_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.county_flows_joja_validation_summary` | `: String`
`Benton: Float64`
`Linn: Float64`
`Marion: Float64`
`Total: Float64` | - |
-| `county_flows_validation_summary` | `county_flows_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.county_flows_validation_summary` | `: String`
`Albany: Float64`
`Corvallis: Float64`
`Lebanon: Float64`
`Philomath: Float64`
`Total: Float64` | - |
+| `county_commuting_flows_validation_summary` | `county_commuting_flows_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.county_commuting_flows_validation_summary` | `: String`
`Benton: Float64`
`Linn: Float64`
`Marion: Float64`
`Total: Float64` | - |
+| `district_commuting_flows_validation_summary` | `district_commuting_flows_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.district_commuting_flows_validation_summary` | `: String`
`Albany: Float64`
`Corvallis: Float64`
`Lebanon: Float64`
`Philomath: Float64`
`Total: Float64` | - |
| `daily_activity_pattern_by_person_type` | `daily_activity_pattern_by_person_type.csv` | `processor.summarize.summaries.daily_travel_activity.dap_summary` | `person_type: String`
`daily_activity_pattern: String`
`person_count: Float64` | per: `person_type`, `cdap_activity`, `finalweight` |
| `escorted_tour_totals` | `escorted_tour_totals.csv` | `processor.summarize.summaries.daily_travel_escort_counts.total_escorted_tours` | `tour_count: Float64` | tours: `school_esc_outbound`, `school_esc_inbound`, `finalweight` |
| `external_nonmandatory_tour_locations` | `external_nonmandatory_tour_locations.csv` | `processor.summarize.summaries.tour_geography.ext_non_mand_tour_loc` | `geography_type: String`
`geography_id: String`
`external_nonmandatory_tour_count: Float64` | tours: `tour_category`, `is_external_tour`, `destination`, `finalweight` |
diff --git a/wiki/31-dashboard-pages.md b/wiki/31-dashboard-pages.md
index 8796af5..4112c55 100644
--- a/wiki/31-dashboard-pages.md
+++ b/wiki/31-dashboard-pages.md
@@ -138,7 +138,7 @@ Total registered pages: **27**
| `traffic` | Traffic Validation | Validation Summaries | yes | `none` | `screenline_flow_comparisons` | `link_validation_summary`, `count_location_counts_validation_summary`, `count_location_volumes_validation_summary`, `count_location_scatter_validation_summary`, `count_location_fit_validation_summary` | - |
| `transit` | Transit Validation | Validation Summaries | yes | `none` | `transit_boardings_by_operator_and_technology`, `transit_transfer_rate` | - | - |
| `vmt` | VMT Validation | Validation Summaries | yes | `none` | `auto_vmt_by_home_geography_income_hhsize_time_period`, `non_motorized_vmt_by_home_geography_income_hhsize_time_period`, `bicycle_vmt_by_facility_type` | `commercial_vehicle_validation_summary`, `commercial_vehicle_vmt_validation_summary`, `external_trip_validation_summary`, `external_vmt_validation_summary` | - |
-| `regional_validation` | Regional Validation | Validation Summaries | no | `none` | - | `county_flows_validation_summary`, `county_flows_joja_validation_summary`, `commuting_flows` | - |
+| `regional_validation` | Regional Validation | Validation Summaries | no | `none` | - | `district_commuting_flows_validation_summary`, `county_commuting_flows_validation_summary`, `commuting_flows` | - |
| `raw_trip_demo` | Prepared Trip Demo | - | no | `required` | - | - | `trips` |
## Registered Page Groups
From d905835869f3e16d37baff92db9a3bd98dac5793 Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Tue, 4 Aug 2026 14:53:42 -0400
Subject: [PATCH 03/27] Add RMSPE equation and update calculation notes
---
dashboard/calculation_notes.yaml | 24 +++++++++++++++++--
.../pages/validation/_traffic/transforms.py | 12 ++++++++++
dashboard/rendering/figures.py | 21 ++++++++--------
tests/test_calculation_notes.py | 11 +++++++++
tests/test_dashboard_live.py | 1 +
tests/test_figure_builders.py | 20 +++++++++++++++-
tests/test_summary_cache.py | 16 +++++++------
7 files changed, 85 insertions(+), 20 deletions(-)
diff --git a/dashboard/calculation_notes.yaml b/dashboard/calculation_notes.yaml
index a839ba5..3703e85 100644
--- a/dashboard/calculation_notes.yaml
+++ b/dashboard/calculation_notes.yaml
@@ -609,6 +609,7 @@ notes:
method_text: Duplicate origin-destination pairs are summed within the observed and modeled sources, then the two matrices are joined by origin and destination before the selected cell-by-cell comparison is calculated.
sources: [district_commuting_flows_validation_summary, county_commuting_flows_validation_summary, commuting_flows]
summary: Observed and modeled origin-destination flow matrices are aligned by geography pair and displayed beside a comparison matrix.
+ formula: difference = modeled - observed; % difference = (modeled - observed) / observed * 100; absolute % difference = absolute value of % difference
source_filters:
- Modeled flows include only workers with known home and workplace zones.
- Flow rows without both an origin and destination geography are excluded.
@@ -618,6 +619,7 @@ notes:
- Duplicate origin-destination pairs are summed before the two matrices are joined.
Comparison:
- The selected metric is computed cell by cell from aligned observed and modeled values.
+ - Percent differences are blank when the observed flow is zero.
- Include Totals adds origin, destination, and grand-total cells to the matrices.
transit_validation.boardings:
@@ -650,14 +652,20 @@ notes:
traffic.facility_summary:
method: aligned_comparison
- method_text: Daily observed and modeled volumes are paired by count-location ID and grouped by facility type. Location count, percent RMSE, and R-squared are calculated from those paired daily values.
+ method_text: Daily observed and modeled volumes are paired by count-location ID and grouped by facility type. Location count, RMSE, RMSPE, and R-squared are calculated from those paired daily values.
sources: [count_location_counts_validation_summary, count_location_volumes_validation_summary, count_location_scatter_validation_summary, count_location_fit_validation_summary]
summary: The table summarizes daily count-location validation statistics by facility type.
+ formula: >-
+ % difference = (sum(modeled_i) - sum(observed_i)) / sum(observed_i) * 100;
+ RMSE = sqrt(sum((modeled_i - observed_i)^2) / n);
+ RMSPE = sqrt(mean(((observed_i - modeled_i) / observed_i)^2)) * 100;
+ R^2 = 1 - sum((modeled_i - fitted_i)^2) / sum((modeled_i - mean(modeled_i))^2)
details:
Aggregation:
- Daily observed and modeled volumes are paired by count location and grouped by facility type.
- - The table reports location count, percent RMSE, and R-squared using the prepared fit summary when available.
+ - RMSE and RMSPE are calculated from the paired points; R-squared uses the prepared fit summary when available and otherwise is calculated from the paired points.
Important details:
+ - RMSPE is blank when any observed count in the facility group is zero.
- This overview always uses unfiltered daily totals; the controls below apply to other traffic charts.
traffic.count_locations:
@@ -665,6 +673,12 @@ notes:
method_text: After period and facility filters are applied, one observed count and one modeled volume are paired by count-location ID; each successfully paired location becomes one scatter point.
sources: [count_location_counts_validation_summary, count_location_volumes_validation_summary, count_location_scatter_validation_summary, count_location_fit_validation_summary]
summary: The scatter plot pairs observed traffic counts with modeled count-location volumes for the selected period and facility type.
+ formula: >-
+ x = observed count; y = modeled volume;
+ slope = sum((x_i - mean(x)) * (y_i - mean(y))) / sum((x_i - mean(x))^2);
+ intercept = mean(y) - slope * mean(x); fitted_i = slope * x_i + intercept;
+ R^2 = 1 - sum((y_i - fitted_i)^2) / sum((y_i - mean(y))^2);
+ one-to-one reference = y = x
details:
Aggregation:
- Observed and modeled values are joined by count-location identifier after applying the selected filters.
@@ -701,6 +715,12 @@ notes:
method_text: Records are first summed by screenline, direction, count period, and facility type; observed and modeled totals with matching keys are then paired for the scatter plot and fitted with an ordinary least-squares trendline.
sources: [screenline_flow_comparisons]
summary: The scatter plot compares observed and modeled screenline flows.
+ formula: >-
+ x = observed flow; y = modeled flow;
+ slope = sum((x_i - mean(x)) * (y_i - mean(y))) / sum((x_i - mean(x))^2);
+ intercept = mean(y) - slope * mean(x); fitted_i = slope * x_i + intercept;
+ R^2 = 1 - sum((y_i - fitted_i)^2) / sum((y_i - mean(y))^2);
+ one-to-one reference = y = x
source_filters:
- Screenline records without an identifier, direction, count period, or observed volume are excluded before matching modeled flows.
details:
diff --git a/dashboard/pages/validation/_traffic/transforms.py b/dashboard/pages/validation/_traffic/transforms.py
index 7d292a0..ac18b45 100644
--- a/dashboard/pages/validation/_traffic/transforms.py
+++ b/dashboard/pages/validation/_traffic/transforms.py
@@ -392,6 +392,17 @@ def demo_facility_comparison_table(
rmse = math.sqrt(
sum(difference**2 for difference in differences) / len(differences)
)
+ if any(value == 0.0 for value in observed):
+ rmspe = ""
+ else:
+ squared_percentage_errors = [
+ ((observe - model) / observe) ** 2
+ for observe, model in zip(observed, modeled)
+ ]
+ rmspe_value = math.sqrt(
+ sum(squared_percentage_errors) / len(squared_percentage_errors)
+ ) * 100.0
+ rmspe = f"{rmspe_value:.2f}%"
percent_value = (
None
if total_observed == 0.0
@@ -411,6 +422,7 @@ def demo_facility_comparison_table(
"Total Modeled Count": total_modeled,
"% Difference": percent_difference,
"RMSE": rmse,
+ "RMSPE": rmspe,
"R^2": r_squared_lookup.get(raw_facility_type)
if raw_facility_type in r_squared_lookup
else _r_squared_from_points(facility_points),
diff --git a/dashboard/rendering/figures.py b/dashboard/rendering/figures.py
index 579ad04..affc152 100644
--- a/dashboard/rendering/figures.py
+++ b/dashboard/rendering/figures.py
@@ -299,17 +299,18 @@ def scatter_figure(
if frame.is_empty() or x not in frame.columns or y not in frame.columns:
continue
color = context.color(str(label), label_indices.get(str(label), index))
+ trace_name = f"{label} fit"
+ if fit_annotation in frame.columns:
+ annotation = str(frame[fit_annotation][0] or "").strip()
+ label_prefix = f"{label}
"
+ if annotation.startswith(label_prefix):
+ annotation = annotation[len(label_prefix):]
+ if annotation:
+ trace_name = f"{trace_name}
{annotation}"
figure.add_trace(go.Scatter(
- name=f"{label} fit", x=frame[x].to_list(), y=frame[y].to_list(),
+ name=trace_name, x=frame[x].to_list(), y=frame[y].to_list(),
mode="lines", line=dict(color=color, width=2),
))
- if fit_annotation in frame.columns and str(frame[fit_annotation][0] or "").strip():
- figure.add_annotation(
- text=str(frame[fit_annotation][0]), xref="paper", yref="paper",
- x=0.02, y=max(0.05, 0.98 - 0.12 * index), showarrow=False,
- font=dict(color=color, size=12), bgcolor="rgba(255,255,255,0.75)",
- bordercolor=color, borderwidth=1,
- )
if one_to_one:
if axis_values:
minimum = min(axis_values)
@@ -323,7 +324,7 @@ def scatter_figure(
figure.add_trace(go.Scatter(
name="1:1 line", x=[minimum, maximum], y=[minimum, maximum], mode="lines",
line=dict(color="#BDBDBD", width=1.5, dash="dash"),
- hoverinfo="skip", showlegend=False,
+ hoverinfo="skip", showlegend=True,
))
_layout(figure, title=title, x_title=x_title, y_title=y_title, height=height)
if one_to_one:
@@ -343,6 +344,6 @@ def scatter_figure(
y=1.0,
yanchor="top",
),
- margin=dict(l=60, r=180, t=90, b=90),
+ margin=dict(l=60, r=240, t=90, b=90),
)
return figure
diff --git a/tests/test_calculation_notes.py b/tests/test_calculation_notes.py
index 7e3c06c..6823353 100644
--- a/tests/test_calculation_notes.py
+++ b/tests/test_calculation_notes.py
@@ -94,6 +94,17 @@ def test_calculation_note_renderer_escapes_configured_text() -> None:
assert rendered.endswith("")
+def test_validation_notes_expose_comparison_and_error_formulas() -> None:
+ regional = get_calculation_note("regional_validation.flows")
+ facility = get_calculation_note("traffic.facility_summary")
+
+ assert "difference = modeled - observed" in regional.formula
+ assert (
+ "RMSPE = sqrt(mean(((observed_i - modeled_i) / observed_i)^2)) * 100"
+ in facility.formula
+ )
+
+
def test_calculation_note_is_collapsed_html_pane_exported_without_conversion() -> None:
pane = calculation_note("traffic.link_volume")
diff --git a/tests/test_dashboard_live.py b/tests/test_dashboard_live.py
index 564b8cc..97c5c98 100644
--- a/tests/test_dashboard_live.py
+++ b/tests/test_dashboard_live.py
@@ -469,6 +469,7 @@ def test_external_traffic_helpers_filter_period_and_facility_type(
"Total Modeled Count": 440.0,
"% Difference": "10.00%",
"RMSE": 22.360679774997898,
+ "RMSPE": "10.00%",
"R^2": 0.875,
}
]
diff --git a/tests/test_figure_builders.py b/tests/test_figure_builders.py
index 71dd44a..4fab66d 100644
--- a/tests/test_figure_builders.py
+++ b/tests/test_figure_builders.py
@@ -109,16 +109,34 @@ def test_scatter_chart_can_add_one_to_one_reference_line() -> None:
[("Base", pl.DataFrame({"observed": [10.0, 20.0], "modeled": [12.0, 25.0]}))],
x="observed",
y="modeled",
+ fit_overlays=[
+ (
+ "Base",
+ pl.DataFrame(
+ {
+ "observed": [0.0, 100.0],
+ "modeled": [-100.0, 200.0],
+ "annotation": [
+ "Base
y = 3.00x - 100.00
R^2 = 0.90
n = 2"
+ ]
+ * 2,
+ }
+ ),
+ )
+ ],
one_to_one=True,
)
reference_line = chart.object.data[-1]
+ fit_line = chart.object.data[-2]
+ assert fit_line.name == "Base fit
y = 3.00x - 100.00
R^2 = 0.90
n = 2"
+ assert not chart.object.layout.annotations
assert reference_line.name == "1:1 line"
assert list(reference_line.x) == [10.0, 25.0]
assert list(reference_line.y) == [10.0, 25.0]
assert reference_line.line.color == "#BDBDBD"
assert reference_line.line.dash == "dash"
- assert reference_line.showlegend is False
+ assert reference_line.showlegend is True
assert list(chart.object.layout.xaxis.range) == [10.0, 25.0]
assert list(chart.object.layout.yaxis.range) == [10.0, 25.0]
assert chart.object.layout.xaxis.constrain == "domain"
diff --git a/tests/test_summary_cache.py b/tests/test_summary_cache.py
index bc4ca6b..3c044ed 100644
--- a/tests/test_summary_cache.py
+++ b/tests/test_summary_cache.py
@@ -5415,6 +5415,7 @@ def test_traffic_validation_external_volume_table_compares_observed_and_modeled(
"Total Modeled Count",
"% Difference",
"RMSE",
+ "RMSPE",
"R^2",
]
assert facility_table.to_dict("records") == [
@@ -5425,6 +5426,7 @@ def test_traffic_validation_external_volume_table_compares_observed_and_modeled(
"Total Modeled Count": "210",
"% Difference": "5.00%",
"RMSE": "10",
+ "RMSPE": "5.00%",
"R^2": None,
},
{
@@ -5434,6 +5436,7 @@ def test_traffic_validation_external_volume_table_compares_observed_and_modeled(
"Total Modeled Count": "110",
"% Difference": "10.00%",
"RMSE": "10",
+ "RMSPE": "10.00%",
"R^2": None,
},
]
@@ -5477,7 +5480,7 @@ def test_traffic_validation_external_volume_table_compares_observed_and_modeled(
assert list(reference_line.y) == [10.0, 11.0]
assert reference_line.line.color == "#BDBDBD"
assert reference_line.line.dash == "dash"
- assert reference_line.showlegend is False
+ assert reference_line.showlegend is True
assert list(count_plot.object.layout.xaxis.range) == [10.0, 11.0]
assert list(count_plot.object.layout.yaxis.range) == [10.0, 11.0]
assert count_plot.object.layout.xaxis.constrain == "domain"
@@ -5495,12 +5498,11 @@ def test_traffic_validation_external_volume_table_compares_observed_and_modeled(
assert bar_plot.object.data[0].name == "Base"
assert plot_titles[-1] == "Screenline Observed vs Modeled - AM"
screenline_plot = _collect_plotly_panes(page._screenline_body)[0]
- assert [trace.name for trace in screenline_plot.object.data] == [
- "Base",
- "Base fit",
- "1:1 line",
- ]
- assert "R^2" in screenline_plot.object.layout.annotations[0].text
+ assert screenline_plot.object.data[0].name == "Base"
+ assert screenline_plot.object.data[-1].name == "1:1 line"
+ assert "Base fit
" in screenline_plot.object.data[1].name
+ assert "R^2" in screenline_plot.object.data[1].name
+ assert not screenline_plot.object.layout.annotations
assert screenline_plot.object.layout.yaxis.scaleanchor == "x"
assert screenline_plot.object.layout.legend.x == 1.02
assert "Traffic Count Comparisons" not in plot_titles
From 4ebb979a75c45e1350ac136a8794d69715704f26 Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Tue, 4 Aug 2026 15:36:11 -0400
Subject: [PATCH 04/27] Made best fit line details a hover feature
---
dashboard/calculation_notes.yaml | 3 +-
.../pages/validation/_traffic/features.py | 12 ++--
.../pages/validation/_traffic/transforms.py | 50 ++++++++++------
dashboard/rendering/figures.py | 13 +++--
tests/test_dashboard_live.py | 14 +++--
tests/test_figure_builders.py | 58 ++++++++++++++++++-
tests/test_summary_cache.py | 16 ++++-
7 files changed, 127 insertions(+), 39 deletions(-)
diff --git a/dashboard/calculation_notes.yaml b/dashboard/calculation_notes.yaml
index 3703e85..5132112 100644
--- a/dashboard/calculation_notes.yaml
+++ b/dashboard/calculation_notes.yaml
@@ -685,6 +685,7 @@ notes:
- A one-to-one reference line is always shown; a fitted regression line is added when fit coefficients are available.
Display:
- Each point is one validated count location.
+ - Hovering over a fitted line shows its run, equation, R-squared, and number of locations without reducing the plot area.
traffic.link_volume:
method: supplied_aggregation
@@ -728,7 +729,7 @@ notes:
- Flow records are grouped by screenline, direction, count period, and facility type before observed and modeled volumes are paired.
Display:
- The time-period and facility-type selectors filter the paired records before fitting.
- - Each run displays its fitted equation, R-squared value, and number of comparable records alongside a one-to-one reference line.
+ - Hovering over a fitted line shows its run, equation, R-squared, and number of comparable records; the plot also includes a one-to-one reference line.
vmt.overview:
method: vmt
diff --git a/dashboard/pages/validation/_traffic/features.py b/dashboard/pages/validation/_traffic/features.py
index 9ab85ff..401719e 100644
--- a/dashboard/pages/validation/_traffic/features.py
+++ b/dashboard/pages/validation/_traffic/features.py
@@ -40,8 +40,8 @@ def render_screenline_flow_section(self):
x="observed_volume",
y="modeled_volume",
title=f"Screenline Observed vs Modeled - {period}",
- x_title="Observed Screenline Flow",
- y_title="Modeled Screenline Flow",
+ x_title="Observed Screenline Flow (vehicles)",
+ y_title="Modeled Screenline Flow (vehicles)",
fit_overlays=fit_data,
one_to_one=True,
legend_on_right=True,
@@ -158,8 +158,8 @@ def render_demo_traffic_section(self) -> list[pn.viewable.Viewable]:
x="observed_volume",
y="modeled_volume",
title=f"Count Location Observed vs Modeled - {period}",
- x_title="Observed Count",
- y_title="Modeled Volume",
+ x_title="Observed Count (vehicles)",
+ y_title="Modeled Volume (vehicles)",
fit_overlays=fit_data,
one_to_one=True,
legend_on_right=True,
@@ -181,8 +181,8 @@ def render_demo_traffic_section(self) -> list[pn.viewable.Viewable]:
x="observed_volume",
y="modeled_volume",
title=f"Count Location Observed vs Modeled - {period}",
- x_title="Observed Count",
- y_title="Modeled Volume",
+ x_title="Observed Count (vehicles)",
+ y_title="Modeled Volume (vehicles)",
one_to_one=True,
legend_on_right=True,
panel_aspect_ratio=1.0,
diff --git a/dashboard/pages/validation/_traffic/transforms.py b/dashboard/pages/validation/_traffic/transforms.py
index ac18b45..8388d69 100644
--- a/dashboard/pages/validation/_traffic/transforms.py
+++ b/dashboard/pages/validation/_traffic/transforms.py
@@ -163,6 +163,26 @@ def demo_count_scatter_data_from_sources(
return out
+def _fit_line_frame(
+ *,
+ observed_min: float,
+ observed_max: float,
+ slope: float,
+ intercept: float,
+ annotation: str,
+) -> pl.DataFrame:
+ point_count = 101
+ step = (observed_max - observed_min) / (point_count - 1)
+ observed = [observed_min + step * index for index in range(point_count)]
+ return pl.DataFrame(
+ {
+ "observed_volume": observed,
+ "modeled_volume": [slope * value + intercept for value in observed],
+ "annotation": [annotation] * point_count,
+ }
+ )
+
+
def demo_count_fit_line_data(
fit_list: list[tuple[str, pl.DataFrame]] | None,
*,
@@ -214,15 +234,12 @@ def demo_count_fit_line_data(
out.append(
(
label,
- pl.DataFrame(
- {
- "observed_volume": [observed_min, observed_max],
- "modeled_volume": [
- slope * observed_min + intercept,
- slope * observed_max + intercept,
- ],
- "annotation": [annotation, annotation],
- }
+ _fit_line_frame(
+ observed_min=observed_min,
+ observed_max=observed_max,
+ slope=slope,
+ intercept=intercept,
+ annotation=annotation,
),
)
)
@@ -280,15 +297,12 @@ def screenline_fit_line_data(
out.append(
(
label,
- pl.DataFrame(
- {
- "observed_volume": [observed_min, observed_max],
- "modeled_volume": [
- slope * observed_min + intercept,
- slope * observed_max + intercept,
- ],
- "annotation": [annotation, annotation],
- }
+ _fit_line_frame(
+ observed_min=observed_min,
+ observed_max=observed_max,
+ slope=slope,
+ intercept=intercept,
+ annotation=annotation,
),
)
)
diff --git a/dashboard/rendering/figures.py b/dashboard/rendering/figures.py
index affc152..a9e2f20 100644
--- a/dashboard/rendering/figures.py
+++ b/dashboard/rendering/figures.py
@@ -294,22 +294,23 @@ def scatter_figure(
figure.add_trace(go.Scatter(
name=str(label), x=x_values, y=y_values, mode="markers",
marker=dict(color=context.color(str(label), index), size=8, line=dict(width=0.4)),
+ hovertemplate=(
+ f"{x_title or x}: %{{x}}
{y_title or y}: %{{y}}"
+ f"{label}"
+ ),
))
for index, (label, frame) in enumerate(fit_overlays or []):
if frame.is_empty() or x not in frame.columns or y not in frame.columns:
continue
color = context.color(str(label), label_indices.get(str(label), index))
trace_name = f"{label} fit"
+ annotation = ""
if fit_annotation in frame.columns:
annotation = str(frame[fit_annotation][0] or "").strip()
- label_prefix = f"{label}
"
- if annotation.startswith(label_prefix):
- annotation = annotation[len(label_prefix):]
- if annotation:
- trace_name = f"{trace_name}
{annotation}"
figure.add_trace(go.Scatter(
name=trace_name, x=frame[x].to_list(), y=frame[y].to_list(),
mode="lines", line=dict(color=color, width=2),
+ hovertemplate=(f"{annotation}" if annotation else None),
))
if one_to_one:
if axis_values:
@@ -344,6 +345,6 @@ def scatter_figure(
y=1.0,
yanchor="top",
),
- margin=dict(l=60, r=240, t=90, b=90),
+ margin=dict(l=60, r=180, t=90, b=90),
)
return figure
diff --git a/tests/test_dashboard_live.py b/tests/test_dashboard_live.py
index 97c5c98..45213b5 100644
--- a/tests/test_dashboard_live.py
+++ b/tests/test_dashboard_live.py
@@ -500,11 +500,15 @@ def test_demo_count_fit_line_helper_builds_plot_data() -> None:
facility_type="4",
)
- assert fit_lines[0][1].select("observed_volume", "modeled_volume").to_dicts() == [
- {"observed_volume": 20.0, "modeled_volume": 67.0},
- {"observed_volume": 40.0, "modeled_volume": 127.0},
- ]
- assert "y = 3.00x + 7.00" in fit_lines[0][1]["annotation"][0]
+ fit_frame = fit_lines[0][1]
+ assert fit_frame.height == 101
+ assert fit_frame.select("observed_volume", "modeled_volume").row(
+ 0, named=True
+ ) == {"observed_volume": 20.0, "modeled_volume": 67.0}
+ assert fit_frame.select("observed_volume", "modeled_volume").row(
+ -1, named=True
+ ) == {"observed_volume": 40.0, "modeled_volume": 127.0}
+ assert "y = 3.00x + 7.00" in fit_frame["annotation"][0]
def test_external_vmt_helper_reshapes_wide_tod_table() -> None:
diff --git a/tests/test_figure_builders.py b/tests/test_figure_builders.py
index 4fab66d..96b2435 100644
--- a/tests/test_figure_builders.py
+++ b/tests/test_figure_builders.py
@@ -124,13 +124,23 @@ def test_scatter_chart_can_add_one_to_one_reference_line() -> None:
),
)
],
+ x_title="Observed Count (vehicles)",
+ y_title="Modeled Volume (vehicles)",
one_to_one=True,
)
+ point_trace = chart.object.data[0]
reference_line = chart.object.data[-1]
fit_line = chart.object.data[-2]
- assert fit_line.name == "Base fit
y = 3.00x - 100.00
R^2 = 0.90
n = 2"
+ assert point_trace.hovertemplate == (
+ "Observed Count (vehicles): %{x}
"
+ "Modeled Volume (vehicles): %{y}Base"
+ )
+ assert fit_line.name == "Base fit"
+ assert "y = 3.00x - 100.00" in fit_line.hovertemplate
assert not chart.object.layout.annotations
+ assert chart.object.layout.height == 400
+ assert chart.object.layout.margin.t == 90
assert reference_line.name == "1:1 line"
assert list(reference_line.x) == [10.0, 25.0]
assert list(reference_line.y) == [10.0, 25.0]
@@ -143,3 +153,49 @@ def test_scatter_chart_can_add_one_to_one_reference_line() -> None:
assert chart.object.layout.yaxis.constrain == "domain"
assert chart.object.layout.yaxis.scaleanchor == "x"
assert chart.object.layout.yaxis.scaleratio == 1.0
+
+
+def test_scatter_fit_details_are_hover_only_for_multiple_runs() -> None:
+ labels = [f"Run {index}" for index in range(4)]
+ scatter_data = [
+ (
+ label,
+ pl.DataFrame({"observed": [10.0, 20.0], "modeled": [12.0, 25.0]}),
+ )
+ for label in labels
+ ]
+ fit_data = [
+ (
+ label,
+ pl.DataFrame(
+ {
+ "observed": [0.0, 100.0],
+ "modeled": [-100.0, 200.0],
+ "annotation": [
+ f"{label}
y = 3.00x - 100.00
R^2 = 0.90
n = 2"
+ ]
+ * 2,
+ }
+ ),
+ )
+ for label in labels
+ ]
+
+ chart = Plotter(RenderContext()).scatter(
+ scatter_data,
+ x="observed",
+ y="modeled",
+ x_title="Observed Count (vehicles)",
+ y_title="Modeled Volume (vehicles)",
+ fit_overlays=fit_data,
+ one_to_one=True,
+ panel_aspect_ratio=1.0,
+ )
+
+ assert not chart.object.layout.annotations
+ assert chart.object.layout.height == 400
+ assert chart.object.layout.margin.t == 90
+ assert all("R^2 = 0.90" in trace.hovertemplate for trace in chart.object.data[4:8])
+ assert list(chart.object.layout.xaxis.range) == [10.0, 25.0]
+ assert list(chart.object.layout.yaxis.range) == [10.0, 25.0]
+ assert chart.aspect_ratio == 1.0
diff --git a/tests/test_summary_cache.py b/tests/test_summary_cache.py
index 3c044ed..37f3b4e 100644
--- a/tests/test_summary_cache.py
+++ b/tests/test_summary_cache.py
@@ -5490,6 +5490,8 @@ def test_traffic_validation_external_volume_table_compares_observed_and_modeled(
assert count_plot.object.layout.legend.x == 1.02
assert count_plot.sizing_mode == "scale_width"
assert count_plot.aspect_ratio == 1.0
+ assert "Observed Count (vehicles): %{x}" in count_plot.object.data[0].hovertemplate
+ assert "Modeled Volume (vehicles): %{y}" in count_plot.object.data[0].hovertemplate
assert list(bar_plot.object.data[0].x) == [
"Minor Arterial",
"Principal Arterial",
@@ -5500,9 +5502,19 @@ def test_traffic_validation_external_volume_table_compares_observed_and_modeled(
screenline_plot = _collect_plotly_panes(page._screenline_body)[0]
assert screenline_plot.object.data[0].name == "Base"
assert screenline_plot.object.data[-1].name == "1:1 line"
- assert "Base fit
" in screenline_plot.object.data[1].name
- assert "R^2" in screenline_plot.object.data[1].name
+ assert screenline_plot.object.data[1].name == "Base fit"
+ assert len(screenline_plot.object.data[1].x) == 101
+ assert "R^2" in screenline_plot.object.data[1].hovertemplate
+ assert "y = 1.30x - 5.50" in screenline_plot.object.data[1].hovertemplate
assert not screenline_plot.object.layout.annotations
+ assert (
+ "Observed Screenline Flow (vehicles): %{x}"
+ in screenline_plot.object.data[0].hovertemplate
+ )
+ assert (
+ "Modeled Screenline Flow (vehicles): %{y}"
+ in screenline_plot.object.data[0].hovertemplate
+ )
assert screenline_plot.object.layout.yaxis.scaleanchor == "x"
assert screenline_plot.object.layout.legend.x == 1.02
assert "Traffic Count Comparisons" not in plot_titles
From fa180fccead42df29e22d3fa1a8bb4b96db9fe25 Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Tue, 4 Aug 2026 15:51:30 -0400
Subject: [PATCH 05/27] replace ^2 with superscript 2; put run name inside
scatterplot hover boxes
---
dashboard/calculation_notes.yaml | 14 +++++++-------
dashboard/pages/validation/_traffic/features.py | 4 ++--
dashboard/pages/validation/_traffic/transforms.py | 4 ++--
dashboard/rendering/figures.py | 4 ++--
processor/summarize/validation_derived.py | 2 +-
tests/test_calculation_notes.py | 2 +-
tests/test_dashboard_live.py | 4 ++--
tests/test_figure_builders.py | 10 +++++-----
tests/test_summary_cache.py | 10 +++++-----
9 files changed, 27 insertions(+), 27 deletions(-)
diff --git a/dashboard/calculation_notes.yaml b/dashboard/calculation_notes.yaml
index 5132112..a0c20f6 100644
--- a/dashboard/calculation_notes.yaml
+++ b/dashboard/calculation_notes.yaml
@@ -657,9 +657,9 @@ notes:
summary: The table summarizes daily count-location validation statistics by facility type.
formula: >-
% difference = (sum(modeled_i) - sum(observed_i)) / sum(observed_i) * 100;
- RMSE = sqrt(sum((modeled_i - observed_i)^2) / n);
- RMSPE = sqrt(mean(((observed_i - modeled_i) / observed_i)^2)) * 100;
- R^2 = 1 - sum((modeled_i - fitted_i)^2) / sum((modeled_i - mean(modeled_i))^2)
+ RMSE = sqrt(sum((modeled_i - observed_i)²) / n);
+ RMSPE = sqrt(mean(((observed_i - modeled_i) / observed_i)²)) * 100;
+ R² = 1 - sum((modeled_i - fitted_i)²) / sum((modeled_i - mean(modeled_i))²)
details:
Aggregation:
- Daily observed and modeled volumes are paired by count location and grouped by facility type.
@@ -675,9 +675,9 @@ notes:
summary: The scatter plot pairs observed traffic counts with modeled count-location volumes for the selected period and facility type.
formula: >-
x = observed count; y = modeled volume;
- slope = sum((x_i - mean(x)) * (y_i - mean(y))) / sum((x_i - mean(x))^2);
+ slope = sum((x_i - mean(x)) * (y_i - mean(y))) / sum((x_i - mean(x))²);
intercept = mean(y) - slope * mean(x); fitted_i = slope * x_i + intercept;
- R^2 = 1 - sum((y_i - fitted_i)^2) / sum((y_i - mean(y))^2);
+ R² = 1 - sum((y_i - fitted_i)²) / sum((y_i - mean(y))²);
one-to-one reference = y = x
details:
Aggregation:
@@ -718,9 +718,9 @@ notes:
summary: The scatter plot compares observed and modeled screenline flows.
formula: >-
x = observed flow; y = modeled flow;
- slope = sum((x_i - mean(x)) * (y_i - mean(y))) / sum((x_i - mean(x))^2);
+ slope = sum((x_i - mean(x)) * (y_i - mean(y))) / sum((x_i - mean(x))²);
intercept = mean(y) - slope * mean(x); fitted_i = slope * x_i + intercept;
- R^2 = 1 - sum((y_i - fitted_i)^2) / sum((y_i - mean(y))^2);
+ R² = 1 - sum((y_i - fitted_i)²) / sum((y_i - mean(y))²);
one-to-one reference = y = x
source_filters:
- Screenline records without an identifier, direction, count period, or observed volume are excluded before matching modeled flows.
diff --git a/dashboard/pages/validation/_traffic/features.py b/dashboard/pages/validation/_traffic/features.py
index 401719e..d093fbd 100644
--- a/dashboard/pages/validation/_traffic/features.py
+++ b/dashboard/pages/validation/_traffic/features.py
@@ -110,8 +110,8 @@ def render_demo_facility_summary_section(self) -> list[pn.viewable.Viewable]:
data_table(
facility_comparison,
title="Count Location Summary by Facility Type",
- numeric_precision_by_column={"RMSE": 3, "R^2": 3},
- column_sorters={"n": "number", "RMSE": "number", "R^2": "number"},
+ numeric_precision_by_column={"RMSE": 3, "R²": 3},
+ column_sorters={"n": "number", "RMSE": "number", "R²": "number"},
)
]
diff --git a/dashboard/pages/validation/_traffic/transforms.py b/dashboard/pages/validation/_traffic/transforms.py
index 8388d69..ec4a860 100644
--- a/dashboard/pages/validation/_traffic/transforms.py
+++ b/dashboard/pages/validation/_traffic/transforms.py
@@ -292,7 +292,7 @@ def screenline_fit_line_data(
sign = "+" if intercept >= 0 else "-"
annotation = (
f"{label}
y = {slope:.2f}x {sign} {abs(intercept):.2f}"
- f"
R^2 = {r_squared:.2f}
n = {points.height}"
+ f"
R² = {r_squared:.2f}
n = {points.height}"
)
out.append(
(
@@ -437,7 +437,7 @@ def demo_facility_comparison_table(
"% Difference": percent_difference,
"RMSE": rmse,
"RMSPE": rmspe,
- "R^2": r_squared_lookup.get(raw_facility_type)
+ "R²": r_squared_lookup.get(raw_facility_type)
if raw_facility_type in r_squared_lookup
else _r_squared_from_points(facility_points),
}
diff --git a/dashboard/rendering/figures.py b/dashboard/rendering/figures.py
index a9e2f20..439bf5c 100644
--- a/dashboard/rendering/figures.py
+++ b/dashboard/rendering/figures.py
@@ -295,8 +295,8 @@ def scatter_figure(
name=str(label), x=x_values, y=y_values, mode="markers",
marker=dict(color=context.color(str(label), index), size=8, line=dict(width=0.4)),
hovertemplate=(
- f"{x_title or x}: %{{x}}
{y_title or y}: %{{y}}"
- f"{label}"
+ f"{label}
{x_title or x}: %{{x}}
"
+ f"{y_title or y}: %{{y}}"
),
))
for index, (label, frame) in enumerate(fit_overlays or []):
diff --git a/processor/summarize/validation_derived.py b/processor/summarize/validation_derived.py
index e83f6e3..4f60b98 100644
--- a/processor/summarize/validation_derived.py
+++ b/processor/summarize/validation_derived.py
@@ -149,7 +149,7 @@ def _fit_group(
intercept=float(intercept),
r_squared=float(r_squared),
equation_label=_equation_label(slope, intercept),
- r_squared_label=f"R^2 = {r_squared:.2f}",
+ r_squared_label=f"R² = {r_squared:.2f}",
)
return base
diff --git a/tests/test_calculation_notes.py b/tests/test_calculation_notes.py
index 6823353..db0f082 100644
--- a/tests/test_calculation_notes.py
+++ b/tests/test_calculation_notes.py
@@ -100,7 +100,7 @@ def test_validation_notes_expose_comparison_and_error_formulas() -> None:
assert "difference = modeled - observed" in regional.formula
assert (
- "RMSPE = sqrt(mean(((observed_i - modeled_i) / observed_i)^2)) * 100"
+ "RMSPE = sqrt(mean(((observed_i - modeled_i) / observed_i)²)) * 100"
in facility.formula
)
diff --git a/tests/test_dashboard_live.py b/tests/test_dashboard_live.py
index 45213b5..878c8ba 100644
--- a/tests/test_dashboard_live.py
+++ b/tests/test_dashboard_live.py
@@ -470,7 +470,7 @@ def test_external_traffic_helpers_filter_period_and_facility_type(
"% Difference": "10.00%",
"RMSE": 22.360679774997898,
"RMSPE": "10.00%",
- "R^2": 0.875,
+ "R²": 0.875,
}
]
@@ -491,7 +491,7 @@ def test_demo_count_fit_line_helper_builds_plot_data() -> None:
"observed_min": [10.0, 20.0],
"observed_max": [30.0, 40.0],
"equation_label": ["y = 2.00x + 5.00", "y = 3.00x + 7.00"],
- "r_squared_label": ["R^2 = 1.00", "R^2 = 0.90"],
+ "r_squared_label": ["R² = 1.00", "R² = 0.90"],
}
),
)
diff --git a/tests/test_figure_builders.py b/tests/test_figure_builders.py
index 96b2435..998fc11 100644
--- a/tests/test_figure_builders.py
+++ b/tests/test_figure_builders.py
@@ -117,7 +117,7 @@ def test_scatter_chart_can_add_one_to_one_reference_line() -> None:
"observed": [0.0, 100.0],
"modeled": [-100.0, 200.0],
"annotation": [
- "Base
y = 3.00x - 100.00
R^2 = 0.90
n = 2"
+ "Base
y = 3.00x - 100.00
R² = 0.90
n = 2"
]
* 2,
}
@@ -133,8 +133,8 @@ def test_scatter_chart_can_add_one_to_one_reference_line() -> None:
reference_line = chart.object.data[-1]
fit_line = chart.object.data[-2]
assert point_trace.hovertemplate == (
- "Observed Count (vehicles): %{x}
"
- "Modeled Volume (vehicles): %{y}Base"
+ "Base
Observed Count (vehicles): %{x}
"
+ "Modeled Volume (vehicles): %{y}"
)
assert fit_line.name == "Base fit"
assert "y = 3.00x - 100.00" in fit_line.hovertemplate
@@ -172,7 +172,7 @@ def test_scatter_fit_details_are_hover_only_for_multiple_runs() -> None:
"observed": [0.0, 100.0],
"modeled": [-100.0, 200.0],
"annotation": [
- f"{label}
y = 3.00x - 100.00
R^2 = 0.90
n = 2"
+ f"{label}
y = 3.00x - 100.00
R² = 0.90
n = 2"
]
* 2,
}
@@ -195,7 +195,7 @@ def test_scatter_fit_details_are_hover_only_for_multiple_runs() -> None:
assert not chart.object.layout.annotations
assert chart.object.layout.height == 400
assert chart.object.layout.margin.t == 90
- assert all("R^2 = 0.90" in trace.hovertemplate for trace in chart.object.data[4:8])
+ assert all("R² = 0.90" in trace.hovertemplate for trace in chart.object.data[4:8])
assert list(chart.object.layout.xaxis.range) == [10.0, 25.0]
assert list(chart.object.layout.yaxis.range) == [10.0, 25.0]
assert chart.aspect_ratio == 1.0
diff --git a/tests/test_summary_cache.py b/tests/test_summary_cache.py
index 37f3b4e..b4c3995 100644
--- a/tests/test_summary_cache.py
+++ b/tests/test_summary_cache.py
@@ -5416,7 +5416,7 @@ def test_traffic_validation_external_volume_table_compares_observed_and_modeled(
"% Difference",
"RMSE",
"RMSPE",
- "R^2",
+ "R²",
]
assert facility_table.to_dict("records") == [
{
@@ -5427,7 +5427,7 @@ def test_traffic_validation_external_volume_table_compares_observed_and_modeled(
"% Difference": "5.00%",
"RMSE": "10",
"RMSPE": "5.00%",
- "R^2": None,
+ "R²": None,
},
{
"Facility Type": "Principal Arterial",
@@ -5437,14 +5437,14 @@ def test_traffic_validation_external_volume_table_compares_observed_and_modeled(
"% Difference": "10.00%",
"RMSE": "10",
"RMSPE": "10.00%",
- "R^2": None,
+ "R²": None,
},
]
assert facility_tables[0]._configuration == {
"columns": [
{"field": "n", "sorter": "number"},
{"field": "RMSE", "sorter": "number"},
- {"field": "R^2", "sorter": "number"},
+ {"field": "R²", "sorter": "number"},
]
}
assert any(
@@ -5504,7 +5504,7 @@ def test_traffic_validation_external_volume_table_compares_observed_and_modeled(
assert screenline_plot.object.data[-1].name == "1:1 line"
assert screenline_plot.object.data[1].name == "Base fit"
assert len(screenline_plot.object.data[1].x) == 101
- assert "R^2" in screenline_plot.object.data[1].hovertemplate
+ assert "R²" in screenline_plot.object.data[1].hovertemplate
assert "y = 1.30x - 5.50" in screenline_plot.object.data[1].hovertemplate
assert not screenline_plot.object.layout.annotations
assert (
From 9234a4d40328d0464b4e1951bb9f9693f00bf937 Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Tue, 4 Aug 2026 16:16:59 -0400
Subject: [PATCH 06/27] Added label wrapping for long run names; unbolded run
name in scatterplot hover box
---
dashboard/export/assets/export.css | 14 +++
dashboard/export/assets/export_runtime.js | 13 +-
dashboard/export/js_runtime/dom.js | 3 +
.../export/js_runtime/plotly_lifecycle.js | 8 +-
.../export/js_runtime/renderers/tables.js | 1 +
dashboard/export/js_runtime/renderers/tabs.js | 1 +
dashboard/export/serializer.py | 50 +++++---
dashboard/export/types.py | 12 +-
dashboard/pages/overview.py | 6 +-
dashboard/pages/validation/regional.py | 39 +++++-
dashboard/rendering/figures.py | 61 ++++++++--
dashboard/rendering/labels.py | 111 ++++++++++++++++++
dashboard/rendering/tables.py | 42 ++++++-
tests/test_dashboard_helpers_phase1.py | 16 +++
tests/test_export_runtime_contract.py | 9 ++
tests/test_export_serializer.py | 25 ++++
tests/test_figure_builders.py | 40 ++++++-
17 files changed, 411 insertions(+), 40 deletions(-)
create mode 100644 dashboard/rendering/labels.py
diff --git a/dashboard/export/assets/export.css b/dashboard/export/assets/export.css
index 56b80e8..f3d446a 100644
--- a/dashboard/export/assets/export.css
+++ b/dashboard/export/assets/export.css
@@ -163,6 +163,13 @@ body {
color: #334155;
}
+.local-tab-button {
+ max-width: 260px;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+}
+
.page-tab-button.active,
.local-tab-button.active {
background: var(--accent);
@@ -452,6 +459,13 @@ table.export-table thead th {
text-align: left;
}
+.export-table-sort-label {
+ min-width: 0;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+}
+
.export-table-sort-indicator {
color: var(--muted);
font-size: 12px;
diff --git a/dashboard/export/assets/export_runtime.js b/dashboard/export/assets/export_runtime.js
index e6ed08a..9b22b98 100644
--- a/dashboard/export/assets/export_runtime.js
+++ b/dashboard/export/assets/export_runtime.js
@@ -91,6 +91,9 @@
});
button.type = "button";
button.disabled = !!config.disabled;
+ if (config.title) {
+ button.title = String(config.title);
+ }
if (!config.disabled && typeof config.onClick === "function") {
button.addEventListener("click", config.onClick);
}
@@ -940,7 +943,13 @@
getTraceFieldLength(trace && trace.x),
getTraceFieldLength(trace && trace.y)
);
- const traceName = trace && trace.name ? trace.name : "trace_" + String(traceIndex + 1);
+ const traceName = (
+ trace && trace.meta && trace.meta.run_name
+ ? trace.meta.run_name
+ : trace && trace.name
+ ? trace.name
+ : "trace_" + String(traceIndex + 1)
+ );
for (let pointIndex = 0; pointIndex < pointCount; pointIndex += 1) {
rows.push([
traceName,
@@ -1359,6 +1368,7 @@
type: "button",
"data-column": column,
"aria-sort": "none",
+ title: (node.column_tooltips || {})[column] || column,
},
}, [
el("span", { className: "export-table-sort-label", text: column }),
@@ -1395,6 +1405,7 @@
tabRow.appendChild(
makeButton({
label: tab.title,
+ title: tab.full_title || tab.title,
active: index === activeIndex,
onClick: () => {
activeIndex = index;
diff --git a/dashboard/export/js_runtime/dom.js b/dashboard/export/js_runtime/dom.js
index 54ae314..394e475 100644
--- a/dashboard/export/js_runtime/dom.js
+++ b/dashboard/export/js_runtime/dom.js
@@ -75,6 +75,9 @@
});
button.type = "button";
button.disabled = !!config.disabled;
+ if (config.title) {
+ button.title = String(config.title);
+ }
if (!config.disabled && typeof config.onClick === "function") {
button.addEventListener("click", config.onClick);
}
diff --git a/dashboard/export/js_runtime/plotly_lifecycle.js b/dashboard/export/js_runtime/plotly_lifecycle.js
index b69d6f3..5283c91 100644
--- a/dashboard/export/js_runtime/plotly_lifecycle.js
+++ b/dashboard/export/js_runtime/plotly_lifecycle.js
@@ -182,7 +182,13 @@
getTraceFieldLength(trace && trace.x),
getTraceFieldLength(trace && trace.y)
);
- const traceName = trace && trace.name ? trace.name : "trace_" + String(traceIndex + 1);
+ const traceName = (
+ trace && trace.meta && trace.meta.run_name
+ ? trace.meta.run_name
+ : trace && trace.name
+ ? trace.name
+ : "trace_" + String(traceIndex + 1)
+ );
for (let pointIndex = 0; pointIndex < pointCount; pointIndex += 1) {
rows.push([
traceName,
diff --git a/dashboard/export/js_runtime/renderers/tables.js b/dashboard/export/js_runtime/renderers/tables.js
index 6143057..1689886 100644
--- a/dashboard/export/js_runtime/renderers/tables.js
+++ b/dashboard/export/js_runtime/renderers/tables.js
@@ -114,6 +114,7 @@
type: "button",
"data-column": column,
"aria-sort": "none",
+ title: (node.column_tooltips || {})[column] || column,
},
}, [
el("span", { className: "export-table-sort-label", text: column }),
diff --git a/dashboard/export/js_runtime/renderers/tabs.js b/dashboard/export/js_runtime/renderers/tabs.js
index cacbc1c..3629b77 100644
--- a/dashboard/export/js_runtime/renderers/tabs.js
+++ b/dashboard/export/js_runtime/renderers/tabs.js
@@ -14,6 +14,7 @@
tabRow.appendChild(
makeButton({
label: tab.title,
+ title: tab.full_title || tab.title,
active: index === activeIndex,
onClick: () => {
activeIndex = index;
diff --git a/dashboard/export/serializer.py b/dashboard/export/serializer.py
index f952361..149a3fb 100644
--- a/dashboard/export/serializer.py
+++ b/dashboard/export/serializer.py
@@ -142,23 +142,28 @@ def _container_css_classes(viewable: Any) -> list[str]:
"css_classes": _container_css_classes(obj),
}
if isinstance(obj, pn.Tabs):
+ full_titles = tuple(getattr(obj, "_run_label_full_titles", ()))
+ serialized_tabs = []
+ for index, (title, child) in enumerate(iter_tabs(obj)):
+ if _is_hidden_view(child):
+ continue
+ tab = {
+ "title": title,
+ "content": serialize_viewable(
+ child,
+ disable_widgets=disable_widgets,
+ widget_metadata=widget_metadata,
+ region_nodes_by_id=region_nodes_by_id,
+ hidden_widget_ids=hidden_widget_ids,
+ hidden_view_ids=hidden_view_ids,
+ ),
+ }
+ if index < len(full_titles) and full_titles[index] != title:
+ tab["full_title"] = full_titles[index]
+ serialized_tabs.append(tab)
return {
"kind": "tabs",
- "tabs": [
- {
- "title": title,
- "content": serialize_viewable(
- child,
- disable_widgets=disable_widgets,
- widget_metadata=widget_metadata,
- region_nodes_by_id=region_nodes_by_id,
- hidden_widget_ids=hidden_widget_ids,
- hidden_view_ids=hidden_view_ids,
- ),
- }
- for title, child in iter_tabs(obj)
- if not _is_hidden_view(child)
- ],
+ "tabs": serialized_tabs,
}
if isinstance(obj, pn.pane.Plotly):
figure = obj.object.to_plotly_json()
@@ -174,7 +179,17 @@ def _container_css_classes(viewable: Any) -> list[str]:
}
columns = [str(column) for column in frame.columns]
display_columns = [title_map.get(column, column) for column in columns]
- return {
+ header_tooltips = {
+ str(column): str(tooltip)
+ for column, tooltip in (obj.header_tooltips or {}).items()
+ if tooltip is not None
+ }
+ column_tooltips = {
+ display_column: header_tooltips[column]
+ for column, display_column in zip(columns, display_columns)
+ if column in header_tooltips
+ }
+ table = {
"kind": "table",
"columns": display_columns,
"rows": [
@@ -185,6 +200,9 @@ def _container_css_classes(viewable: Any) -> list[str]:
for row in frame.to_dict(orient="records")
],
}
+ if column_tooltips:
+ table["column_tooltips"] = column_tooltips
+ return table
if isinstance(obj, pn.widgets.RadioButtonGroup):
if id(obj) in hidden_widget_ids:
return {"kind": "spacer", "height": 0, "width": 0}
diff --git a/dashboard/export/types.py b/dashboard/export/types.py
index efa4f24..7d2c7e4 100644
--- a/dashboard/export/types.py
+++ b/dashboard/export/types.py
@@ -91,7 +91,11 @@ class CardNode(TypedDict):
children: list["ExportNode"]
-class TabPayload(TypedDict):
+class OptionalTabPayload(TypedDict, total=False):
+ full_title: str
+
+
+class TabPayload(OptionalTabPayload):
title: str
content: "ExportNode"
@@ -106,7 +110,11 @@ class PlotlyNode(TypedDict):
figure: dict[str, Any]
-class TableNode(TypedDict):
+class OptionalTableNode(TypedDict, total=False):
+ column_tooltips: dict[str, str]
+
+
+class TableNode(OptionalTableNode):
kind: Literal["table"]
columns: list[str]
rows: list[dict[str, Any]]
diff --git a/dashboard/pages/overview.py b/dashboard/pages/overview.py
index 16b7e88..bf93607 100644
--- a/dashboard/pages/overview.py
+++ b/dashboard/pages/overview.py
@@ -7,10 +7,10 @@
from dashboard.rendering import (
to_pandas,
- column_titles,
drop_index_columns,
format_numeric_frame,
)
+from dashboard.rendering.tables import column_title_metadata
from dashboard.helpers.comparison_helpers import (
build_base_run_percent_difference_table,
)
@@ -182,11 +182,13 @@ def render_percent_difference_table(
drop_index_columns(pct_df),
numeric_precision=2,
)
+ titles, header_tooltips = column_title_metadata(display_df.columns)
return pn.widgets.Tabulator(
to_pandas(display_df),
sizing_mode="stretch_width",
height=260,
- titles=column_titles(display_df.columns),
+ titles=titles,
+ header_tooltips=header_tooltips,
show_index=False,
)
diff --git a/dashboard/pages/validation/regional.py b/dashboard/pages/validation/regional.py
index c5fffba..9c8b263 100644
--- a/dashboard/pages/validation/regional.py
+++ b/dashboard/pages/validation/regional.py
@@ -10,6 +10,11 @@
from dashboard.rendering import selector_row
from dashboard.helpers.category_helpers import nonempty
+from dashboard.rendering.labels import (
+ attach_full_tab_titles,
+ display_label_map,
+ hover_label,
+)
from dashboard import DashboardPage, dashboard_page
TOTAL_FLOW_LABELS = {"total", "all", "all_geographies"}
@@ -325,7 +330,11 @@ def flow_heatmap(
title: str,
) -> pn.viewable.Viewable:
tabs = pn.Tabs()
- for label, df in nonempty(data_list):
+ runs = nonempty(data_list)
+ full_labels = [str(label) for label, _ in runs]
+ display_labels = display_label_map(full_labels)
+ for label, df in runs:
+ full_label = str(label)
matrix = normalize_flow_matrix(df, include_totals=include_totals)
destinations = [column for column in matrix.columns if column != "Origin"]
z = matrix.select(destinations).to_numpy().tolist()
@@ -340,7 +349,8 @@ def flow_heatmap(
y=matrix["Origin"].cast(pl.Utf8).to_list(),
colorscale="Blues",
hovertemplate=(
- "Origin: %{y}
Destination: %{x}
Flow: %{z:,.0f}"
+ f"{hover_label(full_label)}
Origin: %{{y}}
"
+ "Destination: %{x}
Flow: %{z:,.0f}"
),
)
)
@@ -352,7 +362,13 @@ def flow_heatmap(
margin=dict(l=70, r=20, t=80, b=70),
font=dict(family="Inter, Segoe UI, Arial, sans-serif", size=12),
)
- tabs.append((label, pn.pane.Plotly(fig, sizing_mode="stretch_width")))
+ tabs.append(
+ (
+ display_labels[full_label],
+ pn.pane.Plotly(fig, sizing_mode="stretch_width"),
+ )
+ )
+ attach_full_tab_titles(tabs, full_labels)
return tabs
@@ -365,7 +381,11 @@ def flow_comparison_heatmap(
"""Render aligned observed/modeled flow comparisons as heatmaps."""
value_col = FLOW_VALUE_COLUMNS[metric]
tabs = pn.Tabs()
- for label, df in nonempty(data_list):
+ runs = nonempty(data_list)
+ full_labels = [str(label) for label, _ in runs]
+ display_labels = display_label_map(full_labels)
+ for label, df in runs:
+ full_label = str(label)
if df.is_empty():
continue
origins = _flow_label_order(df["Origin"].to_list(), include_totals=True)
@@ -407,7 +427,8 @@ def flow_comparison_heatmap(
"y": origins,
"colorscale": colorscale,
"hovertemplate": (
- "Origin: %{y}
Destination: %{x}
"
+ f"{hover_label(full_label)}
Origin: %{{y}}
"
+ "Destination: %{x}
"
f"{metric}: %{{text}}"
),
}
@@ -422,7 +443,13 @@ def flow_comparison_heatmap(
margin=dict(l=70, r=20, t=80, b=70),
font=dict(family="Inter, Segoe UI, Arial, sans-serif", size=12),
)
- tabs.append((label, pn.pane.Plotly(fig, sizing_mode="stretch_width")))
+ tabs.append(
+ (
+ display_labels[full_label],
+ pn.pane.Plotly(fig, sizing_mode="stretch_width"),
+ )
+ )
+ attach_full_tab_titles(tabs, full_labels)
return tabs
diff --git a/dashboard/rendering/figures.py b/dashboard/rendering/figures.py
index 439bf5c..773b1a9 100644
--- a/dashboard/rendering/figures.py
+++ b/dashboard/rendering/figures.py
@@ -11,6 +11,7 @@
from dashboard.data_access import RunTableData, RunTables
from dashboard.rendering.context import RenderContext
+from dashboard.rendering.labels import display_label_map, hover_label
ChartTables = RunTables | RunTableData
ChartValueMode = Literal["dashboard", "count", "share"]
@@ -81,11 +82,20 @@ def _point_hover(
share: bool,
) -> str:
return (
- f"{label}
{x_title or x}: {x_value}"
+ f"{hover_label(label)}
{x_title or x}: {x_value}"
f"
{_y_title(y_title or y, share)}: {_hover_value(y_value, y_title, share)}"
)
+def _run_labels(context: RenderContext, data: ChartTables) -> list[str]:
+ labels = list(context.run_labels)
+ for label, _ in data:
+ text = str(label)
+ if text not in labels:
+ labels.append(text)
+ return labels
+
+
def bar_figure(
context: RenderContext,
data: ChartTables,
@@ -103,9 +113,11 @@ def bar_figure(
show_legend: bool | None = None,
) -> go.Figure:
"""Build a grouped/stacked categorical figure."""
+ data = list(data)
_require_columns(data, "bar", x, y)
share = _share_mode(context, value_mode)
figure = go.Figure()
+ legend_labels = display_label_map(_run_labels(context, data))
observed_order: list[object] = []
for index, (label, frame) in enumerate(data):
if frame.is_empty():
@@ -135,8 +147,9 @@ def bar_figure(
]
figure.add_trace(
go.Bar(
- name=str(label), x=x_values, y=y_values,
+ name=legend_labels[str(label)], x=x_values, y=y_values,
marker_color=context.color(str(label), index),
+ meta={"run_name": str(label)},
hovertemplate="%{customdata}", customdata=hover,
)
)
@@ -166,16 +179,29 @@ def line_figure(
value_mode: ChartValueMode = "dashboard",
height: int = 350,
) -> go.Figure:
+ data = list(data)
_require_columns(data, "line", x, y)
share = _share_mode(context, value_mode)
figure = go.Figure()
+ legend_labels = display_label_map(_run_labels(context, data))
for index, (label, frame) in enumerate(data):
+ x_values = frame[x].to_list()
values = np.asarray(frame[y].to_list(), dtype=float)
if share and values.sum() > 0:
values = values / values.sum() * 100.0
+ y_values = values.tolist()
+ hover = [
+ _point_hover(
+ str(label), x_value, y_value, x=x, y=y,
+ x_title=x_title, y_title=y_title, share=share,
+ )
+ for x_value, y_value in zip(x_values, y_values)
+ ]
figure.add_trace(go.Scatter(
- name=str(label), x=frame[x].to_list(), y=values.tolist(), mode="lines",
+ name=legend_labels[str(label)], x=x_values, y=y_values, mode="lines",
line=dict(color=context.color(str(label), index), width=2),
+ meta={"run_name": str(label)},
+ hovertemplate="%{customdata}", customdata=hover,
))
_layout(figure, title=title, x_title=x_title, y_title=_y_title(y_title, share), height=height)
return figure
@@ -212,9 +238,11 @@ def density_figure(
tick_text: list[str] | None = None,
hover_x_title: str | None = None,
) -> go.Figure:
+ data = list(data)
_require_columns(data, "density", x, y)
share = _share_mode(context, value_mode)
figure = go.Figure()
+ legend_labels = display_label_map(_run_labels(context, data))
observed_x: list[object] = []
for index, (label, frame) in enumerate(data):
x_values = frame[x].to_list()
@@ -230,8 +258,9 @@ def density_figure(
for xv, yv in zip(x_values, y_values)
]
figure.add_trace(go.Scatter(
- name=str(label), x=x_values, y=y_values, mode="lines",
+ name=legend_labels[str(label)], x=x_values, y=y_values, mode="lines",
line=dict(color=color, width=2), fill="tozeroy",
+ meta={"run_name": str(label)},
hovertemplate="%{customdata}", customdata=hover,
))
_layout(figure, title=title, x_title=x_title, y_title=_y_title(y_title, share), height=height)
@@ -279,8 +308,17 @@ def scatter_figure(
one_to_one: bool = False,
legend_on_right: bool = False,
) -> go.Figure:
+ data = list(data)
+ fit_overlays = list(fit_overlays or [])
_require_columns(data, "scatter", x, y)
figure = go.Figure()
+ all_run_labels = _run_labels(context, data)
+ for label, _ in fit_overlays:
+ if str(label) not in all_run_labels:
+ all_run_labels.append(str(label))
+ legend_labels = display_label_map(
+ [*all_run_labels, *(f"{label} fit" for label in all_run_labels)]
+ )
label_indices = {str(label): index for index, (label, _) in enumerate(data)}
axis_values: list[float] = []
for index, (label, frame) in enumerate(data):
@@ -292,24 +330,31 @@ def scatter_figure(
if one_to_one:
axis_values.extend(_finite([*x_values, *y_values]))
figure.add_trace(go.Scatter(
- name=str(label), x=x_values, y=y_values, mode="markers",
+ name=legend_labels[str(label)], x=x_values, y=y_values, mode="markers",
marker=dict(color=context.color(str(label), index), size=8, line=dict(width=0.4)),
+ legendgroup=str(label),
+ meta={"run_name": str(label)},
hovertemplate=(
- f"{label}
{x_title or x}: %{{x}}
"
+ f"{hover_label(label)}
{x_title or x}: %{{x}}
"
f"{y_title or y}: %{{y}}"
),
))
- for index, (label, frame) in enumerate(fit_overlays or []):
+ for index, (label, frame) in enumerate(fit_overlays):
if frame.is_empty() or x not in frame.columns or y not in frame.columns:
continue
color = context.color(str(label), label_indices.get(str(label), index))
- trace_name = f"{label} fit"
+ trace_name = legend_labels[f"{label} fit"]
annotation = ""
if fit_annotation in frame.columns:
annotation = str(frame[fit_annotation][0] or "").strip()
+ prefix = f"{label}
"
+ if annotation.startswith(prefix):
+ annotation = f"{hover_label(label)}
{annotation[len(prefix):]}"
figure.add_trace(go.Scatter(
name=trace_name, x=frame[x].to_list(), y=frame[y].to_list(),
mode="lines", line=dict(color=color, width=2),
+ legendgroup=str(label),
+ meta={"run_name": f"{label} fit"},
hovertemplate=(f"{annotation}" if annotation else None),
))
if one_to_one:
diff --git a/dashboard/rendering/labels.py b/dashboard/rendering/labels.py
new file mode 100644
index 0000000..3e6635f
--- /dev/null
+++ b/dashboard/rendering/labels.py
@@ -0,0 +1,111 @@
+"""Presentation-only shortening and wrapping for run labels."""
+
+from __future__ import annotations
+
+import html
+import textwrap
+from collections.abc import Iterable
+from typing import Any
+
+
+MAX_DISPLAY_LABEL_LENGTH = 30
+HOVER_LABEL_LINE_LENGTH = 36
+
+
+def _truncate_middle(label: str, max_length: int) -> str:
+ if len(label) <= max_length:
+ return label
+ if max_length <= 1:
+ return "…"[:max_length]
+
+ words = label.split()
+ if len(words) >= 3 and len(words[0]) + len(words[-1]) + 1 <= max_length:
+ leading_words = [words[0]]
+ trailing_words = [words[-1]]
+ leading_index = 1
+ trailing_index = len(words) - 2
+ while leading_index <= trailing_index:
+ changed = False
+ leading_candidate = (
+ " ".join([*leading_words, words[leading_index]])
+ + "…"
+ + " ".join(trailing_words)
+ )
+ if len(leading_candidate) <= max_length:
+ leading_words.append(words[leading_index])
+ leading_index += 1
+ changed = True
+ trailing_candidate = (
+ " ".join(leading_words)
+ + "…"
+ + " ".join([words[trailing_index], *trailing_words])
+ )
+ if leading_index <= trailing_index and len(trailing_candidate) <= max_length:
+ trailing_words.insert(0, words[trailing_index])
+ trailing_index -= 1
+ changed = True
+ if not changed:
+ break
+ return " ".join(leading_words) + "…" + " ".join(trailing_words)
+
+ available = max_length - 1
+ leading_length = (available * 2 + 2) // 3
+ trailing_length = available - leading_length
+ leading = label[:leading_length].rstrip()
+ trailing = label[-trailing_length:].lstrip() if trailing_length else ""
+ return f"{leading}…{trailing}"
+
+
+def display_label_map(
+ labels: Iterable[object],
+ *,
+ max_length: int = MAX_DISPLAY_LABEL_LENGTH,
+) -> dict[str, str]:
+ """Return stable, unique display labels without changing full identities."""
+ full_labels = list(dict.fromkeys(str(label) for label in labels))
+ candidates = {
+ label: _truncate_middle(label, max_length)
+ for label in full_labels
+ }
+ groups: dict[str, list[str]] = {}
+ for label, candidate in candidates.items():
+ groups.setdefault(candidate, []).append(label)
+
+ output: dict[str, str] = {}
+ used: set[str] = set()
+ for label in full_labels:
+ candidate = candidates[label]
+ duplicates = groups[candidate]
+ if len(duplicates) == 1 and candidate not in used:
+ output[label] = candidate
+ used.add(candidate)
+ continue
+
+ index = duplicates.index(label) + 1
+ while True:
+ suffix = f" [{index}]"
+ unique_candidate = (
+ _truncate_middle(label, max_length - len(suffix)) + suffix
+ )
+ if unique_candidate not in used:
+ output[label] = unique_candidate
+ used.add(unique_candidate)
+ break
+ index += 1
+ return output
+
+
+def hover_label(label: object) -> str:
+ """Return the escaped full label with line breaks suitable for Plotly."""
+ lines = textwrap.wrap(
+ str(label),
+ width=HOVER_LABEL_LINE_LENGTH,
+ break_long_words=True,
+ break_on_hyphens=False,
+ ) or [""]
+ return "
".join(html.escape(line) for line in lines)
+
+
+def attach_full_tab_titles(tabs: Any, labels: Iterable[object]) -> None:
+ """Attach full titles for the standalone-export serializer."""
+ tabs._run_label_full_titles = tuple(str(label) for label in labels)
diff --git a/dashboard/rendering/tables.py b/dashboard/rendering/tables.py
index 362ec46..1f0b030 100644
--- a/dashboard/rendering/tables.py
+++ b/dashboard/rendering/tables.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import html
import math
import numpy as np
@@ -9,6 +10,7 @@
import polars as pl
from dashboard.data_access import RunTableData, RunTables
+from dashboard.rendering.labels import attach_full_tab_titles, display_label_map
TableData = RunTables | RunTableData
@@ -125,6 +127,24 @@ def column_titles(columns: list[object] | tuple[object, ...]) -> dict[str, str]:
return titles
+def column_title_metadata(
+ columns: list[object] | tuple[object, ...],
+) -> tuple[dict[str, str], dict[str, str]]:
+ """Return compact column titles and full tooltips for truncated titles."""
+ full_titles = column_titles(columns)
+ display_titles = display_label_map(full_titles.values())
+ titles = {
+ column: display_titles[full_title]
+ for column, full_title in full_titles.items()
+ }
+ tooltips = {
+ column: full_title
+ for column, full_title in full_titles.items()
+ if titles[column] != full_title
+ }
+ return titles, tooltips
+
+
def data_table(
data: TableData,
title: str = "",
@@ -133,10 +153,14 @@ def data_table(
numeric_precision_by_column: dict[str, int] | None = None,
column_sorters: dict[str, str] | None = None,
) -> pn.viewable.Viewable:
+ data = list(data)
tabs = pn.Tabs()
+ full_labels = [str(label) for label, frame in data if not frame.is_empty()]
+ display_labels = display_label_map(full_labels)
for label, frame in data:
if frame.is_empty():
continue
+ full_label = str(label)
display = format_numeric_frame(
drop_index_columns(frame),
numeric_precision=numeric_precision,
@@ -149,11 +173,23 @@ def data_table(
for column, sorter in column_sorters.items()
if str(column) in display.columns
]
- tabs.append((label, pn.widgets.Tabulator(
+ titles, header_tooltips = column_title_metadata(display.columns)
+ table = pn.widgets.Tabulator(
to_pandas(display), height=height, sizing_mode="stretch_width",
- theme="simple", titles=column_titles(display.columns),
+ theme="simple", titles=titles, header_tooltips=header_tooltips,
show_index=False, configuration=configuration,
- )))
+ )
+ tab_content: pn.viewable.Viewable = table
+ if display_labels[full_label] != full_label:
+ tab_content = pn.Column(
+ pn.pane.HTML(
+ f'Run: '
+ f"{html.escape(full_label)}
"
+ ),
+ table,
+ )
+ tabs.append((display_labels[full_label], tab_content))
+ attach_full_tab_titles(tabs, full_labels)
return pn.Column(pn.pane.Markdown(f"### {title}"), tabs) if title else tabs
diff --git a/tests/test_dashboard_helpers_phase1.py b/tests/test_dashboard_helpers_phase1.py
index fa9da15..73acf66 100644
--- a/tests/test_dashboard_helpers_phase1.py
+++ b/tests/test_dashboard_helpers_phase1.py
@@ -474,6 +474,22 @@ def test_data_table_drops_index_columns_and_hides_pandas_index() -> None:
assert tabulator.titles == {"metric": "Metric", "value": "Value"}
+def test_data_table_shortens_long_run_tabs_and_column_titles() -> None:
+ run_label = "Regional Transportation Scenario Baseline 2050 North"
+ column = "regional_transportation_scenario_comparison_measure"
+ table = data_table([(run_label, pl.DataFrame({column: [1.0]}))])
+
+ assert table._names == ["Regional Transportation…North"]
+ assert table._run_label_full_titles == (run_label,)
+ content = table.objects[0]
+ assert run_label in content.objects[0].object
+ tabulator = content.objects[1]
+ assert len(tabulator.titles[column]) <= 30
+ assert tabulator.header_tooltips == {
+ column: "Regional Transportation Scenario Comparison Measure"
+ }
+
+
def test_column_titles_for_display_humanizes_machine_column_names() -> None:
titles = column_titles(
[
diff --git a/tests/test_export_runtime_contract.py b/tests/test_export_runtime_contract.py
index 4e03d4a..f9e084f 100644
--- a/tests/test_export_runtime_contract.py
+++ b/tests/test_export_runtime_contract.py
@@ -218,6 +218,7 @@ def test_runtime_asset_contains_explicit_context_action_and_region_helpers() ->
assert "function createRuntimeContext(config)" in runtime_js
assert "function createRuntimeActions(context)" in runtime_js
assert "function makeButton(config)" in runtime_js
+ assert "button.title = String(config.title);" in runtime_js
assert "function buildRegionVariantKey(selectorValues)" in runtime_js
assert "const PLOT_RESIZE_RETRY_DELAYS_MS = [60, 180, 320];" in runtime_js
assert 'displayModeBar: "hover"' in runtime_js
@@ -227,6 +228,13 @@ def test_runtime_asset_contains_explicit_context_action_and_region_helpers() ->
assert "modeBarButtonsToAdd: [makePlotCsvDownloadButton(figure)]" in runtime_js
+def test_runtime_asset_exposes_full_table_and_tab_titles_as_tooltips() -> None:
+ runtime_js = load_export_runtime_js()
+
+ assert "title: tab.full_title || tab.title" in runtime_js
+ assert "(node.column_tooltips || {})[column] || column" in runtime_js
+
+
def test_runtime_asset_contains_plot_csv_export_helpers() -> None:
runtime_js = load_export_runtime_js()
@@ -239,6 +247,7 @@ def test_runtime_asset_contains_plot_csv_export_helpers() -> None:
assert '"y"' in runtime_js
assert '"trace_index"' not in runtime_js
assert '"customdata"' not in runtime_js
+ assert "trace.meta.run_name" in runtime_js
assert '"-" + valueMode + ".csv"' in runtime_js
assert 'return normalized || "plot-data";' in runtime_js
diff --git a/tests/test_export_serializer.py b/tests/test_export_serializer.py
index 6620064..bb82549 100644
--- a/tests/test_export_serializer.py
+++ b/tests/test_export_serializer.py
@@ -240,6 +240,31 @@ def test_sanitize_export_payload_removes_nan_and_infinity() -> None:
}
+def test_serialize_viewable_preserves_shortened_tab_and_column_tooltips() -> None:
+ tabs = pn.Tabs(
+ (
+ "Regional Transportat…050 North",
+ pn.widgets.Tabulator(
+ pd.DataFrame({"long_column": [1]}),
+ titles={"long_column": "Long Column…Title"},
+ header_tooltips={"long_column": "Long Column Full Title"},
+ ),
+ )
+ )
+ tabs._run_label_full_titles = (
+ "Regional Transportation Scenario Baseline 2050 North",
+ )
+
+ payload = serialize_viewable(tabs, disable_widgets=True)
+
+ assert payload["tabs"][0]["full_title"] == (
+ "Regional Transportation Scenario Baseline 2050 North"
+ )
+ assert payload["tabs"][0]["content"]["column_tooltips"] == {
+ "Long Column…Title": "Long Column Full Title"
+ }
+
+
def test_sanitize_export_payload_in_place_retains_existing_containers() -> None:
nested = [1.0, np.float64(2.5), float("nan")]
payload = {
diff --git a/tests/test_figure_builders.py b/tests/test_figure_builders.py
index 998fc11..316b869 100644
--- a/tests/test_figure_builders.py
+++ b/tests/test_figure_builders.py
@@ -7,6 +7,25 @@
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from dashboard.rendering import Plotter, RenderContext
+from dashboard.rendering.labels import display_label_map
+
+
+def test_display_run_labels_preserve_distinct_ends_and_deduplicate_collisions() -> None:
+ labels = [
+ "Regional Transportation Scenario Baseline 2050 North",
+ "Regional Transportation Scenario Baseline 2050 South",
+ f"{'A' * 20} first {'Z' * 12}",
+ f"{'A' * 20} second {'Z' * 12}",
+ ]
+
+ display_labels = display_label_map(labels)
+
+ assert display_labels[labels[0]] == "Regional Transportation…North"
+ assert display_labels[labels[1]] == "Regional Transportation…South"
+ assert len(set(display_labels.values())) == len(labels)
+ assert all(len(label) <= 30 for label in display_labels.values())
+ assert display_labels[labels[2]].endswith("[1]")
+ assert display_labels[labels[3]].endswith("[2]")
def test_figure_first_bar_omits_undeclared_hover_columns() -> None:
@@ -77,6 +96,25 @@ def test_bar_and_density_chart_hover_formatting_matches_units() -> None:
)
+def test_long_run_names_are_short_in_legends_and_full_in_hovers() -> None:
+ label = "Regional Transportation Scenario Baseline 2050 North"
+ data = [(label, pl.DataFrame({"period": [1, 2], "value": [10.0, 20.0]}))]
+ context = RenderContext(run_labels=(label,))
+
+ bar = Plotter(context).figure.bar(data, x="period", y="value")
+ line = Plotter(context).figure.line(data, x="period", y="value")
+ density = Plotter(context).figure.density(data, x="period", y="value")
+ scatter = Plotter(context).figure.scatter(data, x="period", y="value")
+
+ for figure in (bar, line, density, scatter):
+ trace = figure.data[0]
+ assert trace.name == "Regional Transportation…North"
+ assert trace.meta == {"run_name": label}
+ hover = trace.hovertemplate + "".join(map(str, trace.customdata or []))
+ assert "Regional Transportation Scenario
Baseline 2050 North" in hover
+ assert scatter.data[0].legendgroup == label
+
+
def test_bar_chart_uses_configured_all_series_hover_mode() -> None:
data = [
("Base", pl.DataFrame({"mode": ["Walk", "Bike"], "trip_count": [5.0, 1.0]})),
@@ -133,7 +171,7 @@ def test_scatter_chart_can_add_one_to_one_reference_line() -> None:
reference_line = chart.object.data[-1]
fit_line = chart.object.data[-2]
assert point_trace.hovertemplate == (
- "Base
Observed Count (vehicles): %{x}
"
+ "Base
Observed Count (vehicles): %{x}
"
"Modeled Volume (vehicles): %{y}"
)
assert fit_line.name == "Base fit"
From d543222cc1ece1edf0da6e888b4aebf242c603c1 Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Tue, 4 Aug 2026 17:15:29 -0400
Subject: [PATCH 07/27] make sidebar in export collapsible
---
dashboard/export/assets/export.css | 41 +++++++++++++++++-
dashboard/export/assets/export_runtime.js | 44 +++++++++++++++-----
dashboard/export/js_runtime/index.js | 1 +
dashboard/export/js_runtime/renderers/app.js | 43 ++++++++++++++-----
tests/test_export_html_smoke.py | 2 +
tests/test_export_runtime_contract.py | 12 ++++++
6 files changed, 121 insertions(+), 22 deletions(-)
diff --git a/dashboard/export/assets/export.css b/dashboard/export/assets/export.css
index 56b80e8..67af453 100644
--- a/dashboard/export/assets/export.css
+++ b/dashboard/export/assets/export.css
@@ -31,14 +31,43 @@ body {
border-top: 8px solid var(--accent);
}
+.export-header-top {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ gap: 16px;
+}
+
.export-header h1 {
- margin: 0 0 8px;
+ margin: 0;
font-size: 30px;
+ min-width: 0;
}
.export-note {
color: #4b5563;
- margin: 0;
+ margin: 8px 0 0;
+}
+
+.rail-toggle {
+ flex: 0 0 auto;
+ border: 1px solid var(--line);
+ background: var(--surface-soft);
+ color: #334155;
+ border-radius: 10px;
+ padding: 8px 12px;
+ font-weight: 600;
+ cursor: pointer;
+}
+
+.rail-toggle:hover {
+ border-color: var(--accent);
+ color: var(--accent-dark);
+}
+
+.rail-toggle:focus-visible {
+ outline: 3px solid rgba(78, 121, 167, 0.3);
+ outline-offset: 2px;
}
.export-layout {
@@ -48,6 +77,14 @@ body {
align-items: start;
}
+.export-layout.rail-collapsed {
+ grid-template-columns: minmax(0, 1fr);
+}
+
+.export-layout.rail-collapsed .export-rail {
+ display: none;
+}
+
.export-rail,
.export-main {
min-width: 0;
diff --git a/dashboard/export/assets/export_runtime.js b/dashboard/export/assets/export_runtime.js
index e6ed08a..af7372a 100644
--- a/dashboard/export/assets/export_runtime.js
+++ b/dashboard/export/assets/export_runtime.js
@@ -1978,8 +1978,39 @@
}
function renderShell(context, actions) {
+ const railCollapsed = !!context.railCollapsed;
+ const rail = renderRail(context, actions);
+ rail.id = "export-rail";
+
+ const main = el("main", { className: "export-main" }, [
+ renderPageTabs(context, actions),
+ renderPagePanel(context, actions),
+ ]);
+ const layout = el("div", {
+ className: "export-layout" + (railCollapsed ? " rail-collapsed" : ""),
+ }, [rail, main]);
+ const railToggle = el("button", {
+ className: "rail-toggle",
+ text: railCollapsed ? "Show sidebar" : "Hide sidebar",
+ attrs: {
+ "aria-controls": "export-rail",
+ "aria-expanded": String(!railCollapsed),
+ },
+ });
+ railToggle.type = "button";
+ railToggle.addEventListener("click", () => {
+ context.railCollapsed = !context.railCollapsed;
+ layout.classList.toggle("rail-collapsed", context.railCollapsed);
+ railToggle.textContent = context.railCollapsed ? "Show sidebar" : "Hide sidebar";
+ railToggle.setAttribute("aria-expanded", String(!context.railCollapsed));
+ context.plotManager.scheduleResize();
+ });
+
const headerChildren = [
- el("h1", { text: context.payload.title }),
+ el("div", { className: "export-header-top" }, [
+ el("h1", { text: context.payload.title }),
+ railToggle,
+ ]),
];
if (context.payload.client_export_note && String(context.payload.client_export_note).trim()) {
headerChildren.push(
@@ -1990,17 +2021,9 @@
);
}
- const main = el("main", { className: "export-main" }, [
- renderPageTabs(context, actions),
- renderPagePanel(context, actions),
- ]);
-
return el("div", { className: "export-shell" }, [
el("div", { className: "export-header" }, headerChildren),
- el("div", { className: "export-layout" }, [
- renderRail(context, actions),
- main,
- ]),
+ layout,
]);
}
@@ -2039,6 +2062,7 @@
plotManager: config.plotManager,
app: config.app,
renderedRegions: {},
+ railCollapsed: false,
};
}
diff --git a/dashboard/export/js_runtime/index.js b/dashboard/export/js_runtime/index.js
index f637b2f..98af6f4 100644
--- a/dashboard/export/js_runtime/index.js
+++ b/dashboard/export/js_runtime/index.js
@@ -13,6 +13,7 @@
plotManager: config.plotManager,
app: config.app,
renderedRegions: {},
+ railCollapsed: false,
};
}
diff --git a/dashboard/export/js_runtime/renderers/app.js b/dashboard/export/js_runtime/renderers/app.js
index fa37674..2297aec 100644
--- a/dashboard/export/js_runtime/renderers/app.js
+++ b/dashboard/export/js_runtime/renderers/app.js
@@ -253,8 +253,39 @@
}
function renderShell(context, actions) {
+ const railCollapsed = !!context.railCollapsed;
+ const rail = renderRail(context, actions);
+ rail.id = "export-rail";
+
+ const main = el("main", { className: "export-main" }, [
+ renderPageTabs(context, actions),
+ renderPagePanel(context, actions),
+ ]);
+ const layout = el("div", {
+ className: "export-layout" + (railCollapsed ? " rail-collapsed" : ""),
+ }, [rail, main]);
+ const railToggle = el("button", {
+ className: "rail-toggle",
+ text: railCollapsed ? "Show sidebar" : "Hide sidebar",
+ attrs: {
+ "aria-controls": "export-rail",
+ "aria-expanded": String(!railCollapsed),
+ },
+ });
+ railToggle.type = "button";
+ railToggle.addEventListener("click", () => {
+ context.railCollapsed = !context.railCollapsed;
+ layout.classList.toggle("rail-collapsed", context.railCollapsed);
+ railToggle.textContent = context.railCollapsed ? "Show sidebar" : "Hide sidebar";
+ railToggle.setAttribute("aria-expanded", String(!context.railCollapsed));
+ context.plotManager.scheduleResize();
+ });
+
const headerChildren = [
- el("h1", { text: context.payload.title }),
+ el("div", { className: "export-header-top" }, [
+ el("h1", { text: context.payload.title }),
+ railToggle,
+ ]),
];
if (context.payload.client_export_note && String(context.payload.client_export_note).trim()) {
headerChildren.push(
@@ -265,17 +296,9 @@
);
}
- const main = el("main", { className: "export-main" }, [
- renderPageTabs(context, actions),
- renderPagePanel(context, actions),
- ]);
-
return el("div", { className: "export-shell" }, [
el("div", { className: "export-header" }, headerChildren),
- el("div", { className: "export-layout" }, [
- renderRail(context, actions),
- main,
- ]),
+ layout,
]);
}
diff --git a/tests/test_export_html_smoke.py b/tests/test_export_html_smoke.py
index 33f7790..6387b45 100644
--- a/tests/test_export_html_smoke.py
+++ b/tests/test_export_html_smoke.py
@@ -130,6 +130,8 @@ def test_export_runtime_assets_are_loaded_from_source_files() -> None:
assert ".export-shell" in css
assert ".export-error-panel" in css
assert ".export-table-sort" in css
+ assert ".export-layout.rail-collapsed" in css
+ assert ".export-layout.rail-collapsed .export-rail" in css
assert "function validatePayloadSchema(candidate)" in runtime_js
assert "function renderPlot(node, context)" in runtime_js
assert "function renderTable(node)" in runtime_js
diff --git a/tests/test_export_runtime_contract.py b/tests/test_export_runtime_contract.py
index 4e03d4a..7ee4ef0 100644
--- a/tests/test_export_runtime_contract.py
+++ b/tests/test_export_runtime_contract.py
@@ -227,6 +227,18 @@ def test_runtime_asset_contains_explicit_context_action_and_region_helpers() ->
assert "modeBarButtonsToAdd: [makePlotCsvDownloadButton(figure)]" in runtime_js
+def test_runtime_asset_contains_collapsible_export_rail() -> None:
+ runtime_js = load_export_runtime_js()
+
+ assert "railCollapsed: false" in runtime_js
+ assert 'rail.id = "export-rail"' in runtime_js
+ assert '"aria-controls": "export-rail"' in runtime_js
+ assert 'className: "export-layout" + (railCollapsed ? " rail-collapsed" : "")' in runtime_js
+ assert 'railCollapsed ? "Show sidebar" : "Hide sidebar"' in runtime_js
+ assert 'layout.classList.toggle("rail-collapsed", context.railCollapsed)' in runtime_js
+ assert 'context.plotManager.scheduleResize();' in runtime_js
+
+
def test_runtime_asset_contains_plot_csv_export_helpers() -> None:
runtime_js = load_export_runtime_js()
From 37b51f3697006dd364e9aa59eea9139e928dab4b Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Tue, 4 Aug 2026 17:25:39 -0400
Subject: [PATCH 08/27] remove theme toggle
---
dashboard/app.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/dashboard/app.py b/dashboard/app.py
index dc998d8..126ae7a 100644
--- a/dashboard/app.py
+++ b/dashboard/app.py
@@ -151,6 +151,7 @@ def _on_value_change(event) -> None:
sidebar=sidebar_items,
main=[main_content],
theme="default",
+ theme_toggle=False,
accent_base_color="#4E79A7",
header_background="#4E79A7",
sidebar_width=340,
From 6dac8af5d46139759bb2698963d70f9b949e56f0 Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Tue, 4 Aug 2026 17:49:48 -0400
Subject: [PATCH 09/27] Add missing data cards on validation pages
---
.../pages/validation/_traffic/features.py | 47 ++++++++--------
dashboard/pages/validation/_vmt/features.py | 36 +++++--------
dashboard/pages/validation/regional.py | 4 +-
dashboard/pages/validation/transit.py | 4 +-
tests/test_summary_cache.py | 54 +++++++++++++++++++
5 files changed, 96 insertions(+), 49 deletions(-)
diff --git a/dashboard/pages/validation/_traffic/features.py b/dashboard/pages/validation/_traffic/features.py
index d093fbd..a6a6f49 100644
--- a/dashboard/pages/validation/_traffic/features.py
+++ b/dashboard/pages/validation/_traffic/features.py
@@ -17,7 +17,7 @@ def render_screenline_flow_section(self):
if not self.state.run_labels:
return [self.no_runs_message()]
data = self.data.summary("screenline_flow_comparisons", self.weighting_key)
- if data is None:
+ if not data:
return [
self.data_not_available_card(
detail="Screenline flow comparisons are unavailable.",
@@ -66,7 +66,15 @@ def render_demo_facility_summary_section(self) -> list[pn.viewable.Viewable]:
"count_location_fit_validation_summary", self.weighting_key
)
if not any((count_list, volume_list, scatter_list, fit_list)):
- return []
+ return [
+ self.data_not_available_card(
+ detail="Count-location facility summaries are unavailable.",
+ missing_items=[
+ "count_location_counts_validation_summary",
+ "count_location_volumes_validation_summary",
+ ],
+ )
+ ]
# Keep this overview on unfiltered daily totals. The controls below it
# belong only to the Traffic Volume Summaries sections.
@@ -131,8 +139,6 @@ def render_demo_traffic_section(self) -> list[pn.viewable.Viewable]:
fit_list = self.data.summary(
"count_location_fit_validation_summary", self.weighting_key
)
- if not any((count_list, volume_list, scatter_list, fit_list)):
- return []
period = self.demo_period_sel.value
volume_col = DEMO_TRAFFIC_TIME_PERIODS[str(period)]
facility_type = self.selected_facility_type_raw()
@@ -208,7 +214,7 @@ def render_demo_link_volume_section(self) -> list[pn.viewable.Viewable]:
return []
link_list = self.data.summary("link_validation_summary", self.weighting_key)
- if link_list is None:
+ if not link_list:
return [
self.data_not_available_card(
detail="Link validation summaries are unavailable.",
@@ -253,15 +259,12 @@ def render_demo_top_count_section(self) -> list[pn.viewable.Viewable]:
volume_list = self.data.summary(
"count_location_volumes_validation_summary", self.weighting_key
)
- if not any((link_list, count_list, volume_list)):
- return []
-
facility_type = self.selected_facility_type_raw()
top_period = self.demo_top_period_sel.value
top_volume_col = DEMO_TRAFFIC_TIME_PERIODS[str(top_period)]
top_n = int(self.demo_top_n_sel.value)
- if count_list is not None and volume_list is not None:
+ if count_list and volume_list:
volume_comparison = self.query(
lambda: label_category_data(
demo_volume_comparison_table(
@@ -288,17 +291,15 @@ def render_demo_top_count_section(self) -> list[pn.viewable.Viewable]:
column_sorters={"Difference": "number"},
),
]
- if link_list is not None:
- return [
- self.data_not_available_card(
- detail=(
- "Count-location validation counts and volumes are both "
- "required for this comparison table."
- ),
- missing_items=[
- "count_location_counts_validation_summary",
- "count_location_volumes_validation_summary",
- ],
- )
- ]
- return []
+ return [
+ self.data_not_available_card(
+ detail=(
+ "Count-location validation counts and volumes are both "
+ "required for this comparison table."
+ ),
+ missing_items=[
+ "count_location_counts_validation_summary",
+ "count_location_volumes_validation_summary",
+ ],
+ )
+ ]
diff --git a/dashboard/pages/validation/_vmt/features.py b/dashboard/pages/validation/_vmt/features.py
index e645f37..b861fab 100644
--- a/dashboard/pages/validation/_vmt/features.py
+++ b/dashboard/pages/validation/_vmt/features.py
@@ -43,7 +43,17 @@ def render_vmt_overview_section(self) -> list[pn.viewable.Viewable]:
),
)
if not overview_data:
- return []
+ return [
+ self.data_not_available_card(
+ detail="VMT overview summaries are unavailable.",
+ missing_items=[
+ PERSONAL_AUTO_VMT_SUMMARY_ID,
+ NON_MOTORIZED_VMT_SUMMARY_ID,
+ EXTERNAL_VMT_SUMMARY_ID,
+ COMMERCIAL_VMT_SUMMARY_ID,
+ ],
+ )
+ ]
return [
data_table(
overview_data,
@@ -64,7 +74,7 @@ def render_bicycle_chart(self) -> pn.viewable.Viewable:
"bicycle_vmt_by_facility_type",
self.weighting_key,
)
- if bicycle_vmt is None:
+ if not bicycle_vmt:
return self.data_not_available_card(
detail="Bicycle VMT summaries are unavailable.",
missing_items=["bicycle_vmt_by_facility_type"],
@@ -301,15 +311,6 @@ def render_body(self):
def render_commercial_vmt_section(self):
if not self.state.run_labels:
return [self.no_runs_message()]
- summary_ids = [
- "commercial_vehicle_validation_summary",
- "commercial_vehicle_vmt_validation_summary",
- ]
- if not any(
- self.data.summary(summary_id, self.weighting_key)
- for summary_id in summary_ids
- ):
- return []
return [self.render_demo_commercial_chart()]
def render_demo_commercial_chart(self) -> pn.viewable.Viewable:
@@ -319,7 +320,7 @@ def render_demo_commercial_chart(self) -> pn.viewable.Viewable:
else "commercial_vehicle_validation_summary"
)
data = self.data.summary(summary_id, self.weighting_key)
- if data is None:
+ if not data:
return self.data_not_available_card(
detail="Commercial vehicle summaries are unavailable.",
missing_items=[summary_id],
@@ -386,7 +387,7 @@ def render_external_travel_chart(self) -> pn.viewable.Viewable:
else "external_trip_validation_summary"
)
data = self.data.summary(summary_id, self.weighting_key)
- if data is None:
+ if not data:
return self.data_not_available_card(
detail="External travel summaries are unavailable.",
missing_items=[summary_id],
@@ -445,15 +446,6 @@ def render_external_travel_chart(self) -> pn.viewable.Viewable:
)
def render_external_vmt_section(self) -> list[pn.viewable.Viewable]:
- summary_ids = [
- "external_trip_validation_summary",
- "external_vmt_validation_summary",
- ]
- if not any(
- self.data.summary(summary_id, self.weighting_key)
- for summary_id in summary_ids
- ):
- return []
content: list[pn.viewable.Viewable] = [
self.render_external_travel_chart(),
]
diff --git a/dashboard/pages/validation/regional.py b/dashboard/pages/validation/regional.py
index 9c8b263..e39a200 100644
--- a/dashboard/pages/validation/regional.py
+++ b/dashboard/pages/validation/regional.py
@@ -526,7 +526,7 @@ def render_flow_section(self) -> pn.viewable.Viewable:
flow_option.summary_id,
self.weighting_key,
)
- if observed_data is None:
+ if not observed_data:
return self.data_not_available_card(
detail="External regional flow summaries are unavailable.",
missing_items=[flow_option.summary_id],
@@ -550,7 +550,7 @@ def render_flow_section(self) -> pn.viewable.Viewable:
modeled_data,
flow_option.modeled_geography_types,
)
- if modeled_data is None or geography_type is None:
+ if not modeled_data or geography_type is None:
return self.data_not_available_card(
detail=(
"Modeled commuting flows are unavailable for the selected "
diff --git a/dashboard/pages/validation/transit.py b/dashboard/pages/validation/transit.py
index 85d36f8..268bb8e 100644
--- a/dashboard/pages/validation/transit.py
+++ b/dashboard/pages/validation/transit.py
@@ -147,7 +147,7 @@ def render_boardings_chart(
"transit_boardings_by_operator_and_technology",
self.weighting_key,
)
- if boarding_list is None:
+ if not boarding_list:
return self.data_not_available_card(
detail="Transit boarding summaries are unavailable.",
missing_items=["transit_boardings_by_operator_and_technology"],
@@ -171,7 +171,7 @@ def render_transfer_chart(self, operator_values: list[str]) -> pn.viewable.Viewa
"transit_transfer_rate",
self.weighting_key,
)
- if transfer_list is None:
+ if not transfer_list:
return self.data_not_available_card(
detail="Transit transfer summaries are unavailable.",
missing_items=["transit_transfer_rate"],
diff --git a/tests/test_summary_cache.py b/tests/test_summary_cache.py
index b4c3995..8bc0d67 100644
--- a/tests/test_summary_cache.py
+++ b/tests/test_summary_cache.py
@@ -57,6 +57,8 @@
from dashboard.pages.trip_summaries.trip_stop_time import TripStopTimePage
from dashboard.pages.validation.traffic import TrafficValidationPage
from dashboard.pages.validation.transit import TransitValidationPage
+from dashboard.pages.validation.regional import RegionalValidationPage
+from dashboard.pages.validation.vmt import VMTValidationPage
from dashboard.data_access import DashboardPreparedRunProvider
from dashboard.state import DashboardState
from dashboard.page_registry import page_definitions_for_group
@@ -5568,6 +5570,58 @@ def test_transit_validation_places_each_selector_with_its_plot(
assert transfer_plot.object.layout.title.text == "Transit Transfer Rate - walk"
+@pytest.mark.parametrize(
+ ("page_type", "section_names"),
+ [
+ (
+ TrafficValidationPage,
+ (
+ "_facility_summary_body",
+ "_external_volume_body",
+ "_link_volume_body",
+ "_external_top_body",
+ "_screenline_body",
+ ),
+ ),
+ (
+ TransitValidationPage,
+ ("_boardings_body", "_transfer_body"),
+ ),
+ (
+ VMTValidationPage,
+ (
+ "_vmt_overview_body",
+ "_personal_vmt_body",
+ "_non_motorized_vmt_body",
+ "_external_vmt_body",
+ "_body",
+ "_bicycle_body",
+ ),
+ ),
+ (RegionalValidationPage, ("_body",)),
+ ],
+)
+def test_validation_visualizations_render_cards_when_data_is_unavailable(
+ tmp_path: Path,
+ page_type: type,
+ section_names: tuple[str, ...],
+) -> None:
+ config = _write_config(tmp_path)
+ summary_run = _summary_run_with_tables(label="Base", weighted={})
+ state = DashboardState(
+ summary_runs=[summary_run],
+ weighting_modes=config.weighting_modes,
+ )
+
+ page = page_type(state, config)
+ page.refresh(force=True)
+
+ for section_name in section_names:
+ cards = _collect_cards(getattr(page, section_name))
+ assert len(cards) == 1, section_name
+ assert cards[0].title == "Data Not Available"
+
+
def test_tour_distance_chart_casts_distance_bins_consistently_across_runs(
tmp_path: Path,
) -> None:
From a5d9497417466830f930b00aabfe6db6ca57913d Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Tue, 4 Aug 2026 18:26:05 -0400
Subject: [PATCH 10/27] Fixed bug with missing data card; fixed filter
selectors so empty visuals are not presented
---
.../daily_travel/_escorted_tours/contracts.py | 1 -
.../daily_travel/_escorted_tours/domains.py | 2 +-
.../daily_travel/_escorted_tours/features.py | 6 +-
.../daily_travel/daily_activity_pattern.py | 4 +-
.../pages/daily_travel/escorted_tours.py | 3 +-
dashboard/pages/joint_travel.py | 112 +++++++++++------
.../_mandatory_location_choice/domains.py | 2 +-
.../_mandatory_location_choice/features.py | 58 ++++++---
.../long_term_choices/individual_choices.py | 2 +-
.../pages/long_term_choices/shadow_pricing.py | 38 +++---
.../vehicle_ownership_type.py | 16 ++-
dashboard/pages/overview.py | 2 +-
dashboard/pages/skim_summaries/tour_skims.py | 2 +-
dashboard/pages/skim_summaries/trip_skims.py | 2 +-
.../tour_summaries/park_and_ride_location.py | 17 +--
.../pages/tour_summaries/tour_distance.py | 28 +++--
dashboard/pages/tour_summaries/tour_time.py | 2 +-
.../pages/validation/_traffic/features.py | 4 +-
dashboard/pages/validation/_vmt/features.py | 4 +-
tests/test_page_registry_contract.py | 117 ++++++++++++++++++
tests/test_summary_cache.py | 39 +++++-
wiki/24-summary-catalog.md | 2 +-
wiki/31-dashboard-pages.md | 2 +-
23 files changed, 353 insertions(+), 112 deletions(-)
diff --git a/dashboard/pages/daily_travel/_escorted_tours/contracts.py b/dashboard/pages/daily_travel/_escorted_tours/contracts.py
index 5754b9a..9fa28bf 100644
--- a/dashboard/pages/daily_travel/_escorted_tours/contracts.py
+++ b/dashboard/pages/daily_travel/_escorted_tours/contracts.py
@@ -19,7 +19,6 @@
"households_with_school_escorting_by_student_count_and_direction",
"schoolkids_per_escorted_tour_by_student_count_and_direction",
)
-PAGE_SUMMARY_IDS = (*CORE_SUMMARY_IDS, *OPTIONAL_SUMMARY_IDS)
STOP_SEGMENT_LABELS = {
"outbound_before_dropoff": "Adult Escort Stops Before Dropoff - Outbound",
"outbound_after_dropoff": "Adult Escort Stops After Dropoff - Outbound",
diff --git a/dashboard/pages/daily_travel/_escorted_tours/domains.py b/dashboard/pages/daily_travel/_escorted_tours/domains.py
index 5d784a8..686f95f 100644
--- a/dashboard/pages/daily_travel/_escorted_tours/domains.py
+++ b/dashboard/pages/daily_travel/_escorted_tours/domains.py
@@ -30,7 +30,7 @@ def _direction_options(self) -> list[str]:
"school_escorted_tours_by_escort_type_and_direction",
"weighted",
)
- if data is None:
+ if not data:
return ["Both Directions"]
return direction_options(data)
diff --git a/dashboard/pages/daily_travel/_escorted_tours/features.py b/dashboard/pages/daily_travel/_escorted_tours/features.py
index dec42e4..f45e5f1 100644
--- a/dashboard/pages/daily_travel/_escorted_tours/features.py
+++ b/dashboard/pages/daily_travel/_escorted_tours/features.py
@@ -166,7 +166,7 @@ def render_student_school_escort_section(self, summary_data):
def render_student_school_escort_charts(self, summary_data):
"""Build the three student escort status charts when the summary is available."""
- if summary_data is None:
+ if not summary_data:
return None
escort_order = self.config.ordered_values("escort", STUDENT_ESCORT_TYPE_ORDER)
@@ -240,7 +240,7 @@ def render_household_school_escort_charts(
student_count_values: list[str],
):
"""Build household escort count/rate charts for each direction."""
- if denominator_summary is None or numerator_summary is None:
+ if not denominator_summary or not numerator_summary:
return None
charts: list[pn.viewable.Viewable] = []
@@ -321,7 +321,7 @@ def render_schoolkids_per_escorted_tour_charts(
student_count_values: list[str],
):
"""Build average schoolkids-per-tour charts for each direction."""
- if summary_data is None:
+ if not summary_data:
return None
charts: list[pn.viewable.Viewable] = []
diff --git a/dashboard/pages/daily_travel/daily_activity_pattern.py b/dashboard/pages/daily_travel/daily_activity_pattern.py
index 305da91..af02580 100644
--- a/dashboard/pages/daily_travel/daily_activity_pattern.py
+++ b/dashboard/pages/daily_travel/daily_activity_pattern.py
@@ -67,7 +67,7 @@ def _person_type_source_data(self, weighting_key: str):
"""Use the first available person-type summary to seed the selector domain."""
for summary_id in PERSON_TYPE_SUMMARY_IDS:
data = self.data.summary(summary_id, weighting_key)
- if data is not None:
+ if data:
return data
return None
@@ -330,7 +330,7 @@ def render_body(self):
return [self.no_runs_message()]
summaries = self._optional_summaries()
- if not any(data is not None for data in summaries.values()):
+ if not any(summaries.values()):
return [self.summary_only_unavailable_card()]
display_person_type, raw_person_type = self._selected_person_type()
diff --git a/dashboard/pages/daily_travel/escorted_tours.py b/dashboard/pages/daily_travel/escorted_tours.py
index 15afb63..61fd4c5 100644
--- a/dashboard/pages/daily_travel/escorted_tours.py
+++ b/dashboard/pages/daily_travel/escorted_tours.py
@@ -15,7 +15,8 @@
title="Escorted Tours",
group_id="daily_travel",
order=29,
- required_summary_ids=(*PAGE_SUMMARY_IDS,),
+ required_summary_ids=CORE_SUMMARY_IDS,
+ optional_summary_ids=OPTIONAL_SUMMARY_IDS,
)
class EscortedToursPage(
EscortedToursCompositionMixin,
diff --git a/dashboard/pages/joint_travel.py b/dashboard/pages/joint_travel.py
index 33d00b8..c430b2d 100644
--- a/dashboard/pages/joint_travel.py
+++ b/dashboard/pages/joint_travel.py
@@ -85,14 +85,14 @@ def _party_size_options(self) -> list[str]:
"joint_tour_composition_by_party_size",
self.weighting_key,
)
- return party_size_options(data) if data is not None else ["All"]
+ return party_size_options(data) if data else ["All"]
def _household_size_options(self) -> list[str]:
data = self.data.summary(
"household_jtp_by_household_size_and_jtf",
self.weighting_key,
)
- return household_size_options(data) if data is not None else ["All"]
+ return household_size_options(data) if data else ["All"]
def _summaries(self):
return self.data.summaries(*self.required_summary_ids)
@@ -116,8 +116,12 @@ def render_frequency(self):
if not self.state.run_labels:
return [self.no_runs_message()]
summaries = self._summaries()
- if summaries is None:
- return [self.summary_only_unavailable_card()]
+ if not summaries["jtf_distribution"]:
+ return [
+ self.summary_only_unavailable_card(
+ summary_ids=("jtf_distribution",),
+ )
+ ]
return [
selector_row(self.hide_no_joint_tours, height=48),
self.noted_view(
@@ -128,8 +132,6 @@ def render_frequency(self):
def render_joint_tour_detail(self):
summaries = self._summaries()
- if summaries is None:
- return []
party_size = self.party_size_sel.value
joint_tours_hhsize_data = [
(label, df.with_columns(pl.col("household_size").cast(pl.Utf8)))
@@ -160,35 +162,53 @@ def render_joint_tour_detail(self):
summaries["joint_tour_composition_by_party_size"], party_size
)
)
+ household_size_view = (
+ self.render_household_size_chart(
+ complete_joint_household_size_data(
+ joint_tours_hhsize_data,
+ value_col="joint_tour_hh_count",
+ household_size_values=household_size_values,
+ ),
+ household_size_values,
+ )
+ if summaries["joint_tours_by_household_size"]
+ else self.summary_only_unavailable_card(
+ summary_ids=("joint_tours_by_household_size",),
+ )
+ )
+ party_size_view = (
+ self.render_party_size_chart(party_size_data, party_size_values)
+ if summaries["joint_tour_party_size_distribution"]
+ else self.summary_only_unavailable_card(
+ summary_ids=("joint_tour_party_size_distribution",),
+ )
+ )
+ composition_view = (
+ self.render_composition_chart(
+ comp_party_data,
+ composition_label_values,
+ party_size,
+ )
+ if summaries["joint_tour_composition_by_party_size"]
+ else self.summary_only_unavailable_card(
+ summary_ids=("joint_tour_composition_by_party_size",),
+ )
+ )
return [
pn.Column(
selector_row(self.party_size_sel),
pn.Row(
self.noted_view(
"joint_travel.household_size",
- self.render_household_size_chart(
- complete_joint_household_size_data(
- joint_tours_hhsize_data,
- value_col="joint_tour_hh_count",
- household_size_values=household_size_values,
- ),
- household_size_values,
- ),
+ household_size_view,
),
self.noted_view(
"joint_travel.party_size",
- self.render_party_size_chart(
- party_size_data,
- party_size_values,
- ),
+ party_size_view,
),
self.noted_view(
"joint_travel.composition",
- self.render_composition_chart(
- comp_party_data,
- composition_label_values,
- party_size,
- ),
+ composition_view,
),
sizing_mode="stretch_width",
),
@@ -198,8 +218,6 @@ def render_joint_tour_detail(self):
def render_participation(self):
summaries = self._summaries()
- if summaries is None:
- return []
hhsize = self.hhsize_sel.value
person_participation = self.query(
lambda: person_participation_data(
@@ -228,6 +246,35 @@ def render_participation(self):
)
],
)
+ person_view = (
+ self.render_person_participation_chart(
+ complete_joint_household_size_data(
+ person_participation,
+ value_col="person_value",
+ household_size_values=household_size_values,
+ ),
+ household_size_values,
+ )
+ if summaries["person_jtp_by_household_size"]
+ else self.summary_only_unavailable_card(
+ summary_ids=("person_jtp_by_household_size",),
+ )
+ )
+ household_view = (
+ self.render_household_participation_chart(
+ household_participation,
+ jtf_values,
+ hhsize,
+ )
+ if any(not df.is_empty() for _, df in household_participation)
+ else self.summary_only_unavailable_card(
+ summary_ids=("household_jtp_by_household_size_and_jtf",),
+ detail=(
+ "The household joint-tour participation summary has no data "
+ f"for household size `{hhsize}`."
+ ),
+ )
+ )
return [
pn.Column(
pn.Row(
@@ -238,22 +285,11 @@ def render_participation(self):
pn.Row(
self.noted_view(
"joint_travel.person_participation",
- self.render_person_participation_chart(
- complete_joint_household_size_data(
- person_participation,
- value_col="person_value",
- household_size_values=household_size_values,
- ),
- household_size_values,
- ),
+ person_view,
),
self.noted_view(
"joint_travel.household_participation",
- self.render_household_participation_chart(
- household_participation,
- jtf_values,
- hhsize,
- ),
+ household_view,
),
sizing_mode="stretch_width",
),
diff --git a/dashboard/pages/long_term_choices/_mandatory_location_choice/domains.py b/dashboard/pages/long_term_choices/_mandatory_location_choice/domains.py
index 888f310..5e46ca6 100644
--- a/dashboard/pages/long_term_choices/_mandatory_location_choice/domains.py
+++ b/dashboard/pages/long_term_choices/_mandatory_location_choice/domains.py
@@ -200,7 +200,7 @@ def _collect_data(self) -> dict[str, object]:
"average_mandatory_tour_distance_by_purpose_and_geography",
)
- if not any(summary is not None for summary in summaries.values()):
+ if not any(summaries.values()):
return {
"mode": "unavailable",
"geo_opts": [ALL_GEOGRAPHY_TYPES_LABEL],
diff --git a/dashboard/pages/long_term_choices/_mandatory_location_choice/features.py b/dashboard/pages/long_term_choices/_mandatory_location_choice/features.py
index 45ace21..2c9b7fe 100644
--- a/dashboard/pages/long_term_choices/_mandatory_location_choice/features.py
+++ b/dashboard/pages/long_term_choices/_mandatory_location_choice/features.py
@@ -45,18 +45,32 @@ def render_worker_geography_section(self) -> SectionContent:
geography,
)
)
- worker_views.append(
- self.noted_view(
- "mandatory_location.worker_status_table",
- data_table(
- [
- (label, self.render_internal_external_worker_table(df))
- for label, df in internal_external_table
- ],
- "Internal vs. External Workers",
- ),
+ if any(not df.is_empty() for _, df in internal_external_table):
+ worker_views.append(
+ self.noted_view(
+ "mandatory_location.worker_status_table",
+ data_table(
+ [
+ (
+ label,
+ self.render_internal_external_worker_table(df),
+ )
+ for label, df in internal_external_table
+ ],
+ "Internal vs. External Workers",
+ ),
+ )
+ )
+ else:
+ worker_views.append(
+ self.data_not_available_card(
+ detail=(
+ "No internal/external worker data is available for "
+ "the selected geography."
+ ),
+ missing_items=["internal_external_worker_by_geography"],
+ )
)
- )
else:
worker_views.append(
self.data_not_available_card(
@@ -174,8 +188,9 @@ def render_external_workplace_chart(
def render_distance_distribution_section(self) -> SectionContent:
"""Render the three mandatory distance distributions side by side."""
- if self._current_data["mode"] != "ready":
- return []
+ placeholder = self._render_ready_state()
+ if placeholder is not None:
+ return placeholder
geo_level, geography = self._selected_geography()
chart_specs = [
@@ -321,8 +336,9 @@ def render_distance_distribution_chart(
def render_remote_work_section(self) -> SectionContent:
"""Render work-from-home and telecommute summaries."""
- if self._current_data["mode"] != "ready":
- return []
+ placeholder = self._render_ready_state()
+ if placeholder is not None:
+ return placeholder
geo_level, geography = self._selected_geography()
return [
@@ -359,6 +375,13 @@ def render_work_from_home_chart(
geography,
)
)
+ if not any(not df.is_empty() for _, df in wfh_data):
+ return self.data_not_available_card(
+ detail=(
+ "No work-from-home data is available for the selected geography."
+ ),
+ missing_items=["work_from_home_rate_by_geography"],
+ )
return self.plot.bar(
wfh_data,
x="geography_label",
@@ -426,8 +449,9 @@ def render_telecommute_chart(
def render_mandatory_distance_table_section(self) -> SectionContent:
"""Render the percent-difference table for average mandatory tour distance."""
- if self._current_data["mode"] != "ready":
- return []
+ placeholder = self._render_ready_state()
+ if placeholder is not None:
+ return placeholder
geo_level, geography = self._selected_geography()
average_distance = self._current_data["average_distance"]
diff --git a/dashboard/pages/long_term_choices/individual_choices.py b/dashboard/pages/long_term_choices/individual_choices.py
index 1e4b48a..57cbe50 100644
--- a/dashboard/pages/long_term_choices/individual_choices.py
+++ b/dashboard/pages/long_term_choices/individual_choices.py
@@ -113,7 +113,7 @@ def _summary_or_placeholder(
*,
detail: str,
) -> list[tuple[str, pl.DataFrame]] | pn.Card:
- summary = self.data.summary(summary_name, required=False)
+ summary = self.data.summary(summary_name)
if summary:
return summary
return self.data_not_available_card(detail=detail, missing_items=[summary_name])
diff --git a/dashboard/pages/long_term_choices/shadow_pricing.py b/dashboard/pages/long_term_choices/shadow_pricing.py
index 3446160..a85cfd5 100644
--- a/dashboard/pages/long_term_choices/shadow_pricing.py
+++ b/dashboard/pages/long_term_choices/shadow_pricing.py
@@ -227,20 +227,16 @@ def _collect_data(self) -> dict[str, object]:
}
workplace_summary = normalize_geography_data(
- self.data.summary("workplace_shadow_pricing_residuals", required=False)
+ self.data.summary("workplace_shadow_pricing_residuals")
)
school_summary = normalize_geography_data(
- self.data.summary("school_shadow_pricing_residuals", required=False)
+ self.data.summary("school_shadow_pricing_residuals")
)
workplace_hist = normalize_geography_data(
- self.data.summary(
- "workplace_shadow_pricing_residual_histogram", required=False
- )
+ self.data.summary("workplace_shadow_pricing_residual_histogram")
)
school_hist = normalize_geography_data(
- self.data.summary(
- "school_shadow_pricing_residual_histogram", required=False
- )
+ self.data.summary("school_shadow_pricing_residual_histogram")
)
geo_opts, geo_raw_by_label = geography_type_options(
workplace_hist or school_hist or workplace_summary or school_summary,
@@ -310,12 +306,17 @@ def render_workplace_plot_section(self) -> SectionContent:
def render_workplace_table_section(self) -> SectionContent:
"""Render the workplace residual table."""
- if self._current_data["mode"] != "ready":
- return []
+ if self._current_data["mode"] == "no_runs":
+ return [self.no_runs_message()]
workplace_summary = self._current_data["workplace_summary"]
if workplace_summary is None:
- return []
+ return [
+ self.data_not_available_card(
+ detail="The workplace employment residual summary is unavailable.",
+ missing_items=["workplace_shadow_pricing_residuals"],
+ )
+ ]
if self._maz_tables_disabled():
return [
self.data_not_available_card(
@@ -360,8 +361,8 @@ def render_workplace_table(self, df: pl.DataFrame) -> pl.DataFrame:
def render_school_plot_section(self) -> SectionContent:
"""Render the school residual distribution for one student type."""
- if self._current_data["mode"] != "ready":
- return []
+ if self._current_data["mode"] == "no_runs":
+ return [self.no_runs_message()]
school_hist = self._current_data["school_hist"]
if school_hist is None:
@@ -408,12 +409,17 @@ def render_school_plot_section(self) -> SectionContent:
def render_school_table_section(self) -> SectionContent:
"""Render school residuals for the selected geography level and student type."""
- if self._current_data["mode"] != "ready":
- return []
+ if self._current_data["mode"] == "no_runs":
+ return [self.no_runs_message()]
school_summary = self._current_data["school_summary"]
if school_summary is None:
- return []
+ return [
+ self.data_not_available_card(
+ detail="The school enrollment residual summary is unavailable.",
+ missing_items=["school_shadow_pricing_residuals"],
+ )
+ ]
if self._maz_tables_disabled():
return [
self.data_not_available_card(
diff --git a/dashboard/pages/long_term_choices/vehicle_ownership_type.py b/dashboard/pages/long_term_choices/vehicle_ownership_type.py
index fce65d2..c399a9b 100644
--- a/dashboard/pages/long_term_choices/vehicle_ownership_type.py
+++ b/dashboard/pages/long_term_choices/vehicle_ownership_type.py
@@ -161,7 +161,7 @@ def render_ownership_summary(self):
def render_vehicle_mix(self):
if not self.state.run_labels:
- return []
+ return [self.no_runs_message()]
summaries = self._optional_summaries()
vehicle_views: list[pn.viewable.Viewable] = []
@@ -216,11 +216,17 @@ def render_auto_ownership_chart(self, summary_data):
missing_items=["auto_ownership_distribution"],
)
household_size = str(self.hhsize_sel.value)
+ chart_data = _auto_ownership_chart_data(summary_data, household_size)
+ if not any(not df.is_empty() for _, df in chart_data):
+ return self.data_not_available_card(
+ detail=(
+ "The auto ownership summary has no data for household size "
+ f"`{household_size}`."
+ ),
+ missing_items=["auto_ownership_distribution"],
+ )
return self.plot.bar(
- _auto_ownership_chart_data(
- summary_data,
- household_size,
- ),
+ chart_data,
x="household_vehicle_count",
y="household_count",
title=f"Auto Ownership by Household Size - {household_size}",
diff --git a/dashboard/pages/overview.py b/dashboard/pages/overview.py
index bf93607..373a10b 100644
--- a/dashboard/pages/overview.py
+++ b/dashboard/pages/overview.py
@@ -305,7 +305,7 @@ def render_kpis(self) -> SectionContent:
def render_demographics(self) -> SectionContent:
"""Render the demographic distribution charts."""
if not self.state.run_labels:
- return []
+ return [self.no_runs_message()]
ptype_result, hhsize_result = self._demographic_results()
return [
diff --git a/dashboard/pages/skim_summaries/tour_skims.py b/dashboard/pages/skim_summaries/tour_skims.py
index deb8206..edd8c3b 100644
--- a/dashboard/pages/skim_summaries/tour_skims.py
+++ b/dashboard/pages/skim_summaries/tour_skims.py
@@ -313,7 +313,7 @@ def render_summary_table(self):
family = self.tour_family_sel.value
direction = self.tour_direction_sel.value
- if tour_stats is None:
+ if not tour_stats:
return self.data_not_available_card(
detail="Tour skim summaries require the precomputed skim tour statistics table.",
missing_items=[TOUR_STATS_SUMMARY_ID],
diff --git a/dashboard/pages/skim_summaries/trip_skims.py b/dashboard/pages/skim_summaries/trip_skims.py
index 349cba1..c06f933 100644
--- a/dashboard/pages/skim_summaries/trip_skims.py
+++ b/dashboard/pages/skim_summaries/trip_skims.py
@@ -263,7 +263,7 @@ def render_summary_section(self):
return [self.no_runs_message()]
trip_stats = self._trip_summaries()
- if trip_stats is None:
+ if not trip_stats:
return [
self.data_not_available_card(
detail="Trip skim summaries require the precomputed skim trip statistics table.",
diff --git a/dashboard/pages/tour_summaries/park_and_ride_location.py b/dashboard/pages/tour_summaries/park_and_ride_location.py
index 43b3305..1855ca5 100644
--- a/dashboard/pages/tour_summaries/park_and_ride_location.py
+++ b/dashboard/pages/tour_summaries/park_and_ride_location.py
@@ -109,12 +109,10 @@ def _collect_data(self) -> dict[str, object]:
}
residuals = normalize_geography_data(
- self.data.summary("park_and_ride_location_residuals", required=False)
+ self.data.summary("park_and_ride_location_residuals")
)
histogram = normalize_geography_data(
- self.data.summary(
- "park_and_ride_location_residual_histogram", required=False
- )
+ self.data.summary("park_and_ride_location_residual_histogram")
)
geo_opts, geo_raw_by_label = geography_type_options(
histogram or residuals,
@@ -167,12 +165,17 @@ def render_plot_section(self) -> SectionContent:
def render_table_section(self) -> SectionContent:
"""Render the residual table for the selected geography level."""
- if self._current_data["mode"] != "ready":
- return []
+ if self._current_data["mode"] == "no_runs":
+ return [self.no_runs_message()]
residuals = self._current_data["residuals"]
if residuals is None:
- return []
+ return [
+ self.data_not_available_card(
+ detail="The park-and-ride residual summary is unavailable.",
+ missing_items=["park_and_ride_location_residuals"],
+ )
+ ]
if self._maz_tables_disabled():
return [
self.data_not_available_card(
diff --git a/dashboard/pages/tour_summaries/tour_distance.py b/dashboard/pages/tour_summaries/tour_distance.py
index be9c146..ddacb6d 100644
--- a/dashboard/pages/tour_summaries/tour_distance.py
+++ b/dashboard/pages/tour_summaries/tour_distance.py
@@ -211,8 +211,6 @@ def _summaries(self) -> dict[str, object] | None:
def _distance_sources(self):
summaries = self._summaries()
- if not summaries:
- return None, None, None
nonmandatory_average = normalize_geography_data(
summaries["average_nonmandatory_tour_distance_by_purpose_and_geography"]
)
@@ -282,8 +280,13 @@ def render_distance_section(self) -> SectionContent:
return [self.no_runs_message()]
summaries = self._summaries()
- if summaries is None:
- return [self.summary_only_unavailable_card()]
+ distance_summary = summaries["tour_distance_by_tour_purpose"]
+ if not distance_summary:
+ return [
+ self.summary_only_unavailable_card(
+ summary_ids=("tour_distance_by_tour_purpose",),
+ )
+ ]
selected_purpose = str(self.tour_purpose_sel.value)
raw_purpose = self._tour_purpose_to_raw.get(
@@ -291,7 +294,7 @@ def render_distance_section(self) -> SectionContent:
)
distance_data = self.query(
lambda: tour_distance_chart_data(
- summaries["tour_distance_by_tour_purpose"],
+ distance_summary,
str(raw_purpose),
)
)
@@ -344,11 +347,20 @@ def render_distance_chart(
def render_average_section(self) -> SectionContent:
"""Render the average non-mandatory distance comparison table."""
summaries = self._summaries()
- if summaries is None:
- return []
+ nonmandatory_summary = summaries[
+ "average_nonmandatory_tour_distance_by_purpose_and_geography"
+ ]
+ if not nonmandatory_summary:
+ return [
+ self.summary_only_unavailable_card(
+ summary_ids=(
+ "average_nonmandatory_tour_distance_by_purpose_and_geography",
+ ),
+ )
+ ]
nonmandatory_average = normalize_geography_data(
- summaries["average_nonmandatory_tour_distance_by_purpose_and_geography"]
+ nonmandatory_summary
)
geo_level = self.selected_geography_level_raw()
geography = self.selected_geography_raw()
diff --git a/dashboard/pages/tour_summaries/tour_time.py b/dashboard/pages/tour_summaries/tour_time.py
index d4ef3a6..a738bcd 100644
--- a/dashboard/pages/tour_summaries/tour_time.py
+++ b/dashboard/pages/tour_summaries/tour_time.py
@@ -98,7 +98,7 @@ def _purpose_options(self) -> list[str]:
"tour_time_of_day_by_tour_purpose",
self.weighting_key,
)
- if data is None:
+ if not data:
self._purpose_to_raw = {self.TOTAL_PURPOSE_LABEL: "all_tour_purposes"}
return [self.TOTAL_PURPOSE_LABEL]
options, self._purpose_to_raw = column_options(
diff --git a/dashboard/pages/validation/_traffic/features.py b/dashboard/pages/validation/_traffic/features.py
index a6a6f49..e6dd29b 100644
--- a/dashboard/pages/validation/_traffic/features.py
+++ b/dashboard/pages/validation/_traffic/features.py
@@ -51,7 +51,7 @@ def render_screenline_flow_section(self):
def render_demo_facility_summary_section(self) -> list[pn.viewable.Viewable]:
if not self.state.run_labels:
- return []
+ return [self.no_runs_message()]
count_list = self.data.summary(
"count_location_counts_validation_summary", self.weighting_key
@@ -211,7 +211,7 @@ def render_demo_traffic_section(self) -> list[pn.viewable.Viewable]:
def render_demo_link_volume_section(self) -> list[pn.viewable.Viewable]:
if not self.state.run_labels:
- return []
+ return [self.no_runs_message()]
link_list = self.data.summary("link_validation_summary", self.weighting_key)
if not link_list:
diff --git a/dashboard/pages/validation/_vmt/features.py b/dashboard/pages/validation/_vmt/features.py
index b861fab..cc3a37c 100644
--- a/dashboard/pages/validation/_vmt/features.py
+++ b/dashboard/pages/validation/_vmt/features.py
@@ -97,7 +97,7 @@ def render_bicycle_section(self):
class SegmentedVmtFeatureMixin:
def render_personal_auto_vmt_section(self) -> list[pn.viewable.Viewable]:
if not self.state.run_labels:
- return []
+ return [self.no_runs_message()]
personal_vmt = self.data.summary(
PERSONAL_AUTO_VMT_SUMMARY_ID,
columns=PERSONAL_AUTO_VMT_REQUIRED_COLUMNS,
@@ -199,7 +199,7 @@ def render_personal_auto_vmt_section(self) -> list[pn.viewable.Viewable]:
def render_non_motorized_vmt_section(self) -> list[pn.viewable.Viewable]:
if not self.state.run_labels:
- return []
+ return [self.no_runs_message()]
non_motorized_vmt = self.data.summary(
NON_MOTORIZED_VMT_SUMMARY_ID,
columns=NON_MOTORIZED_VMT_REQUIRED_COLUMNS,
diff --git a/tests/test_page_registry_contract.py b/tests/test_page_registry_contract.py
index 838d68c..43c3186 100644
--- a/tests/test_page_registry_contract.py
+++ b/tests/test_page_registry_contract.py
@@ -3,20 +3,62 @@
from pathlib import Path
import sys
+import panel as pn
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.path.insert(0, str(Path(__file__).resolve().parent))
from dashboard import DashboardState
+from dashboard.data_access import DashboardPreparedRunProvider
from dashboard.export.protocols import validate_export_page
from dashboard.export.traversal import resolve_page_parts
from dashboard.page_registry import (
+ all_page_definitions,
build_registered_live_pages,
build_registered_export_pages,
+ page_definition_by_id,
+)
+from dashboard.pages.daily_travel._escorted_tours.contracts import (
+ CORE_SUMMARY_IDS,
+ OPTIONAL_SUMMARY_IDS,
)
+from processor.summarize.cache_types import create_summary_run
from _dashboard_expectations import EXPECTED_DEFAULT_LEAF_PAGE_IDS
from test_export_html import _full_summary_run, _write_config
+def _walk(viewable):
+ yield viewable
+ for child in getattr(viewable, "objects", []):
+ yield from _walk(child)
+
+
+def _assert_sections_render_outcomes(page, *, allow_no_runs: bool = False) -> None:
+ for section in page.registered_sections:
+ context = f"{page.page_id()}.{section.section_id}"
+ nodes = list(_walk(section.container))
+ cards = [node for node in nodes if isinstance(node, pn.Card)]
+ plots = [node for node in nodes if isinstance(node, pn.pane.Plotly)]
+ tables = [node for node in nodes if isinstance(node, pn.widgets.Tabulator)]
+ no_run_messages = [
+ node
+ for node in nodes
+ if isinstance(node, pn.pane.Markdown)
+ and str(node.object) == "No runs loaded."
+ ]
+ assert cards or plots or tables or (allow_no_runs and no_run_messages), context
+ assert all(plot.object.data for plot in plots), context
+ assert all(tabs.objects for tabs in nodes if isinstance(tabs, pn.Tabs)), context
+
+
+def _state_for_run(run, config) -> DashboardState:
+ return DashboardState(
+ summary_runs=[run],
+ weighting_modes=config.weighting_modes,
+ prepared_run_provider=DashboardPreparedRunProvider.unavailable(),
+ )
+
+
def test_all_registered_export_pages_satisfy_export_protocol(tmp_path: Path) -> None:
config = _write_config(tmp_path)
state = DashboardState(
@@ -113,3 +155,78 @@ def test_representative_export_pages_keep_expected_runtime_sections(
(part_def.part_id, tuple(part_def.selector_ids))
for part_def, _ in resolved_parts
] == expected
+
+
+def test_escorted_tours_declares_independent_addons_as_optional() -> None:
+ definition = page_definition_by_id("escorted_tours")
+ assert definition is not None
+ assert definition.required_summary_ids == CORE_SUMMARY_IDS
+ assert definition.optional_summary_ids == OPTIONAL_SUMMARY_IDS
+
+
+def test_all_registered_page_sections_explain_no_run_sessions(tmp_path: Path) -> None:
+ config = _write_config(tmp_path)
+
+ for definition in all_page_definitions():
+ state = DashboardState(
+ summary_runs=[],
+ weighting_modes=config.weighting_modes,
+ )
+ page = definition.page_cls(state, config)
+ page.refresh(force=True)
+ _assert_sections_render_outcomes(page, allow_no_runs=True)
+
+
+def test_all_registered_page_selector_states_render_valid_outcomes(
+ tmp_path: Path,
+) -> None:
+ config = _write_config(tmp_path)
+ summary_run = _full_summary_run()
+
+ for definition in all_page_definitions():
+ page = definition.page_cls(_state_for_run(summary_run, config), config)
+ page.refresh(force=True)
+ _assert_sections_render_outcomes(page)
+
+ for selector in page.registered_selectors:
+ widget = selector.widget
+ values = (
+ [False, True]
+ if isinstance(widget, pn.widgets.Checkbox)
+ else list(getattr(widget, "options", []) or [])
+ )
+ for value in values:
+ widget.value = value
+ page.refresh(force=False)
+ _assert_sections_render_outcomes(page)
+
+
+def test_all_registered_pages_handle_each_missing_declared_summary(
+ tmp_path: Path,
+) -> None:
+ config = _write_config(tmp_path)
+ full_run = _full_summary_run()
+
+ for definition in all_page_definitions():
+ summary_ids = (
+ *definition.required_summary_ids,
+ *definition.optional_summary_ids,
+ )
+ for missing_summary_id in summary_ids:
+ summaries_by_mode = {
+ mode: {
+ summary_id: table
+ for summary_id, table in tables.items()
+ if summary_id != missing_summary_id
+ }
+ for mode, tables in full_run.summaries_by_mode.items()
+ }
+ summary_run = create_summary_run(
+ label="Base",
+ run_key="base",
+ summaries_by_mode=summaries_by_mode,
+ source_run_dir="C:/runs/base",
+ )
+ page = definition.page_cls(_state_for_run(summary_run, config), config)
+ page.refresh(force=True)
+ _assert_sections_render_outcomes(page)
diff --git a/tests/test_summary_cache.py b/tests/test_summary_cache.py
index 8bc0d67..5f1eef6 100644
--- a/tests/test_summary_cache.py
+++ b/tests/test_summary_cache.py
@@ -61,7 +61,7 @@
from dashboard.pages.validation.vmt import VMTValidationPage
from dashboard.data_access import DashboardPreparedRunProvider
from dashboard.state import DashboardState
-from dashboard.page_registry import page_definitions_for_group
+from dashboard.page_registry import all_page_definitions, page_definitions_for_group
from processor.models import RunData
from processor.prepare.cache import build_prepared_manifest_identity
from processor.prepare.enrichment.pipeline import prepare_data
@@ -3275,6 +3275,17 @@ def test_escorted_tours_page_renders_core_charts_when_optional_summaries_missing
assert "Adult Escort Stops Before Dropoff - Outbound" in titles
assert "Adult Escort Trip Stop Frequency - Both Directions" not in titles
assert all("Schoolkids Per Escorted Tour" not in title for title in titles)
+ card_text = [
+ str(card.objects[0].object)
+ for card in _collect_cards(page.view)
+ if card.objects
+ ]
+ assert any("student_school_escort_status_by_direction" in text for text in card_text)
+ assert any("student_households_by_student_count" in text for text in card_text)
+ assert any(
+ "schoolkids_per_escorted_tour_by_student_count_and_direction" in text
+ for text in card_text
+ )
def test_escorted_tours_page_uses_configured_escort_labels_for_student_status(
@@ -5622,6 +5633,32 @@ def test_validation_visualizations_render_cards_when_data_is_unavailable(
assert cards[0].title == "Data Not Available"
+def test_all_dashboard_sections_render_missing_data_content(tmp_path: Path) -> None:
+ config = _write_config(tmp_path)
+ summary_run = _summary_run_with_tables(label="Base", weighted={})
+
+ for definition in all_page_definitions():
+ state = DashboardState(
+ summary_runs=[summary_run],
+ weighting_modes=config.weighting_modes,
+ prepared_run_provider=DashboardPreparedRunProvider.unavailable(),
+ )
+ page = definition.page_cls(state, config)
+ page.refresh(force=True)
+
+ for section in page.registered_sections:
+ context = f"{definition.page_id}.{section.section_id}"
+ assert section.container.objects, context
+ cards = _collect_cards(section.container)
+ plots = _collect_plotly_panes(section.container)
+ tables = _collect_tabulators(section.container)
+ assert cards or plots or tables, context
+ for plot in plots:
+ assert plot.object.data, context
+ for tabs in _collect_tabs(section.container):
+ assert tabs.objects, context
+
+
def test_tour_distance_chart_casts_distance_bins_consistently_across_runs(
tmp_path: Path,
) -> None:
diff --git a/wiki/24-summary-catalog.md b/wiki/24-summary-catalog.md
index 4b5423f..7d06632 100644
--- a/wiki/24-summary-catalog.md
+++ b/wiki/24-summary-catalog.md
@@ -44,8 +44,8 @@ Total registered summaries: **100**
| `count_location_scatter_validation_summary` | `count_location_scatter_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.count_location_scatter_validation_summary` | `id: Int64`
`facility_type: String`
`period: String`
`observed_volume: Float64`
`modeled_volume: Float64` | - |
| `count_location_volumes_validation_summary` | `count_location_volumes_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.count_location_volumes_validation_summary` | `id: Int64`
`FACTYPE: Int64`
`am_vol: Float64`
`md_vol: Float64`
`pm_vol: Float64`
`day_vol: Float64` | - |
| `county_commuting_flows_validation_summary` | `county_commuting_flows_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.county_commuting_flows_validation_summary` | `: String`
`Benton: Float64`
`Linn: Float64`
`Marion: Float64`
`Total: Float64` | - |
-| `district_commuting_flows_validation_summary` | `district_commuting_flows_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.district_commuting_flows_validation_summary` | `: String`
`Albany: Float64`
`Corvallis: Float64`
`Lebanon: Float64`
`Philomath: Float64`
`Total: Float64` | - |
| `daily_activity_pattern_by_person_type` | `daily_activity_pattern_by_person_type.csv` | `processor.summarize.summaries.daily_travel_activity.dap_summary` | `person_type: String`
`daily_activity_pattern: String`
`person_count: Float64` | per: `person_type`, `cdap_activity`, `finalweight` |
+| `district_commuting_flows_validation_summary` | `district_commuting_flows_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.district_commuting_flows_validation_summary` | `: String`
`Albany: Float64`
`Corvallis: Float64`
`Lebanon: Float64`
`Philomath: Float64`
`Total: Float64` | - |
| `escorted_tour_totals` | `escorted_tour_totals.csv` | `processor.summarize.summaries.daily_travel_escort_counts.total_escorted_tours` | `tour_count: Float64` | tours: `school_esc_outbound`, `school_esc_inbound`, `finalweight` |
| `external_nonmandatory_tour_locations` | `external_nonmandatory_tour_locations.csv` | `processor.summarize.summaries.tour_geography.ext_non_mand_tour_loc` | `geography_type: String`
`geography_id: String`
`external_nonmandatory_tour_count: Float64` | tours: `tour_category`, `is_external_tour`, `destination`, `finalweight` |
| `external_trip_validation_summary` | `external_trip_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.external_trip_validation_summary` | `tod: String`
`hbcoll: Float64`
`hbo: Float64`
`hbr: Float64`
`hbs: Float64`
`hbsch: Float64`
`hbw: Float64`
`nhbnw: Float64`
`nhbw: Float64`
`truck: Float64`
`Total: Float64` | - |
diff --git a/wiki/31-dashboard-pages.md b/wiki/31-dashboard-pages.md
index 4112c55..7b0d162 100644
--- a/wiki/31-dashboard-pages.md
+++ b/wiki/31-dashboard-pages.md
@@ -115,7 +115,7 @@ Total registered pages: **27**
|---|---|---|---|---|---|---|---|
| `overview` | Overview | - | yes | `none` | `population_totals`, `person_type_distribution`, `household_size_distribution`, `auto_vmt_totals` | - | - |
| `daily_activity_pattern` | Daily Activity Pattern | Daily Travel | yes | `none` | `daily_activity_pattern_by_person_type`, `mandatory_tour_frequency_by_person_type`, `nonmandatory_tour_frequency_by_person_type`, `tour_rates_by_person_type_and_tour_purpose`, `trip_rates_by_person_type_and_trip_purpose` | - | - |
-| `escorted_tours` | Escorted Tours | Daily Travel | yes | `none` | `escorted_tour_totals`, `school_escorted_tours_by_escort_type_and_direction`, `adult_escort_event_stop_distribution`, `adult_escorted_tours_by_person_type_and_direction`, `adult_escorted_tour_distance_distribution_by_direction`, `adult_escorted_trip_distance_distribution_by_direction`, `student_school_escort_status_by_direction`, `student_households_by_student_count`, `households_with_school_escorting_by_student_count_and_direction`, `schoolkids_per_escorted_tour_by_student_count_and_direction` | - | - |
+| `escorted_tours` | Escorted Tours | Daily Travel | yes | `none` | `escorted_tour_totals`, `school_escorted_tours_by_escort_type_and_direction`, `adult_escort_event_stop_distribution`, `adult_escorted_tours_by_person_type_and_direction`, `adult_escorted_tour_distance_distribution_by_direction`, `adult_escorted_trip_distance_distribution_by_direction` | `student_school_escort_status_by_direction`, `student_households_by_student_count`, `households_with_school_escorting_by_student_count_and_direction`, `schoolkids_per_escorted_tour_by_student_count_and_direction` | - |
| `joint_travel` | Joint Travel | - | yes | `none` | `jtf_distribution`, `joint_tours_by_household_size`, `joint_tour_party_size_distribution`, `joint_tour_composition_by_party_size`, `person_jtp_by_household_size`, `household_jtp_by_household_size_and_jtf` | - | - |
| `individual_choices` | Individual Choices | Long-Term Choices | yes | `none` | `license_holding_status_distribution`, `bicycle_comfort_level_distribution`, `transit_pass_ownership_by_person_type`, `transit_subsidy_by_person_type` | - | - |
| `vehicle_ownership_type` | Vehicle Ownership and Type | Long-Term Choices | yes | `none` | `auto_ownership_distribution`, `autonomous_vehicle_ownership_totals`, `vehicle_age_distribution`, `vehicle_fuel_type_distribution`, `vehicle_body_type_distribution` | - | - |
From 289febc14e39e4f6f75589362aeec94be8b3c5c3 Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Wed, 5 Aug 2026 12:54:10 -0400
Subject: [PATCH 11/27] added more description to summary tables section of
wiki; simplified README and basic config; added simor project configs
---
README.md | 647 +++---------------
config.yaml | 300 +-------
scripts/generate_wiki_catalogs.py | 41 +-
simor_configs/metro_configs/metro_config.yaml | 624 +++++++++++++++++
.../metro_configs/metro_skimjoin_config.yaml | 415 +++++++++++
...etro_skimjoin_config_alternate_id_col.yaml | 415 +++++++++++
tests/test_runtime_config_package.py | 4 +-
tests/test_summary_declarations.py | 17 +
wiki/24-summary-catalog.md | 467 ++++++++++---
9 files changed, 1979 insertions(+), 951 deletions(-)
create mode 100644 simor_configs/metro_configs/metro_config.yaml
create mode 100644 simor_configs/metro_configs/metro_skimjoin_config.yaml
create mode 100644 simor_configs/metro_configs/metro_skimjoin_config_alternate_id_col.yaml
diff --git a/README.md b/README.md
index 04b8812..3c49b76 100644
--- a/README.md
+++ b/README.md
@@ -1,619 +1,138 @@
# ActivitySim Visualizer
-`activitysim_visualizer` is a Panel-based dashboard for exploring and comparing [ActivitySim](https://activitysim.github.io/) outputs. It can:
+ActivitySim Visualizer turns [ActivitySim](https://activitysim.github.io/)
+outputs into an interactive dashboard for exploring one model run, comparing
+several runs side by side, or comparing model outputs to survey results.
-- compare multiple model runs side by side
-- build and reuse prepared and summary caches
-- serve a live local dashboard
-- export a standalone HTML version for offline sharing
+It can:
+
+- prepare and summarize ActivitySim household, person, tour, and trip outputs;
+- compare travel patterns, model choices, and validation measures across runs;
+- reuse cached results so subsequent launches are faster; and
+- serve a local dashboard or create a standalone HTML file for sharing.
## Quick Start
-Install dependencies with `uv`:
+### 1. Install the project
-```bash
-uv sync --locked
-```
-
-Notebook tooling is optional; install it only when working with the repository's
-notebooks:
+From the repository root, use `uv` to create the environment and install the
+locked dependencies:
```bash
-uv sync --locked --group notebooks
+uv sync --locked
```
-If `uv sync` fails because of a hardlink issue, retry with:
+If Windows reports a hardlink problem, use:
```bash
uv sync --locked --link-mode=copy
```
-Create a project-specific config:
-
-```bash
-Copy-Item config.yaml local_config.yaml
-```
-
-Edit `local_config.yaml`, then run the app with that config:
-
-```bash
-uv run activitysim-viz --config local_config.yaml
-```
-
-By default, `activitysim-viz` follows `pipeline.steps` from the loaded config when no explicit step flags are supplied. The shipped example config defaults to `summarize` + `dashboard`, so a normal run will reuse summary caches when possible, rebuild them when needed, and then start the live dashboard on [http://localhost:5006](http://localhost:5006).
-
-## Dashboard Pages
-
-Dashboard pages now use one shared authoring model:
-
-- page classes use `@dashboard_page(...)` and subclass `DashboardPage`
-- dropdowns use `select(...)`; custom widgets use `selector(...)`
-- dynamic selectors declare an option provider and default policy
-- refreshable regions are registered with `section(...)`
-- large pages compose related selectors and sections with `feature(...)`
-- repeated chart transforms use `query(...)` without page-authored cache keys
-- live refresh and export metadata both derive from those registrations
-
-The main shared page-helper modules live under `dashboard/helpers/`:
-
-- `category_helpers.py`
-- `geography_helpers.py`
-- `person_type_helpers.py`
-- `time_distance_helpers.py`
-- `comparison_helpers.py`
-
-If you are adding or refactoring a page, start with the
-[dashboard page recipes](wiki/33-dashboard-page-recipes.md) and
-[figures/widgets guide](wiki/32-figures-and-widgets.md). The
-[dashboard extension cookbook](wiki/45-dashboard-extension-cookbook.md) covers
-the complete contributor path.
-
-## Config Setup
-
-The repo ships with `config.yaml` as a template. In practice, most people should:
-
-1. Copy `config.yaml` to `local_config.yaml` or another machine-specific file.
-2. Update the `runs` section to point at real ActivitySim output folders.
-3. Update `prepare.distance_skim`, `zones`, and `files` if your model layout differs from the defaults.
-4. Run with `--config your_file.yaml`.
-
-The canonical config layout is organized around a few top-level sections:
-
-```yaml
-root: artifacts/
-log_level: INFO
-
-pipeline:
- steps: [summarize, dashboard]
- dashboard_mode: live
- overwrite: false
-
-prepare: ...
-summarize: ...
-segment: ...
-dashboard: ...
-display: ...
-skimjoin: ...
-extensions: ...
-```
-
-Removed keys such as `processor.*`, `summaries.*`, `visualizer.*`, top-level
-`dashboard_labels`, and top-level `run_colors` now fail validation and name the
-canonical replacement. Unknown keys also fail instead of being silently ignored.
+### 2. Create a configuration
-The minimum useful config is usually:
+Copy `config.yaml` to `local_config.yaml`. In the new file, update the entries
+under `runs` so they point to your ActivitySim output directories:
```yaml
-root: artifacts/summary_cache
-
-pipeline:
- steps:
- - summarize
- - dashboard
- dashboard_mode: live
-
runs:
- - dir: path\to\run1
+ - dir: C:\models\base\output
label: Base
- - dir: path\to\run2
+ - dir: C:\models\build\output
label: Build
-
-skimjoin:
- distance_skim:
- file: path\to\skims.omx
- matrix: SOV_DIST__MD
-
-zones:
- use_maz: false
- maz_col: zone_id
- taz_col: TAZ
-
-files:
- households: final_households
- persons: final_persons
- tours: final_tours
- trips: final_trips
- joint_tour_participants: final_joint_tour_participants
- land_use: final_land_use
```
-If runs use different raw filenames, keep `files:` as the default mapping and
-override only the differences inside `runs[*].file_map`:
+The default file names are `final_households`, `final_persons`, `final_tours`,
+`final_trips`, `final_joint_tour_participants`, and `final_land_use`. Both CSV
+and Parquet inputs are supported.
-```yaml
-files:
- households: final_households
- persons: final_persons
- tours: final_tours
- trips: final_trips
-
-runs:
- - dir: path\to\run1
- label: Base
- file_map:
- households: final_hh
- trips: trip_linked
-
- - dir: path\to\run2
- label: Build
- file_map:
- households: household
- persons: person
- tours: tour
- trips: trip
-```
-
-If a run should skip raw prepare and use externally managed canonical prepared
-tables instead, point it at those files with `runs[*].prepared_table_map`:
-
-```yaml
-runs:
- - dir: path\to\raw_run
- label: Raw Run
-
- - label: Custom Prepared Run
- prepared_table_map:
- households: path\to\custom\households.parquet
- persons: path\to\custom\persons.csv
- tours: path\to\custom\tours.parquet
- trips: path\to\custom\trips.csv
- joint_tour_participants: path\to\custom\joint_tour_participants.parquet
- land_use: path\to\custom\land_use.csv
-```
-
-`prepared_table_map` is intended for canonical prepared tables that were already
-skimjoined and then optionally filtered or otherwise post-processed outside this
-repo. When a run uses `prepared_table_map`, the workflow loads those prepared
-tables directly and does not rerun raw prepare or integrated skimjoin for that run.
-
-If a run already has dashboard-ready summary tables, point directly at those
-files with `runs[*].summary_table_map`:
-
-```yaml
-runs:
- - label: Summary Only Demo
- summary_table_map:
- population_totals: path\to\summaries\population_totals.csv
- traffic_count_comparisons: path\to\summaries\traffic_count_comparisons.parquet
-```
+For a smaller example configuration and help with nonstandard files or zones,
+see [Getting Started](wiki/10-getting-started.md) and
+[Configuring Your Data](wiki/11-configuring-your-data.md).
-`summary_table_map` uses registered summary IDs as keys, accepts explicit
-`.csv` or `.parquet` paths, and resolves relative paths from the config file
-directory. Mapped summaries are expected to already use the dashboard's canonical
-columns. During summarize they override the listed generated summaries; missing
-summaries can still be generated from raw/prepared inputs when those inputs exist.
-Some registered summary IDs are external/demo-only and are not generated by
-default for raw/prepared runs, which avoids writing `__empty__` cache CSVs just
-to make those IDs available to `summary_table_map`.
+### 3. Start the visualizer
-Integrated skim enrichment can now be selected per run without forcing one
-shared skimjoin config for every skim structure. Keep the explicit skimjoin
-YAML logic in separate files, then choose the file and optional project-input
-overrides per run:
-
-```yaml
-skimjoin:
- defaults:
- config_path: configs/skimjoin_default.yaml
-
-runs:
- - dir: path\to\run_a
- label: Run A
- skimjoin:
- config_path: configs/skimjoin_odot_series15.yaml
- skim_files:
- - path\to\run_a\skims\*.omx
- - path\to\run_a\skims\maz_stop_walk.csv
- network_los_file: path\to\run_a\network_los.yaml
-
- - dir: path\to\run_b
- label: Run B
- skimjoin:
- config_path: configs/skimjoin_combined_walk.yaml
- skim_files:
- - path\to\run_b\skims\*.omx
-```
-
-Skimjoin override rules:
-
-- `runs[*].skimjoin.config_path` overrides global `skimjoin.config_path`.
-- `runs[*].skimjoin.skim_files` overrides the selected skimjoin config's `project.skim_files`.
-- `runs[*].skimjoin.network_los_file` overrides the selected skimjoin config's `project.network_los_file`.
-- `skimjoin.failure_policy` defaults to `record`; use `error` when skimjoin failures must stop a validation or batch run.
-- If a run omits `runs[*].skimjoin`, it uses the global skimjoin settings exactly as before.
-
-Recommended rule of thumb:
-
-- If runs differ only by skim file locations, share one skimjoin config and override `runs[*].skimjoin.skim_files`.
-- If runs differ only by period definitions, share one skimjoin config and override `runs[*].skimjoin.network_los_file`.
-- If runs differ by lookup logic, fallback behavior, combined vs split components, or directional semantics, use different skimjoin config files.
-
-VOT bin preparation stays in `prepare.vot_bins` and remains run-aware by run label.
-
-Skimjoin dimensions are now standardized under `dimensions`, while
-`activitysim` only carries the structural trip/tour fields. The recommended
-integrated-runtime pattern is:
-
-```yaml
-activitysim:
- trip_mode_column: trip_mode
- trip_id_column: trip_id
- tour_mode_column: tour_mode
- tour_id_column: tour_id
- outbound_column: outbound
-
-dimensions:
- PERIOD:
- source_columns:
- trip_source_column: depart_hour
- outbound_tour_source_column: start_hour
- inbound_tour_source_column: first_inbound_trip_depart
- values_from_network_los: true
- values:
- 8: AM
- 17: PM
- VOT:
- source_columns:
- trip_source_column: vot_bin
- outbound_tour_source_column: vot_bin
- inbound_tour_source_column: vot_bin
- values:
- L: L
- M: M
- H: H
+```bash
+uv run activitysim-viz --config local_config.yaml
```
-Period behavior is directional by design:
-
-- trips use `dimensions.PERIOD.source_columns.trip_source_column`
-- outbound tours use `dimensions.PERIOD.source_columns.outbound_tour_source_column`
-- inbound tours use `dimensions.PERIOD.source_columns.inbound_tour_source_column`
+The first run prepares the inputs, builds the summary tables needed by the
+dashboard, and opens a local server at
+[http://localhost:5006](http://localhost:5006). Later runs reuse valid caches.
+Stop the server with `Ctrl+C`.
-In the standard prepare workflow, `first_inbound_trip_depart` is derived from
-the first inbound trip on each tour before integrated skimjoin runs.
+If something is missing or the first run fails, start with
+[Troubleshooting](wiki/90-troubleshooting.md).
-Prepared endpoint columns are also standardized before skimjoin runs:
+## How It Works
-- prepared trips and tours always include `OTAZ` and `DTAZ`
-- when `zones.use_maz: true`, prepare also materializes `o_maz` and `d_maz`
-- inbound tour lookups reuse those same column names, while skimjoin swaps
- their logical direction in the inbound tour context
-
-The normal prepare step can also write prepared caches as CSV when needed:
-
-```yaml
-prepare:
- output:
- file_format: csv
- validation:
- relationship_checks: warn
+```text
+ActivitySim outputs
+ -> prepare canonical tables
+ -> summarize travel measures
+ -> display a live dashboard or export standalone HTML
```
-Important path rules:
-
-- `root` is resolved relative to the config file if you give a relative path.
-- The prepared cache is created automatically next to `root` as `prepared_cache/`.
-- `runs[*].dir` should point at an ActivitySim output directory.
-- `prepare.distance_skim.file` may be absolute, or relative to each run directory.
-- File entries under `files` can be bare stems like `final_trips` or explicit filenames like `final_trips.csv`.
-- `runs[*].file_map` uses the same filename rules as `files`, but applies only to that run.
-- `runs[*].prepared_table_map` must use explicit `.parquet` or `.csv` paths and resolves relative paths from the config file directory.
-- `runs[*].summary_table_map` must use registered summary IDs with explicit `.parquet` or `.csv` paths and resolves relative paths from the config file directory.
-- `prepare.output.file_format` controls how standard prepared caches are written; supported values are `parquet` and `csv`, with `parquet` as the default.
-- `prepare.validation.relationship_checks` controls prepared-table foreign-key validation. Use `warn` to log inconsistencies and continue, `error` to fail the run, or `off` to skip the checks.
-- `dashboard.export.output_path`, when relative, is resolved from `root`.
-
-## Config Reference
+The configuration selects the inputs, workflow steps, output location, and
+dashboard mode. Most users can keep using the same launch command and change
+the YAML when they want a different workflow.
-These are the sections most people need to touch:
-
-| Section | Purpose |
+| Goal | Where to learn more |
|---|---|
-| `root` | Where summary caches are stored |
-| `pipeline` | Default workflow steps, dashboard mode, and overwrite behavior |
-| `runs` | Run directories, display labels, and optional per-run skim, raw file-map, custom prepared-table map, custom summary-table map, and weight overrides |
-| `prepare.distance_skim` | Default distance skim file and matrix name used by summaries |
-| `zones` | MAZ/TAZ settings for skim joins and zone normalization |
-| `files` | Default ActivitySim output file stems or filenames used unless a run overrides them |
-| `columns` | Column aliases when outputs use non-default names |
-| `prepare.output.file_format` | On-disk format for prepared caches written by the normal prepare workflow |
-| `prepare.validation.relationship_checks` | Whether cross-table prepared-key validation is disabled, warns, or errors |
-| `prepare.student_types` | Optional school/university enrollment definitions for shadow pricing pages |
-| `dashboard.title` | Title used in the live dashboard and HTML export |
-| `dashboard.include_notes` | Show per-plot and per-table calculation notes in the live dashboard and HTML export (default: `true`) |
-| `dashboard.live.pages` | Ordered list of live pages/groups to show |
-| `dashboard.export` | Export-only output path, page selection, and selector-state controls |
-| `display.run_colors` | Plot colors by run |
-| `display.labels` | Presentation-only labels and ordering for dashboard/export |
-| `weighting.modes` | Named weighting alternatives backed by household, person, and/or trip columns |
-| `extensions` | Advanced importable weighting calculations and their summary-affecting settings |
-| `summarize.weighting_modes` | Ordered built-in, declarative, or custom weighting-mode IDs to build |
-| `summarize.failure_policy` | `record` keeps failed summaries visible as diagnostics; `error` stops immediately on a builder exception |
-| `summarize.geography` | Optional configured district/county/zone mappings |
-| `summarize.pnr_tour_modes` | Which tour modes count as park-and-ride in summary builders |
-| `summarize.group_*_tour_purposes` | Summary-time purpose regrouping switches |
-| `summarize.category_normalization` | Summary-affecting category normalization/regrouping |
-| `modes` | Optional mode ordering and grouped mode display |
-| `display.labels.person_type` | Optional display labels for `ptype` values |
-
-Weighting rules:
-
-- If a run sets `hh_weight_col`, `person_weight_col`, or `trip_weight_col`, those are used.
-- Otherwise, if a `sample_rate` column is available, weights are derived from it.
-- Otherwise, weights default to `1`.
-- `weighting.modes` can select additional prepared household, person, and trip columns as named alternatives without replacing the primary `weighted` mode.
-
-Geography summary notes:
-
-- Summaries may emit `all_geographies` total rows independently of the geography config.
-- Native prepared geographies such as `home_taz`, `home_county`, and `home_mpo` may appear whenever those columns are available in prepared data, even when `summarize.geography.enabled: false`.
-- `summarize.geography` controls additional mapped geography aggregations, such as `home_geo__school_district`, `work_geo__county`, or `land_use_geo__district`.
-
-Removed config notes:
-
-- Prefer the canonical top-level schema: `root`, `pipeline`, `dashboard`, `display`, `summarize`, `segment`, and `skimjoin`.
-- Older keys such as `processor.root`, `summaries.weighting_modes`, `visualizer.dashboard_pages`, top-level `run_colors`, top-level `summary_categories`, and top-level `student_types` are rejected with their canonical replacement.
-
-Geography note:
-
-- `summarize.geography.enabled: false` disables mapped geography aggregation columns. Set it to `true` for aggregation-based geography summaries.
-
-Category config note:
-
-- Use `summarize.category_normalization` when a mapping changes summary values, grouping membership, or canonical category values.
-- Use `display.labels` when a change is cosmetic and should only affect dashboard/export labels or ordering.
-
-## Live Pages And Export Pages
-
-`dashboard.live.pages` controls the live dashboard only. `dashboard.export` controls what goes into the standalone HTML export.
-
-Current top-level page ids are:
-
-- `overview`
-- `long_term_choices`
-- `daily_travel`
-- `joint_travel`
-- `tour_summaries`
-- `trip_summaries`
-- `validation`
-- `raw_trip_demo`
-
-Grouped page ids support either the whole group or specific child pages. For example:
+| Use raw ActivitySim output folders | [Configuring Your Data](wiki/11-configuring-your-data.md#raw-activitysim-output) |
+| Use already-prepared tables | [Already-Prepared Tables](wiki/11-configuring-your-data.md#already-prepared-tables) |
+| Use dashboard-ready summary tables | [Dashboard-Ready Summary Tables](wiki/11-configuring-your-data.md#dashboard-ready-summary-tables) |
+| Run only the processor | [Processor-Only Workflow](wiki/12-running-workflows.md#configure-a-processor-only-workflow) |
+| Create a standalone HTML dashboard | [HTML Export](wiki/34-html-export.md) |
+| Understand caches and workflow steps | [Running Workflows](wiki/12-running-workflows.md) |
+| Find an exact configuration field | [Configuration Reference](wiki/13-configuration-reference.md) |
+| Understand a summary table or field | [Summary Catalog](wiki/24-summary-catalog.md) |
-```yaml
-dashboard:
- # Set to false to omit all per-plot and per-table calculation notes.
- include_notes: true
- live:
- pages:
- - overview
- - long_term_choices:
- - individual_choices
- - mandatory_location_choice
- - shadow_pricing
- - daily_travel: default
- - tour_summaries: all
- - trip_summaries:
- - trip_mode
- - trip_stop_time
-```
-
-Notes:
-
-- `default` means "the group's default enabled children".
-- `all` means every child page in the group.
-- A plain group id like `tour_summaries` behaves like the group's default selection.
-- `raw_trip_demo` is disabled by default and requests prepared trip tables, so keep it out unless you explicitly want that behavior.
-
-For HTML export, start with the live page set and override selector states or
-parts as needed:
-
-```yaml
-dashboard:
- export:
- dashboard:
- weighting: [unweighted]
- values: [percent]
- exclude_groups: [validation]
- pages:
- long_term_choices:
- shadow_pricing:
- geography_level: [all]
- student_type: [all]
- parts:
- workplace_table:
- enabled: false
- school_table:
- enabled: false
-```
-
-Rules worth remembering:
-
-- If `dashboard.live.pages` is omitted, the app uses its built-in default page set.
-- Export always starts from the live page set. Entries under
- `dashboard.export.pages` modify matching pages; they are not an allow-list.
-- Export selector requests accept `default`, `all`, or a list of explicit values.
-- Set a page override's `enabled` to `false`, or use
- `dashboard.export.exclude_pages` / `exclude_groups`, to remove pages from
- export without changing the live dashboard.
-
-## Run Modes
-
-The CLI exposes three workflow steps:
-
-1. `prepare`
-2. `summarize`
-3. `dashboard`
-
-Common commands:
+## Documentation
-| Command | What it does |
-|---|---|
-| `python run.py --config local_config.yaml` | Reuse or build summaries, then start the live dashboard |
-| `python run.py --config local_config.yaml --prepare-only` | Build prepared caches and exit |
-| `python run.py --config local_config.yaml --summarize` | Reuse or build summary caches and exit |
-| `python run.py --config local_config.yaml --summarize --dashboard` | Explicit form of the default live workflow |
-| `python run.py --config local_config.yaml --dashboard` | Start the dashboard from existing summary caches for the configured runs |
-| `python run.py --config local_config.yaml --prepare --summarize --dashboard` | Force the full prepare -> summarize -> dashboard chain in one run |
-| `python run.py --config local_config.yaml --from-csvs` | Start the dashboard from existing summary caches only |
-| `python run.py --config local_config.yaml --from-csvs --export-html output.html` | Build a standalone HTML export from existing summary caches |
-| `python run.py --config local_config.yaml --summarize --write-csvs` | Rebuild summaries and write fresh cache files |
-| `python run.py --config local_config.yaml --summarize --skip-summary-cache-write` | Build summaries for this run without writing cache updates |
-| `python run.py --config local_config.yaml --summarize --refresh-summary-cache` | Delete and rebuild summary caches for the selected runs |
-| `python run.py --config local_config.yaml --summarize --refresh-prepared-cache` | Rebuild summaries from freshly prepared tables instead of prepared-cache hits |
-| `python run.py --config local_config.yaml --prepare --summarize --refresh-caches` | Delete and rebuild both prepared and summary caches for the selected runs |
-
-Behavior details:
-
-- `--from-csvs` is cache-only: it reads visualizer summary-cache directories with manifests, not loose summary CSVs.
-- `--from-csvs path\to\cache1 path\to\cache2` lets you point directly at specific summary cache directories.
-- Use `runs[*].summary_table_map` when you have loose dashboard-ready summary files instead of visualizer cache directories.
-- `--dashboard` by itself is valid when summary caches already exist for the configured runs.
-- During summarize, the app will reuse prepared cache when possible and rebuild from raw outputs only when needed.
-- `--refresh-prepared-cache` deletes the selected runs' prepared-cache directories first, then disables prepared-cache reuse for that invocation.
-- `--refresh-summary-cache` deletes the selected runs' summary-cache directories first, then disables summary-cache reuse for that invocation.
-- `--refresh-caches` is shorthand for both refresh flags together.
-
-## Cache Layout
-
-Prepared caches are written automatically next to the summary cache root:
+The [wiki home](wiki/00-home.md) is the main documentation index.
-```text
-/
- prepared_cache/
- /
- manifest.json
- households.parquet|csv
- persons.parquet|csv
- tours.parquet|csv
- trips.parquet|csv
- joint_tour_participants.parquet|csv
- land_use.parquet|csv
-```
+For normal use, these three chapters cover the usual path:
-Summary caches are written under `root`:
+1. [Getting Started](wiki/10-getting-started.md)
+2. [Configuring Your Data](wiki/11-configuring-your-data.md)
+3. [Running Workflows](wiki/12-running-workflows.md)
-```text
-/
- /
- manifest.json
- weighted/
- unweighted/
-```
+Additional user references:
-Both cache layers validate manifests before reuse. Cache invalidation is driven by:
+- [Output Visualizer](wiki/30-output-visualizer.md) explains the dashboard.
+- [Dashboard Pages](wiki/31-dashboard-pages.md) lists the available analyses.
+- [HTML Export](wiki/34-html-export.md) covers offline sharing.
+- [Summary Catalog](wiki/24-summary-catalog.md) documents every summary table.
+- [Glossary](wiki/99-glossary.md) defines project terminology.
+- [Troubleshooting](wiki/90-troubleshooting.md) covers common failures.
-- the run inputs
-- the prepare and summary config digests
-- the prepared-manifest identity used to build summary caches
-- per-summary summary digests inside the summary-cache manifest
+## For Contributors
-That means presentation-only config changes usually do not force summary rebuilds, and adding a newly requested summary can backfill just that table instead of rebuilding the entire summary bundle.
+Start with [Architecture](wiki/01-architecture.md) and
+[Developer Workflows](wiki/40-developer-workflows.md). Task-specific guides are
+available for:
-## CLI Overrides
+- [extending prepared data](wiki/41-data-extension-cookbook.md);
+- [adding a summary function](wiki/44-summary-function-cookbook.md);
+- [adding dashboard pages, figures, or widgets](wiki/45-dashboard-extension-cookbook.md);
+- [changing configuration, columns, or labels](wiki/42-config-column-label-cookbook.md);
+- [skim enrichment](wiki/22-skimjoin.md); and
+- [testing](wiki/46-testing.md).
-You can override runs on the command line instead of putting them in the config:
+Run focused tests while developing. The standard full test command is:
```bash
-python run.py --config local_config.yaml ^
- --run C:\path\to\run1 "Base" ^
- --run C:\path\to\run2 "Build"
+uv run pytest --basetemp .pytest_tmp
```
-Optional per-run skim overrides can be supplied in the same order:
+After changing summary declarations or dashboard page definitions, regenerate
+the code-backed wiki catalogs:
```bash
-python run.py --config local_config.yaml ^
- --run C:\path\to\run1 "Base" ^
- --run C:\path\to\run2 "Build" ^
- --run-skim C:\path\to\base_skims.omx C:\path\to\build_skims.omx
+uv run python scripts/generate_wiki_catalogs.py
```
-Use `null`, `None`, or an empty string in `--run-skim` to fall back to the configured `prepare.distance_skim.file`.
-
-## Codebase Map
-
-```text
-activitysim_visualizer/
-|-- run.py
-|-- runtime/
-| |-- workflows/
-|-- runtime/
-| `-- config/
-|-- processor/
-| |-- prepare/
-| |-- summarize/
-| `-- models.py
-|-- dashboard/
-| |-- app.py
-| |-- export/
-| |-- page_base.py
-| |-- page_declarations.py
-| |-- page_diagnostics.py
-| |-- page_features.py
-| |-- page_lifecycle.py
-| |-- page_navigation.py
-| |-- page_definitions.py
-| |-- page_registry.py
-| |-- state.py
-| `-- pages/
-`-- tests/
-```
-
-## Documentation
+## License
-The main user and contributor documentation lives in the
-[`wiki/`](wiki/00-home.md) chapter set. Start with:
-
-- [Getting Started](wiki/10-getting-started.md)
-- [Architecture](wiki/01-architecture.md)
-- [Configuration Reference](wiki/13-configuration-reference.md)
-- [Output Processor](wiki/20-output-processor.md)
-- [Output Visualizer](wiki/30-output-visualizer.md)
-- [Developer Workflows](wiki/40-developer-workflows.md)
-- [Data Extension Cookbook](wiki/41-data-extension-cookbook.md)
-- [Config, Columns, and Labels](wiki/42-config-column-label-cookbook.md)
-- [Weighting and Hosting Extensions](wiki/43-weighting-hosting-extensions.md)
-- [Summary Function Cookbook](wiki/44-summary-function-cookbook.md)
-- [Dashboard Extension Cookbook](wiki/45-dashboard-extension-cookbook.md)
-- [Testing](wiki/46-testing.md)
-- [Troubleshooting](wiki/90-troubleshooting.md)
-
-The wiki is the sole documentation source. Add or revise a wiki chapter instead
-of creating a parallel documentation tree.
-
-## Documentation Maintenance Checklist
-
-When behavior changes, update docs in the same change:
-
-- New config key or config behavior: update chapters 11 and 13.
-- New summary declaration or contract: update chapter 23 and regenerate catalogs.
-- New page, selector, or plotting behavior: update chapters 31 through 33 and regenerate catalogs.
-- New export payload/runtime behavior: update chapter 34.
-- Architecture or runtime-flow changes: update chapters 12, 20, and 30 as applicable.
-
-## Tests
-
-See [Developer Workflows](wiki/40-developer-workflows.md) for the normal test
-loop and [Testing](wiki/46-testing.md) for the fast/full split and
-offline-export boundary.
+This project is licensed under the GNU General Public License v3.0. See
+[`LICENSE.txt`](LICENSE.txt).
diff --git a/config.yaml b/config.yaml
index ede454f..b4e9e09 100644
--- a/config.yaml
+++ b/config.yaml
@@ -1,42 +1,25 @@
-# ActivitySim Visualizer Configuration
-# Canonical config example using the current schema.
+# ActivitySim Visualizer starter configuration
+#
+# Copy this file to local_config.yaml, then:
+# 1. Replace the two run directories below.
+# 2. Confirm the zone settings match your model.
+# 3. Run: uv run activitysim-viz --config local_config.yaml
+#
+# Standard ActivitySim final_* CSV or Parquet files work without other changes.
+# For other input types or advanced options, start at wiki/00-home.md.
-name: "Example ActivitySim Visualizer"
+name: ActivitySim Run Comparison
root: artifacts
-log_level: INFO
+# Summarize prepares raw inputs automatically when no valid prepared cache
+# exists, then the dashboard opens at http://localhost:5006.
pipeline:
- steps:
- - summarize
- - dashboard
- # Add `prepare`, `skimjoin`, or `segment` when those explicit stages are
- # needed. Summarize automatically prepares data when no valid cache exists.
- dashboard_mode: live # none | live | export | host
+ steps: [summarize, dashboard]
+ dashboard_mode: live
overwrite: false
-# Optional named alternatives backed by columns retained in prepared tables.
-# weighting:
-# modes:
-# calibrated:
-# label: Calibrated
-# columns:
-# households: calibrated_hh_weight
-# persons: calibrated_person_weight
-# trips: calibrated_trip_weight
-
-# Advanced trusted calculations only. Each importable module defines
-# register_weighting_modes(registry); settings are available on Config and enter
-# summary cache identity.
-# extensions:
-# modules: [my_project.weighting]
-# settings:
-# calibrated:
-# multiplier: 1.0
-
-# ---------------------------------------------------------------------------
-# ActivitySim output file names
-# Use stems (no extension) for automatic format detection.
-# ---------------------------------------------------------------------------
+# These are the default ActivitySim file stems. Edit only names that differ in
+# your model outputs; the visualizer accepts either .csv or .parquet files.
files:
households: final_households
persons: final_persons
@@ -45,261 +28,22 @@ files:
joint_tour_participants: final_joint_tour_participants
land_use: final_land_use
-# Optional shared fallback files for optional inputs that may be missing in
-# some run folders. These must be explicit .csv or .parquet paths.
-# fallback_files:
-# land_use: C:\path\to\shared\land_use.csv
-
-# ---------------------------------------------------------------------------
-# Runs to compare
-# `runs[*].skimjoin` overrides the global `skimjoin.defaults` settings.
-# `prepared_table_map` entries must be explicit .csv or .parquet paths.
-#
-# Use `file_map` when using non-standard input filenames
-#
-# use `prepared_table_map` if you have prepared tables you want to use other
-# than the cached prepared tables. For example, you could run the `prepare`
-# step of the pipeline, perform your own filtering operations on the
-# prepared tables, then read your filtered prepared tables into the
-# `summarize` and `dashboard` steps.
-# ---------------------------------------------------------------------------
+# Required: replace these example directories. Add or remove runs as needed.
runs:
- - dir: path\to\activitysim\output\run1
+ - dir: C:\path\to\base\output
label: Base
- # skimjoin:
- # config_path: example_skimjoin_config.yaml
- # skim_files:
- # - C:\path\to\model_skims\*.omx
- # - C:\path\to\model_skims\maz_stop_walk.csv
- # network_los_file: C:\path\to\model_skims\network_los.yaml
-
- - dir: path\to\activitysim\output\run2
+ - dir: C:\path\to\build\output
label: Build
- # file_map:
- # households: household
- # persons: person
- # tours: tour
- # trips: trip_linked
- # joint_tour_participants: joint_tour_participants
- # land_use: land_use
- # prepared_table_map:
- # households: path\to\prepared\households.parquet
- # persons: path\to\prepared\persons.parquet
- # tours: path\to\prepared\tours.parquet
- # trips: path\to\prepared\trips.parquet
- # joint_tour_participants: path\to\prepared\joint_tour_participants.parquet
- # land_use: path\to\prepared\land_use.parquet
-# ---------------------------------------------------------------------------
-# Zone system
-# Set use_maz: false for TAZ-only models.
-# ---------------------------------------------------------------------------
+# Confirm these fields before the first run. For a TAZ-only model, keep
+# use_maz false. For a MAZ/TAZ model, set it to true and name both columns.
zones:
use_maz: false
maz_col: zone_id
taz_col: TAZ
-# ---------------------------------------------------------------------------
-# Column names in the ActivitySim output files
-# Alias-capable fields may be a single string or an ordered list of candidates.
-# ---------------------------------------------------------------------------
-columns:
- ptype: ptype
- hhsize: hhsize
- auto_ownership: auto_ownership
- num_workers: num_workers
- num_adults: num_adults
- # sample_rate: sample_rate
- # household_id: [household_id, hh_id]
- # person_id: [person_id, pid]
- # tour_id: [tour_id, tid]
- # trip_id: [trip_id, tripid]
- # tour_purpose: [tour_purpose, primary_purpose, purpose]
- # trip_purpose: [trip_purpose, purpose]
- # tour_mode: [tour_mode, mode]
- # trip_mode: [trip_mode, mode]
-
-
-# ---------------------------------------------------------------------------
-# Settings for the `prepare` step.
-# ---------------------------------------------------------------------------
-prepare:
- output:
- file_format: parquet # parquet | csv
- validation:
- relationship_checks: warn # off | warn | error
- distance_skim:
- file: path\to\skims.omx
- matrix: SOV_DIST__MD
- # auto_sufficiency_basis: licensed_drivers # licensed_drivers | workers | adults
- # vot_bins:
- # source_column: income_segment
- # output_column: vot_bin
- # fallback_value: M
- # mappings:
- # base:
- # 1: L
- # 2: M
- # 3: H
-
-# ---------------------------------------------------------------------------
-# Settings for the `skimjoin` step, to be used as defaults.
-# ---------------------------------------------------------------------------
-skimjoin:
- failure_policy: record # record | error
- create_hypothetical_skim_tables: false
- defaults:
- config_path: example_skimjoin_config.yaml
- skim_files:
- - C:\path\to\model_skims\*.omx
- - C:\path\to\model_skims\maz_stop_walk.csv
- - C:\path\to\model_skims\maz_maz_walk.csv
- network_los_file: C:\path\to\model_skims\network_los.yaml
-
-# ---------------------------------------------------------------------------
-# Settings for the `segment` step.
-# ---------------------------------------------------------------------------
-segment:
- dashboard:
- segmentation_type: signup_platform
- visibility: segments_only # full_only | segments_only | full_and_segments
- definitions:
- signup_platform:
- include_full: true
- persist_segmented_prepared_tables: false
- allow_overlapping: false
- on_empty_segment: warn
- source:
- type: prepared_column
- source_table: hh
- column: signup_platform
- segments:
- - id: rmove
- label: RMove
- values: ["rmove"]
- - id: browser
- label: Browser
- values: ["browser"]
- - id: call
- label: Call
- values: ["call"]
-
-# ---------------------------------------------------------------------------
-# Settings for the `summarize` step.
-# ---------------------------------------------------------------------------
summarize:
weighting_modes: [weighted, unweighted]
- failure_policy: record # record | error
- pnr_tour_modes:
- - PNR_TRANSIT
- # Controls additional mapped geography aggregations only. Summaries may still
- # emit all_geographies totals, and native prepared home geographies such as
- # home_taz, home_county, and home_mpo can appear when those columns exist.
- geography:
- enabled: false
- # Configured mappings create columns such as home_geo__district,
- # work_geo__county, or land_use_geo__district.
- # landuse_col: COUNTY
- # mapping:
- # 1: County 1
- # aggregations:
- # district:
- # source_zone_system: maz
- # file: C:\path\to\land_use.csv
- # zone_id_col: MAZ
- # geography_col: DISTRICT
-# ---------------------------------------------------------------------------
-# Settings for the `dashboard` step.
-# ---------------------------------------------------------------------------
dashboard:
- title: "ActivitySim Comparison Visualizer"
- include_notes: true
- enable_maz_geographies: false
- live:
- pages:
- - overview
- - long_term_choices
- - daily_travel
- - joint_travel
- - tour_summaries
- - trip_summaries
- - validation
- export:
- output_path: exports/dashboard.html
- # `pages` overrides matching live pages; it is not an inclusion list.
- # dashboard:
- # weighting: [unweighted]
- # pages:
- # long_term_choices:
- # shadow_pricing:
- # geography_level: [all]
- # student_type: [all]
- # parts:
- # workplace_table:
- # enabled: false
- # school_table:
- # enabled: false
- # Reserved for a future hosted-dashboard implementation. These settings are
- # validated but intentionally ignored by the current runtime.
- # host:
- # account: my-connect-cloud-account
- # app_id: 12345
- # title: ActivitySim Comparison Visualizer
- # verify: true
-
-display:
- missing_data_display: card # card | blank
- # bar_hover_mode: all # closest | all
- # density_hover_mode: all # closest | all
- labels:
- person_type:
- mapping:
- all_person_types: All Person Types
- 1: Full-time worker
- 2: Part-time worker
- 3: University student
- 4: Non-worker adult
- 5: Retired
- 6: Driving-age student
- 7: Non-driving-age student
- 8: Preschool
-
- geography:
- mapping:
- all_geographies: All Geographies
- county: County
-
- tour_purpose:
- mapping:
- all_tour_purposes: All Tour Purposes
- work: Work
- school: School
- escort: Escort
- shopping: Shopping
- othmaint: Other Maintenance
- eatout: Eat Out
- social: Social
- othdiscr: Other Discretionary
- atwork: At-Work
- joint: Joint
-
- mode:
- mapping:
- SOV: Drive Alone
- HOV2: Shared Ride 2
- HOV3: Shared Ride 3+
- WALK: Walk
- BIKE: Bike
- WALK_TRANSIT: Walk-Transit
- PNR_TRANSIT: PNR-Transit
- KNR_TRANSIT: KNR-Transit
- TNC_SINGLE: TNC-Single
- TNC_SHARED: TNC-Pool
-
- run_colors:
- - "#298c8c"
- - "#a00000"
- - "#b8b8b8"
- - "#384860"
- - "#ff7f0e"
+ title: ActivitySim Run Comparison
diff --git a/scripts/generate_wiki_catalogs.py b/scripts/generate_wiki_catalogs.py
index 0bccbce..385cd12 100644
--- a/scripts/generate_wiki_catalogs.py
+++ b/scripts/generate_wiki_catalogs.py
@@ -54,8 +54,8 @@ def build_summary_catalog() -> str:
"",
f"Total registered summaries: **{len(SUMMARY_DEFINITIONS)}**",
"",
- "| Summary ID | Filename | Builder | Output schema | Required inputs |",
- "|---|---|---|---|---|",
+ "| Summary ID | Filename | Default build | Builder | Output schema | Required inputs |",
+ "|---|---|---|---|---|---|",
]
for definition in sorted(
@@ -85,6 +85,7 @@ def build_summary_catalog() -> str:
[
f"`{_escape_cell(definition.summary_id)}`",
f"`{_escape_cell(definition.filename)}.csv`",
+ "yes" if definition.build_by_default else "no",
f"`{_escape_cell(builder_name)}`",
schema,
required,
@@ -96,6 +97,41 @@ def build_summary_catalog() -> str:
return "\n".join(lines)
+def _validate_summary_reference() -> None:
+ """Keep the hand-written analytical reference aligned with declarations."""
+ from processor.summarize.catalog import SUMMARY_DEFINITIONS
+
+ path = WIKI / "24-summary-catalog.md"
+ reference = path.read_text(encoding="utf-8").split(
+ "",
+ 1,
+ )[0]
+ missing: list[str] = []
+ for definition in SUMMARY_DEFINITIONS:
+ prefix = f"| `{definition.summary_id}` |"
+ row = next(
+ (line for line in reference.splitlines() if line.startswith(prefix)),
+ "",
+ )
+ if not row:
+ missing.append(definition.summary_id)
+ continue
+ absent_fields = [
+ field_name
+ for field_name in definition.contract.schema
+ if field_name and f"`{field_name}`" not in row
+ ]
+ if absent_fields:
+ missing.append(
+ f"{definition.summary_id} fields: {', '.join(absent_fields)}"
+ )
+
+ if missing:
+ raise ValueError(
+ "Summary analytical reference is incomplete: " + "; ".join(missing)
+ )
+
+
def build_dashboard_page_catalog() -> str:
from dashboard.page_registry import all_group_definitions, all_page_definitions
@@ -166,6 +202,7 @@ def build_dashboard_page_catalog() -> str:
def main() -> None:
+ _validate_summary_reference()
_replace_generated_section(
WIKI / "24-summary-catalog.md",
marker="SUMMARY-CATALOG",
diff --git a/simor_configs/metro_configs/metro_config.yaml b/simor_configs/metro_configs/metro_config.yaml
new file mode 100644
index 0000000..c1a5695
--- /dev/null
+++ b/simor_configs/metro_configs/metro_config.yaml
@@ -0,0 +1,624 @@
+# ActivitySim Visualizer Configuration
+# Adapt this file for each model deployment.
+# Only the sections you need are required — see comments for which are optional.
+
+name: "Metro Settings"
+root: ../../simor_project_outputs/metro_outputs # relative to the config's location
+log_level: INFO
+
+pipeline:
+ steps:
+ - prepare
+ - skimjoin
+ # - segment
+ - summarize # will automatically overwrite summaries with a stale cache
+ # - dashboard
+ dashboard_mode: live # live | export | host
+ overwrite: true # overwrite ALL prepared tables / summaries
+
+# ---------------------------------------------------------------------------
+# ActivitySim output file names
+# Use stems (no extension) for automatic format detection: the reader will try
+# .parquet first, then .csv. You may also specify an explicit extension to
+# force a particular format (e.g. final_households.csv or final_tours.parquet).
+# ---------------------------------------------------------------------------
+files:
+ households: household
+ persons: person
+ day: day
+ tours: tour
+ trips: trip_linked
+ vehicles: vehicles
+ joint_tour_participants: joint_tour_participants
+ land_use: land_use
+
+
+
+# Optional shared fallback files for optional inputs that may be missing in some
+# run folders. These must be explicit .csv or .parquet paths.
+# fallback_files:
+# land_use: C:\Users\wesley.darling\Downloads\dataset_2026-05-21\unfiltered\land_use.csv
+
+# ---------------------------------------------------------------------------
+# Runs to compare
+# Each entry specifies a run directory, a display label, and optional overrides.
+#
+# Weight columns (all optional — leave as null or omit to use default rules):
+# hh_weight_col: explicit household weight column in final_households
+# person_weight_col: explicit person weight column in final_persons
+# trip_weight_col: explicit trip weight column in final_trips
+# (tour weight = average of its trips' weights when set)
+#
+# If none of the above are set and sample_rate is not in columns, all weights = 1.
+# ---------------------------------------------------------------------------
+runs:
+ # - dir: C:\Users\wesley.darling\Downloads\dataset_2026-05-21\unfiltered
+ # label: Unfiltered
+ # skimjoin:
+ # config_path: will_skimjoin_config_alternate_id_col.yaml
+ # # Synthetic assignment-side summaries exercise validation-page wiring.
+ # # Replace these paths with real summaries without changing dashboard code.
+ # summary_table_map:
+ # link_validation_summary: ../outside_summary_tables/estimated_fixtures/unfiltered/link_validation_summary.csv
+ # count_location_counts_validation_summary: ../outside_summary_tables/countLocCounts.csv
+ # count_location_volumes_validation_summary: ../outside_summary_tables/estimated_fixtures/unfiltered/count_location_volumes_validation_summary.csv
+ # screenline_flow_comparisons: ../outside_summary_tables/estimated_fixtures/unfiltered/screenline_flow_comparisons.csv
+ # commuting_flows: ../outside_summary_tables/estimated_fixtures/unfiltered/commuting_flows.csv
+ # transit_boardings_by_operator_and_technology: ../outside_summary_tables/estimated_fixtures/unfiltered/transit_boardings_by_operator_and_technology.csv
+ # transit_transfer_rate: ../outside_summary_tables/estimated_fixtures/unfiltered/transit_transfer_rate.csv
+ # bicycle_vmt_by_facility_type: ../outside_summary_tables/estimated_fixtures/unfiltered/bicycle_vmt_by_facility_type.csv
+ # commercial_vehicle_validation_summary: ../outside_summary_tables/estimated_fixtures/unfiltered/commercial_vehicle_validation_summary.csv
+ # commercial_vehicle_vmt_validation_summary: ../outside_summary_tables/estimated_fixtures/unfiltered/commercial_vehicle_vmt_validation_summary.csv
+ # external_trip_validation_summary: ../outside_summary_tables/estimated_fixtures/unfiltered/external_trip_validation_summary.csv
+ # external_vmt_validation_summary: ../outside_summary_tables/estimated_fixtures/unfiltered/external_vmt_validation_summary.csv
+ # # skim_files:
+ # - C:\Users\wesley.darling\project_data\odot_skims\*.omx
+ # - C:\Users\wesley.darling\project_data\odot_skims\maz_stop_walk.csv
+ # - C:\Users\wesley.darling\project_data\odot_skims\maz_maz_walk.csv
+ # network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
+
+ # - dir: C:\Users\wesley.darling\Downloads\dataset_2026-05-21\filtered
+ # label: Filtered
+ # skimjoin:
+ # config_path: will_skimjoin_config_alternate_id_col.yaml
+ # summary_table_map:
+ # link_validation_summary: ../outside_summary_tables/estimated_fixtures/filtered/link_validation_summary.csv
+ # count_location_counts_validation_summary: ../outside_summary_tables/countLocCounts.csv
+ # count_location_volumes_validation_summary: ../outside_summary_tables/estimated_fixtures/filtered/count_location_volumes_validation_summary.csv
+ # screenline_flow_comparisons: ../outside_summary_tables/estimated_fixtures/filtered/screenline_flow_comparisons.csv
+ # commuting_flows: ../outside_summary_tables/estimated_fixtures/filtered/commuting_flows.csv
+ # transit_boardings_by_operator_and_technology: ../outside_summary_tables/estimated_fixtures/filtered/transit_boardings_by_operator_and_technology.csv
+ # transit_transfer_rate: ../outside_summary_tables/estimated_fixtures/filtered/transit_transfer_rate.csv
+ # bicycle_vmt_by_facility_type: ../outside_summary_tables/estimated_fixtures/filtered/bicycle_vmt_by_facility_type.csv
+ # commercial_vehicle_validation_summary: ../outside_summary_tables/estimated_fixtures/filtered/commercial_vehicle_validation_summary.csv
+ # commercial_vehicle_vmt_validation_summary: ../outside_summary_tables/estimated_fixtures/filtered/commercial_vehicle_vmt_validation_summary.csv
+ # external_trip_validation_summary: ../outside_summary_tables/estimated_fixtures/filtered/external_trip_validation_summary.csv
+ # external_vmt_validation_summary: ../outside_summary_tables/estimated_fixtures/filtered/external_vmt_validation_summary.csv
+ # # skim_files:
+ # - C:\Users\wesley.darling\project_data\odot_skims\*.omx
+ # - C:\Users\wesley.darling\project_data\odot_skims\maz_stop_walk.csv
+ # - C:\Users\wesley.darling\project_data\odot_skims\maz_maz_walk.csv
+ # network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
+
+ # - dir: C:\Users\wesley.darling\Downloads\dataset_2026-05-21\override
+ # label: Override
+ # summary_table_map:
+ # link_validation_summary: ../outside_summary_tables/estimated_fixtures/override/link_validation_summary.csv
+ # count_location_counts_validation_summary: ../outside_summary_tables/countLocCounts.csv
+ # count_location_volumes_validation_summary: ../outside_summary_tables/estimated_fixtures/override/count_location_volumes_validation_summary.csv
+ # screenline_flow_comparisons: ../outside_summary_tables/estimated_fixtures/override/screenline_flow_comparisons.csv
+ # commuting_flows: ../outside_summary_tables/estimated_fixtures/override/commuting_flows.csv
+ # transit_boardings_by_operator_and_technology: ../outside_summary_tables/estimated_fixtures/override/transit_boardings_by_operator_and_technology.csv
+ # transit_transfer_rate: ../outside_summary_tables/estimated_fixtures/override/transit_transfer_rate.csv
+ # bicycle_vmt_by_facility_type: ../outside_summary_tables/estimated_fixtures/override/bicycle_vmt_by_facility_type.csv
+ # commercial_vehicle_validation_summary: ../outside_summary_tables/estimated_fixtures/override/commercial_vehicle_validation_summary.csv
+ # commercial_vehicle_vmt_validation_summary: ../outside_summary_tables/estimated_fixtures/override/commercial_vehicle_vmt_validation_summary.csv
+ # external_trip_validation_summary: ../outside_summary_tables/estimated_fixtures/override/external_trip_validation_summary.csv
+ # external_vmt_validation_summary: ../outside_summary_tables/estimated_fixtures/override/external_vmt_validation_summary.csv
+ # # skimjoin:
+ # config_path: will_skimjoin_config.yaml
+ # skim_files:
+ # - C:\Users\wesley.darling\project_data\odot_skims\*.omx
+ # - C:\Users\wesley.darling\project_data\odot_skims\maz_stop_walk.csv
+ # - C:\Users\wesley.darling\project_data\odot_skims\maz_maz_walk.csv
+ # network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
+
+ - dir: C:\Users\wesley.darling\Downloads\dataset_2026-05-21\output
+ label: Estimation Output
+ summary_table_map:
+ link_validation_summary: ../outside_summary_tables/estimated_fixtures/estimation-output/link_validation_summary.csv
+ count_location_counts_validation_summary: ../outside_summary_tables/countLocCounts.csv
+ count_location_volumes_validation_summary: ../outside_summary_tables/estimated_fixtures/estimation-output/count_location_volumes_validation_summary.csv
+ screenline_flow_comparisons: ../outside_summary_tables/estimated_fixtures/estimation-output/screenline_flow_comparisons.csv
+ commuting_flows: ../outside_summary_tables/estimated_fixtures/estimation-output/commuting_flows.csv
+ transit_boardings_by_operator_and_technology: ../outside_summary_tables/estimated_fixtures/estimation-output/transit_boardings_by_operator_and_technology.csv
+ transit_transfer_rate: ../outside_summary_tables/estimated_fixtures/estimation-output/transit_transfer_rate.csv
+ bicycle_vmt_by_facility_type: ../outside_summary_tables/estimated_fixtures/estimation-output/bicycle_vmt_by_facility_type.csv
+ commercial_vehicle_validation_summary: ../outside_summary_tables/estimated_fixtures/estimation-output/commercial_vehicle_validation_summary.csv
+ commercial_vehicle_vmt_validation_summary: ../outside_summary_tables/estimated_fixtures/estimation-output/commercial_vehicle_vmt_validation_summary.csv
+ external_trip_validation_summary: ../outside_summary_tables/estimated_fixtures/estimation-output/external_trip_validation_summary.csv
+ external_vmt_validation_summary: ../outside_summary_tables/estimated_fixtures/estimation-output/external_vmt_validation_summary.csv
+ # # skimjoin:
+ # config_path: will_skimjoin_config.yaml
+ # skim_files:
+ # - C:\Users\wesley.darling\project_data\odot_skims\*.omx
+ # - C:\Users\wesley.darling\project_data\odot_skims\maz_stop_walk.csv
+ # - C:\Users\wesley.darling\project_data\odot_skims\maz_maz_walk.csv
+ # network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
+
+ # - dir: C:\Users\wesley.darling\Downloads\viz\viz\new_override
+ # label: New Override
+
+ # - dir: C:\Users\wesley.darling\Downloads\viz\viz\new_estimation_5-20
+ # label: New Estimation Output 20
+ # file_map:
+ # households: final_households
+ # persons: final_persons
+ # tours: final_tours
+ # trips: final_trips
+ # joint_tour_participants: final_joint_tour_participants
+ # land_use: final_land_use
+ # vehicles: final_vehicles
+
+ # - label: Will Prepared Tables
+ # prepared_table_map:
+ # households: C:/Users/wesley.darling/projects/activitysim_visualizer/will_prepared_tables/households.parquet
+ # persons: C:/Users/wesley.darling/projects/activitysim_visualizer/will_prepared_tables/persons.parquet
+ # tours: C:/Users/wesley.darling/projects/activitysim_visualizer/will_prepared_tables/tours.parquet
+ # trips: C:/Users/wesley.darling/projects/activitysim_visualizer/will_prepared_tables/trips.parquet
+ # joint_tour_participants: C:/Users/wesley.darling/projects/activitysim_visualizer/will_prepared_tables/joint_tour_participants.parquet
+ # land_use: C:/Users/wesley.darling/projects/activitysim_visualizer/will_prepared_tables/land_use.parquet
+
+ # - label: Demo Validation Data
+ # summary_table_map:
+ # link_validation_summary: ../outside_summary_tables/allLinkSummary.csv
+ # count_location_counts_validation_summary: ../outside_summary_tables/countLocCounts.csv
+ # count_location_volumes_validation_summary: ../outside_summary_tables/countLocVolumes.csv
+ # district_commuting_flows_validation_summary: ../outside_summary_tables/countyFlows.csv
+ # county_commuting_flows_validation_summary: ../outside_summary_tables/countyFlows_JoJa.csv
+ # commercial_vehicle_validation_summary: ../outside_summary_tables/cvm_summary.csv
+ # commercial_vehicle_vmt_validation_summary: ../outside_summary_tables/cvm_vmt_summary.csv
+ # external_trip_validation_summary: ../outside_summary_tables/ext_summary.csv
+ # external_vmt_validation_summary: ../outside_summary_tables/ext_vmt_summary.csv
+ # auto_vmt_validation_summary: ../outside_summary_tables/vmtSummary.csv
+ # work_from_home_validation_summary: ../outside_summary_tables/wfh_summary.csv
+ # transit_boardings_by_operator_and_technology: ../outside_summary_tables/estimated_fixtures/observed/transit_boardings_by_operator_and_technology.csv
+ # transit_transfer_rate: ../outside_summary_tables/estimated_fixtures/observed/transit_transfer_rate.csv
+ # bicycle_vmt_by_facility_type: ../outside_summary_tables/estimated_fixtures/observed/bicycle_vmt_by_facility_type.csv
+
+# ---------------------------------------------------------------------------
+# Zone system
+# Set use_maz: false for TAZ-only models (zone IDs in outputs are already TAZs).
+# For 2-zone (MAZ+TAZ) models, specify which columns in final_land_use
+# hold the MAZ and TAZ IDs.
+# ---------------------------------------------------------------------------
+zones:
+ use_maz: true # false = TAZ-only model, no MAZ→TAZ conversion needed
+ maz_col: [MAZ, zone_id] # column in final_land_use containing MAZ IDs
+ taz_col: TAZ # column in final_land_use containing TAZ IDs
+
+
+# ---------------------------------------------------------------------------
+# Column names in the ActivitySim output files
+# Change only if your outputs use non-standard column names.
+# ---------------------------------------------------------------------------
+columns:
+ ptype: ptype # person type column in persons file
+ hhsize: hhsize # HH size column in households file
+ auto_ownership: auto_ownership # vehicle count column in households file
+ num_workers: num_workers # workers per HH in households file
+ num_adults: num_adults # adults per HH in households file
+ # income_segment: [income_segment, income_broad] # OPTIONAL household income segment/level column for trip enrichment
+ # Prefer a readable purpose column per run. Some survey-style outputs store
+ # numeric codes in primary_purpose/tour_purpose and labels in tour_type.
+ tour_purpose: [primary_purpose, tour_type, purpose]
+ # sample_rate: sample_rate # OPTIONAL override; if omitted, code auto-detects
+ # a households column literally named 'sample_rate'.
+ # finalweight = 1/sample_rate (unless explicit run weight columns are provided)
+ # school_esc_outbound: [school_esc_outbound, out_escort_type]
+ # school_esc_inbound: [school_esc_inbound, inb_escort_type]
+
+prepare:
+ output:
+ file_format: parquet
+ validation:
+ relationship_checks: warn
+ distance_skim:
+ file: C:\Users\wesley.darling\project_data\new_odot_skims\Metro_skims\autoSkims__MD.omx # path relative to run directory, or absolute
+ matrix: SOV_H_DIST__MD
+ non_motorized_distance_skim:
+ file: C:\Users\wesley.darling\project_data\odot_skims\maz_maz_walk.csv
+ matrix: null
+ time_periods:
+ network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
+ trip_period_number_column: depart
+ tour_start_period_number_column: start
+ tour_end_period_number_column: end
+ auto_sufficiency_basis: licensed_drivers # licensed_drivers | workers | adults
+ vot_bins:
+ source_column: income_segment
+ output_column: vot_bin
+ fallback_value: M
+ mappings:
+ unfiltered:
+ 1: L
+ 2: L
+ 3: M
+ 4: M
+ 5: H
+ 6: H
+ new-filtered:
+ 1: L
+ 2: L
+ 3: M
+ 4: M
+ 5: H
+ 6: H
+ new-override:
+ 1: L
+ 2: L
+ 3: M
+ 4: M
+ 5: H
+ 6: H
+ new-estimation-output:
+ 1: L
+ 2: M
+ 3: M
+ 4: H
+
+skimjoin:
+ # Create optional long-form trip/tour tables with skim values for alternate modes.
+ create_hypothetical_skim_tables: true
+ defaults:
+ config_path: will_skimjoin_config.yaml
+ skim_files:
+ - C:\Users\wesley.darling\project_data\new_odot_skims\Metro_skims\*.omx
+ - C:\Users\wesley.darling\project_data\odot_skims\maz_stop_walk.csv
+ - C:\Users\wesley.darling\project_data\odot_skims\maz_maz_walk.csv
+ network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
+
+segment:
+ dashboard:
+ segmentation_type: person_sex
+ visibility: segments_only # full_only | segments_only | full_and_segments
+ definitions:
+ signup_platform:
+ include_full: true
+ persist_segmented_prepared_tables: false
+ allow_overlapping: false
+ on_empty_segment: warn
+ source:
+ type: prepared_column
+ source_table: hh
+ column: signup_platform
+ segments:
+ - id: rmove
+ label: RMove
+ values: ["rmove"]
+ - id: browser
+ label: Browser
+ values: ["browser"]
+ - id: call
+ label: Call
+ values: ["call"]
+
+ person_sex:
+ include_full: false
+ persist_segmented_prepared_tables: false
+ allow_overlapping: false
+ on_empty_segment: warn
+ source:
+ type: prepared_column
+ source_table: per
+ column: SEX
+ segments:
+ - id: male
+ label: Male
+ values: [1]
+ - id: female
+ label: Female
+ values: [2]
+
+summarize:
+ weighting_modes: [unweighted, weighted]
+ pnr_tour_modes:
+ - PNR_TRANSIT
+ # OPTIONAL summary-time grouping for outputs with a tour_purpose dimension
+ # group_joint_tour_purposes: true
+ # group_atwork_tour_purposes: true
+ # group_school_tour_purposes: true
+ # Controls additional mapped geography aggregations only. Summaries may still
+ # emit all_geographies totals, and native prepared home geographies such as
+ # home_taz, home_county, and home_mpo can appear when those columns exist.
+ geography:
+ enabled: false
+ # Configured mappings create columns such as home_geo__school_district,
+ # work_geo__county, or land_use_geo__district.
+ aggregations:
+ school_district:
+ source_zone_system: maz
+ file: C:\Users\wesley.darling\Downloads\viz\viz\new_output\land_use.csv
+ zone_id_col: zone_id
+ geography_col: DIST_9to12
+
+dashboard:
+ title: "Estimation Mode Comparison Visualizer"
+ enable_maz_geographies: false
+ live:
+ pages:
+ - overview
+ - long_term_choices
+ - daily_travel
+ - joint_travel
+ - tour_summaries
+ - trip_summaries
+ - skim_summaries
+ - validation:
+ - traffic
+ - transit
+ - vmt
+ - regional_validation
+ export:
+ output_path: exports/dashboard.html
+ # summary series are baked into one exported HTML file.
+ # `dashboard.export.pages` modifies matching live pages; it is not an
+ # inclusion list. Omit selector overrides to export all widget states.
+ # Export segmentation defaults to segmentation.dashboard when omitted.
+ # Use full_only, segments_only, or full_and_segments to choose which
+ dashboard:
+ weighting: [unweighted]
+ # segmentation_type: signup_platform
+ # segmentation_visibility: segments_only
+ pages:
+ long_term_choices:
+ mandatory_location_choice:
+ geography_level:
+ - All Geography Types
+ - County
+ - MPO
+ # omit TAZ
+ # parts:
+ # commuting_flows:
+ # enabled: false
+
+ # shadow_pricing:
+ # enabled: true
+ # geography_level: [all]
+ # student_type: [all]
+ # parts:
+ # workplace_table:
+ # enabled: false
+ # school_table:
+ # enabled: false
+ # tour_summaries:
+ # internal_external_tours:
+ # enabled: false
+ validation:
+ vmt:
+ personal_auto_vmt_breakdown: all
+ personal_auto_vmt_geography_type: all
+ personal_auto_vmt_geography: default
+ personal_auto_vmt_time_period: default
+ personal_auto_vmt_mode: default
+ personal_auto_vmt_income_segment: default
+ personal_auto_vmt_household_size: default
+ non_motorized_vmt_breakdown: all
+ non_motorized_vmt_geography_type: all
+ non_motorized_vmt_geography: default
+ non_motorized_vmt_time_period: default
+ non_motorized_vmt_mode: default
+ non_motorized_vmt_income_segment: default
+ non_motorized_vmt_household_size: default
+
+ external_travel_metric: all
+ external_travel_breakdown: all
+ external_travel_trip_purpose: default
+ external_travel_time_period: default
+
+ demo_commercial_metric: all
+ demo_commercial_breakdown: all
+ demo_commercial_vehicle_type: default
+ demo_commercial_time_period: default
+ host:
+ account: my-connect-cloud-account
+ # app_id: 12345
+ # title: Estimation Mode Comparison Visualizer
+ # First hosted publish may open a browser for Posit Connect Cloud login.
+ # Later runs usually reuse saved local rsconnect metadata.
+ verify: true
+
+
+# Summary-affecting regrouping/normalization belongs under summarize.category_normalization.
+# Cosmetic labels and ordering belong under display.labels.
+display:
+ bar_hover_mode: all # closest | all
+ density_hover_mode: all # closest | all
+ labels:
+ person_type:
+ mapping:
+ all_person_types: All Person Types
+ 1: Full-time worker
+ 2: Part-time worker
+ 3: University student
+ 4: Non-worker adult
+ 5: Retired
+ 6: Student
+ 7: Preschool
+
+ transit_subsidy:
+ mapping:
+ 0: No subsidy
+ 1: Discounted
+ 2: Free
+
+ license_holding_status:
+ mapping:
+ has_license: Has License
+ no_license: No License
+
+ transit_pass_ownership_status:
+ mapping:
+ has_transit_pass: Has Transit Pass
+ no_transit_pass: No Transit Pass
+
+ telecommute_frequency:
+ mapping:
+ 1_day_week: 1 Day per Week
+ 2_3_days_week: 2-3 Days per Week
+ 4_days_week: 4+ Days per Week
+ No_Telecommute: No Telecommute
+
+ mode:
+ mapping:
+ SOV: Drive Alone
+ HOV2: Shared Ride 2
+ HOV3: Shared Ride 3+
+ WALK: Walk
+ BIKE: Bike
+ EBIKE: E-Bike
+ ESCOOTER: E-Scooter
+ WALK_TRANSIT: Walk-Transit
+ BIKE_TRANSIT: Bike-Transit
+ PNR_TRANSIT: PNR-Transit
+ KNR_TRANSIT: KNR-Transit
+ TAXI: Taxi
+ TNC_SINGLE: TNC-Single
+ TNC_SHARED: TNC-Pool
+ SCHOOLBUS: School Bus
+
+
+ # escort:
+ # mapping:
+ # not_escorted: No Escort
+ # pure_escort: Pure Escort
+ # ride_share: Ride Share
+ # 0: No Escort
+ # 1: Pure Escort
+ # 2: Ride Share
+ # "": No Escort
+
+ tour_purpose:
+ mapping:
+ all_tour_purposes: All Tour Purposes
+ work: Work
+ school: School
+ joint: Joint
+ escort: Escort
+ shopping: Shopping
+ othmaint: Other Maintenance
+ eatout: Eat Out
+ social: Social
+ othdiscr: Other Discretionary
+ atwork: At-Work
+ loop: Loop
+
+ trip_purpose:
+ mapping:
+ home: Home
+ work: Work
+ school: School
+ escort: Escort
+ shopping: Shopping
+ othmaint: Other Maintenance
+ eatout: Eat Out
+ social: Social
+ othdiscr: Other Discretionary
+ loop: Loop
+
+ stop_purpose:
+ mapping:
+ work: Work
+ school: School
+ escort: Escort
+ shopping: Shopping
+ othmaint: Other Maintenance
+ eatout: Eat Out
+ social: Social
+ othdiscr: Other Discretionary
+
+ tour_composition:
+ mapping:
+ adults: Adults
+ mixed: Mixed
+ children: Children
+
+ tour_category:
+ mapping:
+ mandatory: Mandatory
+ joint: Joint
+ non_mandatory: Non-Mandatory
+ atwork: At-Work
+
+ atwork_subtour_frequency_category:
+ mapping:
+ no_subtours: None
+ eat: 1 Eating Out
+ business1: 1 Business
+ maint: 1 Maintenance
+ business2: 2 Business
+ eat_business: 1 Eating Out + 1 Business
+
+ geography:
+ mapping:
+ all_geographies: All Geographies
+ county: County
+ home_county: County
+ home_mpo: MPO
+ home_taz: TAZ
+ district: District
+ taz: TAZ
+ maz: MAZ
+ school_district: School District
+
+ mandatory_tour_frequency:
+ # mapping:
+ # work1: Work
+ # school1: School
+ # work2: 2 Work
+ # school2: 2 School
+ # work_and_school: 1 Work + 1 School
+ mapping:
+ 1: Work
+ 3: School
+ 2: 2 Work
+ 4: 2 School
+ 5: 1 Work + 1 School
+
+ daily_activity_pattern:
+ mapping:
+ M : Mandatory
+ N : Non-Mandatory
+ H : At-Home
+
+ facility_type:
+ mapping:
+ 1: "Interstate"
+ 3: "Principal Arterial"
+ 4: "Minor Arterial"
+ 5: "Major Collector"
+ 6: "Minor Collector"
+ 7: "Local Road"
+ 30: "Ramp"
+ 998: "Bike/Walk"
+
+ commercial_vehicle_type:
+ mapping:
+ car: Car
+ su: Single-Unit Truck
+ mu: Multi-Unit Truck
+
+ # run_colors:
+ # - "#298c8c" # Teal
+ # - "#a00000" # Red
+ # - "#b8b8b8" # Light gray
+ # - "#384860" # Dark blue-gray
+ # - "#ff7f0e" # Orange
+ # - "#1f77b4"
+ # - "#ff7f0e"
+ # - "#2ca02c"
+ # - "#d62728"
+ # - "#9467bd"
+ # - "#8c564b"
+ # - "#e377c2"
+ # - "#7f7f7f"
diff --git a/simor_configs/metro_configs/metro_skimjoin_config.yaml b/simor_configs/metro_configs/metro_skimjoin_config.yaml
new file mode 100644
index 0000000..79ffea1
--- /dev/null
+++ b/simor_configs/metro_configs/metro_skimjoin_config.yaml
@@ -0,0 +1,415 @@
+# project:
+# skim_files:
+# - C:\Users\wesley.darling\project_data\odot_skims\*.omx
+# - C:\Users\wesley.darling\project_data\odot_skims\maz_stop_walk.csv
+# - C:\Users\wesley.darling\project_data\odot_skims\maz_maz_walk.csv
+
+# network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
+
+ # project.trips_table / project.tours_table / project.output_dir are used by the
+ # standalone skimjoin CLI only. Integrated runtime skimjoin reads prepared
+ # in-memory trips/tours from the processor pipeline instead.
+ # trips_table: C:\Users\wesley.darling\Downloads\viz\viz\output\trip_linked.csv
+ # trips_table: C:\Users\wesley.darling\projects\activitysim_visualizer\artifacts\prepared_cache\override\trips.parquet
+ # tours_table: C:\Users\wesley.darling\Downloads\viz\viz\output\tour.csv
+ # tours_table: C:\Users\wesley.darling\projects\activitysim_visualizer\artifacts\prepared_cache\override\tours.parquet
+ # output_dir: will_output
+
+
+activitysim:
+ trip_mode_column: trip_mode
+ trip_id_column: trip_id
+ tour_mode_column: tour_mode
+ tour_id_column: tour_id
+ outbound_column: outbound
+
+
+defaults:
+ origin: OTAZ
+ destination: DTAZ
+ output_prefix: skim_
+ sentinel_values:
+ - 9999
+ - 999999
+
+zone_mapping:
+ lookup_name: taz
+ file_lookup_names:
+ fares.omx: zone_number
+ missing_zone_policy: error
+
+dimensions:
+ PERIOD:
+ source_columns:
+ trip_source_column: trip_period
+ outbound_tour_source_column: start_period
+ inbound_tour_source_column: first_inbound_trip_period
+ VOT:
+ source_columns:
+ trip_source_column: vot_bin
+ outbound_tour_source_column: vot_bin
+ inbound_tour_source_column: vot_bin
+ values:
+ L: L
+ M: M
+ H: H
+
+ignore_modes:
+ - ESCOOTER
+ - EBIKE
+ - BIKE_TRANSIT
+ - MISSING
+ - OTHER
+
+modes:
+ SOV:
+ output_prefix: skim_auto_
+ time: "SOV_{VOT}_TIME__{PERIOD}"
+ cost: "SOV_{VOT}_COST__{PERIOD}"
+ distance: "SOV_{VOT}_DIST__{PERIOD}"
+ HOV2:
+ output_prefix: skim_auto_
+ time: "SR2_{VOT}_TIME__{PERIOD}"
+ cost: "SR2_{VOT}_COST__{PERIOD}"
+ distance: "SR2_{VOT}_DIST__{PERIOD}"
+ HOV3:
+ output_prefix: skim_auto_
+ time: "SR3_{VOT}_TIME__{PERIOD}"
+ cost: "SR3_{VOT}_COST__{PERIOD}"
+ distance: "SR3_{VOT}_DIST__{PERIOD}"
+ KNR_TRANSIT:
+ output_prefix: skim_
+ segment_on: outbound
+ segments:
+ true:
+ auto_time: "KTW_ACC__{PERIOD}"
+ auto_distance: "KTW_TDD__{PERIOD}"
+ transit_auxiliary_walk_time: "KTW_AUX__{PERIOD}"
+ transit_brt_ivtt: "KTW_BRT__{PERIOD}"
+ transit_bus_ivtt: "KTW_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "KTW_CRT__{PERIOD}"
+ walk_time: "KTW_EGR__{PERIOD}"
+ transit_first_wait_time: "KTW_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "KTW_LRT__{PERIOD}"
+ transit_tiv: "KTW_TIV__{PERIOD}"
+ transit_num_transfers: "KTW_XFR__{PERIOD}"
+ transit_transfer_wait_time: "KTW_XWT__{PERIOD}"
+ transit_fare: "fare__{PERIOD}"
+ d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ false:
+ walk_time: "WTK_ACC__{PERIOD}"
+ transit_auxiliary_walk_time: "WTK_AUX__{PERIOD}"
+ transit_brt_ivtt: "WTK_BRT__{PERIOD}"
+ transit_bus_ivtt: "WTK_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "WTK_CRT__{PERIOD}"
+ auto_time: "WTK_EGR__{PERIOD}"
+ auto_distance: "WTK_TDD__{PERIOD}"
+ transit_first_wait_time: "WTK_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "WTK_LRT__{PERIOD}"
+ transit_tiv: "WTK_TIV__{PERIOD}"
+ transit_num_transfers: "WTK_XFR__{PERIOD}"
+ transit_transfer_wait_time: "WTK_XWT__{PERIOD}"
+ transit_fare: "fare__{PERIOD}"
+ o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ PNR_TRANSIT:
+ output_prefix: skim_
+ segment_on: outbound
+ segments:
+ true:
+ auto_time:
+ matrix: "SOV_{VOT}_TIME__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ auto_distance:
+ matrix: "SOV_{VOT}_DIST__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ auto_cost:
+ matrix: "SOV_{VOT}_COST__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_auxiliary_walk_time:
+ matrix: "WTW_AUX__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_brt_ivtt:
+ matrix: "WTW_BRT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_bus_ivtt:
+ matrix: "WTW_BUS__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_commuter_rail_ivtt:
+ matrix: "WTW_CRT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_first_wait_time:
+ matrix: "WTW_FWT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_light_rail_ivtt:
+ matrix: "WTW_LRT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_tiv:
+ matrix: "WTW_TIV__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_num_transfers:
+ matrix: "WTW_XFR__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_transfer_wait_time:
+ matrix: "WTW_XWT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_fare:
+ matrix: "fare__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: pnr_zone_id
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: pnr_zone_id
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ false:
+ auto_time:
+ matrix: "SOV_{VOT}_TIME__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ auto_distance:
+ matrix: "SOV_{VOT}_DIST__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ auto_cost:
+ matrix: "SOV_{VOT}_COST__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_auxiliary_walk_time:
+ matrix: "WTW_AUX__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_brt_ivtt:
+ matrix: "WTW_BRT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_bus_ivtt:
+ matrix: "WTW_BUS__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_commuter_rail_ivtt:
+ matrix: "WTW_CRT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_first_wait_time:
+ matrix: "WTW_FWT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_light_rail_ivtt:
+ matrix: "WTW_LRT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_tiv:
+ matrix: "WTW_TIV__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_num_transfers:
+ matrix: "WTW_XFR__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_transfer_wait_time:
+ matrix: "WTW_XWT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_fare:
+ matrix: "fare__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: pnr_zone_id
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: pnr_zone_id
+ matrix: maz_stop_walk__walk_dist_premium_transit
+
+ SCHOOLBUS:
+ output_prefix: skim_
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
+ transit_bus_ivtt: "WTW_BUS__{PERIOD}"
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ transit_first_wait_time: "WTW_FWT__{PERIOD}"
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ fallbacks:
+ - matrix: "WTW_ACC__{PERIOD}"
+ origin: OTAZ
+ destination: DTAZ
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ fallbacks:
+ - matrix: "WTW_EGR__{PERIOD}"
+ origin: OTAZ
+ destination: DTAZ
+ TAXI:
+ output_prefix: skim_auto_
+ time: "SOV_{VOT}_TIME__{PERIOD}"
+ cost: "SOV_{VOT}_COST__{PERIOD}"
+ distance: "SOV_{VOT}_DIST__{PERIOD}"
+ TNC_SHARED:
+ output_prefix: skim_auto_
+ time: "SR2_{VOT}_TIME__{PERIOD}"
+ cost: "SR2_{VOT}_COST__{PERIOD}"
+ distance: "SR2_{VOT}_DIST__{PERIOD}"
+ TNC_SINGLE:
+ output_prefix: skim_auto_
+ time: "SOV_{VOT}_TIME__{PERIOD}"
+ cost: "SOV_{VOT}_COST__{PERIOD}"
+ distance: "SOV_{VOT}_DIST__{PERIOD}"
+ WALK:
+ output_prefix: skim_walk_
+ distance: WLK_DIST
+ maz_walk_distance:
+ output: skim_walk_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: maz_maz_walk__DISTWALK
+ maz_walk_actual:
+ output: skim_walk_maz_actual
+ origin: o_maz
+ destination: d_maz
+ matrix: maz_maz_walk__actual
+ WALK_TRANSIT:
+ output_prefix: skim_
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
+ transit_brt_ivtt: "WTW_BRT__{PERIOD}"
+ transit_bus_ivtt: "WTW_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "WTW_CRT__{PERIOD}"
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ transit_first_wait_time: "WTW_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "WTW_LRT__{PERIOD}"
+ transit_tiv: "WTW_TIV__{PERIOD}"
+ transit_num_transfers: "WTW_XFR__{PERIOD}"
+ transit_transfer_wait_time: "WTW_XWT__{PERIOD}"
+ transit_fare: "fare__{PERIOD}"
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ BIKE:
+ output_prefix: skim_bike_
+ distance: WLK_DIST
+ maz_bike_distance:
+ output: skim_bike_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: maz_maz_walk__DISTWALK
+
+
diff --git a/simor_configs/metro_configs/metro_skimjoin_config_alternate_id_col.yaml b/simor_configs/metro_configs/metro_skimjoin_config_alternate_id_col.yaml
new file mode 100644
index 0000000..fcd58c2
--- /dev/null
+++ b/simor_configs/metro_configs/metro_skimjoin_config_alternate_id_col.yaml
@@ -0,0 +1,415 @@
+project:
+ skim_files:
+ - C:\Users\wesley.darling\project_data\new_odot_skims\Metro_skims\*.omx
+ - C:\Users\wesley.darling\project_data\odot_skims\maz_stop_walk.csv
+ - C:\Users\wesley.darling\project_data\odot_skims\maz_maz_walk.csv
+ network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
+ # project.trips_table / project.tours_table / project.output_dir are used by the
+ # standalone skimjoin CLI only. Integrated runtime skimjoin reads prepared
+ # in-memory trips/tours from the processor pipeline instead.
+ # trips_table: C:\Users\wesley.darling\Downloads\viz\viz\output\trip_linked.csv
+ # trips_table: C:\Users\wesley.darling\projects\activitysim_visualizer\artifacts\prepared_cache\override\trips.parquet
+ # tours_table: C:\Users\wesley.darling\Downloads\viz\viz\output\tour.csv
+ # tours_table: C:\Users\wesley.darling\projects\activitysim_visualizer\artifacts\prepared_cache\override\tours.parquet
+ # output_dir: will_output
+
+
+activitysim:
+ # activitysim now only defines structural trip/tour fields used by skimjoin.
+ trip_mode_column: trip_mode
+ trip_id_column: linked_trip_id
+ tour_mode_column: tour_mode
+ tour_id_column: tour_id
+ outbound_column: outbound
+
+# Integrated runtime currently supports OMX skim inputs only.
+
+defaults:
+ origin: OTAZ
+ destination: DTAZ
+ output_prefix: skim_
+ sentinel_values:
+ - 9999
+ - 999999
+
+zone_mapping:
+ lookup_name: taz
+ file_lookup_names:
+ fares.omx: zone_number
+ missing_zone_policy: error
+
+dimensions:
+ PERIOD:
+ source_columns:
+ trip_source_column: trip_period
+ outbound_tour_source_column: start_period
+ inbound_tour_source_column: first_inbound_trip_period
+ VOT:
+ source_columns:
+ trip_source_column: vot_bin
+ outbound_tour_source_column: vot_bin
+ inbound_tour_source_column: vot_bin
+ values:
+ L: L
+ M: M
+ H: H
+
+ignore_modes:
+ - ESCOOTER
+ - EBIKE
+ - BIKE_TRANSIT
+ - MISSING
+ - OTHER
+
+modes:
+ SOV:
+ output_prefix: skim_auto_
+ time: "SOV_{VOT}_TIME__{PERIOD}"
+ cost: "SOV_{VOT}_COST__{PERIOD}"
+ distance: "SOV_{VOT}_DIST__{PERIOD}"
+ HOV2:
+ output_prefix: skim_auto_
+ time: "SR2_{VOT}_TIME__{PERIOD}"
+ cost: "SR2_{VOT}_COST__{PERIOD}"
+ distance: "SR2_{VOT}_DIST__{PERIOD}"
+ HOV3:
+ output_prefix: skim_auto_
+ time: "SR3_{VOT}_TIME__{PERIOD}"
+ cost: "SR3_{VOT}_COST__{PERIOD}"
+ distance: "SR3_{VOT}_DIST__{PERIOD}"
+ KNR_TRANSIT:
+ output_prefix: skim_
+ segment_on: outbound
+ segments:
+ true:
+ auto_time: "KTW_ACC__{PERIOD}"
+ auto_distance: "KTW_TDD__{PERIOD}"
+ transit_auxiliary_walk_time: "KTW_AUX__{PERIOD}"
+ transit_brt_ivtt: "KTW_BRT__{PERIOD}"
+ transit_bus_ivtt: "KTW_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "KTW_CRT__{PERIOD}"
+ walk_time: "KTW_EGR__{PERIOD}"
+ transit_first_wait_time: "KTW_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "KTW_LRT__{PERIOD}"
+ transit_tiv: "KTW_TIV__{PERIOD}"
+ transit_num_transfers: "KTW_XFR__{PERIOD}"
+ transit_transfer_wait_time: "KTW_XWT__{PERIOD}"
+ transit_fare: "fare__{PERIOD}"
+ d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ false:
+ walk_time: "WTK_ACC__{PERIOD}"
+ transit_auxiliary_walk_time: "WTK_AUX__{PERIOD}"
+ transit_brt_ivtt: "WTK_BRT__{PERIOD}"
+ transit_bus_ivtt: "WTK_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "WTK_CRT__{PERIOD}"
+ auto_time: "WTK_EGR__{PERIOD}"
+ auto_distance: "WTK_TDD__{PERIOD}"
+ transit_first_wait_time: "WTK_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "WTK_LRT__{PERIOD}"
+ transit_tiv: "WTK_TIV__{PERIOD}"
+ transit_num_transfers: "WTK_XFR__{PERIOD}"
+ transit_transfer_wait_time: "WTK_XWT__{PERIOD}"
+ transit_fare: "fare__{PERIOD}"
+ o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ PNR_TRANSIT:
+ output_prefix: skim_
+ segment_on: outbound
+ segments:
+ true:
+ auto_time:
+ matrix: "SOV_{VOT}_TIME__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ auto_distance:
+ matrix: "SOV_{VOT}_DIST__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ auto_cost:
+ matrix: "SOV_{VOT}_COST__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_auxiliary_walk_time:
+ matrix: "WTW_AUX__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_brt_ivtt:
+ matrix: "WTW_BRT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_bus_ivtt:
+ matrix: "WTW_BUS__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_commuter_rail_ivtt:
+ matrix: "WTW_CRT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_first_wait_time:
+ matrix: "WTW_FWT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_light_rail_ivtt:
+ matrix: "WTW_LRT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_tiv:
+ matrix: "WTW_TIV__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_num_transfers:
+ matrix: "WTW_XFR__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_transfer_wait_time:
+ matrix: "WTW_XWT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_fare:
+ matrix: "fare__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: pnr_zone_id
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: pnr_zone_id
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ false:
+ auto_time:
+ matrix: "SOV_{VOT}_TIME__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ auto_distance:
+ matrix: "SOV_{VOT}_DIST__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ auto_cost:
+ matrix: "SOV_{VOT}_COST__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_auxiliary_walk_time:
+ matrix: "WTW_AUX__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_brt_ivtt:
+ matrix: "WTW_BRT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_bus_ivtt:
+ matrix: "WTW_BUS__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_commuter_rail_ivtt:
+ matrix: "WTW_CRT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_first_wait_time:
+ matrix: "WTW_FWT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_light_rail_ivtt:
+ matrix: "WTW_LRT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_tiv:
+ matrix: "WTW_TIV__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_num_transfers:
+ matrix: "WTW_XFR__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_transfer_wait_time:
+ matrix: "WTW_XWT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_fare:
+ matrix: "fare__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: pnr_zone_id
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: pnr_zone_id
+ matrix: maz_stop_walk__walk_dist_premium_transit
+
+ SCHOOLBUS:
+ output_prefix: skim_
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
+ transit_bus_ivtt: "WTW_BUS__{PERIOD}"
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ transit_first_wait_time: "WTW_FWT__{PERIOD}"
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ fallbacks:
+ - matrix: "WTW_ACC__{PERIOD}"
+ origin: OTAZ
+ destination: DTAZ
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ fallbacks:
+ - matrix: "WTW_EGR__{PERIOD}"
+ origin: OTAZ
+ destination: DTAZ
+ TAXI:
+ output_prefix: skim_auto_
+ time: "SOV_{VOT}_TIME__{PERIOD}"
+ cost: "SOV_{VOT}_COST__{PERIOD}"
+ distance: "SOV_{VOT}_DIST__{PERIOD}"
+ TNC_SHARED:
+ output_prefix: skim_auto_
+ time: "SR2_{VOT}_TIME__{PERIOD}"
+ cost: "SR2_{VOT}_COST__{PERIOD}"
+ distance: "SR2_{VOT}_DIST__{PERIOD}"
+ TNC_SINGLE:
+ output_prefix: skim_auto_
+ time: "SOV_{VOT}_TIME__{PERIOD}"
+ cost: "SOV_{VOT}_COST__{PERIOD}"
+ distance: "SOV_{VOT}_DIST__{PERIOD}"
+ WALK:
+ output_prefix: skim_walk_
+ distance: WLK_DIST
+ maz_walk_distance:
+ output: skim_walk_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: maz_maz_walk__DISTWALK
+ maz_walk_actual:
+ output: skim_walk_maz_actual
+ origin: o_maz
+ destination: d_maz
+ matrix: maz_maz_walk__actual
+ WALK_TRANSIT:
+ output_prefix: skim_
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
+ transit_brt_ivtt: "WTW_BRT__{PERIOD}"
+ transit_bus_ivtt: "WTW_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "WTW_CRT__{PERIOD}"
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ transit_first_wait_time: "WTW_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "WTW_LRT__{PERIOD}"
+ transit_tiv: "WTW_TIV__{PERIOD}"
+ transit_num_transfers: "WTW_XFR__{PERIOD}"
+ transit_transfer_wait_time: "WTW_XWT__{PERIOD}"
+ transit_fare: "fare__{PERIOD}"
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ BIKE:
+ output_prefix: skim_bike_
+ distance: WLK_DIST
+ maz_bike_distance:
+ output: skim_bike_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: maz_maz_walk__DISTWALK
+
+
diff --git a/tests/test_runtime_config_package.py b/tests/test_runtime_config_package.py
index a547fab..75047d1 100644
--- a/tests/test_runtime_config_package.py
+++ b/tests/test_runtime_config_package.py
@@ -65,9 +65,7 @@ def test_repository_example_configs_match_current_schemas() -> None:
assert config.pipeline.steps == ("summarize", "dashboard")
assert config.pipeline.dashboard_mode == "live"
assert config.skimjoin.enabled is False
- assert config.skimjoin.config_path == str(
- (ROOT / "example_skimjoin_config.yaml").resolve()
- )
+ assert config.skimjoin.config_path is None
assert config.include_notes is True
assert config.missing_data_display == "card"
diff --git a/tests/test_summary_declarations.py b/tests/test_summary_declarations.py
index 7fba1c0..76df1de 100644
--- a/tests/test_summary_declarations.py
+++ b/tests/test_summary_declarations.py
@@ -12,6 +12,10 @@
from processor.models import RunData
from processor.summarize.catalog import build_summary_catalog
from processor.summarize.contracts import SummaryResultError, summary
+from scripts.generate_wiki_catalogs import (
+ _validate_summary_reference,
+ build_summary_catalog as build_wiki_summary_catalog,
+)
def _run(**tables) -> RunData:
@@ -100,3 +104,16 @@ def second(run, config):
with pytest.raises(ValueError, match="Duplicate summary id 'duplicate'"):
build_summary_catalog((module,))
+
+
+def test_wiki_summary_catalog_documents_build_status_and_all_fields() -> None:
+ _validate_summary_reference()
+
+ catalog = build_wiki_summary_catalog()
+
+ assert "| Summary ID | Filename | Default build |" in catalog
+ assert "| `population_totals` | `population_totals.csv` | yes |" in catalog
+ assert (
+ "| `auto_vmt_validation_summary` | "
+ "`auto_vmt_validation_summary.csv` | no |"
+ ) in catalog
diff --git a/wiki/24-summary-catalog.md b/wiki/24-summary-catalog.md
index 7d06632..0381f58 100644
--- a/wiki/24-summary-catalog.md
+++ b/wiki/24-summary-catalog.md
@@ -1,7 +1,266 @@
# 24 - Summary Catalog
-This page is generated from the `@summary(...)` declarations collected by
-`processor.summarize.catalog`.
+This page is the analytical data dictionary for the summary CSV tables exposed
+by the Output Processor. It covers all registered summary tables, explains the
+observation represented by each row, identifies practical travel-analysis uses,
+and defines every output field. The generated developer inventory at the end of
+the page remains the authoritative list of filenames, schemas, builders, and
+input prerequisites.
+
+## How to Interpret the Tables
+
+- Count, volume, mileage, and boarding fields are numeric measures. In a
+ weighted cache they are sums of `finalweight`; in an unweighted cache the
+ workflow substitutes unit weights. A `Float64` count can therefore be a
+ fractional population estimate, not a literal row count.
+- Rate, percentage, mean, standard-deviation, median, and percentile fields are
+ calculated from the weighted observations described for that table.
+- Values such as `all_geographies`, `all_person_types`, `all_tour_purposes`,
+ `all_tour_modes`, `All Modes`, `All Auto`, and `Daily` are rollups. Do not add
+ a rollup to its component rows; choose either the rollup or the detail level.
+- `geography_type` names the configured spatial system, such as MAZ, TAZ,
+ county, MPO, or a custom geography. `geography_id` is the identifier within
+ that system. Home, work, school, destination, and parking geography are
+ stated in each table description.
+- Distance values use the units of the prepared distance or skim fields,
+ normally miles. Integer distance bins use the truncated mile value unless a
+ table explicitly says distances are rounded; terminal bins such as `40+`,
+ `20+`, or numeric bin `51` include all larger values.
+- `time_bin` is the prepared ActivitySim period index: 1--24 for hourly inputs
+ or 1--48 for half-hour-period inputs. Named `time_period` and `count_period`
+ values come from configured or supplied period labels.
+- Category codes and labels come from prepared ActivitySim values and the
+ configured category mappings. Analysts should retain the code field for
+ joins and use its label field for presentation.
+- A valid calculation can produce an empty CSV. That is distinct from a summary
+ marked unavailable because an input table or field was absent; consult the
+ summary manifest when the distinction matters.
+
+## Build Status
+
+The normal summarize workflow builds **85** tables. The other **15** registered
+tables have `Default build = no`: the two skim ECDF tables are optional
+on-demand products, while the 13 validation contracts are supplied through
+`summary_table_map` rather than calculated from `RunData`. All 100 contracts are
+documented below and appear in the generated inventory.
+
+## Analytical Table Reference
+
+The fields listed for a table are the complete persisted output schema. See the
+generated inventory for physical data types and mechanical input requirements.
+
+### Population and Demographics
+
+| Summary table | Information and analytical use | Fields |
+|---|---|---|
+| `household_size_distribution` | Household totals by modeled household size. Use it to profile household composition, calculate size shares, and compare population-synthesis results across runs. | `household_size`: number of people in the household.
`household_count`: weighted households in that size category. |
+| `person_type_distribution` | Person totals by ActivitySim person type, with a display label. Use it to compare demographic market segments and as a denominator for person-type travel rates. | `person_type`: stable person-type code.
`person_type_label`: configured readable label for the code.
`person_count`: weighted people of that type. |
+| `population_totals` | One run-level control-total row for people, households, tours, trips, and intermediate stops. Use it for reasonableness checks and top-level comparisons; the measures use their own table weights and are not additive to one another. | `person_count`: weighted persons.
`household_count`: weighted households.
`tour_count`: weighted tours.
`trip_count`: weighted trips.
`stop_count`: weighted trip records flagged as intermediate stops. |
+
+### Person Attributes and Long-Term Choices
+
+| Summary table | Information and analytical use | Fields |
+|---|---|---|
+| `license_holding_status_distribution` | Licensed and unlicensed people age 16 or older by person type, including an all-person-types rollup. Use it to assess access to driving and explain auto-mode availability. | `person_type`: person-type code or `all_person_types` rollup.
`license_holding_status`: `has_license` or `no_license`.
`person_type_label`: configured readable person-type label.
`person_count`: weighted people in the group. |
+| `bicycle_comfort_level_distribution` | Bicycle comfort categories by person type, including an all-person-types rollup. Use it to understand the population assumed willing to use different bicycle facilities. | `person_type`: person-type code or rollup.
`bicycle_comfort_level`: prepared bicycle-comfort category.
`person_type_label`: configured readable person-type label.
`person_count`: weighted people in the group. |
+| `transit_pass_ownership_by_person_type` | Transit-pass ownership status by person type, including an all-person-types rollup. Use it to evaluate transit market eligibility and pass-ownership model results. | `person_type`: person-type code or rollup.
`transit_pass_ownership_status`: `has_transit_pass` or `no_transit_pass`.
`person_type_label`: configured readable person-type label.
`person_count`: weighted people in the group. |
+| `transit_subsidy_by_person_type` | Transit-pass subsidy alternatives for workers, by person type and with an all-person-types rollup. Use it to examine employer or institutional transit-benefit assumptions. | `person_type`: person-type code or rollup.
`transit_subsidy_status`: prepared subsidy alternative code.
`transit_subsidy_label`: configured readable subsidy label.
`person_type_label`: configured readable person-type label.
`person_count`: weighted eligible workers in the group. |
+| `telecommute_frequency_distribution` | Non-work-from-home workers by telecommute-frequency category and home geography, plus an all-geographies rollup. Use it to analyze recurring telecommuting among workers who still have an external workplace. | `geography_type`: home-geography system or rollup.
`geography_id`: home-geography identifier or rollup value.
`telecommute_frequency`: prepared telecommute-frequency alternative.
`person_count`: weighted workers in the group. |
+
+### Household Vehicles and Vehicle Characteristics
+
+| Summary table | Information and analytical use | Fields |
+|---|---|---|
+| `autonomous_vehicle_ownership_totals` | One run-level total of households modeled as owning an autonomous vehicle. Use it to report scenario penetration and compare AV-ownership assumptions. | `household_with_autonomous_vehicle_count`: weighted households with `av_ownership` true. |
+| `auto_ownership_distribution` | Household totals jointly classified by household size and vehicle count; household sizes of five or more are grouped as `5+`. Use it to assess motorization and auto sufficiency. | `household_size`: household-size category, with `5+` as the terminal group.
`household_vehicle_count`: vehicles available to the household.
`household_count`: weighted households in the joint category. |
+| `vehicle_age_distribution` | Household vehicles by age, with ages 20 and older grouped as `20+`. Use it for fleet turnover, emissions, and technology analyses. | `age`: vehicle age in years or `20+`.
`vehicle_count`: weighted vehicles in the age category. |
+| `vehicle_fuel_type_distribution` | Household vehicles by prepared fuel or powertrain type. Use it for fleet composition, energy, and emissions analysis. | `fuel_type`: prepared vehicle fuel/powertrain category.
`vehicle_count`: weighted vehicles of that type. |
+| `vehicle_body_type_distribution` | Household vehicles by prepared body type. Use it to characterize the light-duty fleet and support occupancy or emissions comparisons. | `body_type`: prepared vehicle body-style category.
`vehicle_count`: weighted vehicles of that type. |
+
+### Long-Term Geography, Location, and Shadow Pricing
+
+| Summary table | Information and analytical use | Fields |
+|---|---|---|
+| `work_from_home_rate_by_geography` | All workers and work-from-home workers by home geography, plus a regional rollup. Divide the WFH count by the worker count to calculate the WFH rate and map its spatial pattern. | `geography_type`: home-geography system or rollup.
`geography_id`: home-geography identifier or rollup value.
`worker_count`: weighted workers living in the geography.
`work_from_home_worker_count`: weighted workers flagged as working from home. |
+| `internal_external_worker_by_geography` | Internal and external workers by home geography, plus a regional rollup. Use it to understand external-worker incidence and its residential distribution. | `geography_type`: home-geography system or rollup.
`geography_id`: home-geography identifier or rollup value.
`internal_worker_count`: weighted workers with an internal workplace.
`external_worker_count`: weighted workers classified as external. |
+| `external_worker_workplace_locations` | External workers by their external workplace geography, plus a regional rollup. Use it to analyze external commute orientation; the all-worker denominator is repeated to support share calculations. | `geography_type`: external-workplace geography system or rollup.
`geography_id`: external-workplace geography identifier or rollup value.
`external_worker_count`: weighted external workers assigned to that destination.
`all_worker_count`: weighted workers in the full run, repeated on every row. |
+| `workplace_location_employment_comparison` | Land-use employment and modeled worker workplace choices aligned by workplace geography. Use it to compare attraction targets with assigned workers and diagnose location-choice balance. | `geography_type`: workplace-geography system.
`geography_id`: workplace-geography identifier.
`employment_count`: employment opportunities from land use.
`worker_count`: weighted workers assigned to the geography. |
+| `commuting_flows` | Worker flows from home geography to workplace geography at matching configured geography levels, plus a regional total. Use it as an origin-destination matrix for commute sheds, self-containment, and interjurisdictional flows. | `origin_geography_type`: home-geography system.
`origin_geography_id`: home-geography identifier.
`destination_geography_type`: workplace-geography system.
`destination_geography_id`: workplace-geography identifier.
`commuter_count`: weighted workers in the OD pair. |
+| `school_location_enrollment_comparison` | Land-use enrollment and modeled student school locations aligned by geography and student type. Use it to compare school-location targets with assigned students. | `geography_type`: school-geography system.
`geography_id`: school-geography identifier.
`student_type`: prepared school/enrollment market segment.
`enrollment_count`: target enrollment from land use.
`student_count`: weighted students assigned to the geography and type. |
+| `workplace_shadow_pricing_residuals` | Zone-level workplace target-versus-modeled residuals. Use positive residuals to find over-assigned workplace geographies and negative residuals to find under-assigned ones. | `geography_type`: workplace-geography system.
`geography_id`: workplace-geography identifier.
`target_count`: land-use employment target.
`modeled_count`: weighted assigned workers.
`residual_count`: `modeled_count - target_count`.
`absolute_residual_count`: absolute residual magnitude.
`percent_error`: residual divided by target, times 100; null when target is zero. |
+| `school_shadow_pricing_residuals` | Zone-level school target-versus-modeled residuals by student type. Use it to diagnose school-location shadow-pricing convergence and segment-specific imbalance. | `geography_type`: school-geography system.
`geography_id`: school-geography identifier.
`student_type`: school/enrollment market segment.
`target_count`: land-use enrollment target.
`modeled_count`: weighted assigned students.
`residual_count`: modeled minus target.
`absolute_residual_count`: absolute residual magnitude.
`percent_error`: residual divided by target, times 100; null for zero targets. |
+| `workplace_shadow_pricing_residual_histogram` | Distribution of workplace residuals by geography system. Use it to assess convergence across all zones without inspecting each zone separately; zero residuals receive their own zero-width bin. | `geography_type`: workplace-geography system.
`bin_start`: inclusive lower residual bound.
`bin_end`: upper residual bound; both bounds are zero for the exact-zero bin.
`geography_count`: number of geography records in the bin. |
+| `school_shadow_pricing_residual_histogram` | Distribution of school residuals by geography system and student type. Use it to compare convergence across student markets. | `geography_type`: school-geography system.
`student_type`: school/enrollment market segment.
`bin_start`: lower residual bound.
`bin_end`: upper residual bound, or zero for the exact-zero bin.
`geography_count`: number of geography/student-type records in the bin. |
+| `park_and_ride_location_residuals` | Modeled park-and-ride tour use compared with lot capacity by lot geography. Use it to identify over-capacity or underused PNR locations. | `geography_type`: PNR-lot geography system.
`geography_id`: PNR-lot geography identifier.
`pnr_tour_count`: weighted PNR tours assigned to the location.
`pnr_lot_capacity`: supplied lot capacity target.
`residual_count`: tours minus capacity.
`absolute_residual_count`: absolute residual magnitude.
`percent_error`: residual divided by capacity, times 100; null for zero capacity. |
+| `park_and_ride_location_residual_histogram` | Distribution of PNR use-minus-capacity residuals by geography system. Use it for systemwide capacity-fit assessment. | `geography_type`: PNR-lot geography system.
`bin_start`: lower residual bound.
`bin_end`: upper residual bound, or zero for the exact-zero bin.
`geography_count`: number of PNR geography records in the bin. |
+| `free_parking_eligibility_by_workplace_geography` | Workers with and without free workplace parking by workplace geography. Use it to analyze parking-cost exposure and its effect on commute mode choice. | `geography_type`: workplace-geography system.
`geography_id`: workplace-geography identifier.
`workers_without_free_parking_count`: weighted workers not eligible for free parking.
`workers_with_free_parking_count`: weighted workers eligible for free parking. |
+
+### Long-Term Location Distance
+
+These three tables contain a dense 0--51 distribution for each geography; bin
+51 contains distances of 51 or more. Missing distance values are treated as
+zero by these distribution builders.
+
+| Summary table | Information and analytical use | Fields |
+|---|---|---|
+| `work_location_distance_distribution_by_geography` | Workers with valid internal workplace locations by home geography and truncated home-to-work distance. Use it for commute-length distributions and spatial comparisons. | `distance_bin`: integer distance category from 0 through 51, with 51 terminal.
`geography_type`: home-geography system or rollup.
`geography_id`: home-geography identifier or rollup value.
`person_count`: weighted workers in the bin. |
+| `university_location_distance_distribution_by_geography` | University students, identified by person type 3, by home geography and truncated home-to-school distance. Use it to examine university travel markets and campus catchments. | `distance_bin`: integer distance category from 0 through 51.
`geography_type`: home-geography system or rollup.
`geography_id`: home-geography identifier or rollup value.
`person_count`: weighted university students in the bin. |
+| `school_location_distance_distribution_by_geography` | School students, identified by person types 6 and higher, by home geography and truncated home-to-school distance. Use it for K--12 travel-distance and school-catchment analysis. | `distance_bin`: integer distance category from 0 through 51.
`geography_type`: home-geography system or rollup.
`geography_id`: home-geography identifier or rollup value.
`person_count`: weighted school students in the bin. |
+
+### Daily Activity Patterns and Travel Rates
+
+| Summary table | Information and analytical use | Fields |
+|---|---|---|
+| `daily_activity_pattern_by_person_type` | Daily activity pattern alternatives by person type, including an all-person-types rollup. Use it to compare mandatory, nonmandatory, and home-stay behavior. | `person_type`: person-type code or rollup.
`daily_activity_pattern`: prepared CDAP/activity-pattern category.
`person_count`: weighted people in the pattern. |
+| `mandatory_tour_frequency_by_person_type` | Positive mandatory-tour-frequency choice by person type, plus an all-person-types rollup. Use it to analyze how many mandatory tours travelers make; people with a choice of zero are excluded. | `person_type`: person-type code or rollup.
`mandatory_tour_frequency`: prepared positive mandatory-tour frequency alternative.
`person_count`: weighted people choosing that frequency. |
+| `nonmandatory_tour_frequency_by_person_type` | Count of individual nonmandatory tours plus joint-tour participation per person, grouped as 0, 1, 2, or 3+, by person type and for all types. Use it to compare discretionary travel propensity. | `person_type`: person-type code or rollup.
`nonmandatory_tour_frequency`: combined nonmandatory-tour category `0`, `1`, `2`, or `3+`.
`person_count`: weighted people in the category. |
+| `tour_rates_by_person_type_and_tour_purpose` | Tours per weighted person-day by person type and tour purpose, plus all-person-types rates. Use it to compare tour-generation rates while controlling for population composition. | `person_type`: person-type code or rollup.
`tour_purpose`: prepared tour-purpose category.
`tour_rate`: weighted tours divided by weighted persons for the applicable person type. |
+| `trip_rates_by_person_type_and_trip_purpose` | Trips per weighted person by person type and trip purpose, plus all-person-types rates. Use it to compare trip-generation rates across demographic markets. | `person_type`: person-type code or rollup.
`trip_purpose`: destination purpose of the trip.
`trip_rate`: weighted trips divided by weighted persons for the applicable person type. |
+
+### School Escorting
+
+`direction` values distinguish outbound and inbound tour halves. Some tables
+also include `both`, which counts tours or households escorted in both halves,
+or `all_directions`, which sums directional escort incidences and can count the
+same tour twice. These values are not interchangeable.
+
+| Summary table | Information and analytical use | Fields |
+|---|---|---|
+| `escorted_tour_totals` | One run-level total of adult-side tours with an outbound or inbound school-escort condition. Use it as the top-level escorted-tour control total. | `tour_count`: weighted distinct eligible tours with at least one escorted direction. |
+| `school_escorted_tours_by_escort_type_and_direction` | Adult-side escorted tours by escort arrangement and direction, with an `all_directions` incidence rollup. Use it to compare ride-share and pure-escort patterns. | `escort_type`: prepared escort arrangement label.
`direction`: `outbound`, `inbound`, or `all_directions`.
`tour_count`: weighted escorted-tour incidences. |
+| `adult_escorted_tour_purposes_by_direction` | Purposes of the adult tours that perform school escorting, by direction and with an all-directions incidence rollup. Use it to see how escorting is linked with work or other adult activities. | `tour_purpose`: adult tour's primary purpose.
`direction`: escorted half or `all_directions`.
`tour_count`: weighted escorted-tour incidences. |
+| `adult_escorted_tours_by_person_type_and_direction` | Adult-side escorted tours by the adult traveler’s person type and escorted direction. Use it to identify who performs school escorting. | `person_type`: adult traveler person-type code.
`direction`: `outbound`, `inbound`, or `both`.
`tour_count`: weighted tours meeting that directional condition. |
+| `student_school_escort_status_by_direction` | Student school tours classified by normalized escort type for each direction and for tours escorted both ways. Use it to measure the student-side escort experience. | `direction`: `outbound`, `inbound`, or `both`.
`escort_type`: normalized escort arrangement, including unescorted alternatives where present.
`tour_count`: weighted student school tours in the group. |
+| `student_households_by_student_count` | Households by the number of school-age/student household members recognized by the escort logic. Use it as a denominator for household escort participation. | `student_count`: students in the household.
`household_count`: weighted households with that count. |
+| `households_with_school_escorting_by_student_count_and_direction` | Unique households with at least one escorted student school tour, by number of students and directional condition. Use it to calculate escort-participation rates by household composition. | `student_count`: students in the household.
`direction`: `outbound`, `inbound`, or `both`.
`household_count`: weighted unique households meeting the condition. |
+| `schoolkids_per_escorted_tour_by_student_count_and_direction` | Average number of escorted children on adult-side escorted tours by household student count and direction. Use it to analyze escorting efficiency and child grouping. | `student_count`: students in the adult traveler’s household.
`direction`: `outbound`, `inbound`, or `both`.
`avg_schoolkids_per_tour`: weighted mean number of escortees per eligible tour.
`tour_count`: weighted eligible tours used as the mean denominator. |
+| `adult_escorted_tour_distance_distribution_by_direction` | Adult-side escorted tours by rounded tour distance and directional escort condition. Use it to compare the length of outbound-only, inbound-only, and both-way escort tours. | `distance_bin`: rounded tour-distance label from `0` to `39` or `40+`.
`direction`: `outbound`, `inbound`, or `both`.
`tour_count`: weighted eligible tours in the bin. |
+| `adult_escorted_trip_distance_distribution_by_direction` | Trips belonging to explicitly escorted adult tours, filtered to the corresponding outbound or inbound half, by rounded trip distance. Use it to examine the trip-leg burden of escorting. | `distance_bin`: rounded trip-distance label from `0` to `39` or `40+`.
`direction`: `outbound`, `inbound`, or `both` condition.
`trip_count`: weighted eligible trips in the bin. |
+| `adult_escort_event_stop_distribution` | Number of intermediate stops before and after the school drop-off or pickup event on explicitly escorted adult tours. Use it to analyze chaining around escort events. | `segment`: one of `outbound_before_dropoff`, `outbound_after_dropoff`, `inbound_before_pickup`, or `inbound_after_pickup`.
`stop_count`: prepared count of stops in that segment.
`tour_count`: weighted escort-event records with that stop count. |
+| `adult_escort_trip_stop_frequency` | Adult-side escorted tours jointly classified by purpose and outbound, inbound, and total stop counts. Use it to compare stop-making complexity on escort tours. | `tour_purpose`: adult tour purpose.
`outbound_stop_count`: outbound stops capped at 3.
`inbound_stop_count`: inbound stops capped at 3.
`total_stop_count`: total stops capped at 6.
`tour_count`: weighted escorted tours in the combination. |
+
+### Joint Travel
+
+| Summary table | Information and analytical use | Fields |
+|---|---|---|
+| `jtf_distribution` | Households across 21 joint-tour-frequency alternatives describing zero, one, or two joint tours and their purpose combination. Use it to validate joint-tour generation. The current builder uses a simplified purpose-slot coding, so confirm local purpose ordering before treating codes as formal ActivitySim alternatives. | `jtf_code`: integer alternative 1--21.
`jtf_label`: readable frequency/purpose-combination label.
`household_count`: weighted households assigned to the alternative. |
+| `joint_tours_by_household_size` | All households and households making at least one joint tour by household size. Use the two counts to calculate joint-tour participation rates. | `household_size`: number of household members.
`household_count`: weighted households of that size.
`joint_tour_hh_count`: weighted unique households of that size with a joint tour. |
+| `joint_tour_party_size_distribution` | Joint tours by number of household participants, with parties of five or more stored in bin 5. Use it to assess joint-tour occupancy. | `party_size`: household participants; value 5 represents `5+`.
`joint_tour_count`: weighted joint tours in the party-size bin. |
+| `joint_tour_composition_distribution` | Joint tours by prepared party-composition category. Use it to compare adult-only, child-inclusive, and other modeled compositions. | `tour_composition`: prepared joint-party composition.
`joint_tour_count`: weighted joint tours in the category. |
+| `joint_tour_composition_by_party_size` | Joint tours jointly classified by party composition and exact participant count. Use it to study how household makeup and group size interact. | `tour_composition`: prepared party-composition category.
`party_size`: number of tour participants.
`joint_tour_count`: weighted joint tours in the combination. |
+| `person_jtp_by_household_size` | All people and people participating in one or more joint tours by household size. Use the two counts to calculate person-level participation rates. | `household_size`: size of the person’s household.
`joint_tour_person_count`: weighted people with `num_joint_tours > 0`.
`total_person_count`: weighted people in households of that size. |
+| `household_jtp_by_household_size_and_jtf` | For households of size two or more, percentage distribution across 0, 1, and 2+ joint tours within each household size. Use it to compare joint-tour propensity independent of household-size totals. | `jtf`: joint-tour count category `0`, `1`, or `2+`.
`household_size`: household size as a category.
`household_percent`: percent of households of that size in the JTF category. |
+
+### Basic Tour Distributions
+
+| Summary table | Information and analytical use | Fields |
+|---|---|---|
+| `tour_category_distribution` | Tours by ActivitySim category, such as mandatory, nonmandatory, at-work, or joint. Use it for high-level tour-system composition. | `tour_category`: prepared tour category.
`tour_count`: weighted tours in the category. |
+| `tour_purpose_distribution` | Tours by configured summary purpose. Use it to compare the volume and share of work, school, escort, shopping, and other travel. | `tour_purpose`: canonical summary tour purpose.
`tour_count`: weighted tours for that purpose. |
+
+### Vehicles Allocated to Tours
+
+These tables decode the vehicle-type strings allocated under occupancy
+conditions 1, 2, and 3+. They describe modeled allocation incidences, not the
+unique household vehicle inventory.
+
+| Summary table | Information and analytical use | Fields |
+|---|---|---|
+| `allocated_vehicle_age_by_occupancy` | Allocated vehicle ages by occupancy condition. Use it to analyze how fleet age is associated with single- and shared-occupant travel. | `age`: decoded vehicle age, with `20+` terminal.
`occupancy`: allocation condition `1`, `2`, or `3+`.
`vehicle_count`: weighted tour allocation incidences. |
+| `allocated_vehicle_fuel_type_by_occupancy` | Allocated vehicle fuel/powertrain type by occupancy condition. Use it for energy or emissions segmentation of auto travel. | `fuel_type`: decoded fuel/powertrain category.
`occupancy`: allocation condition `1`, `2`, or `3+`.
`vehicle_count`: weighted tour allocation incidences. |
+| `allocated_vehicle_body_type_by_occupancy` | Allocated vehicle body type by occupancy condition. Use it to relate party size to the modeled vehicle used. | `body_type`: decoded vehicle body-style category.
`occupancy`: allocation condition `1`, `2`, or `3+`.
`vehicle_count`: weighted tour allocation incidences. |
+
+### Tour Mode, Stops, Time, and Distance
+
+| Summary table | Information and analytical use | Fields |
+|---|---|---|
+| `tour_mode_by_tour_purpose_and_auto_sufficiency` | Tour-mode counts by purpose and household auto sufficiency, with all-purpose rows. Joint tours are expanded by household participants for this summary. Use it to compare mode choice across vehicle-availability markets. | `tour_mode`: prepared tour mode.
`tour_purpose`: tour purpose or `all_tour_purposes`.
`tour_count_zero_auto`: weighted tours from zero-auto households.
`tour_count_auto_deficient`: weighted tours from households with fewer autos than workers.
`tour_count_auto_sufficient`: weighted tours from auto-sufficient households.
`tour_count_all_households`: sum of the three auto-sufficiency counts. |
+| `tour_stop_frequency_by_tour_purpose` | Tours jointly classified by purpose and outbound, inbound, and total intermediate-stop counts. Use it to measure tour complexity and stop-generation patterns. | `tour_purpose`: canonical tour purpose.
`outbound_stop_count`: outbound stops capped at 3.
`inbound_stop_count`: inbound stops capped at 3.
`total_stop_count`: total stops capped at 6.
`tour_count`: weighted tours in the combination. |
+| `atwork_subtour_frequency_distribution` | Mandatory work tours by their at-work-subtour-frequency alternative. Use it to validate subtour generation from the workplace. | `atwork_subtour_frequency_category`: prepared at-work subtour-frequency category.
`atwork_subtour_count`: weighted parent work tours choosing the category. |
+| `tour_time_of_day_by_tour_purpose` | Dense departure, arrival, and duration profiles by tour purpose plus all-purpose totals. Joint tours are participant-expanded. Use it to compare scheduling and duration distributions. | `time_bin`: ActivitySim period index.
`tour_purpose`: tour purpose or `all_tour_purposes`.
`departure_tour_count`: weighted tours starting in the bin.
`arrival_tour_count`: weighted tours ending in the bin.
`duration_tour_count`: weighted tours whose prepared duration falls in the bin. |
+| `tour_distance_by_tour_purpose` | Tours by rounded skim distance and purpose, plus all-purpose totals. Joint-tour weights are multiplied by participant count. Use it for purpose-specific length-frequency distributions. | `distance_bin`: rounded distance `0`--`39` or `40+`.
`tour_purpose`: purpose or `all_tour_purposes`.
`tour_count`: weighted, participant-adjusted tours in the bin. |
+
+### Tour Geography
+
+| Summary table | Information and analytical use | Fields |
+|---|---|---|
+| `average_mandatory_tour_distance_by_purpose_and_geography` | Weighted average home-to-work or home-to-school distance for workers, university students, and school students, by home geography and regionwide. Use it to compare mandatory destination accessibility. | `mandatory_tour_purpose`: `work`, `university`, or `school`.
`geography_type`: home-geography system or rollup.
`geography_id`: home-geography identifier or rollup.
`average_tour_distance`: finalweight-weighted mean person-level mandatory distance.
`person_count`: weighted people contributing to the mean. |
+| `average_nonmandatory_tour_distance_by_purpose_and_geography` | Weighted average skim distance for individual nonmandatory tours by purpose and traveler home geography, plus regional rows. Use it to compare discretionary travel reach. | `nonmandatory_tour_purpose`: nonmandatory purpose.
`geography_type`: home-geography system or rollup.
`geography_id`: home-geography identifier or rollup.
`average_tour_distance`: finalweight-weighted mean tour skim distance.
`tour_count`: weighted tours contributing to the mean. |
+| `internal_external_nonmandatory_tour_frequency_by_home_geography` | Internal and external nonmandatory tours by traveler home geography, plus regional totals. Use it to calculate external-tour shares and locate households producing external discretionary travel. | `geography_type`: home-geography system or rollup.
`geography_id`: home-geography identifier or rollup.
`internal_nonmandatory_tour_count`: weighted internal nonmandatory tours.
`external_nonmandatory_tour_count`: weighted external nonmandatory tours. |
+| `external_nonmandatory_tour_locations` | External nonmandatory tours by destination geography, plus a regional total. Use it to analyze external destination orientation and gateway demand. | `geography_type`: destination-geography system or rollup.
`geography_id`: destination-geography identifier or rollup.
`external_nonmandatory_tour_count`: weighted external nonmandatory tours ending there. |
+
+### Trip Purpose, Mode, and Parking
+
+| Summary table | Information and analytical use | Fields |
+|---|---|---|
+| `trip_purpose_distribution` | Trip destination purpose cross-classified by parent tour purpose, with all-tour-purpose rows. Use it to analyze activity chains within different kinds of tours. | `tour_purpose`: parent tour purpose or `all_tour_purposes`.
`trip_purpose`: destination purpose of the trip leg.
`trip_count`: weighted trips in the combination. |
+| `stop_destination_purpose_by_tour_purpose` | Intermediate-stop destination purposes by parent tour purpose. Use it to understand what activities are chained into tours. | `stop_destination_purpose`: destination purpose of trip records flagged as stops.
`tour_purpose`: parent tour purpose.
`stop_count`: weighted intermediate stops in the combination. |
+| `trip_mode_by_tour_purpose_and_tour_mode` | Trip-mode counts by parent tour purpose and main tour mode, including all-purpose, all-tour-mode, and grand rollups. Use it to examine access/egress and mode combinations within tours. | `tour_purpose`: parent purpose or `all_tour_purposes`.
`tour_mode`: main tour mode or `all_tour_modes`.
`trip_mode`: mode of the individual trip leg.
`trip_count`: weighted trips in the combination. |
+| `parking_locations` | Auto-trip parking events by configured parking geography. Use it to map modeled parking demand and compare locations across runs. | `geography_type`: parking-geography system.
`geography_id`: valid positive parking-zone identifier at that geography.
`trip_count`: weighted trips parking there. |
+
+### Trip Time and Distance
+
+| Summary table | Information and analytical use | Fields |
+|---|---|---|
+| `trip_departure_time_by_purpose` | Dense departure-period profiles for all trips and for intermediate stops, by parent tour purpose and for all purposes. Use it to compare trip timing with stop timing. | `tour_purpose`: parent purpose or `all_tour_purposes`.
`time_bin`: prepared trip departure period index.
`departure_trip_count`: weighted trips departing in the bin.
`departure_stop_count`: weighted departing trip records flagged as intermediate stops. |
+| `trip_distance_by_purpose` | Trips by rounded OD distance and parent tour purpose, plus all-purpose totals. Weights are multiplied by tour participants, so joint travel is person-trip-like. Use it for purpose-specific trip length distributions. | `distance_bin`: rounded trip distance `0`--`39` or `40+`.
`tour_purpose`: parent purpose or `all_tour_purposes`.
`trip_count`: weighted participant-adjusted trips in the bin. |
+| `stop_out_of_direction_distance_by_tour_purpose` | Intermediate stops by truncated out-of-direction distance, with a dense 0--40 distribution for each purpose and all purposes; bin 40 is terminal. Use it to quantify detour burden from stop-making. | `distance_bin`: truncated out-of-direction distance 0--40, with 40 meaning 40 or more.
`tour_purpose`: parent purpose or `all_tour_purposes`.
`stop_count`: weighted intermediate stops in the bin. |
+
+### Skimjoin Diagnostics
+
+`skim_scenario` distinguishes values for the chosen mode from hypothetical
+mode/scenario sidecars; `all_records` is used for hypothetical values evaluated
+over all applicable records. Each table also includes an all-modes group.
+
+| Summary table | Information and analytical use | Fields |
+|---|---|---|
+| `skimjoin_trip_component_stats` | Weighted descriptive statistics for every numeric `skim_` component on trips, by trip mode and skim scenario. Use it to QA joined time, distance, and cost values and identify missing or zero-heavy components. | `skim_scenario`: chosen or hypothetical evaluation scenario.
`trip_mode`: trip mode or `All Modes`.
`component`: numeric skim column name.
`n_total`: total trip weight eligible for the component/mode group.
`n_valid`: trip weight with non-null component values.
`mean`: weighted mean.
`std`: weighted population standard deviation.
`min`: minimum observed value.
`max`: maximum observed value.
`median`: weighted 50th percentile.
`mode`: value with greatest total weight, using the smaller value on ties.
`zero_share`: valid weight at exactly zero divided by `n_valid`.
`missing_share`: missing weight divided by `n_total`. |
+| `skimjoin_trip_component_ecdf` | Optional 0th-through-100th weighted percentile curves for trip skim components by mode and scenario. Use it for distribution comparison when the compact stats table is insufficient. | `skim_scenario`: chosen or hypothetical scenario.
`trip_mode`: trip mode or `All Modes`.
`component`: numeric skim column name.
`percentile`: cumulative probability from 0.00 through 1.00 in 0.01 steps.
`value`: weighted quantile at that probability.
`n_valid`: total valid trip weight behind the curve. |
+| `skimjoin_tour_component_stats` | Weighted descriptive statistics for numeric tour `skim_` components by tour mode and scenario. Use it to QA tour-level round-trip or composite skims. | `skim_scenario`: chosen or hypothetical evaluation scenario.
`tour_mode`: tour mode or `All Modes`.
`component`: numeric skim column name.
`n_total`: total tour weight in scope.
`n_valid`: tour weight with a value.
`mean`: weighted mean.
`std`: weighted population standard deviation.
`min`: minimum value.
`max`: maximum value.
`median`: weighted median.
`mode`: highest-weight value, smaller on ties.
`zero_share`: valid weight at zero divided by `n_valid`.
`missing_share`: missing weight divided by `n_total`. |
+| `skimjoin_tour_component_ecdf` | Optional weighted percentile curves for tour skim components by mode and scenario. Use it to compare complete tour-level distributions across runs. | `skim_scenario`: chosen or hypothetical scenario.
`tour_mode`: tour mode or `All Modes`.
`component`: numeric skim column name.
`percentile`: probability from 0.00 through 1.00.
`value`: weighted quantile at that probability.
`n_valid`: valid tour weight behind the curve. |
+
+### Processor-Built Validation Summaries
+
+Several assignment-based tables accept optional tables attached to `RunData`.
+They remain valid but empty when those optional assignment inputs are absent.
+
+| Summary table | Information and analytical use | Fields |
+|---|---|---|
+| `traffic_count_comparisons` | Observed and modeled traffic counts matched at count-location, direction, and period level. Use it for count scatterplots, percent differences, RMSE, and facility calibration. Only keys present in both sources are retained. | `count_location_id`: traffic-count station/location identifier.
`direction`: observed/modeled direction label.
`count_period`: count time-period label.
`observed_volume`: summed observed count for the key.
`modeled_volume`: summed assigned volume for the matching key. |
+| `screenline_flow_comparisons` | Observed and modeled screenline flows matched by screenline, direction, and period, with a representative facility type. Use it for corridor-level flow validation and regression analysis. | `screenline_id`: screenline/cutline identifier.
`direction`: flow direction.
`count_period`: comparison period.
`facility_type`: supplied facility class, or `All` if absent.
`observed_volume`: summed observed flow.
`modeled_volume`: summed modeled flow for the matching key. |
+| `transit_boardings_by_operator_and_technology` | Assigned transit boardings summed by operator and transit technology. Use it to compare ridership scale across agencies and modes. | `operator`: supplied transit operator identifier or name.
`technology`: supplied transit mode/technology category.
`boardings`: total assigned unlinked passenger boardings. |
+| `transit_transfer_rate` | Assigned boardings divided by linked transit trips by operator, technology, and access mode. The value is boardings per linked trip, so values above one indicate transfers; subtract one if a transfers-per-trip measure is needed. | `operator`: transit operator.
`technology`: transit technology/mode.
`access_mode`: mode used to access transit.
`transfer_rate`: assigned boardings divided by linked trips; null for a zero linked-trip denominator. |
+| `auto_vmt_totals` | One run-level personal-auto VMT total. Each auto trip contributes weighted OD distance divided by participants when occupancy is available. Use it for overview controls and scenario comparison. | `auto_vmt`: total personal-auto vehicle miles traveled. |
+| `auto_vmt_by_home_geography_income_hhsize_time_period` | Personal-auto VMT and trip counts by home geography, income, household size, time period, and auto mode, with daily rows derived from detailed periods. Use it for equity, temporal, modal, and spatial VMT analysis. | `geography_type`: home-geography system or `all_geographies`.
`geography_id`: home-geography identifier or rollup.
`income_segment`: prepared household income segment or fallback rollup.
`household_size`: prepared household size or fallback rollup.
`time_period`: configured period or `Daily`.
`mode`: trip mode or `All Auto` fallback.
`auto_vmt`: sum of distance times weight divided by occupancy.
`trip_count`: weighted auto trips.
`distance_source`: provenance of the distance used, such as a skim or OD-distance field.
`time_period_source`: provenance of the time-period assignment. |
+| `non_motorized_vmt_by_home_geography_income_hhsize_time_period` | Walk, bicycle, and e-bike weighted miles and trip counts by home geography, income, household size, period, and mode, including derived daily rows. Use it for active-travel exposure and equity analysis. | `geography_type`: home-geography system or rollup.
`geography_id`: home-geography identifier or rollup.
`income_segment`: household income segment or fallback rollup.
`household_size`: household size or fallback rollup.
`time_period`: configured period or `Daily`.
`mode`: `WALK`, `BIKE`, or `EBIKE` as available.
`non_motorized_vmt`: distance times final trip weight; despite the VMT name, this is weighted traveler mileage.
`trip_count`: weighted eligible trips.
`distance_source`: prepared or skim distance source used for the mode.
`time_period_source`: provenance of the period assignment. |
+| `commercial_vmt_totals` | Commercial-vehicle VMT by vehicle type, split between internal and external travel. Use it for freight VMT totals and internal/external shares. | `commercial_vehicle_type`: supplied commercial vehicle/truck class.
`external_vmt`: VMT from records classified as external.
`internal_vmt`: VMT from records classified as internal. |
+| `bicycle_vmt_by_facility_type` | Bicycle VMT by facility type, read directly when supplied or calculated as assigned bicycle trips times link distance. Use it to evaluate bicycle use by facility class. | `facility_type`: supplied bicycle/network facility category.
`bicycle_vmt`: summed bicycle vehicle/traveler miles on that facility type. |
+
+### Externally Supplied Validation Contracts
+
+The following 13 tables are registered so externally prepared CSVs can be
+loaded consistently. Their no-op builders do not calculate values. The stated
+meaning is therefore the contract expected by the dashboard; the supplying
+workflow is responsible for units, period definitions, and internal
+consistency.
+
+| Summary table | Information and analytical use | Fields |
+|---|---|---|
+| `link_validation_summary` | Modeled network link volumes with link endpoints and facility class. Use it to aggregate modeled flow by facility or inspect high-volume links. | `id`: link identifier.
`From_Node`: upstream node identifier.
`To_Node`: downstream node identifier.
`FACTYPE`: facility-type code.
`am_vol`: AM-period modeled link volume.
`md_vol`: midday modeled link volume.
`pm_vol`: PM-period modeled link volume.
`day_vol`: daily modeled link volume. |
+| `count_location_counts_validation_summary` | Observed traffic-count volumes by count location and facility class. Use it as the observed side of location-level modeled-versus-observed comparisons. | `id`: count-location identifier.
`FACTYPE`: facility-type code.
`am_vol`: observed AM volume.
`md_vol`: observed midday volume.
`pm_vol`: observed PM volume.
`day_vol`: observed daily volume. |
+| `count_location_volumes_validation_summary` | Modeled volumes at the traffic-count locations. Use it as the modeled side of location-level count comparisons. | `id`: count-location identifier matching the observed table.
`FACTYPE`: facility-type code.
`am_vol`: modeled AM volume.
`md_vol`: modeled midday volume.
`pm_vol`: modeled PM volume.
`day_vol`: modeled daily volume. |
+| `count_location_scatter_validation_summary` | Long-form observed/modeled point pairs already prepared for count scatterplots. Use it when the source workflow supplies paired values directly. | `id`: count-location identifier.
`facility_type`: facility class code or label.
`period`: comparison period.
`observed_volume`: observed traffic volume.
`modeled_volume`: modeled traffic volume. |
+| `count_location_fit_validation_summary` | Precomputed linear-fit diagnostics for observed-versus-modeled counts by facility type and period. Use it to draw regression lines and report calibration fit. | `facility_type`: facility class used for the fit.
`period`: time period used for the fit.
`slope`: fitted slope for modeled volume as a function of observed volume.
`intercept`: fitted modeled-volume intercept.
`r_squared`: coefficient of determination.
`n_locations`: paired count locations in the fit.
`observed_min`: minimum observed volume in the fitting data.
`observed_max`: maximum observed volume.
`equation_label`: preformatted regression-equation text.
`r_squared_label`: preformatted R-squared text. |
+| `district_commuting_flows_validation_summary` | Supplied district-to-district commute-flow matrix for Albany, Corvallis, Lebanon, and Philomath. Use it as a local validation/control matrix. | empty-name column `""`: origin district or row label.
`Albany`: commuters to Albany.
`Corvallis`: commuters to Corvallis.
`Lebanon`: commuters to Lebanon.
`Philomath`: commuters to Philomath.
`Total`: row total across destinations. |
+| `county_commuting_flows_validation_summary` | Supplied county-to-county commute-flow matrix for Benton, Linn, and Marion counties. Use it as a regional commute-flow validation/control matrix. | empty-name column `""`: origin county or row label.
`Benton`: commuters to Benton County.
`Linn`: commuters to Linn County.
`Marion`: commuters to Marion County.
`Total`: row total across destinations. |
+| `commercial_vehicle_validation_summary` | Supplied commercial-vehicle trip totals by time of day and vehicle class. Use it to compare commercial demand composition and daily profiles. | `tod`: time-of-day row label.
`car`: commercial-car/light-vehicle trips.
`mu`: multi-unit truck trips.
`su`: single-unit truck trips.
`Total`: total commercial trips across classes. |
+| `commercial_vehicle_vmt_validation_summary` | Supplied commercial-vehicle VMT by time of day and vehicle class. Use it to compare freight mileage composition and temporal patterns. | `tod`: time-of-day row label.
`car`: commercial-car/light-vehicle VMT.
`mu`: multi-unit truck VMT.
`su`: single-unit truck VMT.
`Total`: total commercial VMT across classes. |
+| `external_trip_validation_summary` | Supplied external trip totals by time of day and purpose/class. Use it to analyze gateway demand by travel market. | `tod`: time-of-day row label.
`hbcoll`: home-based college trips.
`hbo`: home-based other trips.
`hbr`: home-based recreation trips.
`hbs`: home-based shopping trips.
`hbsch`: home-based school trips.
`hbw`: home-based work trips.
`nhbnw`: non-home-based non-work trips.
`nhbw`: non-home-based work trips.
`truck`: truck trips.
`Total`: total external trips across purposes/classes. |
+| `external_vmt_validation_summary` | Supplied external VMT by time of day and purpose/class. Use it to identify which external markets contribute mileage. | `tod`: time-of-day row label.
`hbcoll`: home-based college VMT.
`hbo`: home-based other VMT.
`hbr`: home-based recreation VMT.
`hbs`: home-based shopping VMT.
`hbsch`: home-based school VMT.
`hbw`: home-based work VMT.
`nhbnw`: non-home-based non-work VMT.
`nhbw`: non-home-based work VMT.
`truck`: truck VMT.
`Total`: total external VMT across purposes/classes. |
+| `auto_vmt_validation_summary` | Supplied auto and truck VMT by time of day and occupancy class. Use it as an independent control for modeled VMT. | `TOD`: time-of-day row label.
`SOV`: single-occupant-vehicle VMT.
`HOV2`: two-occupant shared-ride VMT.
`HOV3`: three-or-more-occupant shared-ride VMT.
`Truck`: truck VMT.
`Total`: total VMT across listed classes. |
+| `work_from_home_validation_summary` | Supplied worker and work-from-home controls by district. Use it to compare modeled WFH counts or rates with external targets. | `District`: district name or identifier.
`Workers`: total workers in the district.
`WFH`: workers who work from home. |
+
+## Generated Developer Inventory
Regenerate it with:
@@ -14,106 +273,106 @@ _Generated from `processor.summarize.catalog.SUMMARY_DEFINITIONS`._
Total registered summaries: **100**
-| Summary ID | Filename | Builder | Output schema | Required inputs |
-|---|---|---|---|---|
-| `adult_escort_event_stop_distribution` | `adult_escort_event_stop_distribution.csv` | `processor.summarize.summaries.daily_travel_escort_distributions.adult_escort_event_stop_distribution` | `segment: String`
`stop_count: Int32`
`tour_count: Float64` | tours: `tour_id`, `school_esc_outbound`, `school_esc_inbound`
trips: `tour_id`, `escort_event_role`, `escort_stops_before_event`, `escort_stops_after_event`, `finalweight` |
-| `adult_escort_trip_stop_frequency` | `adult_escort_trip_stop_frequency.csv` | `processor.summarize.summaries.daily_travel_escort_distributions.adult_escort_trip_stop_frequency` | `tour_purpose: String`
`outbound_stop_count: Int32`
`inbound_stop_count: Int32`
`total_stop_count: Int32`
`tour_count: Float64` | tours: `tour_purpose`, `school_esc_outbound`, `school_esc_inbound`, `num_ob_stops`, `num_ib_stops`, `num_tot_stops`, `finalweight` |
-| `adult_escorted_tour_distance_distribution_by_direction` | `adult_escorted_tour_distance_distribution_by_direction.csv` | `processor.summarize.summaries.daily_travel_escort_distributions.adult_escorted_tour_distance_distribution_by_direction` | `distance_bin: String`
`direction: String`
`tour_count: Float64` | tours: `SKIMDIST`, `school_esc_outbound`, `school_esc_inbound`, `finalweight` |
-| `adult_escorted_tour_purposes_by_direction` | `adult_escorted_tour_purposes_by_direction.csv` | `processor.summarize.summaries.daily_travel_escort_counts.adult_escorted_tour_purposes_by_direction` | `tour_purpose: String`
`direction: String`
`tour_count: Float64` | tours: `tour_purpose`, `school_esc_outbound`, `school_esc_inbound`, `finalweight` |
-| `adult_escorted_tours_by_person_type_and_direction` | `adult_escorted_tours_by_person_type_and_direction.csv` | `processor.summarize.summaries.daily_travel_escort_counts.adult_escorted_tours_by_person_type_and_direction` | `person_type: String`
`direction: String`
`tour_count: Float64` | tours: `school_esc_outbound`, `school_esc_inbound`, `finalweight` |
-| `adult_escorted_trip_distance_distribution_by_direction` | `adult_escorted_trip_distance_distribution_by_direction.csv` | `processor.summarize.summaries.daily_travel_escort_distributions.adult_escorted_trip_distance_distribution_by_direction` | `distance_bin: String`
`direction: String`
`trip_count: Float64` | tours: `tour_id`, `school_esc_outbound`, `school_esc_inbound`
trips: `tour_id`, `od_dist`, `finalweight` |
-| `allocated_vehicle_age_by_occupancy` | `allocated_vehicle_age_by_occupancy.csv` | `processor.summarize.summaries.tour_vehicles.allocated_vehicle_age` | `age: String`
`occupancy: String`
`vehicle_count: Float64` | tours: `vehicle_occup_1`, `vehicle_occup_2`, `vehicle_occup_3.5`, `finalweight` |
-| `allocated_vehicle_body_type_by_occupancy` | `allocated_vehicle_body_type_by_occupancy.csv` | `processor.summarize.summaries.tour_vehicles.allocated_vehicle_body` | `body_type: String`
`occupancy: String`
`vehicle_count: Float64` | tours: `vehicle_occup_1`, `vehicle_occup_2`, `vehicle_occup_3.5`, `finalweight` |
-| `allocated_vehicle_fuel_type_by_occupancy` | `allocated_vehicle_fuel_type_by_occupancy.csv` | `processor.summarize.summaries.tour_vehicles.allocated_vehicle_fuel` | `fuel_type: String`
`occupancy: String`
`vehicle_count: Float64` | tours: `vehicle_occup_1`, `vehicle_occup_2`, `vehicle_occup_3.5`, `finalweight` |
-| `atwork_subtour_frequency_distribution` | `atwork_subtour_frequency_distribution.csv` | `processor.summarize.summaries.tour_profiles.at_work_sub_tour_freq` | `atwork_subtour_frequency_category: String`
`atwork_subtour_count: Float64` | tours: `tour_purpose`, `tour_category`, `atwork_subtour_frequency`, `finalweight` |
-| `auto_ownership_distribution` | `auto_ownership_distribution.csv` | `processor.summarize.summaries.long_term_vehicle.auto_ownership` | `household_size: String`
`household_vehicle_count: Int64`
`household_count: Float64` | hh: `HHSIZE`, `HHVEH`, `finalweight` |
-| `auto_vmt_by_home_geography_income_hhsize_time_period` | `auto_vmt_by_home_geography_income_hhsize_time_period.csv` | `processor.summarize.summaries.validation.auto_vmt_by_home_geography_income_hhsize_time_period` | `geography_type: String`
`geography_id: String`
`income_segment: String`
`household_size: String`
`time_period: String`
`mode: String`
`auto_vmt: Float64`
`trip_count: Float64`
`distance_source: String`
`time_period_source: String` | trips: `finalweight` |
-| `auto_vmt_totals` | `auto_vmt_totals.csv` | `processor.summarize.summaries.validation.auto_vmt_totals` | `auto_vmt: Float64` | trips: `trip_mode`, `od_dist`, `finalweight` |
-| `auto_vmt_validation_summary` | `auto_vmt_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.auto_vmt_validation_summary` | `TOD: String`
`SOV: Float64`
`HOV2: Float64`
`HOV3: Float64`
`Truck: Float64`
`Total: Float64` | - |
-| `autonomous_vehicle_ownership_totals` | `autonomous_vehicle_ownership_totals.csv` | `processor.summarize.summaries.long_term_vehicle.av_ownership` | `household_with_autonomous_vehicle_count: Float64` | hh: `av_ownership`, `finalweight` |
-| `average_mandatory_tour_distance_by_purpose_and_geography` | `average_mandatory_tour_distance_by_purpose_and_geography.csv` | `processor.summarize.summaries.tour_geography.avg_mand_tour_distance` | `mandatory_tour_purpose: String`
`geography_type: String`
`geography_id: String`
`average_tour_distance: Float64`
`person_count: Float64` | per: `finalweight` |
-| `average_nonmandatory_tour_distance_by_purpose_and_geography` | `average_nonmandatory_tour_distance_by_purpose_and_geography.csv` | `processor.summarize.summaries.tour_geography.avg_non_mand_tour_distance` | `nonmandatory_tour_purpose: String`
`geography_type: String`
`geography_id: String`
`average_tour_distance: Float64`
`tour_count: Float64` | per: `person_id`, `home_zone_id`
tours: `person_id`, `tour_category`, `tour_purpose`, `SKIMDIST`, `finalweight` |
-| `bicycle_comfort_level_distribution` | `bicycle_comfort_level_distribution.csv` | `processor.summarize.summaries.long_term_person.bicycle_comfort_level` | `person_type: String`
`bicycle_comfort_level: String`
`person_type_label: String`
`person_count: Float64` | per: `person_type`, `bike_comfort`, `finalweight` |
-| `bicycle_vmt_by_facility_type` | `bicycle_vmt_by_facility_type.csv` | `processor.summarize.summaries.validation.bicycle_vmt_by_facility` | `facility_type: String`
`bicycle_vmt: Float64` | - |
-| `commercial_vehicle_validation_summary` | `commercial_vehicle_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.commercial_vehicle_validation_summary` | `tod: String`
`car: Float64`
`mu: Float64`
`su: Float64`
`Total: Float64` | - |
-| `commercial_vehicle_vmt_validation_summary` | `commercial_vehicle_vmt_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.commercial_vehicle_vmt_validation_summary` | `tod: String`
`car: Float64`
`mu: Float64`
`su: Float64`
`Total: Float64` | - |
-| `commercial_vmt_totals` | `commercial_vmt_totals.csv` | `processor.summarize.summaries.validation.commercial_vehicle_vmt` | `commercial_vehicle_type: String`
`external_vmt: Float64`
`internal_vmt: Float64` | - |
-| `commuting_flows` | `commuting_flows.csv` | `processor.summarize.summaries.long_term_geography.commuting_flows` | `origin_geography_type: String`
`origin_geography_id: String`
`destination_geography_type: String`
`destination_geography_id: String`
`commuter_count: Float64` | per: `home_zone_id`, `workplace_zone_id`, `is_worker`, `finalweight` |
-| `count_location_counts_validation_summary` | `count_location_counts_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.count_location_counts_validation_summary` | `id: Int64`
`FACTYPE: Int64`
`am_vol: Float64`
`md_vol: Float64`
`pm_vol: Float64`
`day_vol: Float64` | - |
-| `count_location_fit_validation_summary` | `count_location_fit_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.count_location_fit_validation_summary` | `facility_type: String`
`period: String`
`slope: Float64`
`intercept: Float64`
`r_squared: Float64`
`n_locations: Int64`
`observed_min: Float64`
`observed_max: Float64`
`equation_label: String`
`r_squared_label: String` | - |
-| `count_location_scatter_validation_summary` | `count_location_scatter_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.count_location_scatter_validation_summary` | `id: Int64`
`facility_type: String`
`period: String`
`observed_volume: Float64`
`modeled_volume: Float64` | - |
-| `count_location_volumes_validation_summary` | `count_location_volumes_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.count_location_volumes_validation_summary` | `id: Int64`
`FACTYPE: Int64`
`am_vol: Float64`
`md_vol: Float64`
`pm_vol: Float64`
`day_vol: Float64` | - |
-| `county_commuting_flows_validation_summary` | `county_commuting_flows_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.county_commuting_flows_validation_summary` | `: String`
`Benton: Float64`
`Linn: Float64`
`Marion: Float64`
`Total: Float64` | - |
-| `daily_activity_pattern_by_person_type` | `daily_activity_pattern_by_person_type.csv` | `processor.summarize.summaries.daily_travel_activity.dap_summary` | `person_type: String`
`daily_activity_pattern: String`
`person_count: Float64` | per: `person_type`, `cdap_activity`, `finalweight` |
-| `district_commuting_flows_validation_summary` | `district_commuting_flows_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.district_commuting_flows_validation_summary` | `: String`
`Albany: Float64`
`Corvallis: Float64`
`Lebanon: Float64`
`Philomath: Float64`
`Total: Float64` | - |
-| `escorted_tour_totals` | `escorted_tour_totals.csv` | `processor.summarize.summaries.daily_travel_escort_counts.total_escorted_tours` | `tour_count: Float64` | tours: `school_esc_outbound`, `school_esc_inbound`, `finalweight` |
-| `external_nonmandatory_tour_locations` | `external_nonmandatory_tour_locations.csv` | `processor.summarize.summaries.tour_geography.ext_non_mand_tour_loc` | `geography_type: String`
`geography_id: String`
`external_nonmandatory_tour_count: Float64` | tours: `tour_category`, `is_external_tour`, `destination`, `finalweight` |
-| `external_trip_validation_summary` | `external_trip_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.external_trip_validation_summary` | `tod: String`
`hbcoll: Float64`
`hbo: Float64`
`hbr: Float64`
`hbs: Float64`
`hbsch: Float64`
`hbw: Float64`
`nhbnw: Float64`
`nhbw: Float64`
`truck: Float64`
`Total: Float64` | - |
-| `external_vmt_validation_summary` | `external_vmt_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.external_vmt_validation_summary` | `tod: String`
`hbcoll: Float64`
`hbo: Float64`
`hbr: Float64`
`hbs: Float64`
`hbsch: Float64`
`hbw: Float64`
`nhbnw: Float64`
`nhbw: Float64`
`truck: Float64`
`Total: Float64` | - |
-| `external_worker_workplace_locations` | `external_worker_workplace_locations.csv` | `processor.summarize.summaries.long_term_geography.external_workplace_loc` | `geography_type: String`
`geography_id: String`
`external_worker_count: Float64`
`all_worker_count: Float64` | per: `is_worker`, `is_external_worker`, `external_workplace_zone_id`, `finalweight` |
-| `free_parking_eligibility_by_workplace_geography` | `free_parking_eligibility_by_workplace_geography.csv` | `processor.summarize.summaries.long_term_geography.free_parking` | `geography_type: String`
`geography_id: String`
`workers_without_free_parking_count: Float64`
`workers_with_free_parking_count: Float64` | per: `is_worker`, `free_parking_at_work`, `workplace_zone_id`, `finalweight` |
-| `household_jtp_by_household_size_and_jtf` | `household_jtp_by_household_size_and_jtf.csv` | `processor.summarize.summaries.joint_travel.jtf_by_hhsize` | `jtf: String`
`household_size: String`
`household_percent: Float64` | hh: `household_id`, `HHSIZE`, `finalweight`
tours: `tour_category`, `household_id` |
-| `household_size_distribution` | `household_size_distribution.csv` | `processor.summarize.summaries.demographics.hh_size` | `household_size: Int64`
`household_count: Float64` | hh: `HHSIZE`, `finalweight` |
-| `households_with_school_escorting_by_student_count_and_direction` | `households_with_school_escorting_by_student_count_and_direction.csv` | `processor.summarize.summaries.daily_travel_escort_counts.households_with_school_escorting_by_student_count_and_direction` | `student_count: Int64`
`direction: String`
`household_count: Float64` | hh: `household_id`, `finalweight`
per: `household_id`, `person_type`
tours: `tour_purpose`, `school_esc_outbound`, `school_esc_inbound`, `finalweight` |
-| `internal_external_nonmandatory_tour_frequency_by_home_geography` | `internal_external_nonmandatory_tour_frequency_by_home_geography.csv` | `processor.summarize.summaries.tour_geography.int_vs_ext_non_mand_tour_freq` | `geography_type: String`
`geography_id: String`
`internal_nonmandatory_tour_count: Float64`
`external_nonmandatory_tour_count: Float64` | per: `person_id`, `home_zone_id`
tours: `person_id`, `tour_category`, `is_external_tour`, `finalweight` |
-| `internal_external_worker_by_geography` | `internal_external_worker_by_geography.csv` | `processor.summarize.summaries.long_term_geography.internal_vs_external` | `geography_type: String`
`geography_id: String`
`internal_worker_count: Float64`
`external_worker_count: Float64` | per: `is_worker`, `is_external_worker`, `home_zone_id`, `finalweight` |
-| `joint_tour_composition_by_party_size` | `joint_tour_composition_by_party_size.csv` | `processor.summarize.summaries.joint_travel.joint_composition_by_party_size` | `tour_composition: String`
`party_size: Int64`
`joint_tour_count: Float64` | tours: `tour_category`, `composition`, `number_of_participants`, `finalweight` |
-| `joint_tour_composition_distribution` | `joint_tour_composition_distribution.csv` | `processor.summarize.summaries.joint_travel.joint_composition` | `tour_composition: String`
`joint_tour_count: Float64` | tours: `tour_category`, `finalweight` |
-| `joint_tour_party_size_distribution` | `joint_tour_party_size_distribution.csv` | `processor.summarize.summaries.joint_travel.joint_party_size` | `party_size: Int32`
`joint_tour_count: Float64` | tours: `tour_category`, `NUMBER_HH`, `finalweight` |
-| `joint_tours_by_household_size` | `joint_tours_by_household_size.csv` | `processor.summarize.summaries.joint_travel.joint_tours_hhsize` | `household_size: Int32`
`household_count: Float64`
`joint_tour_hh_count: Float64` | hh: `household_id`, `HHSIZE`, `finalweight`
tours: `tour_category`, `household_id` |
-| `jtf_distribution` | `jtf_distribution.csv` | `processor.summarize.summaries.joint_travel.joint_tour_freq` | `jtf_code: Int32`
`jtf_label: String`
`household_count: Float64` | hh: `household_id`, `finalweight` |
-| `license_holding_status_distribution` | `license_holding_status_distribution.csv` | `processor.summarize.summaries.long_term_person.license_holding_status` | `person_type: String`
`license_holding_status: String`
`person_type_label: String`
`person_count: Float64` | per: `person_type`, `has_license`, `finalweight`, `age` |
-| `link_validation_summary` | `link_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.link_validation_summary` | `id: Int64`
`From_Node: Int64`
`To_Node: Int64`
`FACTYPE: Int64`
`am_vol: Float64`
`md_vol: Float64`
`pm_vol: Float64`
`day_vol: Float64` | - |
-| `mandatory_tour_frequency_by_person_type` | `mandatory_tour_frequency_by_person_type.csv` | `processor.summarize.summaries.daily_travel_activity.mandatory_tour_freq` | `person_type: String`
`mandatory_tour_frequency: Int32`
`person_count: Float64` | per: `person_type`, `imf_choice`, `finalweight` |
-| `non_motorized_vmt_by_home_geography_income_hhsize_time_period` | `non_motorized_vmt_by_home_geography_income_hhsize_time_period.csv` | `processor.summarize.summaries.validation.non_motorized_vmt_by_home_geography_income_hhsize_time_period` | `geography_type: String`
`geography_id: String`
`income_segment: String`
`household_size: String`
`time_period: String`
`mode: String`
`non_motorized_vmt: Float64`
`trip_count: Float64`
`distance_source: String`
`time_period_source: String` | trips: `finalweight`, `trip_mode` |
-| `nonmandatory_tour_frequency_by_person_type` | `nonmandatory_tour_frequency_by_person_type.csv` | `processor.summarize.summaries.daily_travel_activity.indiv_nm_summary` | `person_type: String`
`nonmandatory_tour_frequency: String`
`person_count: Float64` | joint_participants: `person_id`
per: `person_id`, `person_type`, `finalweight`
tours: `person_id`, `tour_category` |
-| `park_and_ride_location_residual_histogram` | `park_and_ride_location_residual_histogram.csv` | `processor.summarize.summaries.long_term_geography.park_and_ride_location_residual_histogram` | `geography_type: String`
`bin_start: Float64`
`bin_end: Float64`
`geography_count: Float64` | land_use: -
tours: `tour_mode`, `finalweight` |
-| `park_and_ride_location_residuals` | `park_and_ride_location_residuals.csv` | `processor.summarize.summaries.long_term_geography.park_and_ride_location_residuals` | `geography_type: String`
`geography_id: String`
`pnr_tour_count: Float64`
`pnr_lot_capacity: Float64`
`residual_count: Float64`
`absolute_residual_count: Float64`
`percent_error: Float64` | land_use: -
tours: `tour_mode`, `finalweight` |
-| `parking_locations` | `parking_locations.csv` | `processor.summarize.summaries.trip.parking_locations` | `geography_type: String`
`geography_id: String`
`trip_count: Float64` | trips: `parking_zone`, `finalweight` |
-| `person_jtp_by_household_size` | `person_jtp_by_household_size.csv` | `processor.summarize.summaries.joint_travel.joint_participation_person_by_hhsize` | `household_size: Int64`
`joint_tour_person_count: Float64`
`total_person_count: Float64` | hh: `household_id`, `hhsize`
per: `household_id`, `num_joint_tours`, `finalweight` |
-| `person_type_distribution` | `person_type_distribution.csv` | `processor.summarize.summaries.demographics.person_type` | `person_type: String`
`person_type_label: String`
`person_count: Float64` | per: `person_type`, `finalweight` |
-| `population_totals` | `population_totals.csv` | `processor.summarize.summaries.demographics.population_totals` | `person_count: Float64`
`household_count: Float64`
`tour_count: Float64`
`trip_count: Float64`
`stop_count: Float64` | hh: `finalweight`
per: `finalweight`
tours: `finalweight`
trips: `finalweight`, `stops` |
-| `school_escorted_tours_by_escort_type_and_direction` | `school_escorted_tours_by_escort_type_and_direction.csv` | `processor.summarize.summaries.daily_travel_escort_counts.escorted_tours_to_from_school` | `escort_type: String`
`direction: String`
`tour_count: Float64` | tours: `tour_purpose`, `school_esc_outbound`, `school_esc_inbound`, `finalweight` |
-| `school_location_distance_distribution_by_geography` | `school_location_distance_distribution_by_geography.csv` | `processor.summarize.summaries.long_term_distance.schl_tlfd` | `distance_bin: Int32`
`geography_type: String`
`geography_id: String`
`person_count: Float64` | per: `distance_to_school`, `finalweight` |
-| `school_location_enrollment_comparison` | `school_location_enrollment_comparison.csv` | `processor.summarize.summaries.long_term_geography.school_loc_vs_land_use_enrollment` | `geography_type: String`
`geography_id: String`
`student_type: String`
`enrollment_count: Float64`
`student_count: Float64` | land_use: `MAZ`, `enrollment_count`, `student_type`
per: `school_zone_id`, `is_student`, `finalweight` |
-| `school_shadow_pricing_residual_histogram` | `school_shadow_pricing_residual_histogram.csv` | `processor.summarize.summaries.long_term_geography.school_shadow_pricing_residual_histogram` | `geography_type: String`
`student_type: String`
`bin_start: Float64`
`bin_end: Float64`
`geography_count: Float64` | land_use: `MAZ`, `enrollment_count`, `student_type`
per: `school_zone_id`, `is_student`, `finalweight`, `student_type` |
-| `school_shadow_pricing_residuals` | `school_shadow_pricing_residuals.csv` | `processor.summarize.summaries.long_term_geography.school_shadow_pricing_residuals` | `geography_type: String`
`geography_id: String`
`student_type: String`
`target_count: Float64`
`modeled_count: Float64`
`residual_count: Float64`
`absolute_residual_count: Float64`
`percent_error: Float64` | land_use: `MAZ`, `enrollment_count`, `student_type`
per: `school_zone_id`, `is_student`, `finalweight`, `student_type` |
-| `schoolkids_per_escorted_tour_by_student_count_and_direction` | `schoolkids_per_escorted_tour_by_student_count_and_direction.csv` | `processor.summarize.summaries.daily_travel_escort_counts.schoolkids_per_escorted_tour_by_student_count_and_direction` | `student_count: Int64`
`direction: String`
`avg_schoolkids_per_tour: Float64`
`tour_count: Float64` | hh: `household_id`, `finalweight`
per: `household_id`, `person_type`
tours: `school_esc_outbound`, `school_esc_inbound`, `num_escortees`, `finalweight` |
-| `screenline_flow_comparisons` | `screenline_flow_comparisons.csv` | `processor.summarize.summaries.validation.screenline_flow_comparisons` | `screenline_id: String`
`direction: String`
`count_period: String`
`facility_type: String`
`observed_volume: Float64`
`modeled_volume: Float64` | - |
-| `skimjoin_tour_component_ecdf` | `skimjoin_tour_component_ecdf.csv` | `processor.summarize.summaries.skimjoin.tour_skim_component_ecdf` | `skim_scenario: String`
`tour_mode: String`
`component: String`
`percentile: Float64`
`value: Float64`
`n_valid: Float64` | tours: `tour_mode`, `finalweight` |
-| `skimjoin_tour_component_stats` | `skimjoin_tour_component_stats.csv` | `processor.summarize.summaries.skimjoin.tour_skim_component_stats` | `skim_scenario: String`
`tour_mode: String`
`component: String`
`n_total: Float64`
`n_valid: Float64`
`mean: Float64`
`std: Float64`
`min: Float64`
`max: Float64`
`median: Float64`
`mode: Float64`
`zero_share: Float64`
`missing_share: Float64` | tours: `tour_mode`, `finalweight` |
-| `skimjoin_trip_component_ecdf` | `skimjoin_trip_component_ecdf.csv` | `processor.summarize.summaries.skimjoin.trip_skim_component_ecdf` | `skim_scenario: String`
`trip_mode: String`
`component: String`
`percentile: Float64`
`value: Float64`
`n_valid: Float64` | trips: `trip_mode`, `finalweight` |
-| `skimjoin_trip_component_stats` | `skimjoin_trip_component_stats.csv` | `processor.summarize.summaries.skimjoin.trip_skim_component_stats` | `skim_scenario: String`
`trip_mode: String`
`component: String`
`n_total: Float64`
`n_valid: Float64`
`mean: Float64`
`std: Float64`
`min: Float64`
`max: Float64`
`median: Float64`
`mode: Float64`
`zero_share: Float64`
`missing_share: Float64` | trips: `trip_mode`, `finalweight` |
-| `stop_destination_purpose_by_tour_purpose` | `stop_destination_purpose_by_tour_purpose.csv` | `processor.summarize.summaries.trip.stop_purpose_by_tour_purpose` | `stop_destination_purpose: String`
`tour_purpose: String`
`stop_count: Float64` | trips: `stops`, `tour_purpose`, `trip_purpose`, `finalweight` |
-| `stop_out_of_direction_distance_by_tour_purpose` | `stop_out_of_direction_distance_by_tour_purpose.csv` | `processor.summarize.summaries.trip_distributions.stop_ood_distance` | `distance_bin: Int32`
`tour_purpose: String`
`stop_count: Float64` | trips: `stops`, `out_dir_dist`, `tour_purpose`, `finalweight` |
-| `student_households_by_student_count` | `student_households_by_student_count.csv` | `processor.summarize.summaries.daily_travel_escort_counts.student_households_by_student_count` | `student_count: Int64`
`household_count: Float64` | hh: `household_id`, `finalweight`
per: `household_id`, `person_type` |
-| `student_school_escort_status_by_direction` | `student_school_escort_status_by_direction.csv` | `processor.summarize.summaries.daily_travel_escort_counts.student_school_escort_status_by_direction` | `direction: String`
`escort_type: String`
`tour_count: Float64` | tours: `tour_purpose`, `school_esc_outbound`, `school_esc_inbound`, `finalweight` |
-| `telecommute_frequency_distribution` | `telecommute_frequency_distribution.csv` | `processor.summarize.summaries.long_term_person.telecommute` | `geography_type: String`
`geography_id: String`
`telecommute_frequency: String`
`person_count: Float64` | per: `telecommute_frequency`, `finalweight`, `is_worker`, `work_from_home`, `home_zone_id` |
-| `tour_category_distribution` | `tour_category_distribution.csv` | `processor.summarize.summaries.tour.tour_category` | `tour_category: String`
`tour_count: Float64` | tours: `tour_category`, `finalweight` |
-| `tour_distance_by_tour_purpose` | `tour_distance_by_tour_purpose.csv` | `processor.summarize.summaries.tour_profiles.tour_distance` | `distance_bin: String`
`tour_purpose: String`
`tour_count: Float64` | tours: `tour_purpose`, `tour_category`, `number_of_participants`, `SKIMDIST`, `finalweight` |
-| `tour_mode_by_tour_purpose_and_auto_sufficiency` | `tour_mode_by_tour_purpose_and_auto_sufficiency.csv` | `processor.summarize.summaries.tour_profiles.tour_mode` | `tour_mode: String`
`tour_purpose: String`
`tour_count_zero_auto: Float64`
`tour_count_auto_deficient: Float64`
`tour_count_auto_sufficient: Float64`
`tour_count_all_households: Float64` | tours: `tour_mode`, `tour_purpose`, `finalweight`, `AUTOSUFF` |
-| `tour_purpose_distribution` | `tour_purpose_distribution.csv` | `processor.summarize.summaries.tour.tour_purpose` | `tour_purpose: String`
`tour_count: Float64` | tours: `tour_purpose`, `finalweight` |
-| `tour_rates_by_person_type_and_tour_purpose` | `tour_rates_by_person_type_and_tour_purpose.csv` | `processor.summarize.summaries.daily_travel_activity.tour_rate_per_person` | `person_type: String`
`tour_purpose: String`
`tour_rate: Float64` | per: `person_id`, `person_type`, `finalweight`
tours: `person_id`, `tour_purpose` |
-| `tour_stop_frequency_by_tour_purpose` | `tour_stop_frequency_by_tour_purpose.csv` | `processor.summarize.summaries.tour_profiles.stop_freq` | `tour_purpose: String`
`outbound_stop_count: Int32`
`inbound_stop_count: Int32`
`total_stop_count: Int32`
`tour_count: Float64` | tours: `tour_purpose`, `tour_category`, `num_ob_stops`, `num_ib_stops`, `num_tot_stops`, `finalweight` |
-| `tour_time_of_day_by_tour_purpose` | `tour_time_of_day_by_tour_purpose.csv` | `processor.summarize.summaries.tour_profiles.tour_tod` | `time_bin: Int32`
`tour_purpose: String`
`departure_tour_count: Float64`
`arrival_tour_count: Float64`
`duration_tour_count: Float64` | tours: `tour_category`, `tour_purpose`, `finalweight` |
-| `traffic_count_comparisons` | `traffic_count_comparisons.csv` | `processor.summarize.summaries.validation.traffic_count_comparisons` | `count_location_id: String`
`direction: String`
`count_period: String`
`observed_volume: Float64`
`modeled_volume: Float64` | - |
-| `transit_boardings_by_operator_and_technology` | `transit_boardings_by_operator_and_technology.csv` | `processor.summarize.summaries.validation.total_transit_boardings` | `operator: String`
`technology: String`
`boardings: Float64` | - |
-| `transit_pass_ownership_by_person_type` | `transit_pass_ownership_by_person_type.csv` | `processor.summarize.summaries.long_term_person.transit_pass` | `person_type: String`
`transit_pass_ownership_status: String`
`person_type_label: String`
`person_count: Float64` | per: `person_type`, `transit_pass_ownership`, `finalweight` |
-| `transit_subsidy_by_person_type` | `transit_subsidy_by_person_type.csv` | `processor.summarize.summaries.long_term_person.transit_subsidy` | `person_type: String`
`transit_subsidy_status: String`
`transit_subsidy_label: String`
`person_type_label: String`
`person_count: Float64` | per: `person_type`, `transit_pass_subsidy`, `is_worker`, `is_student`, `finalweight` |
-| `transit_transfer_rate` | `transit_transfer_rate.csv` | `processor.summarize.summaries.validation.transit_transfer_rate` | `operator: String`
`technology: String`
`access_mode: String`
`transfer_rate: Float64` | - |
-| `trip_departure_time_by_purpose` | `trip_departure_time_by_purpose.csv` | `processor.summarize.summaries.trip_distributions.trip_stop_tod` | `tour_purpose: String`
`time_bin: Int32`
`departure_trip_count: Float64`
`departure_stop_count: Float64` | trips: `tour_purpose`, `stops`, `finalweight` |
-| `trip_distance_by_purpose` | `trip_distance_by_purpose.csv` | `processor.summarize.summaries.trip_distributions.trip_distance` | `distance_bin: String`
`tour_purpose: String`
`trip_count: Float64` | trips: `tour_purpose`, `od_dist`, `num_participants`, `finalweight` |
-| `trip_mode_by_tour_purpose_and_tour_mode` | `trip_mode_by_tour_purpose_and_tour_mode.csv` | `processor.summarize.summaries.trip.trip_mode` | `tour_purpose: String`
`tour_mode: String`
`trip_mode: String`
`trip_count: Float64` | trips: `tour_purpose`, `tour_mode`, `trip_mode`, `finalweight` |
-| `trip_purpose_distribution` | `trip_purpose_distribution.csv` | `processor.summarize.summaries.trip.trip_purpose` | `tour_purpose: String`
`trip_purpose: String`
`trip_count: Float64` | trips: `tour_purpose`, `trip_purpose`, `finalweight` |
-| `trip_rates_by_person_type_and_trip_purpose` | `trip_rates_by_person_type_and_trip_purpose.csv` | `processor.summarize.summaries.daily_travel_activity.trip_rate_per_person` | `person_type: String`
`trip_purpose: String`
`trip_rate: Float64` | per: `person_id`, `person_type`, `finalweight`
trips: `person_id`, `trip_purpose`, `finalweight` |
-| `university_location_distance_distribution_by_geography` | `university_location_distance_distribution_by_geography.csv` | `processor.summarize.summaries.long_term_distance.univ_tlfd` | `distance_bin: Int32`
`geography_type: String`
`geography_id: String`
`person_count: Float64` | per: `distance_to_school`, `finalweight` |
-| `vehicle_age_distribution` | `vehicle_age_distribution.csv` | `processor.summarize.summaries.long_term_vehicle.vehicle_char_age` | `age: String`
`vehicle_count: Float64` | vehicles: `vehicle_age`, `finalweight` |
-| `vehicle_body_type_distribution` | `vehicle_body_type_distribution.csv` | `processor.summarize.summaries.long_term_vehicle.vehicle_char_body` | `body_type: String`
`vehicle_count: Float64` | vehicles: `body_type`, `finalweight` |
-| `vehicle_fuel_type_distribution` | `vehicle_fuel_type_distribution.csv` | `processor.summarize.summaries.long_term_vehicle.vehicle_char_fuel` | `fuel_type: String`
`vehicle_count: Float64` | vehicles: `fuel_type`, `finalweight` |
-| `work_from_home_rate_by_geography` | `work_from_home_rate_by_geography.csv` | `processor.summarize.summaries.long_term_geography.wfh` | `geography_type: String`
`geography_id: String`
`worker_count: Float64`
`work_from_home_worker_count: Float64` | per: `is_worker`, `home_zone_id`, `finalweight` |
-| `work_from_home_validation_summary` | `work_from_home_validation_summary.csv` | `processor.summarize.summaries.validation_scaffolds.work_from_home_validation_summary` | `District: String`
`Workers: Float64`
`WFH: Float64` | - |
-| `work_location_distance_distribution_by_geography` | `work_location_distance_distribution_by_geography.csv` | `processor.summarize.summaries.long_term_distance.work_tlfd` | `distance_bin: Int32`
`geography_type: String`
`geography_id: String`
`person_count: Float64` | per: `distance_to_work`, `finalweight` |
-| `workplace_location_employment_comparison` | `workplace_location_employment_comparison.csv` | `processor.summarize.summaries.long_term_geography.workplace_vs_land_use_employment` | `geography_type: String`
`geography_id: String`
`employment_count: Float64`
`worker_count: Float64` | land_use: `MAZ`, `employment_count`
per: `workplace_zone_id`, `is_worker`, `finalweight` |
-| `workplace_shadow_pricing_residual_histogram` | `workplace_shadow_pricing_residual_histogram.csv` | `processor.summarize.summaries.long_term_geography.workplace_shadow_pricing_residual_histogram` | `geography_type: String`
`bin_start: Float64`
`bin_end: Float64`
`geography_count: Float64` | land_use: `MAZ`, `employment_count`
per: `workplace_zone_id`, `is_worker`, `finalweight` |
-| `workplace_shadow_pricing_residuals` | `workplace_shadow_pricing_residuals.csv` | `processor.summarize.summaries.long_term_geography.workplace_shadow_pricing_residuals` | `geography_type: String`
`geography_id: String`
`target_count: Float64`
`modeled_count: Float64`
`residual_count: Float64`
`absolute_residual_count: Float64`
`percent_error: Float64` | land_use: `MAZ`, `employment_count`
per: `workplace_zone_id`, `is_worker`, `finalweight` |
+| Summary ID | Filename | Default build | Builder | Output schema | Required inputs |
+|---|---|---|---|---|---|
+| `adult_escort_event_stop_distribution` | `adult_escort_event_stop_distribution.csv` | yes | `processor.summarize.summaries.daily_travel_escort_distributions.adult_escort_event_stop_distribution` | `segment: String`
`stop_count: Int32`
`tour_count: Float64` | tours: `tour_id`, `school_esc_outbound`, `school_esc_inbound`
trips: `tour_id`, `escort_event_role`, `escort_stops_before_event`, `escort_stops_after_event`, `finalweight` |
+| `adult_escort_trip_stop_frequency` | `adult_escort_trip_stop_frequency.csv` | yes | `processor.summarize.summaries.daily_travel_escort_distributions.adult_escort_trip_stop_frequency` | `tour_purpose: String`
`outbound_stop_count: Int32`
`inbound_stop_count: Int32`
`total_stop_count: Int32`
`tour_count: Float64` | tours: `tour_purpose`, `school_esc_outbound`, `school_esc_inbound`, `num_ob_stops`, `num_ib_stops`, `num_tot_stops`, `finalweight` |
+| `adult_escorted_tour_distance_distribution_by_direction` | `adult_escorted_tour_distance_distribution_by_direction.csv` | yes | `processor.summarize.summaries.daily_travel_escort_distributions.adult_escorted_tour_distance_distribution_by_direction` | `distance_bin: String`
`direction: String`
`tour_count: Float64` | tours: `SKIMDIST`, `school_esc_outbound`, `school_esc_inbound`, `finalweight` |
+| `adult_escorted_tour_purposes_by_direction` | `adult_escorted_tour_purposes_by_direction.csv` | yes | `processor.summarize.summaries.daily_travel_escort_counts.adult_escorted_tour_purposes_by_direction` | `tour_purpose: String`
`direction: String`
`tour_count: Float64` | tours: `tour_purpose`, `school_esc_outbound`, `school_esc_inbound`, `finalweight` |
+| `adult_escorted_tours_by_person_type_and_direction` | `adult_escorted_tours_by_person_type_and_direction.csv` | yes | `processor.summarize.summaries.daily_travel_escort_counts.adult_escorted_tours_by_person_type_and_direction` | `person_type: String`
`direction: String`
`tour_count: Float64` | tours: `school_esc_outbound`, `school_esc_inbound`, `finalweight` |
+| `adult_escorted_trip_distance_distribution_by_direction` | `adult_escorted_trip_distance_distribution_by_direction.csv` | yes | `processor.summarize.summaries.daily_travel_escort_distributions.adult_escorted_trip_distance_distribution_by_direction` | `distance_bin: String`
`direction: String`
`trip_count: Float64` | tours: `tour_id`, `school_esc_outbound`, `school_esc_inbound`
trips: `tour_id`, `od_dist`, `finalweight` |
+| `allocated_vehicle_age_by_occupancy` | `allocated_vehicle_age_by_occupancy.csv` | yes | `processor.summarize.summaries.tour_vehicles.allocated_vehicle_age` | `age: String`
`occupancy: String`
`vehicle_count: Float64` | tours: `vehicle_occup_1`, `vehicle_occup_2`, `vehicle_occup_3.5`, `finalweight` |
+| `allocated_vehicle_body_type_by_occupancy` | `allocated_vehicle_body_type_by_occupancy.csv` | yes | `processor.summarize.summaries.tour_vehicles.allocated_vehicle_body` | `body_type: String`
`occupancy: String`
`vehicle_count: Float64` | tours: `vehicle_occup_1`, `vehicle_occup_2`, `vehicle_occup_3.5`, `finalweight` |
+| `allocated_vehicle_fuel_type_by_occupancy` | `allocated_vehicle_fuel_type_by_occupancy.csv` | yes | `processor.summarize.summaries.tour_vehicles.allocated_vehicle_fuel` | `fuel_type: String`
`occupancy: String`
`vehicle_count: Float64` | tours: `vehicle_occup_1`, `vehicle_occup_2`, `vehicle_occup_3.5`, `finalweight` |
+| `atwork_subtour_frequency_distribution` | `atwork_subtour_frequency_distribution.csv` | yes | `processor.summarize.summaries.tour_profiles.at_work_sub_tour_freq` | `atwork_subtour_frequency_category: String`
`atwork_subtour_count: Float64` | tours: `tour_purpose`, `tour_category`, `atwork_subtour_frequency`, `finalweight` |
+| `auto_ownership_distribution` | `auto_ownership_distribution.csv` | yes | `processor.summarize.summaries.long_term_vehicle.auto_ownership` | `household_size: String`
`household_vehicle_count: Int64`
`household_count: Float64` | hh: `HHSIZE`, `HHVEH`, `finalweight` |
+| `auto_vmt_by_home_geography_income_hhsize_time_period` | `auto_vmt_by_home_geography_income_hhsize_time_period.csv` | yes | `processor.summarize.summaries.validation.auto_vmt_by_home_geography_income_hhsize_time_period` | `geography_type: String`
`geography_id: String`
`income_segment: String`
`household_size: String`
`time_period: String`
`mode: String`
`auto_vmt: Float64`
`trip_count: Float64`
`distance_source: String`
`time_period_source: String` | trips: `finalweight` |
+| `auto_vmt_totals` | `auto_vmt_totals.csv` | yes | `processor.summarize.summaries.validation.auto_vmt_totals` | `auto_vmt: Float64` | trips: `trip_mode`, `od_dist`, `finalweight` |
+| `auto_vmt_validation_summary` | `auto_vmt_validation_summary.csv` | no | `processor.summarize.summaries.validation_scaffolds.auto_vmt_validation_summary` | `TOD: String`
`SOV: Float64`
`HOV2: Float64`
`HOV3: Float64`
`Truck: Float64`
`Total: Float64` | - |
+| `autonomous_vehicle_ownership_totals` | `autonomous_vehicle_ownership_totals.csv` | yes | `processor.summarize.summaries.long_term_vehicle.av_ownership` | `household_with_autonomous_vehicle_count: Float64` | hh: `av_ownership`, `finalweight` |
+| `average_mandatory_tour_distance_by_purpose_and_geography` | `average_mandatory_tour_distance_by_purpose_and_geography.csv` | yes | `processor.summarize.summaries.tour_geography.avg_mand_tour_distance` | `mandatory_tour_purpose: String`
`geography_type: String`
`geography_id: String`
`average_tour_distance: Float64`
`person_count: Float64` | per: `finalweight` |
+| `average_nonmandatory_tour_distance_by_purpose_and_geography` | `average_nonmandatory_tour_distance_by_purpose_and_geography.csv` | yes | `processor.summarize.summaries.tour_geography.avg_non_mand_tour_distance` | `nonmandatory_tour_purpose: String`
`geography_type: String`
`geography_id: String`
`average_tour_distance: Float64`
`tour_count: Float64` | per: `person_id`, `home_zone_id`
tours: `person_id`, `tour_category`, `tour_purpose`, `SKIMDIST`, `finalweight` |
+| `bicycle_comfort_level_distribution` | `bicycle_comfort_level_distribution.csv` | yes | `processor.summarize.summaries.long_term_person.bicycle_comfort_level` | `person_type: String`
`bicycle_comfort_level: String`
`person_type_label: String`
`person_count: Float64` | per: `person_type`, `bike_comfort`, `finalweight` |
+| `bicycle_vmt_by_facility_type` | `bicycle_vmt_by_facility_type.csv` | yes | `processor.summarize.summaries.validation.bicycle_vmt_by_facility` | `facility_type: String`
`bicycle_vmt: Float64` | - |
+| `commercial_vehicle_validation_summary` | `commercial_vehicle_validation_summary.csv` | no | `processor.summarize.summaries.validation_scaffolds.commercial_vehicle_validation_summary` | `tod: String`
`car: Float64`
`mu: Float64`
`su: Float64`
`Total: Float64` | - |
+| `commercial_vehicle_vmt_validation_summary` | `commercial_vehicle_vmt_validation_summary.csv` | no | `processor.summarize.summaries.validation_scaffolds.commercial_vehicle_vmt_validation_summary` | `tod: String`
`car: Float64`
`mu: Float64`
`su: Float64`
`Total: Float64` | - |
+| `commercial_vmt_totals` | `commercial_vmt_totals.csv` | yes | `processor.summarize.summaries.validation.commercial_vehicle_vmt` | `commercial_vehicle_type: String`
`external_vmt: Float64`
`internal_vmt: Float64` | - |
+| `commuting_flows` | `commuting_flows.csv` | yes | `processor.summarize.summaries.long_term_geography.commuting_flows` | `origin_geography_type: String`
`origin_geography_id: String`
`destination_geography_type: String`
`destination_geography_id: String`
`commuter_count: Float64` | per: `home_zone_id`, `workplace_zone_id`, `is_worker`, `finalweight` |
+| `count_location_counts_validation_summary` | `count_location_counts_validation_summary.csv` | no | `processor.summarize.summaries.validation_scaffolds.count_location_counts_validation_summary` | `id: Int64`
`FACTYPE: Int64`
`am_vol: Float64`
`md_vol: Float64`
`pm_vol: Float64`
`day_vol: Float64` | - |
+| `count_location_fit_validation_summary` | `count_location_fit_validation_summary.csv` | no | `processor.summarize.summaries.validation_scaffolds.count_location_fit_validation_summary` | `facility_type: String`
`period: String`
`slope: Float64`
`intercept: Float64`
`r_squared: Float64`
`n_locations: Int64`
`observed_min: Float64`
`observed_max: Float64`
`equation_label: String`
`r_squared_label: String` | - |
+| `count_location_scatter_validation_summary` | `count_location_scatter_validation_summary.csv` | no | `processor.summarize.summaries.validation_scaffolds.count_location_scatter_validation_summary` | `id: Int64`
`facility_type: String`
`period: String`
`observed_volume: Float64`
`modeled_volume: Float64` | - |
+| `count_location_volumes_validation_summary` | `count_location_volumes_validation_summary.csv` | no | `processor.summarize.summaries.validation_scaffolds.count_location_volumes_validation_summary` | `id: Int64`
`FACTYPE: Int64`
`am_vol: Float64`
`md_vol: Float64`
`pm_vol: Float64`
`day_vol: Float64` | - |
+| `county_commuting_flows_validation_summary` | `county_commuting_flows_validation_summary.csv` | no | `processor.summarize.summaries.validation_scaffolds.county_commuting_flows_validation_summary` | `: String`
`Benton: Float64`
`Linn: Float64`
`Marion: Float64`
`Total: Float64` | - |
+| `daily_activity_pattern_by_person_type` | `daily_activity_pattern_by_person_type.csv` | yes | `processor.summarize.summaries.daily_travel_activity.dap_summary` | `person_type: String`
`daily_activity_pattern: String`
`person_count: Float64` | per: `person_type`, `cdap_activity`, `finalweight` |
+| `district_commuting_flows_validation_summary` | `district_commuting_flows_validation_summary.csv` | no | `processor.summarize.summaries.validation_scaffolds.district_commuting_flows_validation_summary` | `: String`
`Albany: Float64`
`Corvallis: Float64`
`Lebanon: Float64`
`Philomath: Float64`
`Total: Float64` | - |
+| `escorted_tour_totals` | `escorted_tour_totals.csv` | yes | `processor.summarize.summaries.daily_travel_escort_counts.total_escorted_tours` | `tour_count: Float64` | tours: `school_esc_outbound`, `school_esc_inbound`, `finalweight` |
+| `external_nonmandatory_tour_locations` | `external_nonmandatory_tour_locations.csv` | yes | `processor.summarize.summaries.tour_geography.ext_non_mand_tour_loc` | `geography_type: String`
`geography_id: String`
`external_nonmandatory_tour_count: Float64` | tours: `tour_category`, `is_external_tour`, `destination`, `finalweight` |
+| `external_trip_validation_summary` | `external_trip_validation_summary.csv` | no | `processor.summarize.summaries.validation_scaffolds.external_trip_validation_summary` | `tod: String`
`hbcoll: Float64`
`hbo: Float64`
`hbr: Float64`
`hbs: Float64`
`hbsch: Float64`
`hbw: Float64`
`nhbnw: Float64`
`nhbw: Float64`
`truck: Float64`
`Total: Float64` | - |
+| `external_vmt_validation_summary` | `external_vmt_validation_summary.csv` | no | `processor.summarize.summaries.validation_scaffolds.external_vmt_validation_summary` | `tod: String`
`hbcoll: Float64`
`hbo: Float64`
`hbr: Float64`
`hbs: Float64`
`hbsch: Float64`
`hbw: Float64`
`nhbnw: Float64`
`nhbw: Float64`
`truck: Float64`
`Total: Float64` | - |
+| `external_worker_workplace_locations` | `external_worker_workplace_locations.csv` | yes | `processor.summarize.summaries.long_term_geography.external_workplace_loc` | `geography_type: String`
`geography_id: String`
`external_worker_count: Float64`
`all_worker_count: Float64` | per: `is_worker`, `is_external_worker`, `external_workplace_zone_id`, `finalweight` |
+| `free_parking_eligibility_by_workplace_geography` | `free_parking_eligibility_by_workplace_geography.csv` | yes | `processor.summarize.summaries.long_term_geography.free_parking` | `geography_type: String`
`geography_id: String`
`workers_without_free_parking_count: Float64`
`workers_with_free_parking_count: Float64` | per: `is_worker`, `free_parking_at_work`, `workplace_zone_id`, `finalweight` |
+| `household_jtp_by_household_size_and_jtf` | `household_jtp_by_household_size_and_jtf.csv` | yes | `processor.summarize.summaries.joint_travel.jtf_by_hhsize` | `jtf: String`
`household_size: String`
`household_percent: Float64` | hh: `household_id`, `HHSIZE`, `finalweight`
tours: `tour_category`, `household_id` |
+| `household_size_distribution` | `household_size_distribution.csv` | yes | `processor.summarize.summaries.demographics.hh_size` | `household_size: Int64`
`household_count: Float64` | hh: `HHSIZE`, `finalweight` |
+| `households_with_school_escorting_by_student_count_and_direction` | `households_with_school_escorting_by_student_count_and_direction.csv` | yes | `processor.summarize.summaries.daily_travel_escort_counts.households_with_school_escorting_by_student_count_and_direction` | `student_count: Int64`
`direction: String`
`household_count: Float64` | hh: `household_id`, `finalweight`
per: `household_id`, `person_type`
tours: `tour_purpose`, `school_esc_outbound`, `school_esc_inbound`, `finalweight` |
+| `internal_external_nonmandatory_tour_frequency_by_home_geography` | `internal_external_nonmandatory_tour_frequency_by_home_geography.csv` | yes | `processor.summarize.summaries.tour_geography.int_vs_ext_non_mand_tour_freq` | `geography_type: String`
`geography_id: String`
`internal_nonmandatory_tour_count: Float64`
`external_nonmandatory_tour_count: Float64` | per: `person_id`, `home_zone_id`
tours: `person_id`, `tour_category`, `is_external_tour`, `finalweight` |
+| `internal_external_worker_by_geography` | `internal_external_worker_by_geography.csv` | yes | `processor.summarize.summaries.long_term_geography.internal_vs_external` | `geography_type: String`
`geography_id: String`
`internal_worker_count: Float64`
`external_worker_count: Float64` | per: `is_worker`, `is_external_worker`, `home_zone_id`, `finalweight` |
+| `joint_tour_composition_by_party_size` | `joint_tour_composition_by_party_size.csv` | yes | `processor.summarize.summaries.joint_travel.joint_composition_by_party_size` | `tour_composition: String`
`party_size: Int64`
`joint_tour_count: Float64` | tours: `tour_category`, `composition`, `number_of_participants`, `finalweight` |
+| `joint_tour_composition_distribution` | `joint_tour_composition_distribution.csv` | yes | `processor.summarize.summaries.joint_travel.joint_composition` | `tour_composition: String`
`joint_tour_count: Float64` | tours: `tour_category`, `finalweight` |
+| `joint_tour_party_size_distribution` | `joint_tour_party_size_distribution.csv` | yes | `processor.summarize.summaries.joint_travel.joint_party_size` | `party_size: Int32`
`joint_tour_count: Float64` | tours: `tour_category`, `NUMBER_HH`, `finalweight` |
+| `joint_tours_by_household_size` | `joint_tours_by_household_size.csv` | yes | `processor.summarize.summaries.joint_travel.joint_tours_hhsize` | `household_size: Int32`
`household_count: Float64`
`joint_tour_hh_count: Float64` | hh: `household_id`, `HHSIZE`, `finalweight`
tours: `tour_category`, `household_id` |
+| `jtf_distribution` | `jtf_distribution.csv` | yes | `processor.summarize.summaries.joint_travel.joint_tour_freq` | `jtf_code: Int32`
`jtf_label: String`
`household_count: Float64` | hh: `household_id`, `finalweight` |
+| `license_holding_status_distribution` | `license_holding_status_distribution.csv` | yes | `processor.summarize.summaries.long_term_person.license_holding_status` | `person_type: String`
`license_holding_status: String`
`person_type_label: String`
`person_count: Float64` | per: `person_type`, `has_license`, `finalweight`, `age` |
+| `link_validation_summary` | `link_validation_summary.csv` | no | `processor.summarize.summaries.validation_scaffolds.link_validation_summary` | `id: Int64`
`From_Node: Int64`
`To_Node: Int64`
`FACTYPE: Int64`
`am_vol: Float64`
`md_vol: Float64`
`pm_vol: Float64`
`day_vol: Float64` | - |
+| `mandatory_tour_frequency_by_person_type` | `mandatory_tour_frequency_by_person_type.csv` | yes | `processor.summarize.summaries.daily_travel_activity.mandatory_tour_freq` | `person_type: String`
`mandatory_tour_frequency: Int32`
`person_count: Float64` | per: `person_type`, `imf_choice`, `finalweight` |
+| `non_motorized_vmt_by_home_geography_income_hhsize_time_period` | `non_motorized_vmt_by_home_geography_income_hhsize_time_period.csv` | yes | `processor.summarize.summaries.validation.non_motorized_vmt_by_home_geography_income_hhsize_time_period` | `geography_type: String`
`geography_id: String`
`income_segment: String`
`household_size: String`
`time_period: String`
`mode: String`
`non_motorized_vmt: Float64`
`trip_count: Float64`
`distance_source: String`
`time_period_source: String` | trips: `finalweight`, `trip_mode` |
+| `nonmandatory_tour_frequency_by_person_type` | `nonmandatory_tour_frequency_by_person_type.csv` | yes | `processor.summarize.summaries.daily_travel_activity.indiv_nm_summary` | `person_type: String`
`nonmandatory_tour_frequency: String`
`person_count: Float64` | joint_participants: `person_id`
per: `person_id`, `person_type`, `finalweight`
tours: `person_id`, `tour_category` |
+| `park_and_ride_location_residual_histogram` | `park_and_ride_location_residual_histogram.csv` | yes | `processor.summarize.summaries.long_term_geography.park_and_ride_location_residual_histogram` | `geography_type: String`
`bin_start: Float64`
`bin_end: Float64`
`geography_count: Float64` | land_use: -
tours: `tour_mode`, `finalweight` |
+| `park_and_ride_location_residuals` | `park_and_ride_location_residuals.csv` | yes | `processor.summarize.summaries.long_term_geography.park_and_ride_location_residuals` | `geography_type: String`
`geography_id: String`
`pnr_tour_count: Float64`
`pnr_lot_capacity: Float64`
`residual_count: Float64`
`absolute_residual_count: Float64`
`percent_error: Float64` | land_use: -
tours: `tour_mode`, `finalweight` |
+| `parking_locations` | `parking_locations.csv` | yes | `processor.summarize.summaries.trip.parking_locations` | `geography_type: String`
`geography_id: String`
`trip_count: Float64` | trips: `parking_zone`, `finalweight` |
+| `person_jtp_by_household_size` | `person_jtp_by_household_size.csv` | yes | `processor.summarize.summaries.joint_travel.joint_participation_person_by_hhsize` | `household_size: Int64`
`joint_tour_person_count: Float64`
`total_person_count: Float64` | hh: `household_id`, `hhsize`
per: `household_id`, `num_joint_tours`, `finalweight` |
+| `person_type_distribution` | `person_type_distribution.csv` | yes | `processor.summarize.summaries.demographics.person_type` | `person_type: String`
`person_type_label: String`
`person_count: Float64` | per: `person_type`, `finalweight` |
+| `population_totals` | `population_totals.csv` | yes | `processor.summarize.summaries.demographics.population_totals` | `person_count: Float64`
`household_count: Float64`
`tour_count: Float64`
`trip_count: Float64`
`stop_count: Float64` | hh: `finalweight`
per: `finalweight`
tours: `finalweight`
trips: `finalweight`, `stops` |
+| `school_escorted_tours_by_escort_type_and_direction` | `school_escorted_tours_by_escort_type_and_direction.csv` | yes | `processor.summarize.summaries.daily_travel_escort_counts.escorted_tours_to_from_school` | `escort_type: String`
`direction: String`
`tour_count: Float64` | tours: `tour_purpose`, `school_esc_outbound`, `school_esc_inbound`, `finalweight` |
+| `school_location_distance_distribution_by_geography` | `school_location_distance_distribution_by_geography.csv` | yes | `processor.summarize.summaries.long_term_distance.schl_tlfd` | `distance_bin: Int32`
`geography_type: String`
`geography_id: String`
`person_count: Float64` | per: `distance_to_school`, `finalweight` |
+| `school_location_enrollment_comparison` | `school_location_enrollment_comparison.csv` | yes | `processor.summarize.summaries.long_term_geography.school_loc_vs_land_use_enrollment` | `geography_type: String`
`geography_id: String`
`student_type: String`
`enrollment_count: Float64`
`student_count: Float64` | land_use: `MAZ`, `enrollment_count`, `student_type`
per: `school_zone_id`, `is_student`, `finalweight` |
+| `school_shadow_pricing_residual_histogram` | `school_shadow_pricing_residual_histogram.csv` | yes | `processor.summarize.summaries.long_term_geography.school_shadow_pricing_residual_histogram` | `geography_type: String`
`student_type: String`
`bin_start: Float64`
`bin_end: Float64`
`geography_count: Float64` | land_use: `MAZ`, `enrollment_count`, `student_type`
per: `school_zone_id`, `is_student`, `finalweight`, `student_type` |
+| `school_shadow_pricing_residuals` | `school_shadow_pricing_residuals.csv` | yes | `processor.summarize.summaries.long_term_geography.school_shadow_pricing_residuals` | `geography_type: String`
`geography_id: String`
`student_type: String`
`target_count: Float64`
`modeled_count: Float64`
`residual_count: Float64`
`absolute_residual_count: Float64`
`percent_error: Float64` | land_use: `MAZ`, `enrollment_count`, `student_type`
per: `school_zone_id`, `is_student`, `finalweight`, `student_type` |
+| `schoolkids_per_escorted_tour_by_student_count_and_direction` | `schoolkids_per_escorted_tour_by_student_count_and_direction.csv` | yes | `processor.summarize.summaries.daily_travel_escort_counts.schoolkids_per_escorted_tour_by_student_count_and_direction` | `student_count: Int64`
`direction: String`
`avg_schoolkids_per_tour: Float64`
`tour_count: Float64` | hh: `household_id`, `finalweight`
per: `household_id`, `person_type`
tours: `school_esc_outbound`, `school_esc_inbound`, `num_escortees`, `finalweight` |
+| `screenline_flow_comparisons` | `screenline_flow_comparisons.csv` | yes | `processor.summarize.summaries.validation.screenline_flow_comparisons` | `screenline_id: String`
`direction: String`
`count_period: String`
`facility_type: String`
`observed_volume: Float64`
`modeled_volume: Float64` | - |
+| `skimjoin_tour_component_ecdf` | `skimjoin_tour_component_ecdf.csv` | no | `processor.summarize.summaries.skimjoin.tour_skim_component_ecdf` | `skim_scenario: String`
`tour_mode: String`
`component: String`
`percentile: Float64`
`value: Float64`
`n_valid: Float64` | tours: `tour_mode`, `finalweight` |
+| `skimjoin_tour_component_stats` | `skimjoin_tour_component_stats.csv` | yes | `processor.summarize.summaries.skimjoin.tour_skim_component_stats` | `skim_scenario: String`
`tour_mode: String`
`component: String`
`n_total: Float64`
`n_valid: Float64`
`mean: Float64`
`std: Float64`
`min: Float64`
`max: Float64`
`median: Float64`
`mode: Float64`
`zero_share: Float64`
`missing_share: Float64` | tours: `tour_mode`, `finalweight` |
+| `skimjoin_trip_component_ecdf` | `skimjoin_trip_component_ecdf.csv` | no | `processor.summarize.summaries.skimjoin.trip_skim_component_ecdf` | `skim_scenario: String`
`trip_mode: String`
`component: String`
`percentile: Float64`
`value: Float64`
`n_valid: Float64` | trips: `trip_mode`, `finalweight` |
+| `skimjoin_trip_component_stats` | `skimjoin_trip_component_stats.csv` | yes | `processor.summarize.summaries.skimjoin.trip_skim_component_stats` | `skim_scenario: String`
`trip_mode: String`
`component: String`
`n_total: Float64`
`n_valid: Float64`
`mean: Float64`
`std: Float64`
`min: Float64`
`max: Float64`
`median: Float64`
`mode: Float64`
`zero_share: Float64`
`missing_share: Float64` | trips: `trip_mode`, `finalweight` |
+| `stop_destination_purpose_by_tour_purpose` | `stop_destination_purpose_by_tour_purpose.csv` | yes | `processor.summarize.summaries.trip.stop_purpose_by_tour_purpose` | `stop_destination_purpose: String`
`tour_purpose: String`
`stop_count: Float64` | trips: `stops`, `tour_purpose`, `trip_purpose`, `finalweight` |
+| `stop_out_of_direction_distance_by_tour_purpose` | `stop_out_of_direction_distance_by_tour_purpose.csv` | yes | `processor.summarize.summaries.trip_distributions.stop_ood_distance` | `distance_bin: Int32`
`tour_purpose: String`
`stop_count: Float64` | trips: `stops`, `out_dir_dist`, `tour_purpose`, `finalweight` |
+| `student_households_by_student_count` | `student_households_by_student_count.csv` | yes | `processor.summarize.summaries.daily_travel_escort_counts.student_households_by_student_count` | `student_count: Int64`
`household_count: Float64` | hh: `household_id`, `finalweight`
per: `household_id`, `person_type` |
+| `student_school_escort_status_by_direction` | `student_school_escort_status_by_direction.csv` | yes | `processor.summarize.summaries.daily_travel_escort_counts.student_school_escort_status_by_direction` | `direction: String`
`escort_type: String`
`tour_count: Float64` | tours: `tour_purpose`, `school_esc_outbound`, `school_esc_inbound`, `finalweight` |
+| `telecommute_frequency_distribution` | `telecommute_frequency_distribution.csv` | yes | `processor.summarize.summaries.long_term_person.telecommute` | `geography_type: String`
`geography_id: String`
`telecommute_frequency: String`
`person_count: Float64` | per: `telecommute_frequency`, `finalweight`, `is_worker`, `work_from_home`, `home_zone_id` |
+| `tour_category_distribution` | `tour_category_distribution.csv` | yes | `processor.summarize.summaries.tour.tour_category` | `tour_category: String`
`tour_count: Float64` | tours: `tour_category`, `finalweight` |
+| `tour_distance_by_tour_purpose` | `tour_distance_by_tour_purpose.csv` | yes | `processor.summarize.summaries.tour_profiles.tour_distance` | `distance_bin: String`
`tour_purpose: String`
`tour_count: Float64` | tours: `tour_purpose`, `tour_category`, `number_of_participants`, `SKIMDIST`, `finalweight` |
+| `tour_mode_by_tour_purpose_and_auto_sufficiency` | `tour_mode_by_tour_purpose_and_auto_sufficiency.csv` | yes | `processor.summarize.summaries.tour_profiles.tour_mode` | `tour_mode: String`
`tour_purpose: String`
`tour_count_zero_auto: Float64`
`tour_count_auto_deficient: Float64`
`tour_count_auto_sufficient: Float64`
`tour_count_all_households: Float64` | tours: `tour_mode`, `tour_purpose`, `finalweight`, `AUTOSUFF` |
+| `tour_purpose_distribution` | `tour_purpose_distribution.csv` | yes | `processor.summarize.summaries.tour.tour_purpose` | `tour_purpose: String`
`tour_count: Float64` | tours: `tour_purpose`, `finalweight` |
+| `tour_rates_by_person_type_and_tour_purpose` | `tour_rates_by_person_type_and_tour_purpose.csv` | yes | `processor.summarize.summaries.daily_travel_activity.tour_rate_per_person` | `person_type: String`
`tour_purpose: String`
`tour_rate: Float64` | per: `person_id`, `person_type`, `finalweight`
tours: `person_id`, `tour_purpose` |
+| `tour_stop_frequency_by_tour_purpose` | `tour_stop_frequency_by_tour_purpose.csv` | yes | `processor.summarize.summaries.tour_profiles.stop_freq` | `tour_purpose: String`
`outbound_stop_count: Int32`
`inbound_stop_count: Int32`
`total_stop_count: Int32`
`tour_count: Float64` | tours: `tour_purpose`, `tour_category`, `num_ob_stops`, `num_ib_stops`, `num_tot_stops`, `finalweight` |
+| `tour_time_of_day_by_tour_purpose` | `tour_time_of_day_by_tour_purpose.csv` | yes | `processor.summarize.summaries.tour_profiles.tour_tod` | `time_bin: Int32`
`tour_purpose: String`
`departure_tour_count: Float64`
`arrival_tour_count: Float64`
`duration_tour_count: Float64` | tours: `tour_category`, `tour_purpose`, `finalweight` |
+| `traffic_count_comparisons` | `traffic_count_comparisons.csv` | yes | `processor.summarize.summaries.validation.traffic_count_comparisons` | `count_location_id: String`
`direction: String`
`count_period: String`
`observed_volume: Float64`
`modeled_volume: Float64` | - |
+| `transit_boardings_by_operator_and_technology` | `transit_boardings_by_operator_and_technology.csv` | yes | `processor.summarize.summaries.validation.total_transit_boardings` | `operator: String`
`technology: String`
`boardings: Float64` | - |
+| `transit_pass_ownership_by_person_type` | `transit_pass_ownership_by_person_type.csv` | yes | `processor.summarize.summaries.long_term_person.transit_pass` | `person_type: String`
`transit_pass_ownership_status: String`
`person_type_label: String`
`person_count: Float64` | per: `person_type`, `transit_pass_ownership`, `finalweight` |
+| `transit_subsidy_by_person_type` | `transit_subsidy_by_person_type.csv` | yes | `processor.summarize.summaries.long_term_person.transit_subsidy` | `person_type: String`
`transit_subsidy_status: String`
`transit_subsidy_label: String`
`person_type_label: String`
`person_count: Float64` | per: `person_type`, `transit_pass_subsidy`, `is_worker`, `is_student`, `finalweight` |
+| `transit_transfer_rate` | `transit_transfer_rate.csv` | yes | `processor.summarize.summaries.validation.transit_transfer_rate` | `operator: String`
`technology: String`
`access_mode: String`
`transfer_rate: Float64` | - |
+| `trip_departure_time_by_purpose` | `trip_departure_time_by_purpose.csv` | yes | `processor.summarize.summaries.trip_distributions.trip_stop_tod` | `tour_purpose: String`
`time_bin: Int32`
`departure_trip_count: Float64`
`departure_stop_count: Float64` | trips: `tour_purpose`, `stops`, `finalweight` |
+| `trip_distance_by_purpose` | `trip_distance_by_purpose.csv` | yes | `processor.summarize.summaries.trip_distributions.trip_distance` | `distance_bin: String`
`tour_purpose: String`
`trip_count: Float64` | trips: `tour_purpose`, `od_dist`, `num_participants`, `finalweight` |
+| `trip_mode_by_tour_purpose_and_tour_mode` | `trip_mode_by_tour_purpose_and_tour_mode.csv` | yes | `processor.summarize.summaries.trip.trip_mode` | `tour_purpose: String`
`tour_mode: String`
`trip_mode: String`
`trip_count: Float64` | trips: `tour_purpose`, `tour_mode`, `trip_mode`, `finalweight` |
+| `trip_purpose_distribution` | `trip_purpose_distribution.csv` | yes | `processor.summarize.summaries.trip.trip_purpose` | `tour_purpose: String`
`trip_purpose: String`
`trip_count: Float64` | trips: `tour_purpose`, `trip_purpose`, `finalweight` |
+| `trip_rates_by_person_type_and_trip_purpose` | `trip_rates_by_person_type_and_trip_purpose.csv` | yes | `processor.summarize.summaries.daily_travel_activity.trip_rate_per_person` | `person_type: String`
`trip_purpose: String`
`trip_rate: Float64` | per: `person_id`, `person_type`, `finalweight`
trips: `person_id`, `trip_purpose`, `finalweight` |
+| `university_location_distance_distribution_by_geography` | `university_location_distance_distribution_by_geography.csv` | yes | `processor.summarize.summaries.long_term_distance.univ_tlfd` | `distance_bin: Int32`
`geography_type: String`
`geography_id: String`
`person_count: Float64` | per: `distance_to_school`, `finalweight` |
+| `vehicle_age_distribution` | `vehicle_age_distribution.csv` | yes | `processor.summarize.summaries.long_term_vehicle.vehicle_char_age` | `age: String`
`vehicle_count: Float64` | vehicles: `vehicle_age`, `finalweight` |
+| `vehicle_body_type_distribution` | `vehicle_body_type_distribution.csv` | yes | `processor.summarize.summaries.long_term_vehicle.vehicle_char_body` | `body_type: String`
`vehicle_count: Float64` | vehicles: `body_type`, `finalweight` |
+| `vehicle_fuel_type_distribution` | `vehicle_fuel_type_distribution.csv` | yes | `processor.summarize.summaries.long_term_vehicle.vehicle_char_fuel` | `fuel_type: String`
`vehicle_count: Float64` | vehicles: `fuel_type`, `finalweight` |
+| `work_from_home_rate_by_geography` | `work_from_home_rate_by_geography.csv` | yes | `processor.summarize.summaries.long_term_geography.wfh` | `geography_type: String`
`geography_id: String`
`worker_count: Float64`
`work_from_home_worker_count: Float64` | per: `is_worker`, `home_zone_id`, `finalweight` |
+| `work_from_home_validation_summary` | `work_from_home_validation_summary.csv` | no | `processor.summarize.summaries.validation_scaffolds.work_from_home_validation_summary` | `District: String`
`Workers: Float64`
`WFH: Float64` | - |
+| `work_location_distance_distribution_by_geography` | `work_location_distance_distribution_by_geography.csv` | yes | `processor.summarize.summaries.long_term_distance.work_tlfd` | `distance_bin: Int32`
`geography_type: String`
`geography_id: String`
`person_count: Float64` | per: `distance_to_work`, `finalweight` |
+| `workplace_location_employment_comparison` | `workplace_location_employment_comparison.csv` | yes | `processor.summarize.summaries.long_term_geography.workplace_vs_land_use_employment` | `geography_type: String`
`geography_id: String`
`employment_count: Float64`
`worker_count: Float64` | land_use: `MAZ`, `employment_count`
per: `workplace_zone_id`, `is_worker`, `finalweight` |
+| `workplace_shadow_pricing_residual_histogram` | `workplace_shadow_pricing_residual_histogram.csv` | yes | `processor.summarize.summaries.long_term_geography.workplace_shadow_pricing_residual_histogram` | `geography_type: String`
`bin_start: Float64`
`bin_end: Float64`
`geography_count: Float64` | land_use: `MAZ`, `employment_count`
per: `workplace_zone_id`, `is_worker`, `finalweight` |
+| `workplace_shadow_pricing_residuals` | `workplace_shadow_pricing_residuals.csv` | yes | `processor.summarize.summaries.long_term_geography.workplace_shadow_pricing_residuals` | `geography_type: String`
`geography_id: String`
`target_count: Float64`
`modeled_count: Float64`
`residual_count: Float64`
`absolute_residual_count: Float64`
`percent_error: Float64` | land_use: `MAZ`, `employment_count`
per: `workplace_zone_id`, `is_worker`, `finalweight` |
From 7e8579b326d9189598f135f11d868482068dd706 Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Wed, 5 Aug 2026 14:29:52 -0400
Subject: [PATCH 12/27] Make pipeline cache refresh stage-aware and isolate
skimjoin artifacts
---
config.yaml | 2 +-
processor/cache_identity.py | 25 ++-
processor/prepare/cache.py | 76 ++++++-
processor/prepare/reader.py | 36 +++-
processor/summarize/cache_storage.py | 4 +
run.py | 202 ++++++++++++++++--
runtime/config/loader.py | 4 +
runtime/config/models.py | 11 +-
runtime/config/schema.py | 2 +-
runtime/config/sections.py | 34 ++-
runtime/config/signatures.py | 7 +
runtime/workflows/artifacts.py | 7 +-
runtime/workflows/common.py | 10 +-
runtime/workflows/prepare.py | 129 +++++++++--
runtime/workflows/shared.py | 67 ++++++
runtime/workflows/summarize.py | 19 +-
simor_configs/metro_configs/metro_config.yaml | 6 +-
tests/test_config_refactor_phase1.py | 26 ++-
tests/test_run_cli.py | 102 +++++++++
tests/test_runtime_workflows.py | 109 +++++++++-
wiki/01-architecture.md | 2 +-
wiki/10-getting-started.md | 2 +-
wiki/12-running-workflows.md | 26 ++-
wiki/13-configuration-reference.md | 8 +-
wiki/90-troubleshooting.md | 12 +-
25 files changed, 847 insertions(+), 81 deletions(-)
diff --git a/config.yaml b/config.yaml
index b4e9e09..b94b0a0 100644
--- a/config.yaml
+++ b/config.yaml
@@ -16,7 +16,7 @@ root: artifacts
pipeline:
steps: [summarize, dashboard]
dashboard_mode: live
- overwrite: false
+ refresh: []
# These are the default ActivitySim file stems. Edit only names that differ in
# your model outputs; the visualizer accepts either .csv or .parquet files.
diff --git a/processor/cache_identity.py b/processor/cache_identity.py
index a633086..9bf5d44 100644
--- a/processor/cache_identity.py
+++ b/processor/cache_identity.py
@@ -33,6 +33,8 @@ def build_run_fingerprint(
label: str,
run_dir: str | None,
skim_file: str | None,
+ raw_file_identities: dict[str, dict[str, object] | None] | None = None,
+ skim_file_identity: dict[str, object] | None = None,
skimjoin: dict[str, object] | None = None,
file_map: dict[str, str] | None = None,
fallback_file_map: dict[str, str] | None = None,
@@ -45,6 +47,11 @@ def build_run_fingerprint(
"label": label,
"run_dir": str(run_dir) if run_dir is not None else None,
"skim_file": str(skim_file) if skim_file is not None else None,
+ "skim_file_identity": skim_file_identity,
+ "raw_file_identities": {
+ key: value
+ for key, value in sorted((raw_file_identities or {}).items())
+ },
"skimjoin": dict(sorted((skimjoin or {}).items())) if skimjoin else None,
"file_map": dict(sorted((file_map or {}).items())),
"fallback_file_map": dict(sorted((fallback_file_map or {}).items())),
@@ -65,4 +72,20 @@ def file_identity(path: str | Path) -> dict[str, object]:
}
-__all__ = ["build_run_fingerprint", "build_run_keys", "file_identity", "slugify"]
+def optional_file_identity(path: str | Path | None) -> dict[str, object] | None:
+ """Return a file identity when the configured input currently exists."""
+ if path is None:
+ return None
+ resolved = Path(path).expanduser().resolve()
+ if not resolved.is_file():
+ return None
+ return file_identity(resolved)
+
+
+__all__ = [
+ "build_run_fingerprint",
+ "build_run_keys",
+ "file_identity",
+ "optional_file_identity",
+ "slugify",
+]
diff --git a/processor/prepare/cache.py b/processor/prepare/cache.py
index f8c743a..2d13da6 100644
--- a/processor/prepare/cache.py
+++ b/processor/prepare/cache.py
@@ -222,11 +222,10 @@ def _write_sidecar_tables(
if not sidecar_frames:
return {}
- sidecar_dir = cache_dir / "prepared_tables"
- sidecar_dir.mkdir(parents=True, exist_ok=True)
+ cache_dir.mkdir(parents=True, exist_ok=True)
filenames = _sidecar_file_map(file_format)
for attr_name, frame in sidecar_frames.items():
- path = sidecar_dir / filenames[attr_name]
+ path = cache_dir / filenames[attr_name]
if file_format == "parquet":
frame.write_parquet(path)
elif file_format == "csv":
@@ -244,6 +243,8 @@ def write_prepared_run_cache(
output_root: str | Path | None = None,
run_fingerprint: dict[str, object] | None = None,
file_format: str | None = None,
+ cache_name: str = "prepared_tables",
+ prepare_config_digest: str | None = None,
) -> PreparedRunCacheEntry:
"""Write one prepared run's canonical tables and manifest."""
if file_format is None:
@@ -259,7 +260,7 @@ def write_prepared_run_cache(
)
output_root.mkdir(parents=True, exist_ok=True)
- cache_dir = output_root / run_key / "prepared_tables"
+ cache_dir = output_root / run_key / cache_name
cache_dir.mkdir(parents=True, exist_ok=True)
tables_to_write: dict[str, pl.DataFrame] = {}
@@ -277,7 +278,7 @@ def write_prepared_run_cache(
tables_to_write[stem] = table
write_all(tables_to_write, cache_dir, file_format=file_format)
- sidecar_files = _write_sidecar_tables(cache_dir.parent, rd, file_format=file_format)
+ sidecar_files = _write_sidecar_tables(cache_dir, rd, file_format=file_format)
_write_skimjoin_outputs(cache_dir, rd, config)
manifest = {
@@ -288,10 +289,10 @@ def write_prepared_run_cache(
"run_key": run_key,
"source_run_dir": rd.run_dir,
"config_path": config.config_path,
- "prepare_config_digest": config.prepare_config_digest,
+ "prepare_config_digest": prepare_config_digest or config.prepare_config_digest,
"table_format": file_format,
- "table_root": "prepared_tables",
- "sidecar_root": "prepared_tables",
+ "table_root": cache_name,
+ "sidecar_root": cache_name,
"table_files": _table_file_map(file_format),
"sidecar_files": sidecar_files,
"table_states": {
@@ -336,6 +337,18 @@ def write_prepared_run_cache(
"person_weight_col": rd.person_weight_col,
"trip_weight_col": rd.trip_weight_col,
"run_fingerprint": run_fingerprint or {},
+ "identity": {
+ "raw_inputs": dict((run_fingerprint or {}).get("raw_file_identities", {})),
+ "prepare_config": prepare_config_digest or config.prepare_config_digest,
+ "skimjoin_config": (
+ dict((run_fingerprint or {}).get("skimjoin") or {}).get("config_digest")
+ ),
+ "skim_inputs": (
+ dict((run_fingerprint or {}).get("skimjoin") or {}).get(
+ "resolved_skim_file_identities", []
+ )
+ ),
+ },
"prepare_diagnostics": dict(rd.prepare_diagnostics),
"skimjoin_enabled": bool(rd.skimjoin_manifest.get("skimjoin_enabled", False)),
"skimjoin_config_digest": rd.skimjoin_manifest.get("skimjoin_config_digest"),
@@ -547,6 +560,53 @@ def load_prepared_run_cache(
)
+def inspect_prepared_run_cache(
+ cache_dir: str | Path,
+ *,
+ expected_prepare_config_digest: str | None = None,
+ expected_run_fingerprint: dict[str, object] | None = None,
+ expected_label: str | None = None,
+ expected_run_key: str | None = None,
+) -> dict[str, object]:
+ """Validate a prepared cache identity without loading its tables."""
+ cache_dir = Path(cache_dir)
+ manifest = read_manifest(cache_dir, error_cls=PreparedCacheError)
+ validate_schema_version(
+ cache_dir=cache_dir,
+ manifest=manifest,
+ supported_versions=SUPPORTED_SCHEMA_VERSIONS,
+ error_factory=lambda message: PreparedCacheError(
+ message.replace(
+ "Unsupported cache schema_version",
+ "Unsupported prepared cache schema_version",
+ )
+ ),
+ )
+ if expected_label is not None and manifest.get("label") != expected_label:
+ raise PreparedCacheError(
+ f"Prepared cache label mismatch in {cache_dir}: expected {expected_label!r}, found {manifest.get('label')!r}"
+ )
+ if expected_run_key is not None and manifest.get("run_key") != expected_run_key:
+ raise PreparedCacheError(
+ f"Prepared cache run key mismatch in {cache_dir}: expected {expected_run_key!r}, found {manifest.get('run_key')!r}"
+ )
+ if (
+ expected_prepare_config_digest is not None
+ and manifest.get("prepare_config_digest") != expected_prepare_config_digest
+ ):
+ raise PreparedCacheError(
+ f"Prepared cache config digest mismatch in {cache_dir}; tables were built from a different preparation configuration."
+ )
+ if (
+ expected_run_fingerprint is not None
+ and manifest.get("run_fingerprint") != expected_run_fingerprint
+ ):
+ raise PreparedCacheError(
+ f"Prepared cache run fingerprint mismatch in {cache_dir}; tables were built from different run inputs."
+ )
+ return manifest
+
+
def discover_cache_dirs(root: str | Path) -> list[Path]:
"""Return child prepared-cache directories that contain a manifest."""
root = Path(root)
diff --git a/processor/prepare/reader.py b/processor/prepare/reader.py
index 2418d0f..5fd87d5 100644
--- a/processor/prepare/reader.py
+++ b/processor/prepare/reader.py
@@ -49,6 +49,34 @@ def resolve_run_file_map(
return effective
+def resolve_run_file_paths(
+ run_dir: str | Path,
+ config: Config,
+ run_file_map: dict[str, str] | None = None,
+) -> dict[str, str | None]:
+ """Resolve the concrete raw input files the reader would use."""
+ root = Path(run_dir).expanduser()
+ resolved: dict[str, str | None] = {}
+ for table_id, configured in resolve_run_file_map(config, run_file_map).items():
+ configured_path = Path(configured)
+ suffix = configured_path.suffix.lower()
+ candidates = (
+ [root / configured_path]
+ if suffix in {".csv", ".parquet"}
+ else [
+ root / f"{configured_path.name}.parquet",
+ root / f"{configured_path.name}.csv",
+ ]
+ )
+ selected = next((candidate for candidate in candidates if candidate.is_file()), None)
+ if selected is None:
+ fallback = config.fallback_files.get(table_id)
+ fallback_path = Path(fallback).expanduser() if fallback else None
+ selected = fallback_path if fallback_path is not None and fallback_path.is_file() else None
+ resolved[table_id] = str(selected.resolve()) if selected is not None else None
+ return resolved
+
+
def _find_and_read(run_dir: Path, configured: str) -> pl.DataFrame:
"""Read a table from run_dir, resolving file format."""
path = Path(configured)
@@ -217,4 +245,10 @@ def _read(key: str) -> pl.DataFrame:
)
-__all__ = ["RunData", "read_run", "resolve_run_file_map", "resolve_skim_path"]
+__all__ = [
+ "RunData",
+ "read_run",
+ "resolve_run_file_map",
+ "resolve_run_file_paths",
+ "resolve_skim_path",
+]
diff --git a/processor/summarize/cache_storage.py b/processor/summarize/cache_storage.py
index 38fe0f3..09efc94 100644
--- a/processor/summarize/cache_storage.py
+++ b/processor/summarize/cache_storage.py
@@ -147,6 +147,10 @@ def _summary_manifest(
"summary_digests": summary_digests,
"run_fingerprint": run_fingerprint or {},
"prepared_manifest_identity": prepared_manifest_identity,
+ "identity": {
+ "upstream_prepared": prepared_manifest_identity,
+ "summary_config": config.summary_config_digest,
+ },
}
diff --git a/run.py b/run.py
index cac8221..c87b800 100644
--- a/run.py
+++ b/run.py
@@ -119,6 +119,11 @@ def parse_args() -> argparse.Namespace:
action="store_true",
help="Do not open the dashboard in a browser automatically",
)
+ parser.add_argument(
+ "--explain-cache",
+ action="store_true",
+ help="Print cache decisions for the configured pipeline and exit without running it.",
+ )
return parser.parse_args()
@@ -216,7 +221,7 @@ def resolve_effective_dashboard_mode(
def resolve_effective_plan(args: argparse.Namespace, config) -> WorkflowPlan:
- """Resolve logical steps, runtime steps, dashboard mode, and overwrite policy."""
+ """Resolve logical steps, runtime steps, dashboard mode, and refresh policy."""
logical_steps = resolve_requested_steps(args, config)
dashboard_mode = resolve_effective_dashboard_mode(
args,
@@ -227,16 +232,28 @@ def resolve_effective_plan(args: argparse.Namespace, config) -> WorkflowPlan:
logical_steps = [
step for step in logical_steps if step != "dashboard"
]
- overwrite = bool(config.pipeline.overwrite)
- if args.refresh_caches or args.refresh_prepared_cache or args.refresh_summary_cache:
- overwrite = True
+ refresh_steps = set(config.pipeline.refresh)
+ if args.refresh_caches:
+ refresh_steps.update(
+ step
+ for step in logical_steps
+ if step in {"prepare", "skimjoin", "summarize"}
+ )
+ if args.refresh_prepared_cache:
+ refresh_steps.add("prepare")
+ if args.refresh_summary_cache:
+ refresh_steps.add("summarize")
runtime_steps = tuple(collapse_runtime_steps(logical_steps))
return WorkflowPlan(
logical_steps=tuple(logical_steps),
runtime_steps=runtime_steps,
dashboard_mode=dashboard_mode,
- overwrite=overwrite,
+ refresh_steps=tuple(
+ step
+ for step in ("prepare", "skimjoin", "summarize")
+ if step in refresh_steps
+ ),
)
@@ -245,13 +262,23 @@ def _remove_run_cache_dirs(
root: Path,
run_keys: list[str],
cache_label: str,
+ preserve_names: set[str] | None = None,
) -> None:
"""Remove per-run cache directories before a forced rebuild."""
for run_key in run_keys:
cache_dir = root / run_key
if cache_dir.exists():
LOGGER.info("Refreshing %s cache for run key %r", cache_label, run_key)
- shutil.rmtree(cache_dir)
+ if not preserve_names:
+ shutil.rmtree(cache_dir)
+ continue
+ for child in cache_dir.iterdir():
+ if child.name in preserve_names:
+ continue
+ if child.is_dir():
+ shutil.rmtree(child)
+ else:
+ child.unlink()
def _refresh_requested_caches(
@@ -281,6 +308,7 @@ def _refresh_requested_caches(
root=cache_root,
run_keys=run_keys,
cache_label="summary",
+ preserve_names={"prepared_tables", "base_prepared_tables"},
)
return refresh_prepared, refresh_summary
@@ -292,11 +320,11 @@ def resolve_cache_preferences(
refreshed_summary: bool,
) -> tuple[bool, bool]:
"""Resolve cache reuse preferences after config defaults and CLI refresh overrides."""
- prefer_prepared_cache = not (
- "prepare" in plan.runtime_steps and plan.overwrite
+ prefer_prepared_cache = not any(
+ plan.refreshes(step) for step in ("prepare", "skimjoin")
)
- prefer_summary_cache = not (
- "summarize" in plan.runtime_steps and plan.overwrite
+ prefer_summary_cache = not any(
+ plan.refreshes(step) for step in ("prepare", "skimjoin", "summarize")
)
if refreshed_prepared:
prefer_prepared_cache = False
@@ -336,6 +364,137 @@ def resolve_dashboard_execution_mode(dashboard_mode: str) -> str:
return normalized_mode
+def explain_cache_plan(
+ *,
+ config,
+ plan: WorkflowPlan,
+ prepared_root: Path,
+ cache_root: Path,
+ run_entries: list[dict],
+) -> None:
+ """Print cache decisions without loading tables or writing artifacts."""
+ from processor.prepare.cache import PreparedCacheError, inspect_prepared_run_cache
+ from processor.summarize import builder as summary_builder
+ from processor.summarize import cache as summary_cache
+ from processor.summarize.cache_types import SummaryCacheError
+ from runtime.workflows import prepare as prepare_workflow
+ from runtime.workflows import summarize as summarize_workflow
+
+ effective = runtime_workflows.effective_processor_config(config, plan=plan)
+
+ def decision(action: str, reason: str | None = None) -> str:
+ return action if not reason else f"{action} — {reason}"
+
+ for entry, run_key in runtime_workflows.run_entries_with_keys(run_entries):
+ prepare_metadata = prepare_workflow._run_cache_metadata(
+ entry=entry,
+ run_key=run_key,
+ config=effective,
+ )
+ label = str(prepare_metadata["label"])
+ print(f"Pipeline plan — {label}")
+
+ prepare_action = "DISABLED"
+ prepare_reason = None
+ if "prepare" in plan.runtime_steps or "summarize" in plan.runtime_steps:
+ if plan.refreshes("prepare"):
+ prepare_action = "REBUILD"
+ prepare_reason = "explicitly refreshed"
+ else:
+ base_cache = (
+ prepare_workflow.base_prepared_cache_dir(prepared_root, run_key)
+ if plan.includes("skimjoin")
+ else prepare_workflow.prepared_cache_dir(prepared_root, run_key)
+ )
+ base_digest = (
+ effective.base_prepare_config_digest
+ if plan.includes("skimjoin")
+ else effective.prepare_config_digest
+ )
+ base_fingerprint = dict(
+ prepare_metadata[
+ "base_run_fingerprint"
+ if plan.includes("skimjoin")
+ else "run_fingerprint"
+ ]
+ )
+ try:
+ inspect_prepared_run_cache(
+ base_cache,
+ expected_prepare_config_digest=base_digest,
+ expected_run_fingerprint=base_fingerprint,
+ expected_label=label,
+ expected_run_key=run_key,
+ )
+ prepare_action = "REUSE"
+ except PreparedCacheError as exc:
+ prepare_action = "REBUILD"
+ prepare_reason = str(exc)
+ print(f" prepare {decision(prepare_action, prepare_reason)}")
+
+ if plan.includes("skimjoin"):
+ if plan.refreshes("skimjoin"):
+ skimjoin_action = decision("REBUILD", "explicitly refreshed")
+ elif prepare_action == "REBUILD":
+ skimjoin_action = decision("REBUILD", "upstream prepare will change")
+ else:
+ try:
+ inspect_prepared_run_cache(
+ prepare_workflow.prepared_cache_dir(prepared_root, run_key),
+ expected_prepare_config_digest=effective.prepare_config_digest,
+ expected_run_fingerprint=dict(prepare_metadata["run_fingerprint"]),
+ expected_label=label,
+ expected_run_key=run_key,
+ )
+ skimjoin_action = "REUSE"
+ except PreparedCacheError as exc:
+ skimjoin_action = decision("REBUILD", str(exc))
+ print(f" skimjoin {skimjoin_action}")
+ else:
+ print(" skimjoin DISABLED")
+
+ if "summarize" in plan.runtime_steps:
+ if any(plan.refreshes(step) for step in ("prepare", "skimjoin", "summarize")):
+ summary_action = decision("REBUILD", "explicit or upstream refresh")
+ elif prepare_action == "REBUILD":
+ summary_action = decision("REBUILD", "upstream prepare will change")
+ else:
+ summary_metadata = summarize_workflow._run_cache_metadata(
+ entry=entry,
+ run_key=run_key,
+ config=effective,
+ )
+ try:
+ inspection = summary_cache.inspect_summary_run_bundle(
+ cache_root / run_key,
+ effective,
+ expected_modes=effective.weighting_modes,
+ expected_summary_ids=list(summary_builder.DEFAULT_SUMMARY_IDS),
+ expected_run_fingerprint=dict(summary_metadata["run_fingerprint"]),
+ expected_prepared_manifest_identity=summary_metadata[
+ "prepared_manifest_identity"
+ ],
+ expected_label=label,
+ expected_run_key=run_key,
+ )
+ stale = list(inspection["stale_summary_ids"])
+ summary_action = (
+ decision("REBUILD", f"{len(stale)} summary tables are stale")
+ if stale
+ else "REUSE"
+ )
+ except SummaryCacheError as exc:
+ summary_action = decision("REBUILD", str(exc))
+ print(f" summarize {summary_action}")
+ else:
+ print(" summarize DISABLED")
+
+ print(
+ " dashboard "
+ + ("RUN" if "dashboard" in plan.runtime_steps else "DISABLED")
+ )
+
+
def main() -> None:
t0 = time.perf_counter()
args = parse_args()
@@ -347,10 +506,11 @@ def main() -> None:
sys.exit(1)
config = runtime_workflows.load_runtime_config(args.config)
- log_path = configure_logging(config, level=_resolve_terminal_log_level(config))
- LOGGER.info("Starting ActivitySim Visualizer")
- LOGGER.info("Loading config: %s", args.config)
- LOGGER.info("Logging to %s", log_path)
+ if not args.explain_cache:
+ log_path = configure_logging(config, level=_resolve_terminal_log_level(config))
+ LOGGER.info("Starting ActivitySim Visualizer")
+ LOGGER.info("Loading config: %s", args.config)
+ LOGGER.info("Logging to %s", log_path)
try:
plan = resolve_effective_plan(args, config)
@@ -358,11 +518,12 @@ def main() -> None:
LOGGER.info("Requested workflow steps: %s", ", ".join(steps) if steps else "(none)")
LOGGER.info("Effective dashboard mode: %s", plan.dashboard_mode)
cache_root = runtime_workflows.summary_cache_root(
- config, create="summarize" in steps
+ config, create="summarize" in steps and not args.explain_cache
)
prepared_root = runtime_workflows.prepared_cache_root(
config,
- create="prepare" in steps or "summarize" in steps,
+ create=("prepare" in steps or "summarize" in steps)
+ and not args.explain_cache,
)
run_entries = runtime_workflows.resolve_run_entries(
@@ -371,6 +532,15 @@ def main() -> None:
config=config,
require_runs="prepare" in steps or "summarize" in steps,
)
+ if args.explain_cache:
+ explain_cache_plan(
+ config=config,
+ plan=plan,
+ prepared_root=prepared_root,
+ cache_root=cache_root,
+ run_entries=run_entries,
+ )
+ return
refreshed_prepared, refreshed_summary = _refresh_requested_caches(
args=args,
prepared_root=prepared_root,
diff --git a/runtime/config/loader.py b/runtime/config/loader.py
index 2d8db3a..bce8873 100644
--- a/runtime/config/loader.py
+++ b/runtime/config/loader.py
@@ -265,6 +265,7 @@ def load_config_from_yaml(path: str | Path, *, cls: type[ConfigT] = Config) -> C
config = cls(
config_path=str(config_path),
config_digest=hashlib.sha256(config_bytes).hexdigest(),
+ base_prepare_config_digest="",
prepare_config_digest="",
summary_config_digest="",
presentation_config_digest="",
@@ -317,6 +318,9 @@ def load_config_from_yaml(path: str | Path, *, cls: type[ConfigT] = Config) -> C
)
if not config.pnr_tour_modes:
raise ValueError("summarize.pnr_tour_modes must resolve to at least one mode.")
+ config.base_prepare_config_digest = digest_payload(
+ config.base_prepare_signature_payload()
+ )
config.prepare_config_digest = digest_payload(config.prepare_signature_payload())
config.summary_config_digest = digest_payload(config.summary_signature_payload())
config.presentation_config_digest = digest_payload(
diff --git a/runtime/config/models.py b/runtime/config/models.py
index cc79350..60d51a6 100644
--- a/runtime/config/models.py
+++ b/runtime/config/models.py
@@ -132,11 +132,14 @@ class PipelineSettings:
steps: tuple[str, ...] = ("summarize", "dashboard")
dashboard_mode: Literal["none", "live", "export", "host"] = "live"
- overwrite: bool = False
+ refresh: tuple[str, ...] = ()
def has_step(self, step: str) -> bool:
return step in self.steps
+ def refreshes(self, step: str) -> bool:
+ return step in self.refresh
+
@dataclass(frozen=True)
class SkimjoinSettings:
@@ -341,6 +344,7 @@ class Config:
config_path: str
config_digest: str
+ base_prepare_config_digest: str
prepare_config_digest: str
summary_config_digest: str
presentation_config_digest: str
@@ -458,6 +462,11 @@ def prepare_signature_payload(self) -> dict[str, Any]:
return prepare_signature_payload(self)
+ def base_prepare_signature_payload(self) -> dict[str, Any]:
+ from .signatures import base_prepare_signature_payload
+
+ return base_prepare_signature_payload(self)
+
def summary_signature_payload(self) -> dict[str, Any]:
from .signatures import summary_signature_payload
diff --git a/runtime/config/schema.py b/runtime/config/schema.py
index 9a3d764..2b82fe4 100644
--- a/runtime/config/schema.py
+++ b/runtime/config/schema.py
@@ -251,7 +251,7 @@ def validate_canonical_config(raw: Mapping[str, object]) -> None:
_reject_unknown_keys(
pipeline,
field_name="pipeline",
- allowed={"steps", "dashboard_mode", "overwrite"},
+ allowed={"steps", "dashboard_mode", "refresh", "overwrite"},
)
display = _mapping(raw.get("display"), field_name="display")
diff --git a/runtime/config/sections.py b/runtime/config/sections.py
index 32a5e69..7427cca 100644
--- a/runtime/config/sections.py
+++ b/runtime/config/sections.py
@@ -8,6 +8,7 @@
PIPELINE_STEP_ORDER = ("prepare", "skimjoin", "segment", "summarize", "dashboard")
VALID_PIPELINE_STEPS = set(PIPELINE_STEP_ORDER)
+REFRESHABLE_PIPELINE_STEPS = {"prepare", "skimjoin", "summarize"}
VALID_DASHBOARD_MODES = {"none", "live", "export", "host"}
@@ -60,9 +61,34 @@ def parse_pipeline(raw_value) -> PipelineSettings:
"pipeline.dashboard_mode must be one of none, live, export, or host."
)
- overwrite = raw_value.get("overwrite", False)
- if not isinstance(overwrite, bool):
- raise ValueError("pipeline.overwrite must be true or false when provided.")
+ if "overwrite" in raw_value:
+ raise ValueError(
+ "pipeline.overwrite has been replaced by pipeline.refresh. "
+ "Use refresh: all for a complete rebuild or refresh: [] for normal operation."
+ )
+
+ refresh_raw = raw_value.get("refresh", [])
+ if refresh_raw == "all":
+ refresh = [step for step in steps if step in REFRESHABLE_PIPELINE_STEPS]
+ elif isinstance(refresh_raw, list):
+ refresh = []
+ for idx, raw_step in enumerate(refresh_raw):
+ if not isinstance(raw_step, str):
+ raise ValueError("pipeline.refresh entries must be strings.")
+ step = raw_step.strip()
+ if step not in REFRESHABLE_PIPELINE_STEPS:
+ raise ValueError(
+ f"pipeline.refresh[{idx}] must be one of prepare, skimjoin, or summarize."
+ )
+ if step in refresh:
+ raise ValueError(f"pipeline.refresh contains duplicate step {step!r}.")
+ if step not in steps:
+ raise ValueError(
+ f"pipeline.refresh cannot include disabled step {step!r}."
+ )
+ refresh.append(step)
+ else:
+ raise ValueError("pipeline.refresh must be a list or 'all' when provided.")
if "skimjoin" in steps and "prepare" not in steps:
raise ValueError("pipeline.steps cannot include 'skimjoin' without 'prepare'.")
@@ -74,7 +100,7 @@ def parse_pipeline(raw_value) -> PipelineSettings:
return PipelineSettings(
steps=tuple(steps),
dashboard_mode=dashboard_mode,
- overwrite=overwrite,
+ refresh=tuple(refresh),
)
diff --git a/runtime/config/signatures.py b/runtime/config/signatures.py
index d94e9a5..205ea82 100644
--- a/runtime/config/signatures.py
+++ b/runtime/config/signatures.py
@@ -179,6 +179,13 @@ def prepare_signature_payload(config: Config) -> dict[str, Any]:
}
+def base_prepare_signature_payload(config: Config) -> dict[str, Any]:
+ """Return preparation identity before optional skim enrichment."""
+ payload = prepare_signature_payload(config)
+ payload.pop("skimjoin", None)
+ return payload
+
+
def summary_signature_payload(config: Config) -> dict[str, Any]:
segmentation_payload: dict[str, Any] = {"enabled": config.segmentation.enabled}
if config.segmentation.enabled:
diff --git a/runtime/workflows/artifacts.py b/runtime/workflows/artifacts.py
index c963e41..c59fa8a 100644
--- a/runtime/workflows/artifacts.py
+++ b/runtime/workflows/artifacts.py
@@ -15,7 +15,7 @@ class WorkflowPlan:
logical_steps: tuple[str, ...]
runtime_steps: tuple[str, ...]
dashboard_mode: str = "none"
- overwrite: bool = False
+ refresh_steps: tuple[str, ...] = ()
@classmethod
def from_config(cls, config: Any) -> "WorkflowPlan":
@@ -36,12 +36,15 @@ def for_steps(cls, config: Any, steps: Any) -> "WorkflowPlan":
logical_steps=logical_steps,
runtime_steps=tuple(runtime_steps),
dashboard_mode=str(config.pipeline.dashboard_mode).lower(),
- overwrite=bool(config.pipeline.overwrite),
+ refresh_steps=tuple(config.pipeline.refresh),
)
def includes(self, step: str) -> bool:
return step in self.logical_steps
+ def refreshes(self, step: str) -> bool:
+ return step in self.refresh_steps
+
@dataclass
class PreparedRunsArtifact:
diff --git a/runtime/workflows/common.py b/runtime/workflows/common.py
index 09f490f..1ec3210 100644
--- a/runtime/workflows/common.py
+++ b/runtime/workflows/common.py
@@ -6,13 +6,17 @@
from typing import Any
from runtime.logging import get_logger
-from processor.cache_identity import build_run_fingerprint, build_run_keys
+from processor.cache_identity import (
+ build_run_fingerprint,
+ build_run_keys,
+ optional_file_identity,
+)
from processor.models import (
PreparedTableName,
prune_prepared_runs,
)
from processor.prepare.cache import build_prepared_manifest_identity, prepared_root
-from processor.prepare.reader import resolve_skim_path
+from processor.prepare.reader import resolve_run_file_paths, resolve_skim_path
from processor.summarize import cache as summary_cache
from processor.summarize import builder as summary_builder
from processor.summarize import cache_types as summary_types
@@ -146,6 +150,8 @@ def load_summary_runs_from_cache(
config=config,
build_run_fingerprint_fn=build_run_fingerprint,
resolve_skim_path_fn=resolve_skim_path,
+ resolve_run_file_paths_fn=resolve_run_file_paths,
+ optional_file_identity_fn=optional_file_identity,
build_prepared_manifest_identity_fn=build_prepared_manifest_identity,
)
or {}
diff --git a/runtime/workflows/prepare.py b/runtime/workflows/prepare.py
index 46f7dd6..920d43c 100644
--- a/runtime/workflows/prepare.py
+++ b/runtime/workflows/prepare.py
@@ -6,8 +6,8 @@
from typing import Any, Callable
from runtime.logging import get_logger
-from processor.cache_identity import build_run_fingerprint
-from processor.models import PreparedTableName, RunData
+from processor.cache_identity import build_run_fingerprint, optional_file_identity
+from processor.models import PreparedTableName, RunData, map_run_data_tables
from processor.prepare.availability import (
failed_tables,
has_usable_loaded_tables,
@@ -21,7 +21,7 @@
write_prepared_run_cache,
)
from processor.prepare.enrichment.pipeline import prepare_data
-from processor.prepare.reader import read_run, resolve_skim_path
+from processor.prepare.reader import read_run, resolve_run_file_paths, resolve_skim_path
from processor.prepare.validation import (
PreparedRelationshipValidationError,
validate_prepared_relationships,
@@ -41,6 +41,11 @@ def prepared_cache_dir(prepared_root: Path, run_key: str) -> Path:
return prepared_root / run_key / "prepared_tables"
+def base_prepared_cache_dir(prepared_root: Path, run_key: str) -> Path:
+ """Return the pre-skim preparation cache directory for one run."""
+ return prepared_root / run_key / "base_prepared_tables"
+
+
def _run_cache_metadata(
*,
entry: dict,
@@ -48,14 +53,21 @@ def _run_cache_metadata(
config: Config,
) -> dict[str, object]:
"""Return the stable cache metadata for one resolved run entry."""
- return shared.run_cache_metadata(
+ metadata = shared.run_cache_metadata(
entry=entry,
run_key=run_key,
config=config,
resolve_skim_path_fn=resolve_skim_path,
+ resolve_run_file_paths_fn=resolve_run_file_paths,
+ optional_file_identity_fn=optional_file_identity,
build_run_fingerprint_fn=build_run_fingerprint,
build_prepared_manifest_identity_fn=build_prepared_manifest_identity,
)
+ metadata["base_run_fingerprint"] = {
+ **dict(metadata["run_fingerprint"]),
+ "skimjoin": None,
+ }
+ return metadata
def _log_prepare_table_diagnostics(run_label: str, prepared_run: RunData) -> None:
@@ -139,20 +151,26 @@ def _load_prepared_run_from_cache(
run_key: str,
label: str,
run_fingerprint: dict[str, object],
+ prepare_config_digest: str | None = None,
+ stage: str = "prepare",
) -> tuple[str, RunData] | None:
"""Load one prepared run from cache when valid and usable."""
try:
prepared_run = load_prepared_run_cache(
prepared_dir,
config,
- expected_prepare_config_digest=config.prepare_config_digest,
+ expected_prepare_config_digest=(
+ prepare_config_digest or config.prepare_config_digest
+ ),
expected_run_fingerprint=run_fingerprint,
expected_label=label,
expected_run_key=run_key,
)
- LOGGER.info("Loaded prepared cache for run: %r", label)
+ LOGGER.info("Pipeline decision for %r / %s: REUSE", label, stage)
except PreparedCacheError as exc:
- LOGGER.info("Prepared cache miss for %r: %s", label, exc)
+ LOGGER.info(
+ "Pipeline decision for %r / %s: REBUILD — %s", label, stage, exc
+ )
return None
if not has_usable_loaded_tables(prepared_run):
@@ -177,11 +195,16 @@ def _build_prepared_run(
metadata: dict[str, object],
write_cache: bool,
run_skimjoin: bool,
+ cache_name: str = "prepared_tables",
+ prepare_config_digest: str | None = None,
+ run_fingerprint: dict[str, object] | None = None,
) -> tuple[str, RunData] | None:
"""Read, prepare, skimjoin, and optionally cache one run."""
label = str(metadata["label"])
run_dir = str(metadata["run_dir"])
- run_fingerprint = dict(metadata["run_fingerprint"])
+ resolved_run_fingerprint = dict(
+ run_fingerprint or metadata["run_fingerprint"]
+ )
prepared_table_map = entry.get("prepared_table_map") or None
run_config = (
config if prepared_table_map is not None else config_for_run(config, entry)
@@ -205,6 +228,11 @@ def _build_prepared_run(
LOGGER.info("Prepared run: %r", label)
return (label, prepared_run)
+ LOGGER.info(
+ "Pipeline decision for %r / %s: REBUILD",
+ label,
+ "prepare" if not run_skimjoin else "skimjoin",
+ )
LOGGER.info("Reading run %r from %s", label, run_dir)
prepared_run = read_run(
run_dir,
@@ -235,8 +263,10 @@ def _build_prepared_run(
run_config,
run_key=run_key,
output_root=prepared_root,
- run_fingerprint=run_fingerprint,
+ run_fingerprint=resolved_run_fingerprint,
file_format=config.prepare_output_file_format,
+ cache_name=cache_name,
+ prepare_config_digest=prepare_config_digest,
)
LOGGER.info("Wrote prepared cache for run: %r", label)
else:
@@ -251,7 +281,8 @@ def _resolve_prepared_run(
config: Config,
prepared_root: Path,
existing_prepared_runs_by_key: dict[str, tuple[str, RunData]],
- prefer_cache: bool,
+ prefer_base_cache: bool,
+ prefer_skimjoin_cache: bool,
write_cache: bool,
run_skimjoin: bool,
) -> tuple[str, RunData] | None:
@@ -282,7 +313,20 @@ def _resolve_prepared_run(
existing_prepared_runs_by_key[run_key] = loaded_custom_prepared_run
return loaded_custom_prepared_run
- if prefer_cache:
+ if run_skimjoin and prefer_skimjoin_cache:
+ cached_prepared_run = _load_prepared_run_from_cache(
+ prepared_dir=prepared_dir,
+ config=config,
+ run_key=run_key,
+ label=label,
+ run_fingerprint=run_fingerprint,
+ stage="skimjoin",
+ )
+ if cached_prepared_run is not None:
+ existing_prepared_runs_by_key[run_key] = cached_prepared_run
+ return cached_prepared_run
+
+ if not run_skimjoin and prefer_base_cache:
cached_prepared_run = _load_prepared_run_from_cache(
prepared_dir=prepared_dir,
config=config,
@@ -294,6 +338,56 @@ def _resolve_prepared_run(
existing_prepared_runs_by_key[run_key] = cached_prepared_run
return cached_prepared_run
+ if run_skimjoin:
+ base_fingerprint = dict(metadata["base_run_fingerprint"])
+ base_dir = base_prepared_cache_dir(prepared_root, run_key)
+ base_prepared_run = None
+ if prefer_base_cache:
+ base_prepared_run = _load_prepared_run_from_cache(
+ prepared_dir=base_dir,
+ config=config,
+ run_key=run_key,
+ label=label,
+ run_fingerprint=base_fingerprint,
+ prepare_config_digest=config.base_prepare_config_digest,
+ stage="prepare",
+ )
+ if base_prepared_run is None:
+ base_prepared_run = _build_prepared_run(
+ entry=entry,
+ config=config,
+ run_key=run_key,
+ prepared_root=prepared_root,
+ metadata=metadata,
+ write_cache=write_cache,
+ run_skimjoin=False,
+ cache_name="base_prepared_tables",
+ prepare_config_digest=config.base_prepare_config_digest,
+ run_fingerprint=base_fingerprint,
+ )
+ if base_prepared_run is None:
+ return None
+
+ run_config = config_for_run(config, entry)
+ skimjoined_run = map_run_data_tables(base_prepared_run[1], lambda _name, frame: frame)
+ LOGGER.info("Pipeline decision for %r / skimjoin: REBUILD", label)
+ skimjoined_run = apply_skimjoin(skimjoined_run, run_config)
+ _log_prepare_table_diagnostics(label, skimjoined_run)
+ _validate_prepared_run(label, skimjoined_run, run_config)
+ if write_cache:
+ write_prepared_run_cache(
+ skimjoined_run,
+ run_config,
+ run_key=run_key,
+ output_root=prepared_root,
+ run_fingerprint=run_fingerprint,
+ file_format=config.prepare_output_file_format,
+ )
+ LOGGER.info("Wrote skimjoin cache for run: %r", label)
+ result = (label, skimjoined_run)
+ existing_prepared_runs_by_key[run_key] = result
+ return result
+
rebuilt_prepared_run = _build_prepared_run(
entry=entry,
config=config,
@@ -301,7 +395,7 @@ def _resolve_prepared_run(
prepared_root=prepared_root,
metadata=metadata,
write_cache=write_cache,
- run_skimjoin=run_skimjoin,
+ run_skimjoin=False,
)
if rebuilt_prepared_run is None:
return None
@@ -338,6 +432,14 @@ def run_prepare_workflow(
plan=plan,
)
prepared_root = prepared_root or prepared_cache_root(config, create=write_cache)
+ prefer_base_cache = prefer_cache
+ prefer_skimjoin_cache = prefer_cache
+ if plan.refreshes("skimjoin") and not plan.refreshes("prepare"):
+ prefer_base_cache = True
+ prefer_skimjoin_cache = False
+ if plan.refreshes("prepare"):
+ prefer_base_cache = False
+ prefer_skimjoin_cache = False
(
existing_prepared_runs_by_key,
prepared_runs_by_key,
@@ -364,7 +466,8 @@ def run_prepare_workflow(
config=config,
prepared_root=prepared_root,
existing_prepared_runs_by_key=existing_prepared_runs_by_key,
- prefer_cache=prefer_cache,
+ prefer_base_cache=prefer_base_cache,
+ prefer_skimjoin_cache=prefer_skimjoin_cache,
write_cache=write_cache,
run_skimjoin=config.skimjoin_step_enabled(),
)
diff --git a/runtime/workflows/shared.py b/runtime/workflows/shared.py
index 8f1c694..891c6ef 100644
--- a/runtime/workflows/shared.py
+++ b/runtime/workflows/shared.py
@@ -83,6 +83,9 @@ def effective_processor_config(
)
if effective is config:
return config
+ effective.base_prepare_config_digest = digest_payload(
+ effective.base_prepare_signature_payload()
+ )
effective.prepare_config_digest = digest_payload(effective.prepare_signature_payload())
effective.summary_config_digest = digest_payload(effective.summary_signature_payload())
return effective
@@ -123,6 +126,8 @@ def summary_cache_load_expectations(
config: Any,
build_run_fingerprint_fn: Callable[..., dict[str, object]],
resolve_skim_path_fn: Callable[[str | None, str | None, str | Path], str | None],
+ resolve_run_file_paths_fn: Callable[..., dict[str, str | None]],
+ optional_file_identity_fn: Callable[[str | Path | None], dict[str, object] | None],
build_prepared_manifest_identity_fn: Callable[..., dict[str, object]],
) -> dict[str, object] | None:
"""Return cache-load expectations for a cache dir when raw run inputs exist."""
@@ -146,7 +151,18 @@ def summary_cache_load_expectations(
"config_path": resolved_skimjoin.config_path,
"config_digest": resolved_skimjoin.config_digest,
"resolved_skim_files": list(resolved_skimjoin.resolved_skim_files),
+ "resolved_skim_file_identities": [
+ optional_file_identity_fn(path)
+ for path in resolved_skimjoin.resolved_skim_files
+ ],
"resolved_network_los_file": resolved_skimjoin.resolved_network_los_file,
+ "resolved_network_los_identity": optional_file_identity_fn(
+ resolved_skimjoin.resolved_network_los_file
+ ),
+ "create_hypothetical_skim_tables": (
+ resolved_skimjoin.create_hypothetical_skim_tables
+ ),
+ "failure_policy": resolved_skimjoin.failure_policy,
}
base_run_fingerprint = build_run_fingerprint_fn(
label=expected_label,
@@ -173,6 +189,26 @@ def summary_cache_load_expectations(
else config.fallback_files or None
),
skimjoin=expected_skimjoin,
+ raw_file_identities=(
+ None
+ if (uses_custom_prepared_tables or uses_summary_table_map_only)
+ else {
+ table_id: identity
+ for table_id, path in resolve_run_file_paths_fn(
+ run_dir,
+ config,
+ entry.get("file_map") or None,
+ ).items()
+ if (identity := optional_file_identity_fn(path)) is not None
+ }
+ ),
+ skim_file_identity=(
+ None
+ if (uses_custom_prepared_tables or uses_summary_table_map_only)
+ else optional_file_identity_fn(
+ resolve_skim_path_fn(entry.get("skim_file") or None, config.skim_file, run_dir)
+ )
+ ),
hh_weight_col=None
if (uses_custom_prepared_tables or uses_summary_table_map_only)
else entry.get("hh_weight_col") or None,
@@ -320,6 +356,8 @@ def run_cache_metadata(
run_key: str,
config: Any,
resolve_skim_path_fn: Callable[[str | None, str | None, str | Path], str | None],
+ resolve_run_file_paths_fn: Callable[..., dict[str, str | None]],
+ optional_file_identity_fn: Callable[[str | Path | None], dict[str, object] | None],
build_run_fingerprint_fn: Callable[..., dict[str, object]],
build_prepared_manifest_identity_fn: Callable[..., dict[str, object]],
) -> dict[str, object]:
@@ -340,7 +378,18 @@ def run_cache_metadata(
"config_path": resolved_skimjoin.config_path,
"config_digest": resolved_skimjoin.config_digest,
"resolved_skim_files": list(resolved_skimjoin.resolved_skim_files),
+ "resolved_skim_file_identities": [
+ optional_file_identity_fn(path)
+ for path in resolved_skimjoin.resolved_skim_files
+ ],
"resolved_network_los_file": resolved_skimjoin.resolved_network_los_file,
+ "resolved_network_los_identity": optional_file_identity_fn(
+ resolved_skimjoin.resolved_network_los_file
+ ),
+ "create_hypothetical_skim_tables": (
+ resolved_skimjoin.create_hypothetical_skim_tables
+ ),
+ "failure_policy": resolved_skimjoin.failure_policy,
}
resolved_skim = (
None
@@ -355,6 +404,24 @@ def run_cache_metadata(
else run_dir
),
skim_file=resolved_skim,
+ raw_file_identities=(
+ None
+ if (uses_custom_prepared_tables or uses_summary_table_map_only)
+ else {
+ table_id: identity
+ for table_id, path in resolve_run_file_paths_fn(
+ run_dir,
+ config,
+ entry.get("file_map") or None,
+ ).items()
+ if (identity := optional_file_identity_fn(path)) is not None
+ }
+ ),
+ skim_file_identity=(
+ None
+ if (uses_custom_prepared_tables or uses_summary_table_map_only)
+ else optional_file_identity_fn(resolved_skim)
+ ),
skimjoin=resolved_skimjoin_payload,
file_map=None
if (uses_custom_prepared_tables or uses_summary_table_map_only)
diff --git a/runtime/workflows/summarize.py b/runtime/workflows/summarize.py
index 06a2ba8..1c66b4f 100644
--- a/runtime/workflows/summarize.py
+++ b/runtime/workflows/summarize.py
@@ -81,7 +81,10 @@ def _load_summary_run_from_cache(
config,
expected_modes=config.weighting_modes,
expected_summary_ids=reusable_summary_ids,
- expected_summary_config_digest=config.summary_config_digest,
+ # Per-summary digests were validated by the inspection above.
+ # Requiring the bundle-wide digest here would discard otherwise
+ # reusable tables whenever only one builder changed.
+ expected_summary_config_digest=None,
expected_run_fingerprint=run_fingerprint,
expected_prepared_manifest_identity=prepared_manifest_identity,
expected_label=label,
@@ -91,9 +94,14 @@ def _load_summary_run_from_cache(
else []
)
LOGGER.info(
- "Loaded reusable summary cache tables for run %r: %s",
+ "Pipeline decision for %r / summarize: %s (%s)",
label,
- ", ".join(reusable_summary_ids) if reusable_summary_ids else "(none)",
+ "REUSE" if not stale_summary_ids else "REBUILD",
+ (
+ "all summary tables reusable"
+ if not stale_summary_ids
+ else f"{len(stale_summary_ids)} stale; {len(reusable_summary_ids)} reusable"
+ ),
)
return SummaryCacheInspection(
runs=tuple(cached_runs),
@@ -309,6 +317,11 @@ def run_summary_workflow(
if cached_prepared_run is not None:
prepared_runs_by_key[run_key] = cached_prepared_run
continue
+ else:
+ LOGGER.info(
+ "Pipeline decision for %r / summarize: REBUILD — refresh requested or cache reuse disabled",
+ label,
+ )
cached_summary_runs = list(cached_run.runs) if cached_run else []
summary_ids_to_build = list(summary_builder.DEFAULT_SUMMARY_IDS)
diff --git a/simor_configs/metro_configs/metro_config.yaml b/simor_configs/metro_configs/metro_config.yaml
index c1a5695..883281e 100644
--- a/simor_configs/metro_configs/metro_config.yaml
+++ b/simor_configs/metro_configs/metro_config.yaml
@@ -11,10 +11,10 @@ pipeline:
- prepare
- skimjoin
# - segment
- - summarize # will automatically overwrite summaries with a stale cache
+ - summarize # automatically rebuild stale summaries
# - dashboard
dashboard_mode: live # live | export | host
- overwrite: true # overwrite ALL prepared tables / summaries
+ refresh: [] # list stages here only when a forced rebuild is required
# ---------------------------------------------------------------------------
# ActivitySim output file names
@@ -343,7 +343,7 @@ summarize:
geography_col: DIST_9to12
dashboard:
- title: "Estimation Mode Comparison Visualizer"
+ title: "Metro Estimation Visualizer"
enable_maz_geographies: false
live:
pages:
diff --git a/tests/test_config_refactor_phase1.py b/tests/test_config_refactor_phase1.py
index 79778c8..15e7753 100644
--- a/tests/test_config_refactor_phase1.py
+++ b/tests/test_config_refactor_phase1.py
@@ -128,7 +128,7 @@ def test_new_config_layout_normalizes_to_existing_runtime_fields(tmp_path: Path)
" - summarize",
" - dashboard",
" dashboard_mode: export",
- " overwrite: true",
+ " refresh: [summarize]",
"dashboard:",
' title: "Refactor Dashboard"',
" live:",
@@ -192,7 +192,7 @@ def test_new_config_layout_normalizes_to_existing_runtime_fields(tmp_path: Path)
"dashboard",
)
assert config.pipeline.dashboard_mode == "export"
- assert config.pipeline.overwrite is True
+ assert config.pipeline.refresh == ("summarize",)
assert config.dashboard_title == "Refactor Dashboard"
assert [entry.page_id for entry in config.dashboard_pages or []] == [
"overview",
@@ -293,7 +293,9 @@ def test_dashboard_host_placeholder_rejects_unknown_fields(tmp_path: Path) -> No
(["pipeline:", " steps: [segment, dashboard]"], "without 'summarize'"),
(["pipeline:", " steps: [dashboard, summarize]"], "place 'dashboard' last"),
(["pipeline:", " dashboard_mode: deploy"], "dashboard_mode"),
- (["pipeline:", " overwrite: maybe"], "pipeline.overwrite"),
+ (["pipeline:", " refresh: maybe"], "pipeline.refresh"),
+ (["pipeline:", " refresh: [dashboard]"], "pipeline.refresh"),
+ (["pipeline:", " overwrite: false"], "pipeline.overwrite"),
],
)
def test_pipeline_validation_rejects_invalid_configurations(
@@ -303,3 +305,21 @@ def test_pipeline_validation_rejects_invalid_configurations(
) -> None:
with pytest.raises(ValueError, match=message):
_write_config(tmp_path, [*lines, "runs: []"])
+
+
+def test_pipeline_refresh_all_expands_only_enabled_materialized_steps(
+ tmp_path: Path,
+) -> None:
+ config = _write_config(
+ tmp_path,
+ [
+ "pipeline:",
+ " steps: [prepare, skimjoin, summarize, dashboard]",
+ " refresh: all",
+ "skimjoin:",
+ " defaults:",
+ "runs: []",
+ ],
+ )
+
+ assert config.pipeline.refresh == ("prepare", "skimjoin", "summarize")
diff --git a/tests/test_run_cli.py b/tests/test_run_cli.py
index ef7b7c9..d8cb953 100644
--- a/tests/test_run_cli.py
+++ b/tests/test_run_cli.py
@@ -1021,6 +1021,77 @@ def test_main_refresh_summary_cache_rebuilds_and_rewrites_run_cache(
assert (summary_cache_dir / "manifest.json").exists()
+def test_main_refresh_summary_cache_preserves_prepared_cache(
+ tmp_path: Path,
+ monkeypatch,
+) -> None:
+ run_dir = tmp_path / "run_a"
+ config = _write_cli_config(
+ tmp_path,
+ runs=[{"dir": str(run_dir), "label": "Run A"}],
+ )
+ fingerprint = build_run_fingerprint(
+ label="Run A",
+ run_dir=config.runs[0]["dir"],
+ skim_file=None,
+ hh_weight_col=None,
+ person_weight_col=None,
+ trip_weight_col=None,
+ )
+ prepared_entry = write_prepared_run_cache(
+ _fake_run_data("Run A", str(run_dir)),
+ config,
+ run_key="run-a",
+ run_fingerprint=fingerprint,
+ )
+ write_summary_run_cache(
+ _simple_summary_run("Run A", "run-a"),
+ config,
+ run_fingerprint=fingerprint,
+ prepared_manifest_identity=_prepared_identity(
+ config=config,
+ run_key="run-a",
+ label="Run A",
+ run_dir=config.runs[0]["dir"],
+ ),
+ )
+ summary_build_calls: list[str] = []
+ monkeypatch.setattr(summary_builder, "DEFAULT_SUMMARY_IDS", ["population_totals"])
+ _patch_prepare_pipeline(
+ monkeypatch,
+ read_run=lambda *args, **kwargs: (_ for _ in ()).throw(
+ AssertionError("summary refresh must preserve the prepared cache")
+ ),
+ prepare_data=lambda *args, **kwargs: (_ for _ in ()).throw(
+ AssertionError("summary refresh must not rerun preparation")
+ ),
+ )
+ monkeypatch.setattr(
+ summary_builder,
+ "build_mode_summaries_with_metadata",
+ lambda rd, config, **kwargs: (
+ summary_build_calls.append(rd.label),
+ _simple_summary_mode_build(rd.label, Path(rd.run_dir).name),
+ )[1],
+ )
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ [
+ "activitysim-viz",
+ "--config",
+ str(tmp_path / "config.yaml"),
+ "--summarize",
+ "--refresh-summary-cache",
+ ],
+ )
+
+ run.main()
+
+ assert prepared_entry.cache_dir.exists()
+ assert summary_build_calls == ["Run A"]
+
+
def test_main_refresh_prepared_cache_rebuilds_prepared_tables_before_summarize(
tmp_path: Path,
monkeypatch,
@@ -1091,6 +1162,37 @@ def test_main_refresh_prepared_cache_rebuilds_prepared_tables_before_summarize(
assert (prepared_dir / "run-a" / "prepared_tables" / "manifest.json").exists()
+def test_explain_cache_reports_plan_without_creating_cache_root(
+ tmp_path: Path,
+ monkeypatch,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ run_dir = tmp_path / "run_a"
+ config = _write_cli_config(
+ tmp_path,
+ runs=[{"dir": str(run_dir), "label": "Run A"}],
+ )
+ cache_root = Path(config.summary_root)
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ [
+ "activitysim-viz",
+ "--config",
+ str(tmp_path / "config.yaml"),
+ "--explain-cache",
+ ],
+ )
+
+ run.main()
+
+ output = capsys.readouterr().out
+ assert "Pipeline plan — Run A" in output
+ assert "prepare REBUILD" in output
+ assert "summarize REBUILD" in output
+ assert not cache_root.exists()
+
+
def test_main_uses_cache_hit_for_one_run_and_raw_fallback_for_another(
tmp_path: Path,
monkeypatch,
diff --git a/tests/test_runtime_workflows.py b/tests/test_runtime_workflows.py
index 086e886..f9ca6c0 100644
--- a/tests/test_runtime_workflows.py
+++ b/tests/test_runtime_workflows.py
@@ -754,6 +754,109 @@ def fake_resolve_skimjoin(config, entry):
assert skimjoin_labels == ["Run A", "Run B"]
+def test_refresh_skimjoin_reuses_base_prepared_cache(
+ tmp_path: Path,
+ monkeypatch,
+) -> None:
+ run_dir = tmp_path / "run_a"
+ config = _write_config(
+ tmp_path,
+ runs=[{"dir": str(run_dir), "label": "Run A"}],
+ extra_lines=[
+ "pipeline:",
+ " steps: [prepare, skimjoin]",
+ "skimjoin:",
+ " defaults:",
+ ],
+ )
+ read_labels: list[str] = []
+ prepare_labels: list[str] = []
+ skimjoin_labels: list[str] = []
+
+ def fake_resolve_skimjoin(config, entry):
+ return SkimjoinSettings(
+ enabled=True,
+ config_path="mock_skimjoin.yaml",
+ config_digest="mock-digest",
+ )
+
+ monkeypatch.setattr(
+ "runtime.config.resolve_run_skimjoin_settings",
+ fake_resolve_skimjoin,
+ )
+ monkeypatch.setattr(
+ "runtime.config.normalize_prepare.resolve_run_skimjoin_settings",
+ fake_resolve_skimjoin,
+ )
+ _patch_prepare_pipeline(
+ monkeypatch,
+ read_run=lambda run_dir, config, label=None, **kwargs: (
+ read_labels.append(label or Path(run_dir).name),
+ _fake_run_data(label or Path(run_dir).name, str(run_dir)),
+ )[1],
+ prepare_data=lambda rd, config: (
+ prepare_labels.append(rd.label),
+ rd,
+ )[1],
+ )
+ monkeypatch.setattr(
+ prepare_workflow,
+ "apply_skimjoin",
+ lambda rd, config: (skimjoin_labels.append(rd.label), rd)[1],
+ )
+
+ plan = _workflow_plan(config, skimjoin=True)
+ runtime_workflows.run_prepare_workflow(
+ config=config,
+ prepared_root=Path(config.summary_root),
+ run_entries=config.runs,
+ prefer_cache=False,
+ write_cache=True,
+ plan=plan,
+ )
+ refreshed_plan = replace(plan, refresh_steps=("skimjoin",))
+ runtime_workflows.run_prepare_workflow(
+ config=config,
+ prepared_root=Path(config.summary_root),
+ run_entries=config.runs,
+ prefer_cache=False,
+ write_cache=True,
+ plan=refreshed_plan,
+ )
+
+ assert read_labels == ["Run A"]
+ assert prepare_labels == ["Run A"]
+ assert skimjoin_labels == ["Run A", "Run A"]
+ assert (
+ Path(config.summary_root) / "run-a" / "base_prepared_tables" / "manifest.json"
+ ).exists()
+
+
+def test_raw_input_file_identity_changes_run_fingerprint(tmp_path: Path) -> None:
+ run_dir = tmp_path / "run_a"
+ run_dir.mkdir()
+ households = run_dir / "final_households.csv"
+ households.write_text("household_id\n1\n", encoding="utf-8")
+ config = _write_config(
+ tmp_path,
+ runs=[{"dir": str(run_dir), "label": "Run A"}],
+ )
+
+ first = prepare_workflow._run_cache_metadata(
+ entry=config.runs[0],
+ run_key="run-a",
+ config=config,
+ )["run_fingerprint"]
+ households.write_text("household_id\n1\n2\n", encoding="utf-8")
+ second = prepare_workflow._run_cache_metadata(
+ entry=config.runs[0],
+ run_key="run-a",
+ config=config,
+ )["run_fingerprint"]
+
+ assert first["raw_file_identities"] != second["raw_file_identities"]
+
+
def test_run_summary_workflow_does_not_build_non_default_registered_summaries(
tmp_path: Path,
monkeypatch,
@@ -2340,7 +2443,7 @@ def test_resolve_requested_steps_uses_config_pipeline_defaults(tmp_path: Path) -
]
-def test_resolve_effective_plan_uses_pipeline_dashboard_mode_and_overwrite(
+def test_resolve_effective_plan_uses_pipeline_dashboard_mode_and_refresh(
tmp_path: Path,
) -> None:
config = _write_config(
@@ -2352,7 +2455,7 @@ def test_resolve_effective_plan_uses_pipeline_dashboard_mode_and_overwrite(
" - summarize",
" - dashboard",
" dashboard_mode: export",
- " overwrite: true",
+ " refresh: [summarize]",
],
)
@@ -2375,7 +2478,7 @@ def test_resolve_effective_plan_uses_pipeline_dashboard_mode_and_overwrite(
assert plan.runtime_steps == ("summarize", "dashboard")
assert plan.logical_steps == ("summarize", "dashboard")
assert plan.dashboard_mode == "export"
- assert plan.overwrite is True
+ assert plan.refresh_steps == ("summarize",)
def test_resolve_effective_plan_drops_dashboard_when_config_dashboard_mode_is_none(
diff --git a/wiki/01-architecture.md b/wiki/01-architecture.md
index 85e46b8..8ffc924 100644
--- a/wiki/01-architecture.md
+++ b/wiki/01-architecture.md
@@ -67,7 +67,7 @@ Treat it as the contract for:
- which files are read
- which logical pipeline steps are requested by default
- which dashboard mode is used by default (`none`, `live`, `export`, `host`)
-- whether a run should prefer cache reuse or overwrite behavior by default
+- which materialized stages, if any, should be explicitly refreshed
- how schema aliases are resolved
- which weighting modes exist
- which pages are enabled
diff --git a/wiki/10-getting-started.md b/wiki/10-getting-started.md
index dee4f37..e9f4c07 100644
--- a/wiki/10-getting-started.md
+++ b/wiki/10-getting-started.md
@@ -27,7 +27,7 @@ root: artifacts
pipeline:
steps: [prepare, summarize, dashboard]
dashboard_mode: live
- overwrite: false
+ refresh: []
runs:
- dir: C:\models\base\output
diff --git a/wiki/12-running-workflows.md b/wiki/12-running-workflows.md
index 670a1da..7474efe 100644
--- a/wiki/12-running-workflows.md
+++ b/wiki/12-running-workflows.md
@@ -43,7 +43,7 @@ root: artifacts
pipeline:
steps: [prepare, summarize, dashboard]
dashboard_mode: live
- overwrite: false
+ refresh: []
dashboard:
title: Regional Model Comparison
@@ -67,7 +67,7 @@ root: artifacts
pipeline:
steps: [prepare, summarize, dashboard]
dashboard_mode: export
- overwrite: false
+ refresh: []
dashboard:
export:
@@ -87,7 +87,7 @@ Build prepared tables and summaries without opening or exporting a dashboard:
pipeline:
steps: [prepare, summarize]
dashboard_mode: none
- overwrite: false
+ refresh: []
```
Other focused workflows use the same fields:
@@ -155,6 +155,10 @@ regional_comparison/
summary_tables/
```
+When skimjoin is enabled, `base_prepared_tables/` preserves the prepared input
+before skim enrichment. The enriched tables remain in `prepared_tables/` so
+existing summary and dashboard consumers continue to use the same final path.
+
The run-key directory is a filesystem-safe lowercase slug of the run label.
For example, `Build Scenario` becomes `build-scenario`. Colliding labels receive
ordered suffixes such as `build-1` and `build-2`; avoid duplicate labels because
@@ -164,19 +168,23 @@ Relative paths in `dashboard.export.output_path` resolve below this directory.
Input paths follow the path rules documented in
[Configuration Reference](13-configuration-reference.md#reading-this-reference).
-Valid caches are reused automatically. To deliberately rebuild every cache
-used by the configured steps, temporarily set:
+Valid caches are reused automatically. To deliberately rebuild every
+materialized stage used by the configured steps, temporarily set:
```yaml
pipeline:
steps: [prepare, summarize, dashboard]
dashboard_mode: live
- overwrite: true
+ refresh: all
```
-Return `overwrite` to `false` after the forced rebuild. Presentation-only
-changes such as labels, colors, or enabled pages normally do not require cache
-rebuilding.
+Return `refresh` to `[]` after the forced rebuild. To rebuild summaries while
+preserving prepared and skimjoined data, use `refresh: [summarize]`.
+Presentation-only changes such as labels, colors, or enabled pages normally do
+not require cache rebuilding.
+
+Use `--explain-cache` to print the per-run reuse/rebuild decisions and exit
+without executing the pipeline.
## CLI Overrides
diff --git a/wiki/13-configuration-reference.md b/wiki/13-configuration-reference.md
index cb091a9..bf07e63 100644
--- a/wiki/13-configuration-reference.md
+++ b/wiki/13-configuration-reference.md
@@ -188,15 +188,19 @@ See [Advanced: Custom Weight Calculations](43-weighting-hosting-extensions.md#ad
|---|---|---|---|---|---|
| `steps` | non-empty list of strings | `[summarize, dashboard]` | `prepare`, `skimjoin`, `segment`, `summarize`, `dashboard` | Runtime | Steps must be lowercase, unique, and valid. `skimjoin` requires `prepare`; `segment` requires `summarize`; `dashboard` must be last when present. |
| `dashboard_mode` | string | `live` | `none`, `live`, `export`, `host` | Runtime, Presentation | Controls what the dashboard step does. `host` is reserved and currently warns, then falls back to the ordinary live server; it does not publish an application. |
-| `overwrite` | boolean | `false` | `true`, `false` | Runtime | Bypasses reusable prepared/summary caches for configured processor steps and writes rebuilt artifacts. Return it to `false` after a forced rebuild. |
+| `refresh` | list of strings or `all` | `[]` | `prepare`, `skimjoin`, `summarize`, `all` | Runtime | Forces only the named materialized stages to rebuild. Upstream refreshes invalidate enabled downstream stages. Leave empty for normal cache-aware operation. |
```yaml
pipeline:
steps: [prepare, skimjoin, segment, summarize, dashboard]
dashboard_mode: export
- overwrite: false
+ refresh: []
```
+`segment` is materialized within summary bundles, so use `refresh: [summarize]`
+to rebuild segmented outputs. Dashboard rendering has no persistent processor
+cache and is not a refresh target.
+
## `runs`
Each run entry describes one scenario. `label` is strongly recommended because
diff --git a/wiki/90-troubleshooting.md b/wiki/90-troubleshooting.md
index 6fdab0a..6cabbeb 100644
--- a/wiki/90-troubleshooting.md
+++ b/wiki/90-troubleshooting.md
@@ -9,8 +9,8 @@ Use this chapter when a run, cache, page, or export is not behaving as expected.
3. Check whether the issue appears in prepare, summarize, dashboard, or export.
4. Inspect `//manifest.json` for the affected run (see the
[cache layout](12-running-workflows.md#artifact-and-cache-paths)).
-5. If cache reuse is suspect, temporarily set `pipeline.overwrite: true` for
- the affected configured steps.
+5. Run with `--explain-cache` to inspect reuse and rebuild decisions. If a
+ forced rebuild is needed, list only the affected stage in `pipeline.refresh`.
## Symptoms
@@ -27,16 +27,16 @@ Use this chapter when a run, cache, page, or export is not behaving as expected.
## Cache Problems
-For a reproducible full rebuild, configure the steps and overwrite policy:
+For a reproducible full rebuild, configure the steps and refresh policy:
```yaml
pipeline:
steps: [prepare, summarize, dashboard]
dashboard_mode: live
- overwrite: true
+ refresh: all
```
-Return `overwrite` to `false` after the rebuild. Developers can use targeted
+Return `refresh` to `[]` after the rebuild. Developers can use targeted
one-off refresh flags while diagnosing a specific cache layer:
```bash
@@ -73,7 +73,7 @@ Suppose Trip Mode opens but shows the standard unavailable card:
4. If a required prepared column is missing, inspect the same manifest's
prepared-cache entry and the canonical column settings in `columns`.
5. If the contract recently changed, rebuild the configured summarize step
- with `pipeline.overwrite: true`.
+ with `pipeline.refresh: [summarize]`.
6. If the summary is present and valid, confirm the page's `columns=` request
matches the cached schema and that the selected weighting mode exists.
From da02be80db9ab8cf938e2e5b6c5d97d4618f9750 Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Wed, 5 Aug 2026 15:31:09 -0400
Subject: [PATCH 13/27] Update wiki
---
wiki/01-architecture.md | 103 ++++++++++++++++++++-------
wiki/12-running-workflows.md | 96 ++++++++++++++++++++++---
wiki/13-configuration-reference.md | 70 +++++++++++++++---
wiki/22-skimjoin.md | 26 +++++++
wiki/25-skimjoin-config-reference.md | 3 +-
wiki/31-dashboard-pages.md | 25 +++++++
wiki/32-figures-and-widgets.md | 59 +++++++++++++++
wiki/34-html-export.md | 41 +++++++++--
wiki/35-plotting-reference.md | 43 +++++++++++
wiki/90-troubleshooting.md | 21 ++++--
10 files changed, 431 insertions(+), 56 deletions(-)
diff --git a/wiki/01-architecture.md b/wiki/01-architecture.md
index 8ffc924..daa524e 100644
--- a/wiki/01-architecture.md
+++ b/wiki/01-architecture.md
@@ -34,19 +34,23 @@ run.py
-> resolve_effective_plan() from CLI overrides + config.pipeline defaults
-> zero or more runtime steps:
A. run_prepare_workflow()
- -> processor.prepare.cache.load_prepared_run_cache()
+ -> inspect/load the final prepared cache
+ -> when skimjoin is enabled, inspect/load the base prepared cache separately
-> processor.prepare.reader.read_run()
-> processor.prepare.enrichment.pipeline.prepare_data()
+ -> processor.skimjoin.pipeline.apply_skimjoin() when selected
-> processor.prepare.cache.write_prepared_run_cache()
B. run_summary_workflow()
- -> processor.summarize.cache.load_summary_run_cache()
+ -> inspect reusable/stale tables in the summary bundle
-> run_prepare_workflow() on summary-cache miss
-> processor.summarize.builder.build_mode_summaries_with_metadata()
- -> processor.summarize.cache.write_summary_run_cache()
+ -> merge reusable and rebuilt tables
+ -> processor.summarize.cache.write_summary_run_bundle()
C. load_summary_runs_from_cache() for dashboard-only cache runs
- -> processor.summarize.cache.load_summary_run_cache()
- D. dashboard.app.build_dashboard()
- E. dashboard.export.html.build_export_html_document()
+ -> processor.summarize.cache.load_summary_run_bundle()
+ D. run_dashboard_workflow()
+ -> dashboard.app.build_dashboard() and Panel serve for live mode
+ -> dashboard.export.write_export_html_document() for export mode
```
`WorkflowPlan` is the single resolved execution plan passed into these
@@ -85,8 +89,8 @@ logical step names are:
- `prepare`
- `skimjoin`
-- `summarize`
- `segment`
+- `summarize`
- `dashboard`
The runtime still executes three coarse workflow boundaries (`prepare`,
@@ -172,37 +176,86 @@ small summary and distribution features. This is the reference pattern for
logic reusable within one page family but not broad enough for
`dashboard/helpers/`.
+## Public Python APIs
+
+Import through these facades when extending or embedding the visualizer. Files
+not exported by a facade are implementation details unless a focused cookbook
+explicitly identifies them as an extension point.
+
+| Import surface | Public contract |
+|---|---|
+| `runtime.config` | `Config`, `load_config_from_yaml()`, `config_for_run()`, `resolve_run_skimjoin_settings()`, normalized export/pipeline/prepare/segmentation setting types, and weighting registry types. `Config.from_yaml()` is the equivalent class entry point. |
+| `runtime.workflows` | Config/run resolution; prepared and summary cache roots/loaders; `run_prepare_workflow()`, `run_summary_workflow()`, and `run_dashboard_workflow()`; consumer pruning; `WorkflowPlan`, `PreparedRunsArtifact`, `SummaryRunsArtifact`, and `SummaryCacheInspection`. Workflow functions are keyword-oriented and return artifacts rather than hidden module state. |
+| `processor` | `RunData`, the canonical prepared-run data contract. |
+| `processor.summarize` | `summary`, the declaration decorator for registered summary builders. |
+| `dashboard` | `DashboardPage`, `dashboard_page`, `DashboardState`, `PageData`, `RunTables`, and prepared/summary provider types used by page and embedding code. |
+| `dashboard.page_base` | `GroupedDashboardPage`, `PageFeature`, selector/section declaration types, and `PAGE_SELECTOR_STYLESHEET`, in addition to `DashboardPage`. |
+| `dashboard.rendering` | `RenderContext`, `FigureBuilder`, `Plotter`, table/formatting helpers, selector/control rows, legends, and the standard unavailable card. |
+| `dashboard.export` | `build_export_html_document()` for an in-memory document and `write_export_html_document()` for the streamed file/diagnostics workflow. |
+
+The normalized config value objects exported alongside `Config` are
+`CategorySpec`, `PipelineSettings`, `ExportDashboardSettings`,
+`ExportHTMLSettings`, `ExportSelectorRequest`,
+`PrepareNonMotorizedDistanceSkimSettings`, `SegmentationDefinition`,
+`PreparedColumnSegmentationSource`, `CsvLookupSegmentationSource`, and
+`StudentTypeConfig`. They are read-only runtime contracts; user input still
+enters through YAML normalization rather than by manually assembling a
+`Config`.
+
+The workflow facade also exports `effective_processor_config()`,
+`run_entries_with_keys()`, `prepared_cache_root()`, `summary_cache_root()`,
+`prune_summary_runs()`, and `prune_summary_artifact()` for embedding code that
+needs the same identity and consumer-pruning behavior as `run.py`. The
+dashboard facade exports `DashboardPreparedRunProvider` and
+`DashboardSummarySeries`; the page-base facade exports the typed
+`RegisteredPageSelector`, `RegisteredPageSection`, and `SectionContent`
+declaration records.
+
+The page-facing `PageData`/`RunTables` API is documented in chapter 32, chart
+keywords in chapter 35, the `@summary` contract in chapter 23, and workflow
+arguments and artifacts in the subsystem sections above. Public code should
+pass an explicit `WorkflowPlan` when it needs behavior different from the
+loaded config; the plan records logical steps, collapsed runtime boundaries,
+dashboard mode, and refresh targets.
+
## Repository Map
```text
activitysim_visualizer/
|-- run.py
-|-- runtime/
-| |-- workflows/
|-- config.yaml
|-- runtime/
-| `-- config/
+| |-- config/ # canonical schema, normalizers, models, signatures
+| |-- workflows/ # prepare/summarize/dashboard orchestration and artifacts
+| |-- logging.py
+| `-- weighting.py
|-- processor/
+| |-- analysis_units.py
+| |-- cache_identity.py
+| |-- cache_infra.py
| |-- models.py
+| |-- segmentation.py
| |-- prepare/
-| | |-- __init__.py
| | |-- availability.py
| | |-- cache.py
| | |-- enrichment/
-| | | |-- __init__.py
| | | |-- canonicalize.py
| | | |-- columns.py
| | | |-- domains.py
| | | |-- finalize.py
| | | |-- households_persons.py
+| | | |-- non_motorized_distance.py
| | | |-- pipeline.py
+| | | |-- student_enrollment.py
+| | | |-- time_periods.py
| | | |-- tours.py
| | | |-- trips.py
-| | | |-- types.py
| | | |-- weights.py
| | | `-- zones.py
| | |-- reader.py
+| | |-- validation.py
| | `-- writer.py
+| |-- skimjoin/ # config, inventory, annotation, stores, QA reports, CLI
| `-- summarize/
| |-- builder.py
| |-- cache.py
@@ -211,25 +264,20 @@ activitysim_visualizer/
| |-- catalog.py
| |-- contracts.py
| |-- csv_export.py
+| |-- external.py
| |-- schema.py
+| |-- validation_derived.py
| `-- summaries/
-| |-- daily_travel_activity.py
-| |-- daily_travel_escort_counts.py
-| |-- daily_travel_escort_distributions.py
-| |-- demographics.py
-| |-- joint_travel.py
-| |-- long_term_person.py
-| |-- long_term_vehicle.py
-| |-- long_term_geography.py
-| |-- long_term_distance.py
-| |-- tour.py
-| |-- trip.py
-| `-- validation.py
+| `--
|-- dashboard/
| |-- app.py
+| |-- calculation_notes.py / calculation_notes.yaml
+| |-- data_access.py
+| |-- helpers/
| |-- rendering/
| | |-- context.py
| | |-- figures.py
+| | |-- labels.py
| | |-- plotter.py
| | |-- layout.py
| | `-- tables.py
@@ -242,6 +290,7 @@ activitysim_visualizer/
| | |-- traversal.py
| | |-- runtime_assets.py
| | |-- types.py
+| | |-- js_runtime/
| | `-- assets/
| |-- page_base.py
| |-- page_declarations.py
@@ -253,6 +302,10 @@ activitysim_visualizer/
| |-- page_registry.py
| |-- state.py
| `-- pages/
+|-- scripts/
+| |-- generate_wiki_catalogs.py
+| `-- generate_validation_demo_fixtures.py
+|-- wiki/
`-- tests/
```
diff --git a/wiki/12-running-workflows.md b/wiki/12-running-workflows.md
index 7474efe..e81041e 100644
--- a/wiki/12-running-workflows.md
+++ b/wiki/12-running-workflows.md
@@ -137,8 +137,9 @@ For two runs labeled `Base` and `Build`, the normal layout is:
```text
regional_comparison/
base/
- manifest.json
+ manifest.json # summary-bundle manifest
prepared_tables/
+ manifest.json # final prepared/skimjoin identity
households.parquet
persons.parquet
tours.parquet
@@ -150,14 +151,44 @@ regional_comparison/
unweighted/
.csv
build/
- manifest.json
prepared_tables/
+ manifest.json
summary_tables/
+ manifest.json
```
-When skimjoin is enabled, `base_prepared_tables/` preserves the prepared input
-before skim enrichment. The enriched tables remain in `prepared_tables/` so
-existing summary and dashboard consumers continue to use the same final path.
+The run-level `manifest.json` belongs to the summary bundle. Each prepared
+cache has its own manifest inside its table directory. A prepare-only workflow
+therefore writes `prepared_tables/manifest.json` but does not create the
+run-level summary manifest.
+
+When skimjoin is enabled, `base_prepared_tables/` contains a second prepared
+manifest and the canonical tables before skim enrichment. The enriched tables,
+skimjoin reports, and optional hypothetical sidecars remain under
+`prepared_tables/`, so summary and dashboard consumers continue to use the
+same final path:
+
+```text
+base/
+ base_prepared_tables/
+ manifest.json
+ trips.parquet
+ tours.parquet
+ ...
+ prepared_tables/
+ manifest.json
+ trips.parquet
+ tours.parquet
+ trip_hypothetical_skims.parquet # only when enabled and populated
+ tour_hypothetical_skims.parquet # only when enabled and populated
+ skimjoin/
+ config_normalized.yaml
+ .csv
+```
+
+Segmented summary CSVs are nested below
+`summary_tables//segments///` and
+are described by the run-level summary manifest.
The run-key directory is a filesystem-safe lowercase slug of the run label.
For example, `Build Scenario` becomes `build-scenario`. Colliding labels receive
@@ -183,15 +214,60 @@ preserving prepared and skimjoined data, use `refresh: [summarize]`.
Presentation-only changes such as labels, colors, or enabled pages normally do
not require cache rebuilding.
+The refresh targets are stage-aware:
+
+| Refresh target | Reused | Rebuilt when enabled |
+|---|---|---|
+| `prepare` | nothing upstream of prepare | base prepared data, skimjoin output, summaries |
+| `skimjoin` | `base_prepared_tables` | enriched `prepared_tables`, summaries |
+| `summarize` | final `prepared_tables` | stale/default summaries and segmented summaries |
+
+Normal reuse is also content-aware. Prepared manifests record resolved raw
+input identities, including path, size, and modification time, plus the
+prepare/skimjoin config identity and skim input identities. The summary
+manifest records its upstream prepared-manifest identity, summary config, and
+per-summary declaration digest. Consequently, a changed raw file invalidates
+prepare and downstream output, a changed skim input can rebuild only skimjoin
+and downstream output, and a changed summary declaration can rebuild only the
+affected summary tables while reusing compatible tables in the bundle.
+
Use `--explain-cache` to print the per-run reuse/rebuild decisions and exit
-without executing the pipeline.
+without loading tables, deleting caches, or writing artifacts. The report shows
+`REUSE`, `REBUILD`, `RUN`, or `DISABLED` for prepare, skimjoin, summarize, and
+dashboard, with the cache-validation reason when available.
## CLI Overrides
-CLI step, refresh, export-path, and port flags remain available for developers
-and troubleshooting. They override the configured workflow for that one
-invocation. Users should normally change the YAML and continue running the same
-command so the intended workflow remains reproducible.
+CLI flags override the configured workflow for one invocation. Users should
+normally change YAML so the intended workflow remains reproducible.
+
+| Flag | Behavior |
+|---|---|
+| `--config PATH`, `-c PATH` | Load the named main config. The default is `config.yaml` next to `run.py`. |
+| `--run DIR LABEL` | Replace configured `runs` with one CLI run. Repeat the flag for multiple runs. |
+| `--run-skim PATH ...` | Supply one legacy prepare distance-skim path per `--run`, in order. Use `null` or an empty string to inherit `prepare.distance_skim.file`. |
+| `--prepare` | Select the coarse prepare boundary for this invocation. |
+| `--summarize` | Select the coarse summarize boundary for this invocation. |
+| `--dashboard` | Select the dashboard boundary and force live mode unless `--export-html` is also present. |
+| `--prepare-only` | Select only prepare; it cannot be combined with the three explicit step flags. |
+| `--write-csvs` | Bypass reusable summary tables and force summary CSV/manifest writes; requires summarize. |
+| `--from-csvs [CACHE_DIR ...]` | Run dashboard-only and load completed summary-cache directories explicitly. These are cache bundles with manifests, not loose CSV files. |
+| `--skip-summary-cache-write` | Build summaries in memory without writing missing or stale summary cache entries; requires summarize. |
+| `--refresh-prepared-cache` | Force prepared data and all affected downstream output to rebuild for selected runs. |
+| `--refresh-summary-cache` | Preserve prepared directories and force summary output to rebuild. |
+| `--refresh-caches` | Force both prepared and summary cache layers to rebuild. |
+| `--export-html [PATH]` | Use export mode for a selected dashboard step. An omitted path uses `dashboard.export.output_path`, then `/exported_dashboard.html`. |
+| `--port PORT` | Live-server port; default `5006`. |
+| `--no-show` | Start the live server without opening a browser. |
+| `--explain-cache` | Print the cache plan and exit without executing it. |
+
+If any of `--prepare`, `--summarize`, or `--dashboard` is present, those flags
+replace `pipeline.steps` with the selected coarse boundaries. They do not
+implicitly enable the logical `skimjoin` or `segment` steps. `--from-csvs`
+cannot be combined with processor steps or `--write-csvs`; `--write-csvs` and
+`--skip-summary-cache-write` require summarize. Refresh flags require the
+corresponding processor boundary. If the config omits dashboard, pair
+`--export-html` with `--dashboard` to select it.
## Related Chapters
diff --git a/wiki/13-configuration-reference.md b/wiki/13-configuration-reference.md
index bf07e63..5730766 100644
--- a/wiki/13-configuration-reference.md
+++ b/wiki/13-configuration-reference.md
@@ -18,7 +18,8 @@ Path resolution depends on the field:
| `runs[*].dir` | main config directory | Raw ActivitySim output directory. |
| `files.*`, `runs[*].file_map.*` | the resolved run directory | File stems may omit `.parquet` or `.csv`; Parquet is tried before CSV. |
| `fallback_files.*`, `prepared_table_map.*`, `summary_table_map.*` | main config directory | Values must include `.parquet` or `.csv`. |
-| main-config skim, lookup, and skimjoin override paths | main config directory | Includes `prepare.distance_skim.file`, segmentation CSVs, and `skimjoin.defaults.*`. |
+| `prepare.distance_skim.file`, `runs[*].skim_file` | the resolved run directory | A relative legacy distance-skim path is resolved separately for each run. |
+| other main-config enrichment, lookup, and skimjoin paths | main config directory | Includes `prepare.time_periods.network_los_file`, `prepare.non_motorized_distance_skim.file`, segmentation CSVs, and `skimjoin.defaults.*`. |
| paths inside the standalone skimjoin config | standalone skimjoin config directory | See chapter 25. |
| `dashboard.export.output_path` | resolved `root` | Absolute output paths remain absolute. |
@@ -125,10 +126,10 @@ runs:
| `segment` | mapping | disabled | Summary, Presentation | Optional segmented summaries and dashboard segment controls. |
| `weighting` | mapping | `{}` | Summary, Presentation | Declarative named weighting modes backed by prepared source columns. |
| `summarize` | mapping | weighted and unweighted summaries | Summary | Summary weighting, purpose grouping, geography, and PNR mode behavior. |
-| `dashboard` | mapping | live dashboard defaults | Presentation | Dashboard title, page selection, MAZ geography toggle, and export settings. |
+| `dashboard` | mapping | live dashboard defaults | Presentation | Dashboard title, page selection, calculation notes, MAZ geography toggle, and export settings. |
| `display` | mapping | built-in labels and colors | Presentation | Dashboard labels, category order, and run colors. |
| `extensions` | mapping | `{}` | Summary, Presentation | Advanced importable weighting calculation modules and their settings. Extension code is trusted. |
-| `modes` | mapping | `{}` | Presentation | Optional mode ordering used when `display.labels.mode` is absent. |
+| `modes` | mapping | `{}` | Summary, Presentation | Optional mode ordering and named summary mode groups. |
## `weighting`
@@ -213,7 +214,8 @@ it becomes the display name and helps cache/debug output remain understandable.
| `file_map` | mapping | inherits top-level `files` | Prepare | Per-run raw file stem overrides. Cannot be combined with `prepared_table_map`. |
| `prepared_table_map` | mapping | none | Prepare, Summary | Explicit `.parquet` or `.csv` canonical prepared tables. Skips raw prepare for that run. |
| `summary_table_map` | mapping | none | Summary, Presentation | Registered summary IDs mapped to dashboard-ready `.parquet` or `.csv` files. May be used alone or override generated summaries. |
-| `skimjoin` | mapping | inherits global `skimjoin` | Prepare | Per-run skimjoin `config_path`, `skim_files`, and `network_los_file` overrides. |
+| `skim_file` | path string | `prepare.distance_skim.file` | Prepare, Summary | Per-run legacy distance-skim override. Relative paths resolve from this run's `dir`. |
+| `skimjoin` | mapping | inherits global `skimjoin` | Prepare | Per-run skimjoin path and hypothetical-sidecar overrides. |
| `hh_weight_col` | string | none | Prepare, Summary | Household source for the run's primary `weighted` mode. |
| `person_weight_col` | string | none | Prepare, Summary | Person source for the run's primary `weighted` mode. |
| `trip_weight_col` | string | none | Prepare, Summary | Trip source for the run's primary `weighted` mode. |
@@ -387,7 +389,7 @@ columns:
| `distance_skim.matrix` | string | `SOV_DIST__MD` | matrix name | Prepare, Summary | Matrix read from `distance_skim.file`. |
| `auto_sufficiency_basis` | string | `licensed_drivers` | `licensed_drivers`, `workers`, `adults` | Prepare, Summary | Basis for household auto-sufficiency derivation. |
| `student_types` | list of mappings | `[]` | student-type definitions | Prepare, Summary | School/university enrollment definitions used by prepared fields and shadow-pricing summaries. |
-| `time_periods` | mapping | built-in periods | period definitions or ActivitySim config source | Prepare, Summary | Canonical time-period labels used by prepared tours and trips. |
+| `time_periods` | mapping | disabled | ActivitySim `network_los.yaml` source | Prepare, Summary | Derives canonical period labels for prepared tours and trips. |
| `non_motorized_distance_skim` | mapping | disabled | configured lookup | Prepare, Summary | Optional non-motorized distance enrichment. |
| `vot_bins.source_column` | string | `income_segment` | any source column | Prepare, Skimjoin | Source value used to derive VOT bins. |
| `vot_bins.output_column` | string | `vot_bin` | any output column | Prepare, Skimjoin | Prepared column written for skimjoin dimensions. |
@@ -415,6 +417,33 @@ prepare:
3: H
```
+`prepare.time_periods` accepts:
+
+| Field | Type | Default | Notes |
+|---|---|---|---|
+| `network_los_file` | path string | required | ActivitySim YAML containing `skim_time_periods.periods` and `skim_time_periods.labels`. Relative paths resolve from the main config. |
+| `trip_period_number_column` | string | `depart` | Prepared trip source used to write `trip_period`. |
+| `tour_start_period_number_column` | string | `start` | Prepared tour source used to write `start_period`. |
+| `tour_end_period_number_column` | string | `end` | Prepared tour source used to write `end_period`. |
+
+The period breakpoint list must contain at least two integers, and the label
+list must contain exactly one fewer entry. When trips contain `tour_id`,
+`outbound`, and the derived `trip_period`, prepare also writes each tour's
+`first_inbound_trip_period`. Missing configured source columns are recorded in
+prepare diagnostics rather than invented.
+
+`prepare.non_motorized_distance_skim` accepts:
+
+| Field | Type | Default | Notes |
+|---|---|---|---|
+| `file` | path string | required | `.csv`, `.omx`, `.h5`, or `.hdf5` lookup. Relative paths resolve from the main config. |
+| `matrix` | string or null | required for OMX/HDF5; `DISTWALK` for CSV | OMX matrix name. For CSV, names the value column; a `__` prefix is stripped when present. |
+
+CSV lookup files must contain `OMAZ`, `DMAZ`, and the selected value column;
+prepared trips must contain `o_maz` and `d_maz`. OMX/HDF5 lookup uses prepared
+`OTAZ` and `DTAZ`. Both paths write
+`prepared_non_motorized_distance` and record unresolved lookup diagnostics.
+
## `skimjoin`
The main config `skimjoin` section wires the visualizer runtime to a separate
@@ -441,9 +470,11 @@ Merely providing `skimjoin.defaults.config_path` does not run skimjoin;
| `failure_policy` | string | `record` | Runtime, Prepare | `record` keeps a failed enrichment as diagnostics; `error` stops the run. |
| `create_hypothetical_skim_tables` | boolean | `false` | Prepare | Enables configured hypothetical skim tables. |
-Run-level `runs[*].skimjoin` supports `config_path`, `skim_files`, and
-`network_los_file`. Enable skimjoin by including it in `pipeline.steps`;
-top-level `skimjoin.enabled` and `skimjoin.config_path` are removed keys.
+Run-level `runs[*].skimjoin` supports `config_path`, `skim_files`,
+`network_los_file`, and `create_hypothetical_skim_tables`. The last field
+inherits the global value when omitted. Enable skimjoin by including it in
+`pipeline.steps`; top-level `skimjoin.enabled` and `skimjoin.config_path` are
+removed keys.
Integrated skim files must resolve to `.omx`, `.csv`, `.h5`, or `.hdf5`.
@@ -553,6 +584,7 @@ summarize:
| Field | Type | Default | Allowed values | Impact | Notes |
|---|---|---|---|---|---|
| `title` | string | `ActivitySim Visualizer` | any string | Presentation | Dashboard title. |
+| `include_notes` | boolean | `true` | `true`, `false` | Presentation | Show expandable calculation notes beneath annotated charts and tables. |
| `enable_maz_geographies` | boolean | `false` | `true`, `false` | Presentation | Enables MAZ geography options in dashboard pages that support them. |
| `live.pages` | list | all/default page registry behavior | page or group ids | Presentation | Live dashboard page selection. |
| `export.output_path` | path string | none | HTML path | Presentation | Relative paths resolve under `root`. |
@@ -564,6 +596,11 @@ summarize:
| `export.exclude_pages` | list of strings | `[]` | page ids | Presentation | Pages excluded from export. |
| `export.exclude_groups` | list of strings | `[]` | group ids | Presentation | Groups excluded from export. |
+`dashboard.host` is a reserved configuration block. The schema currently
+accepts `account`, `app_id`, `title`, and `verify`, but these values are not
+normalized or consumed: `pipeline.dashboard_mode: host` logs a warning and
+runs the ordinary live server. See the hosting extension recipe in chapter 43.
+
`live.pages` entries may be strings or group mappings:
```yaml
@@ -638,6 +675,23 @@ display:
- "#a00000"
```
+## `modes`
+
+`modes` supplies legacy mode ordering and summary grouping. Prefer
+`display.labels.mode.mapping` when labels and order should be defined together.
+
+| Field | Type | Default | Impact | Notes |
+|---|---|---|---|---|
+| `order` | list of strings | none | Presentation | Raw mode order used only when `display.labels.mode` is absent. |
+| `groups` | mapping of lists | none | Summary | Named mode groups included in summary cache identity. The `Auto` group explicitly selects auto modes for `auto_vmt_totals` and segmented auto-VMT summaries; without it, those summaries use built-in name matching. |
+
+```yaml
+modes:
+ order: [SOV, HOV2, HOV3, WALK, BIKE, WALK_TRANSIT]
+ groups:
+ Auto: [SOV, HOV2, HOV3, TAXI, TNC_SINGLE, TNC_SHARED]
+```
+
## Advanced Category Config
`summarize.category_normalization` uses the same category shape as
diff --git a/wiki/22-skimjoin.md b/wiki/22-skimjoin.md
index dda6607..bff7d27 100644
--- a/wiki/22-skimjoin.md
+++ b/wiki/22-skimjoin.md
@@ -77,6 +77,32 @@ Set `skimjoin.create_hypothetical_skim_tables: true` (globally or in a run
override) when the configured lookups should also produce hypothetical skim
sidecar tables. This is opt-in because it adds output work and artifacts.
+## Standalone Skimjoin CLI
+
+The integrated pipeline is the normal visualizer path. A standalone CLI is
+also available for inspecting and validating a skimjoin config or producing
+annotated tables without running the full visualizer:
+
+```bash
+uv run python -m processor.skimjoin.cli COMMAND --config skimjoin.yaml
+```
+
+| Command | Additional flags | Output |
+|---|---|---|
+| `inventory` | `--preview` | Writes `skim_inventory.csv` and `inventory_debug.log` under `project.output_dir`. Preview also writes trip/tour column inventories and ActivitySim value counts when the configured tables are available. |
+| `validate` | none | Strictly validates config, inventory, and configured ActivitySim tables; writes `config_normalized.yaml` and `validation_report.txt`. Returns exit code 1 and writes a failure report when validation fails. |
+| `annotate-trips` | `--out PATH`, `--preview` | Writes annotated trips plus validation, lookup-summary, and missing-lookup artifacts. The default table is `/trips_with_skims.parquet`. |
+| `annotate-tours` | `--out PATH`, `--preview` | Writes annotated tours, `tour_aggregation_summary.csv`, and `missing_lookup_report.csv`. The default table is `/tours_with_skims.parquet`. |
+| `run` | `--out-trips PATH`, `--out-tours PATH`, `--preview` | Runs both annotations and writes their validation/QA reports. Defaults to the two filenames above. |
+
+`--config` is required for every command. Output flags are optional only when
+`project.output_dir` is configured. Standalone table inputs come from
+`activitysim.trips_table` and `activitysim.tours_table`, with the legacy
+`project.trips_table`/`project.tours_table` fallback described in chapter 25.
+Input and output tables must be CSV or Parquet. `--preview` on annotation
+commands adds a compact output-column inventory; it does not limit rows or
+make the command a dry run.
+
## Debugging Skimjoin
Start with the skimjoin artifacts on the prepared run:
diff --git a/wiki/25-skimjoin-config-reference.md b/wiki/25-skimjoin-config-reference.md
index 7543637..2d4ae47 100644
--- a/wiki/25-skimjoin-config-reference.md
+++ b/wiki/25-skimjoin-config-reference.md
@@ -538,6 +538,5 @@ Policies:
## Related Chapters
- [13 - Configuration Reference](13-configuration-reference.md#skimjoin)
-- [13 - Configuration Reference](13-configuration-reference.md#skimjoin)
-- [22 - Skimjoin](22-skimjoin.md)
+- [22 - Skimjoin](22-skimjoin.md), including the standalone CLI
- [90 - Troubleshooting](90-troubleshooting.md)
diff --git a/wiki/31-dashboard-pages.md b/wiki/31-dashboard-pages.md
index 7b0d162..5d3a0f7 100644
--- a/wiki/31-dashboard-pages.md
+++ b/wiki/31-dashboard-pages.md
@@ -97,6 +97,31 @@ pages. Page render code remains responsible for the fallback. Standalone HTML
export does not load prepared tables; see chapter 34 for section-level export
rules.
+## Availability And Validation Features
+
+Page selectors are data-aware. Option providers enumerate values present in
+usable runs, and the page lifecycle repairs a selection when an upstream choice
+makes it invalid. A selector should not offer a value whose dependent section
+would be empty merely because that value exists in a hard-coded domain.
+
+When no usable run remains, pages render the standard data-unavailable card for
+the affected feature. Required data can make the page's primary workflow
+unavailable; missing optional data replaces only its independent feature. Set
+`display.missing_data_display: blank` to suppress these cards globally.
+
+The validation group currently provides:
+
+| Page | Current behavior |
+|---|---|
+| Traffic Validation | Observed-versus-modeled count-location fit, traffic volume summaries, top modeled count locations, link tables, and screenline flow comparison. Count-location diagnostics report location count, RMSE, RMSPE, and R-squared by facility group. Scatterplots include a 1:1 line; fitted equations, R-squared, and sample size appear on fit-line hover. Screenlines can be filtered by time period and facility type before a per-run ordinary-least-squares fit is calculated. RMSPE is blank for a group containing a zero observed count. |
+| Transit Validation | Boardings by operator/technology and transfer rates by operator, technology, and access mode, with calculation notes and unavailable states when the supplied contracts cannot be used. |
+| VMT Validation | Overview comparisons plus selector-driven personal-auto and non-motorized VMT. Optional outside tables add external travel/VMT, commercial travel/VMT, and bicycle facility summaries; each optional feature gets its own unavailable state. |
+| Regional Validation | Optional district or county observed flow matrices, modeled `commuting_flows`, and aligned heatmaps for modeled, observed, difference, percent difference, or absolute percent difference. Totals can be included or excluded. Only flow types backed by available inputs appear in the selector. |
+
+Expandable calculation notes beneath these outputs identify source summary IDs,
+filters, formulas, and aggregation details. They are enabled by default and can
+be hidden with `dashboard.include_notes: false`.
+
## Generated Page Catalog
The catalog below is generated from the dashboard page registry. Regenerate it
diff --git a/wiki/32-figures-and-widgets.md b/wiki/32-figures-and-widgets.md
index 06458f5..3e8fb17 100644
--- a/wiki/32-figures-and-widgets.md
+++ b/wiki/32-figures-and-widgets.md
@@ -77,6 +77,65 @@ def render_mode_chart(self):
return self.plot.bar(chart_data, x="trip_mode", y="trip_count")
```
+The page-facing data API is:
+
+| API | Result |
+|---|---|
+| `self.data.summary(id, weighting=None, columns=(), required=None)` | One summary across usable runs. `columns` performs a schema compatibility check. |
+| `self.data.summaries(*ids, columns=None, required=None)` | A dictionary of summary ID to `RunTables`. |
+| `self.data.prepared(table, columns=(), weighting_mode=None)` | One declared prepared table across loaded runs. |
+| `self.data.prepared_runs(weighting_mode=None)` | Specialized `RunData` escape hatch for features that require matrices or other non-table state. |
+| `self.data.summary_series(id, weighting=None)` | Specialized skim-summary view that retains summary-series metadata. |
+
+`RunTables` is iterable and indexable as `(run_label, DataFrame)` pairs. Its
+public fluent/query surface is:
+
+| API | Behavior |
+|---|---|
+| `.where(column=value, ...)` | Equality filter; list, tuple, set, or frozenset values use membership. |
+| `.with_columns(*exprs)` / `.select(*exprs)` / `.sort(*by)` | Apply the corresponding Polars operation to every run. |
+| `.group(by, *aggs, **named_aggs)` | Group and aggregate every run. |
+| `.join(other, on=..., how="left", coalesce=None)` | Join matching run labels and merge availability issues/source IDs. |
+| `.map(transform)` | Apply a DataFrame-to-DataFrame transform to every run. |
+| `.requiring(*columns)` | Keep frames containing all named columns. Prefer the lookup `columns=` check when exclusions should produce schema diagnostics. |
+| `.drop_empty()` | Remove frames made empty by a previous operation. |
+| `.values(column)` | Distinct non-null values in first-seen run order. |
+| `.scalar(column, default=None)` | First value for each usable run. |
+| `.to_list()` | Materialize tuples for an external API that cannot consume `RunTables`. |
+| `.available`, `.partial`, `.issues`, `.source_ids` | Availability and provenance metadata retained through fluent operations. |
+
+## Calculation Notes
+
+Calculation notes are expandable, dependency-free HTML details displayed
+beneath annotated charts and tables. Users can hide all notes with:
+
+```yaml
+dashboard:
+ include_notes: false
+```
+
+Content lives in `dashboard/calculation_notes.yaml`. The top-level `methods`
+mapping contains reusable method explanations; `notes` contains stable note
+IDs. Every note requires `summary`, `method`, and a non-empty `sources` list,
+and may add `label`, `method_text`, `formula`, `source_filters`, and grouped
+`details`. Loading validates unknown fields and method references.
+
+Page authors attach a note to a registered selector-driven section with:
+
+```python
+body = self.section(
+ "comparison",
+ selectors=("facility_type",),
+ render=self.render_comparison,
+)
+return self.noted_section("traffic.observed_model_fit", body)
+```
+
+Use `self.noted_view(note_id, view)` for an individual plot or table that is
+not itself the registered section container. `self.section_note(...)` is the
+lower-level helper and rejects unregistered sections. Notes use the same page
+layout in live mode and HTML export.
+
## Selectors
Declare a normal dropdown with its option domain in one place:
diff --git a/wiki/34-html-export.md b/wiki/34-html-export.md
index 1adae1f..c98a828 100644
--- a/wiki/34-html-export.md
+++ b/wiki/34-html-export.md
@@ -8,7 +8,7 @@ registered dashboard pages
-> export payload
-> serialized Panel nodes
-> embedded CSS, Plotly, and runtime JS
- -> one HTML file
+ -> one HTML file + diagnostics JSON sidecar
```
## When To Use Export
@@ -53,6 +53,15 @@ below `root`; an absolute path writes elsewhere. Change
`pipeline.dashboard_mode` back to `live` when the same config should serve the
dashboard instead.
+It also writes `artifacts/exports/dashboard.diagnostics.json`. The sidecar
+records export warnings and size/state analysis for developers; the HTML does
+not depend on the sidecar when it is opened or shared.
+
+For a one-off override, use `--export-html [PATH]`. With no path, the CLI uses
+the configured output path and then falls back to
+`/exported_dashboard.html`. The dashboard step must still be selected;
+add `--dashboard` when it is absent from `pipeline.steps`.
+
Export begins with the pages resolved by `dashboard.live.pages`. The
`dashboard.export.pages` mapping modifies matching page selectors and parts; it
does not select the included page set. Use a page override with `enabled: false`,
@@ -72,6 +81,11 @@ The export runtime supports a deliberately small set of rendered objects:
- registered regions
- registered selector widgets
+Viewers can collapse and restore the export sidebar with the header button;
+Plotly charts resize after the layout changes. Long run names use compact,
+unique tab and legend labels while their full text remains available in tab
+tooltips and chart hovers.
+
The Python-to-JavaScript contract lives in `dashboard/export/types.py`, and the
browser runtime lives under `dashboard/export/js_runtime/`.
@@ -169,6 +183,23 @@ exported safely.
| `dashboard/export/js_runtime/` | Readable browser runtime source. |
| `dashboard/export/assets/export_runtime.js` | Built browser runtime embedded in exports. |
+## Export Write And Python APIs
+
+`dashboard.export` exposes two entry points:
+
+| API | Behavior |
+|---|---|
+| `build_export_html_document(runs, config, summary_runs=None) -> str` | Build, serialize, and validate a complete HTML document in memory. Useful for tests and callers that need the string. |
+| `write_export_html_document(output_path, runs, config, summary_runs=None) -> Path` | Build the payload, stream JSON into a temporary HTML file, write the diagnostics sidecar through a temporary file, and replace each destination only after that file is complete. This is the normal workflow path. |
+
+Payload construction sanitizes NumPy/Pandas values before JSON encoding;
+non-finite numeric values become JSON `null`, timestamps become ISO strings,
+and closing script tags are escaped. The writer streams the JSON rather than
+materializing a second payload string or final HTML string, which keeps peak
+memory lower for large selector-state exports. A serialization, shell, write,
+or finalization failure raises an `ExportBuildError` naming the failed phase
+and cleans up temporary files.
+
## Changing Export Runtime Behavior
Checklist:
@@ -188,9 +219,11 @@ Checklist:
1. Open the exported HTML in a browser.
2. Open developer tools and check the console.
-3. Look for `ExportRuntimeError` messages.
-4. Try `?debug_export=1` in the URL.
-5. Compare live mode to export mode with the same config and summary caches.
+3. Inspect the adjacent `.diagnostics.json` file for build warnings and
+ size/state analysis.
+4. Look for `ExportRuntimeError` messages.
+5. Try `?debug_export=1` in the URL.
+6. Compare live mode to export mode with the same config and summary caches.
## Related Chapters
diff --git a/wiki/35-plotting-reference.md b/wiki/35-plotting-reference.md
index 6cc4874..0ea9392 100644
--- a/wiki/35-plotting-reference.md
+++ b/wiki/35-plotting-reference.md
@@ -61,6 +61,33 @@ Sort ordered data in the query. For categorical bars, pass
`category_order=[...]` when the configured display order matters or missing
categories must keep a stable axis position.
+### Keyword Reference
+
+All figure builders accept `x`, `y`, `title`, `x_title`, `y_title`, and
+`height`. Additional chart-specific keywords are:
+
+| Chart | Keywords |
+|---|---|
+| `bar` | `barmode="group"`, `share_y=None`, `value_mode="dashboard"`, `category_order=None`, `show_legend=None` |
+| `line` | `value_mode="dashboard"` |
+| `density` | `value_mode="dashboard"`, `x_range=None`, `category_order=None`, `tick_values=None`, `tick_text=None`, `hover_x_title=None` |
+| `scatter` | `drop_zero_y=False`, `fit_overlays=None`, `fit_annotation="annotation"`, `one_to_one=False`, `legend_on_right=False` |
+
+`self.plot.scatter(...)` additionally accepts `panel_aspect_ratio`; this sizes
+the returned Panel pane and is not passed to the Plotly figure builder.
+
+For fitted scatterplots, `fit_overlays` is another `RunTables` or iterable of
+run/frame pairs. Each fit frame must contain the same `x` and `y` columns used
+by the scatter and may contain the column named by `fit_annotation`. That text
+is shown when the fitted line is hovered. `one_to_one=True` adds a dashed 1:1
+line, gives both axes the same range, and locks their scale. The validation
+pages use this API for per-run equations, R-squared values, and sample sizes.
+
+Run labels are presentation-safe without changing their underlying identity.
+Long labels are shortened to unique legend/tab labels, while Plotly hovers and
+exported tab tooltips retain the full label. Scatter point and fit hovers also
+include the owning run name.
+
## Count and share behavior
`value_mode` has three values:
@@ -130,6 +157,22 @@ rows, missing-data cards, legends, and other layout helpers live in
`dashboard.rendering.layout`; numeric and column formatting lives in
`dashboard.rendering.tables`.
+The `dashboard.rendering` facade exports these non-plot helpers:
+
+| API | Purpose |
+|---|---|
+| `data_table()`, `to_pandas()` | Render run-aware tables or convert supported Polars/Pandas input at the presentation boundary. |
+| `format_numeric()`, `format_numeric_frame()` | Apply display-only numeric precision. |
+| `drop_index_columns()`, `column_titles()` | Remove serialized index artifacts and create human-readable column titles. |
+| `standardize_keys()` | Normalize a table iterable to common key/value column names. |
+| `selector_row()`, `control_row()`, `control_row_spacer()` | Build consistent page control layouts. |
+| `data_unavailable_card()` | Render the standard missing-data diagnostic card. |
+| `run_legend_entries()`, `run_legend_panes()` | Build run/color legend metadata or panes. |
+
+`column_title_metadata()` is available from
+`dashboard.rendering.tables` for serializer-aware title metadata, but is not
+part of the package-level facade.
+
## Testing charts
Test the figure instead of constructing a full Panel layout:
diff --git a/wiki/90-troubleshooting.md b/wiki/90-troubleshooting.md
index 6cabbeb..3b134e8 100644
--- a/wiki/90-troubleshooting.md
+++ b/wiki/90-troubleshooting.md
@@ -7,8 +7,9 @@ Use this chapter when a run, cache, page, or export is not behaving as expected.
1. Confirm the config path you ran.
2. Check the selected pipeline steps and dashboard mode in logs.
3. Check whether the issue appears in prepare, summarize, dashboard, or export.
-4. Inspect `//manifest.json` for the affected run (see the
- [cache layout](12-running-workflows.md#artifact-and-cache-paths)).
+4. Inspect `//manifest.json` for summary state and
+ `//prepared_tables/manifest.json` for final prepared/skimjoin
+ state (see the [cache layout](12-running-workflows.md#artifact-and-cache-paths)).
5. Run with `--explain-cache` to inspect reuse and rebuild decisions. If a
forced rebuild is needed, list only the affected stage in `pipeline.refresh`.
@@ -17,7 +18,7 @@ Use this chapter when a run, cache, page, or export is not behaving as expected.
| Symptom | Likely causes | First checks |
|---|---|---|
| Run missing from dashboard | Missing summary cache, label mismatch, config run omitted | `runs`, cache directories, log run keys |
-| Summary cache rebuilds unexpectedly | Input fingerprint changed, config digest changed, summary contract changed | the run manifest's summary-cache entries |
+| Summary cache rebuilds unexpectedly | Input fingerprint changed, upstream prepared identity changed, summary config changed, summary declaration changed | run-level summary manifest and `--explain-cache` |
| Page says data unavailable | Required summary missing, optional raw input absent, prepared column missing | page catalog and summary catalog |
| Counts look wrong | Weighting mode, sample rate, explicit weight columns | `summarize.weighting_modes`, prepared `finalweight` |
| Geography options missing | Geography disabled, land-use columns missing, aggregation config wrong | `zones`, `summarize.geography` |
@@ -46,7 +47,11 @@ uv run activitysim-viz --config local_config.yaml --refresh-caches
```
If only dashboard presentation changed, a refresh usually should not be needed.
-If raw inputs or prepare config changed, refresh both caches.
+Raw-file, skim-file, and relevant config identities are checked automatically;
+use a manual refresh only when deliberately overriding a valid cache decision.
+Prefer `pipeline.refresh` for reproducible runs. A prepare refresh necessarily
+invalidates skimjoin and summary output; a skimjoin refresh preserves
+`base_prepared_tables`; a summary refresh preserves final prepared data.
## Missing Page Data
@@ -70,8 +75,9 @@ Suppose Trip Mode opens but shows the standard unavailable card:
3. Open `//manifest.json` and inspect the summary entry. If the
summary is `unavailable`, read its recorded reason before rebuilding
anything.
-4. If a required prepared column is missing, inspect the same manifest's
- prepared-cache entry and the canonical column settings in `columns`.
+4. If a required prepared column is missing, inspect
+ `//prepared_tables/manifest.json`, the table schema, and the
+ canonical column settings in `columns`.
5. If the contract recently changed, rebuild the configured summarize step
with `pipeline.refresh: [summarize]`.
6. If the summary is present and valid, confirm the page's `columns=` request
@@ -109,7 +115,8 @@ If live mode works but export does not:
`self.selector(...)`.
3. Confirm affected content is registered with `self.section(...)`.
4. Check browser console errors.
-5. Try `?debug_export=1`.
+5. Inspect the adjacent `.diagnostics.json` sidecar.
+6. Try `?debug_export=1`.
Export cannot reproduce arbitrary Python callbacks. It can only switch among
serialized states and registered selector variants.
From 5a6950efe673e5c69539e6f79a6290773bc0bf7b Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Wed, 5 Aug 2026 16:23:44 -0400
Subject: [PATCH 14/27] Simplify wiki and README language throughout
---
README.md | 67 ++++-----
wiki/00-home.md | 30 ++--
wiki/01-architecture.md | 142 +++++++++---------
wiki/10-getting-started.md | 44 +++---
wiki/11-configuring-your-data.md | 58 +++----
wiki/12-running-workflows.md | 150 +++++++++---------
wiki/13-configuration-reference.md | 192 ++++++++++++------------
wiki/20-output-processor.md | 68 ++++-----
wiki/21-prepared-tables.md | 72 ++++-----
wiki/22-skimjoin.md | 75 +++++----
wiki/23-summary-functions.md | 94 ++++++------
wiki/24-summary-catalog.md | 116 +++++++-------
wiki/25-skimjoin-config-reference.md | 121 ++++++++-------
wiki/30-output-visualizer.md | 35 ++---
wiki/31-dashboard-pages.md | 94 ++++++------
wiki/32-figures-and-widgets.md | 144 +++++++++---------
wiki/33-dashboard-page-recipes.md | 47 +++---
wiki/34-html-export.md | 131 ++++++++--------
wiki/35-plotting-reference.md | 86 +++++------
wiki/36-html-export-schema.md | 71 +++++----
wiki/40-developer-workflows.md | 22 +--
wiki/41-data-extension-cookbook.md | 97 ++++++------
wiki/42-config-column-label-cookbook.md | 97 ++++++------
wiki/43-weighting-hosting-extensions.md | 159 ++++++++++----------
wiki/44-summary-function-cookbook.md | 89 ++++++-----
wiki/45-dashboard-extension-cookbook.md | 107 +++++++------
wiki/46-testing.md | 35 +++--
wiki/90-troubleshooting.md | 84 +++++------
wiki/99-glossary.md | 24 +--
29 files changed, 1278 insertions(+), 1273 deletions(-)
diff --git a/README.md b/README.md
index 3c49b76..dd49dbc 100644
--- a/README.md
+++ b/README.md
@@ -1,22 +1,22 @@
# ActivitySim Visualizer
-ActivitySim Visualizer turns [ActivitySim](https://activitysim.github.io/)
-outputs into an interactive dashboard for exploring one model run, comparing
-several runs side by side, or comparing model outputs to survey results.
+Use ActivitySim Visualizer to examine [ActivitySim](https://activitysim.github.io/)
+output in an interactive dashboard. You can examine one model run, compare
+multiple runs, or compare model output with survey results.
-It can:
+ActivitySim Visualizer can:
-- prepare and summarize ActivitySim household, person, tour, and trip outputs;
-- compare travel patterns, model choices, and validation measures across runs;
-- reuse cached results so subsequent launches are faster; and
-- serve a local dashboard or create a standalone HTML file for sharing.
+- prepare and summarize household, person, tour, and trip output;
+- compare travel patterns, model choices, and validation measures for multiple runs;
+- use valid cached results to decrease the start time; and
+- start a local dashboard or create a standalone HTML file.
## Quick Start
### 1. Install the project
-From the repository root, use `uv` to create the environment and install the
-locked dependencies:
+In the repository root, use `uv` to create the environment. This command also
+installs the locked dependencies:
```bash
uv sync --locked
@@ -30,8 +30,8 @@ uv sync --locked --link-mode=copy
### 2. Create a configuration
-Copy `config.yaml` to `local_config.yaml`. In the new file, update the entries
-under `runs` so they point to your ActivitySim output directories:
+Copy `config.yaml` to `local_config.yaml`. In the new file, set each `runs.dir`
+value to an ActivitySim output directory:
```yaml
runs:
@@ -42,11 +42,11 @@ runs:
```
The default file names are `final_households`, `final_persons`, `final_tours`,
-`final_trips`, `final_joint_tour_participants`, and `final_land_use`. Both CSV
-and Parquet inputs are supported.
+`final_trips`, `final_joint_tour_participants`, and `final_land_use`. The
+visualizer accepts CSV and Parquet input files.
-For a smaller example configuration and help with nonstandard files or zones,
-see [Getting Started](wiki/10-getting-started.md) and
+For a small example configuration and instructions for nonstandard files or
+zones, see [Getting Started](wiki/10-getting-started.md) and
[Configuring Your Data](wiki/11-configuring-your-data.md).
### 3. Start the visualizer
@@ -55,12 +55,11 @@ see [Getting Started](wiki/10-getting-started.md) and
uv run activitysim-viz --config local_config.yaml
```
-The first run prepares the inputs, builds the summary tables needed by the
-dashboard, and opens a local server at
-[http://localhost:5006](http://localhost:5006). Later runs reuse valid caches.
-Stop the server with `Ctrl+C`.
+The first execution prepares the input and builds the required summary tables. It
+then starts a local server at [http://localhost:5006](http://localhost:5006).
+Later executions use valid caches. To stop the server, press `Ctrl+C`.
-If something is missing or the first run fails, start with
+If data is missing or the first execution fails, use
[Troubleshooting](wiki/90-troubleshooting.md).
## How It Works
@@ -72,13 +71,13 @@ ActivitySim outputs
-> display a live dashboard or export standalone HTML
```
-The configuration selects the inputs, workflow steps, output location, and
-dashboard mode. Most users can keep using the same launch command and change
-the YAML when they want a different workflow.
+The configuration selects the input, workflow steps, output location, and
+dashboard mode. Use the same start command for each workflow. Change the YAML
+configuration to change the workflow.
| Goal | Where to learn more |
|---|---|
-| Use raw ActivitySim output folders | [Configuring Your Data](wiki/11-configuring-your-data.md#raw-activitysim-output) |
+| Use raw ActivitySim output directories | [Configuring Your Data](wiki/11-configuring-your-data.md#raw-activitysim-output) |
| Use already-prepared tables | [Already-Prepared Tables](wiki/11-configuring-your-data.md#already-prepared-tables) |
| Use dashboard-ready summary tables | [Dashboard-Ready Summary Tables](wiki/11-configuring-your-data.md#dashboard-ready-summary-tables) |
| Run only the processor | [Processor-Only Workflow](wiki/12-running-workflows.md#configure-a-processor-only-workflow) |
@@ -91,17 +90,17 @@ the YAML when they want a different workflow.
The [wiki home](wiki/00-home.md) is the main documentation index.
-For normal use, these three chapters cover the usual path:
+For standard use, read these chapters in sequence:
1. [Getting Started](wiki/10-getting-started.md)
2. [Configuring Your Data](wiki/11-configuring-your-data.md)
3. [Running Workflows](wiki/12-running-workflows.md)
-Additional user references:
+Other user references:
- [Output Visualizer](wiki/30-output-visualizer.md) explains the dashboard.
- [Dashboard Pages](wiki/31-dashboard-pages.md) lists the available analyses.
-- [HTML Export](wiki/34-html-export.md) covers offline sharing.
+- [HTML Export](wiki/34-html-export.md) explains how to create an offline file.
- [Summary Catalog](wiki/24-summary-catalog.md) documents every summary table.
- [Glossary](wiki/99-glossary.md) defines project terminology.
- [Troubleshooting](wiki/90-troubleshooting.md) covers common failures.
@@ -109,8 +108,8 @@ Additional user references:
## For Contributors
Start with [Architecture](wiki/01-architecture.md) and
-[Developer Workflows](wiki/40-developer-workflows.md). Task-specific guides are
-available for:
+[Developer Workflows](wiki/40-developer-workflows.md). Use these task-specific
+guides:
- [extending prepared data](wiki/41-data-extension-cookbook.md);
- [adding a summary function](wiki/44-summary-function-cookbook.md);
@@ -119,14 +118,14 @@ available for:
- [skim enrichment](wiki/22-skimjoin.md); and
- [testing](wiki/46-testing.md).
-Run focused tests while developing. The standard full test command is:
+Execute focused tests during development. To execute all tests, use this command:
```bash
uv run pytest --basetemp .pytest_tmp
```
-After changing summary declarations or dashboard page definitions, regenerate
-the code-backed wiki catalogs:
+If you change summary declarations or dashboard page definitions, regenerate
+the wiki catalogs from the code:
```bash
uv run python scripts/generate_wiki_catalogs.py
@@ -134,5 +133,5 @@ uv run python scripts/generate_wiki_catalogs.py
## License
-This project is licensed under the GNU General Public License v3.0. See
+The GNU General Public License v3.0 applies to this project. See
[`LICENSE.txt`](LICENSE.txt).
diff --git a/wiki/00-home.md b/wiki/00-home.md
index ddf390c..555ac2c 100644
--- a/wiki/00-home.md
+++ b/wiki/00-home.md
@@ -1,12 +1,12 @@
# ActivitySim Visualizer Wiki
-This wiki is the main documentation home for the ActivitySim Visualizer. It is
-written for two audiences:
+This wiki contains the main documentation for ActivitySim Visualizer. Use it
+for these tasks:
-- users who need to run the visualizer on ActivitySim outputs
-- developers who need to extend the processor, summaries, skimjoin, or dashboard
+- run the visualizer with ActivitySim output
+- extend the processor, summaries, skimjoin, or dashboard
-The short mental model:
+The main data flow is:
```text
ActivitySim outputs
@@ -16,24 +16,24 @@ ActivitySim outputs
-> live dashboard or standalone HTML export
```
-For the subsystem boundaries and complete repository map, see
+For the subsystem boundaries and the complete repository map, see
[01 - Architecture](01-architecture.md).
## I Am Using The Visualizer
-You only need three short chapters for normal use:
+For standard use, read these three chapters:
1. [Get a dashboard running](10-getting-started.md).
2. [Choose raw, prepared, or summary inputs](11-configuring-your-data.md).
3. [Configure a live, export, or processor workflow](12-running-workflows.md).
-Use [Troubleshooting](90-troubleshooting.md) when something is missing. The
-[Configuration Reference](13-configuration-reference.md) is there when you need
-an exact field or default; it is not required reading.
+Use [Troubleshooting](90-troubleshooting.md) when data is missing. Use the
+[Configuration Reference](13-configuration-reference.md) to find a field or a
+default value. You do not have to read the complete reference.
## I Am Extending The Visualizer
-| If you want to... | Read |
+| Task | Read |
|---|---|
| Find every main config field and option | [13 - Configuration Reference](13-configuration-reference.md) |
| Understand the Output Processor | [20 - Output Processor](20-output-processor.md) |
@@ -91,14 +91,14 @@ an exact field or default; it is not required reading.
## Generated Pages
-Some wiki sections are generated from code to keep reference material from
-drifting:
+The project generates some wiki sections from code. This process keeps the
+reference material consistent with the code:
- [24 - Summary Catalog](24-summary-catalog.md)
- the generated page catalog in [31 - Dashboard Pages](31-dashboard-pages.md)
-Regenerate them after changing summary declarations/contracts, dashboard page
-definitions, or page data requirements:
+Regenerate these sections after you change a summary declaration, a summary
+contract, a dashboard page definition, or a page data requirement:
```bash
uv run python scripts/generate_wiki_catalogs.py
diff --git a/wiki/01-architecture.md b/wiki/01-architecture.md
index daa524e..38b6fc3 100644
--- a/wiki/01-architecture.md
+++ b/wiki/01-architecture.md
@@ -6,12 +6,12 @@
2. Build and cache summary tables.
3. Render those summaries in a live Panel dashboard or a standalone HTML export.
-The codebase is organized around those jobs rather than around one monolithic app layer.
+The codebase has a separate subsystem for each job.
-The config surface is now intentionally split into top-level domains such as
-`pipeline`, `dashboard`, `display`, `summarize`, `segment`, and `skimjoin`.
-`runtime.config.load_config_from_yaml()` validates that canonical schema before
-any workflow code sees it. Removed and unknown keys fail with a focused error.
+The configuration has top-level sections such as `pipeline`, `dashboard`,
+`display`, `summarize`, `segment`, and `skimjoin`.
+`runtime.config.load_config_from_yaml()` validates the canonical schema before
+the workflow uses it. Removed keys and unknown keys cause a specific error.
## Main Subsystems
@@ -53,36 +53,36 @@ run.py
-> dashboard.export.write_export_html_document() for export mode
```
-`WorkflowPlan` is the single resolved execution plan passed into these
-operations. `run_prepare_workflow()` returns `PreparedRunsArtifact`, and
-`run_summary_workflow()` returns `SummaryRunsArtifact`. Cache policy stays in
-these runtime workflows; processor functions only transform tables.
+The runtime passes one resolved `WorkflowPlan` to these operations.
+`run_prepare_workflow()` returns `PreparedRunsArtifact`.
+`run_summary_workflow()` returns `SummaryRunsArtifact`. The runtime workflows
+control the cache policy. Processor functions only transform tables.
## Core Runtime Contracts
### `Config`
-`runtime.config.Config` is the normalized application configuration. The public
-import surface remains `runtime.config`, while the implementation now lives in
-the `runtime/config/` package.
+`runtime.config.Config` is the normalized application configuration. Import
+the public API from `runtime.config`. The implementation is in the
+`runtime/config/` package.
Treat it as the contract for:
-- which files are read
-- which logical pipeline steps are requested by default
-- which dashboard mode is used by default (`none`, `live`, `export`, `host`)
-- which materialized stages, if any, should be explicitly refreshed
-- how schema aliases are resolved
+- files that the application reads
+- logical pipeline steps that the application requests by default
+- default dashboard mode (`none`, `live`, `export`, `host`)
+- stored stages that require a refresh
+- rules to resolve schema aliases
- which weighting modes exist
-- which pages are enabled
-- how export selector requests are configured
+- enabled pages
+- export selector request configuration
-`dashboard.host` is a reserved placeholder for a future hosting integration.
-The schema accepts `account`, `app_id`, `title`, and `verify`, but the current
-runtime deliberately does not store or act on them.
+`dashboard.host` is reserved for a future hosting integration. The schema
+accepts `account`, `app_id`, `title`, and `verify`. The runtime does not store
+or use these values.
-If a new feature adds a config key or changes config behavior, update the README
-and the relevant wiki chapters in the same change.
+If a feature adds a configuration key or changes configuration behavior,
+update the README and the applicable wiki chapters in the same change.
`Config.pipeline` is the canonical home for workflow defaults. Today the
logical step names are:
@@ -93,43 +93,48 @@ logical step names are:
- `summarize`
- `dashboard`
-The runtime still executes three coarse workflow boundaries (`prepare`,
-`summarize`, `dashboard`). `skimjoin` currently resolves inside the prepare
-workflow, and `segment` currently resolves inside the summarize workflow.
+The runtime executes three main workflow boundaries: `prepare`, `summarize`,
+and `dashboard`. The runtime resolves `skimjoin` in the prepare workflow. It
+resolves `segment` in the summarize workflow.
### `RunData`
-`processor.models.RunData` is the prepared-data contract consumed by summary builders and prepared-data dashboard pages. Summary code should rely on canonical prepared columns rather than guessing raw ActivitySim column names directly. `processor/prepare/` is the layer that materializes those canonical fields and owns prepared-table cache helpers.
+`processor.models.RunData` is the prepared-data contract. Summary builders and
+prepared-data dashboard pages use this contract. Summary code must use
+canonical prepared columns. It must not estimate the names of raw ActivitySim
+columns. The `processor/prepare/` subsystem creates the canonical fields and
+contains the prepared-table cache helpers.
### `@summary` and the summary catalog
-Each persisted summary is declared beside its builder with `@summary(...)`. The
+Declare each persistent summary next to its builder with `@summary(...)`. The
declaration defines:
- the stable summary id used by dashboard pages
-- the CSV filename stem used in cache directories
+- the CSV file-name stem used in cache directories
- its ordered output schema and prepared-input prerequisites
-- whether it is built by default
+- default build status
-`processor.summarize.catalog` imports the owning domain modules explicitly,
-collects those declarations deterministically, and rejects duplicate ids.
-Successful builder results are validated for exact columns, order, and dtypes.
-Unexpected builder exceptions follow `summarize.failure_policy`: `record` keeps
-typed failure metadata for an interactive dashboard, while `error` is the
-fail-fast setting for validation and batch workflows.
+`processor.summarize.catalog` imports the applicable domain modules. It
+collects the declarations in a repeatable order and rejects duplicate IDs. The
+system validates the columns, column order, and data types of each successful
+builder result. The `summarize.failure_policy` setting controls unexpected
+builder exceptions. The `record` value keeps typed failure metadata for an
+interactive dashboard. The `error` value stops validation and batch workflows
+immediately.
### `DashboardPageDefinition` and `DashboardPage`
-Dashboard pages are registered with `@dashboard_page(...)` on the page class in
-`dashboard/pages/`. The decorator holds identity, navigation grouping, ordering,
-and the summary/prepared-data contract through `required_summary_ids`,
+Register a dashboard page with `@dashboard_page(...)` on its page class in
+`dashboard/pages/`. The decorator defines identity, navigation group, order,
+and the summary or prepared-data contract through `required_summary_ids`,
`optional_summary_ids`, `prepared_data_mode`, and `required_prepared_tables`.
`dashboard.page_base` is the small public facade. Lifecycle, declarations,
diagnostics, feature composition, data access, and grouped navigation live in
separate implementation modules.
-Page authors are expected to:
+Page authors must:
- implement `build_page()` to declare selectors, features, sections, and layout
- give selectors an option provider and default policy when their domain is dynamic
@@ -137,10 +142,9 @@ Page authors are expected to:
- memoize chart-ready transformations with `self.query(...)`
- keep section render methods to lookup/query/render
-Large controllers may keep their registered page module as a compatibility
-facade and compose page-local implementation mixins from a private `_/`
-package. This convention, its constraints, and its distinction from
-`PageFeature` are documented in
+Large controllers can keep the registered page module as a compatibility
+facade. They can use page-local implementation mixins from a private `_/`
+package. For the rules and the difference from `PageFeature`, see
[Figures And Widgets](32-figures-and-widgets.md#sections-and-features).
The framework now owns:
@@ -154,10 +158,10 @@ The framework now owns:
- export selector metadata
- export region metadata
-That means live refresh behavior and export behavior both derive from the same selector/section registration graph rather than from separate page metadata declarations.
+The same selector and section registration graph controls live refresh and
+export behavior. Separate page metadata does not control these behaviors.
-The shared helper layer under `dashboard/helpers/` is now part of that page
-authoring model:
+The page authoring model includes the shared helpers in `dashboard/helpers/`:
- `category_helpers.py` centralizes selector domains, labels, and category completion
- `geography_helpers.py` centralizes geography normalization, option discovery, and filters
@@ -165,22 +169,21 @@ authoring model:
- `time_distance_helpers.py` centralizes repeated time-bin and distance-bin behavior
- `comparison_helpers.py` centralizes percent-error formatting and base-run comparisons
-For page-local table shaping, `dashboard.data_access.RunTables` applies one
-fluent query to every run while preserving run labels. Pages should prefer its
+For page-local table changes, `dashboard.data_access.RunTables` applies one
+query to every run and keeps the run labels. Pages must use its
`where`, `with_columns`, `group`, `select`, `sort`, `join`, `requiring`,
-`drop_empty`, and `map` operations over open-coded loops through
-run/dataframe pairs.
+`drop_empty`, and `map` operations when possible. Do not write equivalent loops
+through run and data frame pairs.
-The skim pages share their family-specific model/query service while exposing
-small summary and distribution features. This is the reference pattern for
-logic reusable within one page family but not broad enough for
-`dashboard/helpers/`.
+The skim pages use a model and query service for their page family. Each page
+provides small summary and distribution features. Use this pattern for logic
+that one page family shares. Put more general logic in `dashboard/helpers/`.
## Public Python APIs
-Import through these facades when extending or embedding the visualizer. Files
-not exported by a facade are implementation details unless a focused cookbook
-explicitly identifies them as an extension point.
+Import these facades when you extend or embed the visualizer. A file that a
+facade does not export is an implementation detail. A cookbook can identify an
+exception as an extension point.
| Import surface | Public contract |
|---|---|
@@ -198,25 +201,24 @@ The normalized config value objects exported alongside `Config` are
`ExportHTMLSettings`, `ExportSelectorRequest`,
`PrepareNonMotorizedDistanceSkimSettings`, `SegmentationDefinition`,
`PreparedColumnSegmentationSource`, `CsvLookupSegmentationSource`, and
-`StudentTypeConfig`. They are read-only runtime contracts; user input still
-enters through YAML normalization rather than by manually assembling a
-`Config`.
+`StudentTypeConfig`. They are read-only runtime contracts. YAML normalization
+supplies user input. Do not assemble a `Config` manually.
The workflow facade also exports `effective_processor_config()`,
`run_entries_with_keys()`, `prepared_cache_root()`, `summary_cache_root()`,
-`prune_summary_runs()`, and `prune_summary_artifact()` for embedding code that
-needs the same identity and consumer-pruning behavior as `run.py`. The
+`prune_summary_runs()`, and `prune_summary_artifact()`. Embedding code can use
+them to get the same identity and removal behavior as `run.py`. The
dashboard facade exports `DashboardPreparedRunProvider` and
`DashboardSummarySeries`; the page-base facade exports the typed
`RegisteredPageSelector`, `RegisteredPageSection`, and `SectionContent`
declaration records.
-The page-facing `PageData`/`RunTables` API is documented in chapter 32, chart
-keywords in chapter 35, the `@summary` contract in chapter 23, and workflow
-arguments and artifacts in the subsystem sections above. Public code should
-pass an explicit `WorkflowPlan` when it needs behavior different from the
-loaded config; the plan records logical steps, collapsed runtime boundaries,
-dashboard mode, and refresh targets.
+Chapter 32 describes the page-facing `PageData` and `RunTables` API. Chapter 35
+describes chart keywords. Chapter 23 describes the `@summary` contract. The
+subsystem sections above describe workflow arguments and artifacts. Public code must pass
+an explicit `WorkflowPlan` when the required behavior differs from the loaded
+configuration. The plan records logical steps, runtime boundaries, dashboard
+mode, and refresh targets.
## Repository Map
diff --git a/wiki/10-getting-started.md b/wiki/10-getting-started.md
index e9f4c07..5775013 100644
--- a/wiki/10-getting-started.md
+++ b/wiki/10-getting-started.md
@@ -1,6 +1,6 @@
# 10 - Getting Started
-This is the shortest path from a clone to a local dashboard.
+Use this procedure to start a local dashboard from a repository clone.
## 1. Install
@@ -18,8 +18,8 @@ uv sync --locked --link-mode=copy
## 2. Create A Small Config
-Create `local_config.yaml`. This file defines both the inputs and what the run
-should produce:
+Create `local_config.yaml`. This file defines the input and the required
+output:
```yaml
root: artifacts
@@ -46,18 +46,18 @@ dashboard:
output_path: exports/dashboard.html
```
-Change the two `dir` values to real ActivitySim output folders. The default
+Set the two `dir` values to ActivitySim output directories. The default
input names are `final_households`, `final_persons`, `final_tours`,
-`final_trips`, `final_joint_tour_participants`, and `final_land_use`; each may be
-CSV or Parquet.
+`final_trips`, `final_joint_tour_participants`, and `final_land_use`. Each input
+file can be CSV or Parquet.
If your files have different names, read
[File Names](11-configuring-your-data.md#raw-activitysim-output).
-`root` is the visualizer's artifact location. Summary caches are written below
-it, and relative export paths resolve below it. Keep the export path configured
-even for a live workflow; switching from a live dashboard to an HTML file then
-requires changing only `pipeline.dashboard_mode` from `live` to `export`.
+`root` is the artifact directory. The visualizer writes summary caches in this
+directory. It also resolves relative export paths from this directory. Keep
+the export path in the configuration for a live workflow. You can then create
+HTML by changing only `pipeline.dashboard_mode` from `live` to `export`.
## 3. Run The Config
@@ -65,23 +65,23 @@ requires changing only `pipeline.dashboard_mode` from `live` to `export`.
uv run activitysim-viz --config local_config.yaml
```
-The first run prepares data, builds summaries, and starts the dashboard at
-[http://localhost:5006](http://localhost:5006). Later runs reuse valid caches.
+The first execution prepares data and builds summaries. It then starts the dashboard
+at [http://localhost:5006](http://localhost:5006). Later executions use valid caches.
-Stop the server with `Ctrl+C`.
+To stop the server, press `Ctrl+C`.
-Use this same command for live dashboards, HTML exports, and processor-only
-workflows. Change the `pipeline` and `dashboard` sections in the config instead
-of maintaining different launch commands.
+Use this command for live dashboards, HTML exports, and processor-only
+workflows. Change the `pipeline` and `dashboard` sections to select the
+workflow.
-## If The First Run Fails
+## If the first execution fails
-Check these first:
+Do these checks:
-1. each `runs[*].dir` exists;
-2. the expected tables are present as `.csv` or `.parquet`;
-3. `zones.use_maz`, `maz_col`, and `taz_col` match the model; and
-4. the log names the missing file or column.
+1. Make sure that each `runs[*].dir` exists.
+2. Make sure that each required table is a `.csv` or `.parquet` file.
+3. Make sure that `zones.use_maz`, `maz_col`, and `taz_col` agree with the model.
+4. Find the missing file or column in the log.
Then use [Troubleshooting](90-troubleshooting.md).
diff --git a/wiki/11-configuring-your-data.md b/wiki/11-configuring-your-data.md
index 8390ddb..caa7fa5 100644
--- a/wiki/11-configuring-your-data.md
+++ b/wiki/11-configuring-your-data.md
@@ -1,11 +1,11 @@
# 11 - Configuring Your Data
-Most users only need to choose an input type and name their runs. Use one of the
-three patterns below.
+Select an input type and give each run a label. Use one of these three
+configurations.
## Raw ActivitySim Output
-Use this when you have normal ActivitySim output folders:
+Use this configuration for standard ActivitySim output directories:
```yaml
root: artifacts
@@ -17,7 +17,7 @@ runs:
label: Build
```
-The label is what appears in the dashboard.
+The dashboard shows this label.
### File Names
@@ -33,8 +33,8 @@ files:
land_use: final_land_use
```
-A bare name accepts either `.parquet` or `.csv`. Override one unusual run with
-`file_map`:
+A name without an extension selects a `.parquet` or `.csv` file. Use `file_map`
+to set nonstandard file names for one run:
```yaml
runs:
@@ -49,7 +49,8 @@ runs:
### Column Names
-If a model uses different column names, list the candidates in preferred order:
+If a model uses different column names, list the possible names in order of
+preference:
```yaml
columns:
@@ -58,13 +59,14 @@ columns:
trip_mode: mode
```
-Prepare converts the selected source to the visualizer's canonical column. See
-chapter 13 for the [complete column list](13-configuration-reference.md#columns).
+The prepare step copies the selected source to the canonical visualizer
+column. See the [complete column list](13-configuration-reference.md#columns)
+in chapter 13.
## Already-Prepared Tables
-Use `prepared_table_map` for canonical tables that were prepared, skimjoined,
-or filtered elsewhere:
+Use `prepared_table_map` for canonical tables from a different process. This
+process can prepare, skimjoin, or filter the tables:
```yaml
runs:
@@ -77,14 +79,14 @@ runs:
land_use: prepared/land_use.parquet
```
-Paths must end in `.csv` or `.parquet` and are relative to the config file.
-These tables must already use the canonical prepared columns expected by
-summaries. Raw prepare and integrated skimjoin are skipped for this run.
+Each path must end in `.csv` or `.parquet`. A relative path starts from the
+configuration file directory. The tables must contain the canonical prepared
+columns that the summaries require. The visualizer does not run raw prepare or
+integrated skimjoin for this run.
## Dashboard-Ready Summary Tables
-Use `summary_table_map` when another process has already produced registered
-summary tables:
+Use `summary_table_map` for registered summary tables from a different process:
```yaml
runs:
@@ -94,13 +96,14 @@ runs:
traffic_count_comparisons: summaries/traffic_counts.parquet
```
-Keys must appear in the [Summary Catalog](24-summary-catalog.md). Files must
-match the registered columns exactly. A run may contain only outside summaries,
-or they may override selected summaries generated from raw/prepared data.
+Each key must occur in the [Summary Catalog](24-summary-catalog.md). Each file
+must have the registered columns in the specified order. A run can contain
+only external summaries. External summaries can also replace selected
+summaries from raw or prepared data.
## Weights
-The normal modes are configured with:
+Configure the standard modes with:
```yaml
summarize:
@@ -118,11 +121,12 @@ runs:
trip_weight_col: trip_weight
```
-Otherwise prepare uses a configured sample-rate column when available, then
-falls back to `1.0`.
+If you do not set weight columns, prepare uses the configured sample-rate
+column when it is available. If this column is not available, prepare uses
+`1.0`.
-If the same output tables contain an additional set of weights, add a named
-column mode instead of duplicating the run or writing Python:
+If the output tables contain more weights, add a named column mode. Do not
+duplicate the run or write Python for this configuration:
```yaml
weighting:
@@ -138,8 +142,10 @@ summarize:
weighting_modes: [weighted, unweighted, calibrated]
```
-The named sources are validated and propagated to tours, days, vehicles, and
-skimjoin sidecars as appropriate. See [43 - Weighting And Hosting Extensions](43-weighting-hosting-extensions.md#worked-example-add-a-weighting-mode) for the exact rules.
+The visualizer validates the named sources. It copies the weights to applicable
+tours, days, vehicles, and skimjoin sidecar tables. See
+[43 - Weighting And Hosting Extensions](43-weighting-hosting-extensions.md#worked-example-add-a-weighting-mode)
+for the rules.
## Zones
diff --git a/wiki/12-running-workflows.md b/wiki/12-running-workflows.md
index e81041e..23b3a6c 100644
--- a/wiki/12-running-workflows.md
+++ b/wiki/12-running-workflows.md
@@ -1,14 +1,13 @@
# 12 - Running Workflows
-The normal user experience is config-driven. Keep one launch command:
+The configuration controls the standard workflow. Use one start command:
```bash
uv run activitysim-viz --config local_config.yaml
```
-The config decides which work runs, where artifacts are stored, and whether the
-result is a live dashboard or an HTML file. Command-line flags are intended for
-development and one-off diagnostics, not normal operation.
+The configuration selects the work, the artifact location, and the dashboard
+mode. Use command-line flags only for development or one diagnostic execution.
## The Three Main Steps
@@ -18,22 +17,22 @@ prepare -> summarize -> dashboard
- **Prepare** reads raw outputs and creates canonical prepared tables.
- **Summarize** creates the smaller tables used by dashboard pages.
-- **Dashboard** serves the live application or writes standalone HTML.
+- **Dashboard** starts the live application or writes standalone HTML.
Skimjoin runs inside prepare when selected. Segmentation runs with summarize.
-These are requested workflow boundaries, not isolated commands. In particular,
-`summarize` must have prepared data: it reuses a valid prepared cache or builds
-prepared data from the configured raw/prepared inputs when the cache is missing
-or stale. Adding `prepare` explicitly runs and persists that boundary first;
-the summarize boundary then reuses the in-memory or cached result rather than
-preparing a second time.
+These steps are workflow boundaries, not independent commands. The `summarize`
+step requires prepared data. It uses a valid prepared cache when one is
+available. If the cache is missing or stale, it builds prepared data from the
+configured raw or prepared input. If you add `prepare`, the runtime completes
+and stores that step first. The summarize step then uses the result in memory
+or in the cache. It does not prepare the data a second time.
| Requested step | What it guarantees | Prerequisites resolved automatically |
|---|---|---|
-| `prepare` | Prepared tables are loaded/built and cached. | Raw files or `prepared_table_map`. |
-| `summarize` | Default registered summaries are loaded/built and cached. | Prepared data is loaded/built as needed. |
-| `dashboard` | Existing summary caches are loaded and displayed/exported. | No summaries are built; required caches must exist or come from `summary_table_map`. |
+| `prepare` | The runtime loads or builds prepared tables and writes the cache. | Raw files or `prepared_table_map`. |
+| `summarize` | The runtime loads or builds default registered summaries and writes the cache. | The runtime loads or builds prepared data as necessary. |
+| `dashboard` | The runtime loads existing summary caches and shows or exports them. | The runtime does not build summaries. Required caches must exist or come from `summary_table_map`. |
## Configure A Live Workflow
@@ -56,8 +55,8 @@ dashboard:
- trip_summaries
```
-This builds missing or stale artifacts, reuses valid caches, and starts the
-dashboard. `dashboard.live.pages` controls which page groups are available.
+This workflow builds missing or stale artifacts. It uses valid caches and
+starts the dashboard. `dashboard.live.pages` selects the available page groups.
## Configure An HTML Export
@@ -74,14 +73,14 @@ dashboard:
output_path: exports/dashboard.html
```
-The configured output is `artifacts/exports/dashboard.html`: relative export
-paths resolve below `root`. Use an absolute path when the file must be written
-elsewhere. Page and selector choices are covered in
+This configuration writes `artifacts/exports/dashboard.html`. Relative export
+paths start from `root`. Use an absolute path to write the file to a different
+location. For page and selector choices, see
[HTML Export](34-html-export.md).
## Configure A Processor-Only Workflow
-Build prepared tables and summaries without opening or exporting a dashboard:
+Build prepared tables and summaries without a dashboard:
```yaml
pipeline:
@@ -99,32 +98,32 @@ Other focused workflows use the same fields:
| Open a live dashboard from existing caches | `[dashboard]` | `live` |
| Export HTML from existing caches | `[dashboard]` | `export` |
-For loose dashboard-ready CSV or Parquet inputs, configure
-`runs[*].summary_table_map`; do not treat them as cache directories.
+For dashboard-ready CSV or Parquet files, configure
+`runs[*].summary_table_map`. Do not use the files as cache directories.
## Pipeline Rules
-Available logical steps are `prepare`, `skimjoin`, `segment`, `summarize`, and
-`dashboard`. Dashboard must be last. `skimjoin` requires `prepare`; `segment`
+The logical steps are `prepare`, `skimjoin`, `segment`, `summarize`, and
+`dashboard`. Put `dashboard` last. `skimjoin` requires `prepare`. `segment`
requires `summarize`.
-The default when `pipeline.steps` is omitted is `[summarize, dashboard]`.
-That default still prepares raw inputs when a valid prepared cache is not
-available. Include `prepare` explicitly when prepared-cache creation is itself
-an intended, visible stage or when `skimjoin` is enabled.
+If you omit `pipeline.steps`, the default is `[summarize, dashboard]`. This
+default prepares raw input when a valid prepared cache is not available. Add
+`prepare` when cache creation must be a visible step. Also add `prepare` when
+you enable `skimjoin`.
Dashboard modes:
- `live`: local Panel server;
- `export`: standalone HTML;
- `none`: no dashboard; and
-- `host`: reserved extension point that currently logs a warning and executes
- the normal live server; it does not publish to a hosting provider.
+- `host`: reserved extension point. It writes a warning to the log and starts
+ the standard live server. It does not publish to a hosting provider.
## Artifact And Cache Paths
-Prepared and summary caches live under the configured `root`. Each run has a
-manifest describing its inputs and config identity.
+The visualizer stores prepared and summary caches under the configured `root`.
+Each run has a manifest that describes its input and configuration identity.
Set `root` once for the workflow:
@@ -132,7 +131,7 @@ Set `root` once for the workflow:
root: D:\activitysim_visualizer\regional_comparison
```
-For two runs labeled `Base` and `Build`, the normal layout is:
+For two runs labeled `Base` and `Build`, the standard layout is:
```text
regional_comparison/
@@ -157,16 +156,16 @@ regional_comparison/
manifest.json
```
-The run-level `manifest.json` belongs to the summary bundle. Each prepared
-cache has its own manifest inside its table directory. A prepare-only workflow
-therefore writes `prepared_tables/manifest.json` but does not create the
-run-level summary manifest.
+The run-level `manifest.json` describes the summary bundle. Each prepared cache
+has a manifest in its table directory. Thus, a prepare-only workflow writes
+`prepared_tables/manifest.json`. It does not create the run-level summary
+manifest.
-When skimjoin is enabled, `base_prepared_tables/` contains a second prepared
-manifest and the canonical tables before skim enrichment. The enriched tables,
-skimjoin reports, and optional hypothetical sidecars remain under
-`prepared_tables/`, so summary and dashboard consumers continue to use the
-same final path:
+When you enable skimjoin, `base_prepared_tables/` contains a second prepared
+manifest. It also contains the canonical tables before skim enrichment. The
+visualizer stores enriched tables, skimjoin reports, and optional hypothetical
+sidecar tables under `prepared_tables/`. Thus, summary and dashboard consumers
+use the same final path:
```text
base/
@@ -186,21 +185,21 @@ base/
.csv
```
-Segmented summary CSVs are nested below
-`summary_tables//segments///` and
-are described by the run-level summary manifest.
+The visualizer stores segmented summary CSV files in
+`summary_tables//segments///`. The
+run-level summary manifest describes these files.
-The run-key directory is a filesystem-safe lowercase slug of the run label.
-For example, `Build Scenario` becomes `build-scenario`. Colliding labels receive
-ordered suffixes such as `build-1` and `build-2`; avoid duplicate labels because
-reordering them changes which run receives each suffix.
+The run-key directory uses a lowercase, file-system-safe form of the run label.
+For example, `Build Scenario` becomes `build-scenario`. Duplicate labels get
+ordered suffixes such as `build-1` and `build-2`. Do not use duplicate labels.
+If you change their order, you change the suffix for each run.
Relative paths in `dashboard.export.output_path` resolve below this directory.
Input paths follow the path rules documented in
[Configuration Reference](13-configuration-reference.md#reading-this-reference).
-Valid caches are reused automatically. To deliberately rebuild every
-materialized stage used by the configured steps, temporarily set:
+The visualizer automatically uses valid caches. To rebuild each stored stage
+for the configured steps, temporarily set:
```yaml
pipeline:
@@ -209,10 +208,9 @@ pipeline:
refresh: all
```
-Return `refresh` to `[]` after the forced rebuild. To rebuild summaries while
-preserving prepared and skimjoined data, use `refresh: [summarize]`.
-Presentation-only changes such as labels, colors, or enabled pages normally do
-not require cache rebuilding.
+Set `refresh` to `[]` after the rebuild. To rebuild summaries and keep prepared
+and skimjoined data, use `refresh: [summarize]`. Changes to labels, colors, or
+enabled pages do not usually require a cache rebuild.
The refresh targets are stage-aware:
@@ -222,24 +220,24 @@ The refresh targets are stage-aware:
| `skimjoin` | `base_prepared_tables` | enriched `prepared_tables`, summaries |
| `summarize` | final `prepared_tables` | stale/default summaries and segmented summaries |
-Normal reuse is also content-aware. Prepared manifests record resolved raw
-input identities, including path, size, and modification time, plus the
-prepare/skimjoin config identity and skim input identities. The summary
-manifest records its upstream prepared-manifest identity, summary config, and
-per-summary declaration digest. Consequently, a changed raw file invalidates
-prepare and downstream output, a changed skim input can rebuild only skimjoin
-and downstream output, and a changed summary declaration can rebuild only the
-affected summary tables while reusing compatible tables in the bundle.
+Cache reuse also depends on file content information. Prepared manifests record
+the path, size, and modification time of each raw input. They also record the
+prepare, skimjoin, and skim input identities. The summary manifest records the
+prepared-manifest identity, summary configuration, and declaration digest for
+each summary. A changed raw file invalidates prepare and its later output. A
+changed skim input can rebuild only skimjoin and its later output. A changed
+summary declaration can rebuild only the applicable summary tables. The
+visualizer keeps compatible tables in the bundle.
-Use `--explain-cache` to print the per-run reuse/rebuild decisions and exit
-without loading tables, deleting caches, or writing artifacts. The report shows
-`REUSE`, `REBUILD`, `RUN`, or `DISABLED` for prepare, skimjoin, summarize, and
-dashboard, with the cache-validation reason when available.
+Use `--explain-cache` to print the cache decision for each run. The command
+then exits without table loads, cache deletions, or artifact writes. The report
+shows `REUSE`, `REBUILD`, `RUN`, or `DISABLED` for each workflow step. It also
+shows the cache-validation reason when one is available.
## CLI Overrides
-CLI flags override the configured workflow for one invocation. Users should
-normally change YAML so the intended workflow remains reproducible.
+Command-line flags override the configured workflow for one execution. For standard
+operation, change the YAML so that you can reproduce the workflow.
| Flag | Behavior |
|---|---|
@@ -261,13 +259,13 @@ normally change YAML so the intended workflow remains reproducible.
| `--no-show` | Start the live server without opening a browser. |
| `--explain-cache` | Print the cache plan and exit without executing it. |
-If any of `--prepare`, `--summarize`, or `--dashboard` is present, those flags
-replace `pipeline.steps` with the selected coarse boundaries. They do not
-implicitly enable the logical `skimjoin` or `segment` steps. `--from-csvs`
-cannot be combined with processor steps or `--write-csvs`; `--write-csvs` and
-`--skip-summary-cache-write` require summarize. Refresh flags require the
-corresponding processor boundary. If the config omits dashboard, pair
-`--export-html` with `--dashboard` to select it.
+If you use `--prepare`, `--summarize`, or `--dashboard`, these flags replace
+`pipeline.steps` with the selected main boundaries. They do not enable the
+`skimjoin` or `segment` steps. Do not combine `--from-csvs` with processor steps
+or `--write-csvs`. The `--write-csvs` and `--skip-summary-cache-write` flags
+require summarize. Each refresh flag requires its applicable processor
+boundary. If the configuration omits dashboard, use `--export-html` with
+`--dashboard`.
## Related Chapters
diff --git a/wiki/13-configuration-reference.md b/wiki/13-configuration-reference.md
index 5730766..8879150 100644
--- a/wiki/13-configuration-reference.md
+++ b/wiki/13-configuration-reference.md
@@ -1,16 +1,17 @@
# 13 - Configuration Reference
-This page is the field-by-field reference for the main ActivitySim Visualizer
-config file. For a shorter orientation, start with
-[11 - Configuring Your Data](11-configuring-your-data.md). The canonical
-example is [`config.yaml`](../config.yaml).
+This page describes each field in the main ActivitySim Visualizer configuration
+file. For an introduction, read
+[11 - Configuring Your Data](11-configuring-your-data.md). See
+[`config.yaml`](../config.yaml) for the canonical example.
-This page documents the current canonical config layout. Unknown and removed
-keys fail validation and, where possible, name their canonical replacement.
+This page describes the current canonical configuration. Unknown and removed
+keys cause a validation error. When possible, the error gives the canonical
+replacement.
## Reading This Reference
-Path resolution depends on the field:
+The field type controls the base directory for a relative path:
| Field family | Relative to | Notes |
|---|---|---|
@@ -18,12 +19,12 @@ Path resolution depends on the field:
| `runs[*].dir` | main config directory | Raw ActivitySim output directory. |
| `files.*`, `runs[*].file_map.*` | the resolved run directory | File stems may omit `.parquet` or `.csv`; Parquet is tried before CSV. |
| `fallback_files.*`, `prepared_table_map.*`, `summary_table_map.*` | main config directory | Values must include `.parquet` or `.csv`. |
-| `prepare.distance_skim.file`, `runs[*].skim_file` | the resolved run directory | A relative legacy distance-skim path is resolved separately for each run. |
+| `prepare.distance_skim.file`, `runs[*].skim_file` | the resolved run directory | The loader resolves a relative legacy distance-skim path separately for each run. |
| other main-config enrichment, lookup, and skimjoin paths | main config directory | Includes `prepare.time_periods.network_los_file`, `prepare.non_motorized_distance_skim.file`, segmentation CSVs, and `skimjoin.defaults.*`. |
| paths inside the standalone skimjoin config | standalone skimjoin config directory | See chapter 25. |
| `dashboard.export.output_path` | resolved `root` | Absolute output paths remain absolute. |
-Cache impact uses these labels:
+The Impact columns use these terms:
| Impact | Meaning |
|---|---|
@@ -53,8 +54,8 @@ runs:
### Prepared-Table Workflow
-Use `prepared_table_map` when a run should skip raw prepare and load canonical
-prepared tables directly.
+Use `prepared_table_map` to load canonical prepared tables directly. The run
+does not do the raw prepare step.
```yaml
pipeline:
@@ -158,17 +159,18 @@ summarize:
weighting_modes: [weighted, unweighted, calibrated]
```
-Each definition needs at least one supported source table. Named columns are
-validated against every prepared run. See the [weighting cookbook](43-weighting-hosting-extensions.md#worked-example-add-a-weighting-mode).
+Each definition requires at least one supported source table. The visualizer
+validates named columns for each prepared run. See the
+[weighting cookbook](43-weighting-hosting-extensions.md#worked-example-add-a-weighting-mode).
## `extensions`
-This is the advanced path for calculations that cannot be represented by
-`weighting.modes` column selection.
+Use this advanced method for calculations that `weighting.modes` column
+selection cannot define.
| Field | Type | Default | Impact | Notes |
|---|---|---|---|---|
-| `modules` | list of strings | `[]` | Summary | Importable modules that define `register_weighting_modes(registry)`. Installed weighting entry points are discovered separately. |
+| `modules` | list of strings | `[]` | Summary | Importable modules that define `register_weighting_modes(registry)`. The loader finds installed weighting entry points separately. |
| `settings` | mapping | `{}` | Summary | Arbitrary YAML settings available to transforms as `config.extension_settings`. Included in summary cache identity. |
```yaml
@@ -188,8 +190,8 @@ See [Advanced: Custom Weight Calculations](43-weighting-hosting-extensions.md#ad
| Field | Type | Default | Allowed values | Impact | Notes |
|---|---|---|---|---|---|
| `steps` | non-empty list of strings | `[summarize, dashboard]` | `prepare`, `skimjoin`, `segment`, `summarize`, `dashboard` | Runtime | Steps must be lowercase, unique, and valid. `skimjoin` requires `prepare`; `segment` requires `summarize`; `dashboard` must be last when present. |
-| `dashboard_mode` | string | `live` | `none`, `live`, `export`, `host` | Runtime, Presentation | Controls what the dashboard step does. `host` is reserved and currently warns, then falls back to the ordinary live server; it does not publish an application. |
-| `refresh` | list of strings or `all` | `[]` | `prepare`, `skimjoin`, `summarize`, `all` | Runtime | Forces only the named materialized stages to rebuild. Upstream refreshes invalidate enabled downstream stages. Leave empty for normal cache-aware operation. |
+| `dashboard_mode` | string | `live` | `none`, `live`, `export`, `host` | Runtime, Presentation | Controls the dashboard step. `host` writes a warning and uses the standard live server. It does not publish an application. |
+| `refresh` | list of strings or `all` | `[]` | `prepare`, `skimjoin`, `summarize`, `all` | Runtime | Forces only the named stored stages to rebuild. An upstream refresh invalidates enabled downstream stages. Leave empty for standard cache-aware operation. |
```yaml
pipeline:
@@ -198,43 +200,45 @@ pipeline:
refresh: []
```
-`segment` is materialized within summary bundles, so use `refresh: [summarize]`
-to rebuild segmented outputs. Dashboard rendering has no persistent processor
-cache and is not a refresh target.
+The visualizer stores `segment` output in summary bundles. Use
+`refresh: [summarize]` to rebuild segmented output. Dashboard rendering does
+not have a persistent processor cache. Thus, dashboard is not a refresh target.
## `runs`
-Each run entry describes one scenario. `label` is strongly recommended because
-it becomes the display name and helps cache/debug output remain understandable.
+Each run entry describes one scenario. Always set `label` when possible. This
+value becomes the display name and identifies the run in cache and debug output.
| Field | Type | Default | Impact | Notes |
|---|---|---|---|---|
-| `dir` | path string | none | Prepare | Raw ActivitySim output folder. Required unless the run is supplied by `prepared_table_map`, by `summary_table_map` alone, or both. |
+| `dir` | path string | none | Prepare | Raw ActivitySim output directory. Set this field unless `prepared_table_map` or `summary_table_map` supplies the run. |
| `label` | string | folder name or `run` fallback | Summary, Presentation | Dashboard and cache-facing run name. Keep stable across reruns. |
| `file_map` | mapping | inherits top-level `files` | Prepare | Per-run raw file stem overrides. Cannot be combined with `prepared_table_map`. |
| `prepared_table_map` | mapping | none | Prepare, Summary | Explicit `.parquet` or `.csv` canonical prepared tables. Skips raw prepare for that run. |
-| `summary_table_map` | mapping | none | Summary, Presentation | Registered summary IDs mapped to dashboard-ready `.parquet` or `.csv` files. May be used alone or override generated summaries. |
+| `summary_table_map` | mapping | none | Summary, Presentation | Maps registered summary IDs to dashboard-ready `.parquet` or `.csv` files. Use it alone or to replace generated summaries. |
| `skim_file` | path string | `prepare.distance_skim.file` | Prepare, Summary | Per-run legacy distance-skim override. Relative paths resolve from this run's `dir`. |
| `skimjoin` | mapping | inherits global `skimjoin` | Prepare | Per-run skimjoin path and hypothetical-sidecar overrides. |
| `hh_weight_col` | string | none | Prepare, Summary | Household source for the run's primary `weighted` mode. |
| `person_weight_col` | string | none | Prepare, Summary | Person source for the run's primary `weighted` mode. |
| `trip_weight_col` | string | none | Prepare, Summary | Trip source for the run's primary `weighted` mode. |
-Allowed `file_map` and `prepared_table_map` table ids are:
+You can use these table IDs in `file_map` and `prepared_table_map`:
`households`, `persons`, `day`, `tours`, `trips`, `vehicles`,
`joint_tour_participants`, `land_use`.
-`prepared_table_map` paths must include `.parquet` or `.csv`. Relative paths are
-resolved relative to the config file.
+Each `prepared_table_map` path must include `.parquet` or `.csv`. A relative
+path starts from the configuration file directory.
-`summary_table_map` uses registered IDs from the summary catalog. Its paths must
-also end in `.parquet` or `.csv` and are resolved relative to the config file.
+`summary_table_map` uses registered IDs from the summary catalog. Each path must
+end in `.parquet` or `.csv`. A relative path starts from the configuration file
+directory.
### Run Labels And Run Keys
-`label` is the dashboard name. Its filesystem-safe lowercase slug is the run
-key used by cache directories, manifests, and settings such as
+`label` is the dashboard name. The visualizer converts it to a lowercase,
+file-system-safe run key. Cache directories, manifests, and settings use this
+key. One example is
`prepare.vot_bins.mappings`:
| Label | Run key |
@@ -243,9 +247,9 @@ key used by cache directories, manifests, and settings such as
| `Build Scenario` | `build-scenario` |
| `2026 / Toll Test` | `2026-toll-test` |
-If normalized labels collide, every colliding key receives an ordered numeric
-suffix (`build-1`, `build-2`). Keep labels unique and stable: changing their
-order can change those suffixes and therefore cache/mapping identity.
+If normalized labels are equal, each key gets an ordered numeric suffix
+(`build-1`, `build-2`). Keep labels unique and stable. If you change their
+order, you can change the suffixes and the cache or mapping identity.
```yaml
runs:
@@ -259,9 +263,9 @@ runs:
## `files` And `fallback_files`
-`files` maps logical table ids to raw ActivitySim output file stems. If the value
-has no extension, the reader tries `.parquet` first, then `.csv`, inside each
-run directory.
+`files` maps logical table IDs to raw ActivitySim output file names. If a value
+has no extension, the reader first searches for `.parquet`. It then searches
+for `.csv` in each run directory.
| Table id | Default stem |
|---|---|
@@ -274,9 +278,9 @@ run directory.
| `joint_tour_participants` | `final_joint_tour_participants` |
| `land_use` | `final_land_use` |
-`fallback_files` supports optional table ids only: `day`, `vehicles`,
-`joint_tour_participants`, and `land_use`. Values must be explicit `.parquet` or
-`.csv` paths.
+`fallback_files` supports only these optional table IDs: `day`, `vehicles`,
+`joint_tour_participants`, and `land_use`. Each value must be an explicit
+`.parquet` or `.csv` path.
```yaml
files:
@@ -307,9 +311,9 @@ zones:
## `columns`
-Most `columns` values may be a string or an ordered list of candidate source
-column names. The first available candidate is used. The few scalar fields
-listed first are read as single names.
+Most `columns` values can be one string or an ordered list of possible source
+column names. The visualizer uses the first available column. It reads the
+scalar fields at the start of the table as single names.
| Field | Default | Impact | Purpose |
|---|---|---|---|
@@ -394,7 +398,7 @@ columns:
| `vot_bins.source_column` | string | `income_segment` | any source column | Prepare, Skimjoin | Source value used to derive VOT bins. |
| `vot_bins.output_column` | string | `vot_bin` | any output column | Prepare, Skimjoin | Prepared column written for skimjoin dimensions. |
| `vot_bins.fallback_value` | scalar string | none | any value | Prepare, Skimjoin | Value used when no run-specific mapping applies. |
-| `vot_bins.mappings` | mapping | `{}` | run key to value mapping | Prepare, Skimjoin | Enables VOT bin derivation. Run keys are normalized from run labels. |
+| `vot_bins.mappings` | mapping | `{}` | run key to value mapping | Prepare, Skimjoin | Enables VOT bin calculation. The loader normalizes run keys from run labels. |
```yaml
prepare:
@@ -426,11 +430,11 @@ prepare:
| `tour_start_period_number_column` | string | `start` | Prepared tour source used to write `start_period`. |
| `tour_end_period_number_column` | string | `end` | Prepared tour source used to write `end_period`. |
-The period breakpoint list must contain at least two integers, and the label
-list must contain exactly one fewer entry. When trips contain `tour_id`,
-`outbound`, and the derived `trip_period`, prepare also writes each tour's
-`first_inbound_trip_period`. Missing configured source columns are recorded in
-prepare diagnostics rather than invented.
+The period breakpoint list must contain at least two integers. The label list
+must contain one less entry. If trips contain `tour_id`, `outbound`, and the
+derived `trip_period`, prepare also writes `first_inbound_trip_period` for each
+tour. Prepare records missing configured source columns in the diagnostics. It
+does not create values for missing columns.
`prepare.non_motorized_distance_skim` accepts:
@@ -439,15 +443,16 @@ prepare diagnostics rather than invented.
| `file` | path string | required | `.csv`, `.omx`, `.h5`, or `.hdf5` lookup. Relative paths resolve from the main config. |
| `matrix` | string or null | required for OMX/HDF5; `DISTWALK` for CSV | OMX matrix name. For CSV, names the value column; a `__` prefix is stripped when present. |
-CSV lookup files must contain `OMAZ`, `DMAZ`, and the selected value column;
-prepared trips must contain `o_maz` and `d_maz`. OMX/HDF5 lookup uses prepared
-`OTAZ` and `DTAZ`. Both paths write
-`prepared_non_motorized_distance` and record unresolved lookup diagnostics.
+CSV lookup files must contain `OMAZ`, `DMAZ`, and the selected value column.
+Prepared trips must contain `o_maz` and `d_maz`. OMX and HDF5 lookups use the
+prepared `OTAZ` and `DTAZ` columns. Both methods write
+`prepared_non_motorized_distance`. They record diagnostics for unresolved
+lookups.
## `skimjoin`
-The main config `skimjoin` section wires the visualizer runtime to a separate
-skimjoin config file. See
+The `skimjoin` section connects the visualizer runtime to a separate skimjoin
+configuration file. See
[25 - Skimjoin Config Reference](25-skimjoin-config-reference.md) for the
lookup-rule schema.
@@ -459,8 +464,8 @@ main visualizer config
project/activitysim/defaults/modes: defines the actual lookup rules
```
-Merely providing `skimjoin.defaults.config_path` does not run skimjoin;
-`pipeline.steps` must also contain both `prepare` and `skimjoin`.
+The `skimjoin.defaults.config_path` value does not start skimjoin. The
+`pipeline.steps` list must also contain `prepare` and `skimjoin`.
| Field | Type | Default | Impact | Notes |
|---|---|---|---|---|
@@ -471,23 +476,22 @@ Merely providing `skimjoin.defaults.config_path` does not run skimjoin;
| `create_hypothetical_skim_tables` | boolean | `false` | Prepare | Enables configured hypothetical skim tables. |
Run-level `runs[*].skimjoin` supports `config_path`, `skim_files`,
-`network_los_file`, and `create_hypothetical_skim_tables`. The last field
-inherits the global value when omitted. Enable skimjoin by including it in
-`pipeline.steps`; top-level `skimjoin.enabled` and `skimjoin.config_path` are
-removed keys.
+`network_los_file`, and `create_hypothetical_skim_tables`. If you omit the last
+field, it uses the global value. To enable skimjoin, add it to `pipeline.steps`.
+Do not use the removed `skimjoin.enabled` or `skimjoin.config_path` keys.
Integrated skim files must resolve to `.omx`, `.csv`, `.h5`, or `.hdf5`.
## `segment`
-`segment` config is canonical in user YAML. Internally it is normalized to the
-segmentation runtime settings.
+Use `segment` as the canonical section in user YAML. The loader converts it to
+the segmentation runtime settings.
| Field | Type | Default | Allowed values | Impact | Notes |
|---|---|---|---|---|---|
| `dashboard.segmentation_type` | string | first configured definition | configured definition name | Presentation | Selected segment type shown in dashboard/export. |
| `dashboard.visibility` | string | `full_and_segments` | `full_only`, `segments_only`, `full_and_segments` | Presentation | Whether the dashboard shows full-run outputs, segmented outputs, or both. |
-| `definitions` | mapping | required when segment step is enabled | path-safe lowercase names | Summary | Segment definitions. |
+| `definitions` | mapping | required when you enable the segment step | path-safe lowercase names | Summary | Segment definitions. |
| `definitions.*.include_full` | boolean | `true` | `true`, `false` | Summary | Also build full-run summaries. |
| `definitions.*.persist_segmented_prepared_tables` | boolean | `false` | `true`, `false` | Prepare, Summary | Persist segment-specific prepared tables. |
| `definitions.*.allow_overlapping` | boolean | `false` | `true`, `false` | Summary | Allows one source value to appear in multiple segments. |
@@ -555,7 +559,7 @@ segment:
| `geography.mapping` | mapping | none | raw value to label | Summary, Presentation | Label mapping for geography values. |
| `geography.aggregations` | mapping | none | aggregation definitions | Summary, Presentation | Additional zone-to-geography lookup definitions. |
-Each `geography.aggregations.*` entry requires:
+Each `geography.aggregations.*` entry requires these fields:
| Field | Type | Notes |
|---|---|---|
@@ -596,10 +600,11 @@ summarize:
| `export.exclude_pages` | list of strings | `[]` | page ids | Presentation | Pages excluded from export. |
| `export.exclude_groups` | list of strings | `[]` | group ids | Presentation | Groups excluded from export. |
-`dashboard.host` is a reserved configuration block. The schema currently
-accepts `account`, `app_id`, `title`, and `verify`, but these values are not
-normalized or consumed: `pipeline.dashboard_mode: host` logs a warning and
-runs the ordinary live server. See the hosting extension recipe in chapter 43.
+`dashboard.host` is a reserved configuration block. The schema accepts
+`account`, `app_id`, `title`, and `verify`. The runtime does not normalize or
+use these values. `pipeline.dashboard_mode: host` writes a warning to the log
+and starts the standard live server. See the hosting extension procedure in
+chapter 43.
`live.pages` entries may be strings or group mappings:
@@ -614,15 +619,15 @@ dashboard:
- trip_stop_distance
```
-Export page overrides are keyed by page id or by nested group/page id. Selector
-keys depend on the page. Selector values may be `default`, `all`, a single
-string, or a list of strings. `parts.*.enabled` can hide named export parts.
+Use a page ID or a nested group and page ID as an export page override key.
+Selector keys depend on the page. A selector value can be `default`, `all`, one
+string, or a list of strings. Set `parts.*.enabled` to hide named export parts.
-Export inherits the page set resolved by `dashboard.live.pages`.
-`dashboard.export.pages` is an override mapping, not an allow-list: mentioning
-one page does not remove the others. A page override with `enabled: false`, or
-`exclude_pages` / `exclude_groups`, can narrow the inherited set. Export cannot
-add a page that live configuration did not select. Find valid IDs in:
+Export uses the page set from `dashboard.live.pages`.
+`dashboard.export.pages` is an override mapping, not an allow-list. An entry
+for one page does not remove other pages. To remove pages, set `enabled: false`
+or use `exclude_pages` or `exclude_groups`. Export cannot add a page that the
+live configuration did not select. Find valid IDs in these locations:
- page and group IDs: the generated catalog in chapter 31;
- selector IDs: `self.select(...)` and `self.selector(...)` calls on the page;
@@ -677,8 +682,8 @@ display:
## `modes`
-`modes` supplies legacy mode ordering and summary grouping. Prefer
-`display.labels.mode.mapping` when labels and order should be defined together.
+`modes` supplies legacy mode order and summary groups. Use
+`display.labels.mode.mapping` to define labels and order together.
| Field | Type | Default | Impact | Notes |
|---|---|---|---|---|
@@ -694,9 +699,10 @@ modes:
## Advanced Category Config
-`summarize.category_normalization` uses the same category shape as
-`display.labels`, but changes normalized values written into summary outputs.
-Use it for summary-affecting normalization or grouping, not cosmetic relabeling.
+`summarize.category_normalization` uses the same category format as
+`display.labels`. It changes normalized values in summary output. Use it for
+normalization or groups that change summaries. Do not use it only to change
+display labels.
```yaml
summarize:
@@ -736,19 +742,19 @@ prepare:
is_university: true
```
-Matching rules are deterministic:
+The visualizer applies these rules in sequence:
-1. When `prepare.student_types` is empty, prepare infers `School` from available
+1. When `prepare.student_types` is empty, prepare gets `School` from available
`ENROLLGRADEKto8`/`ENROLLGRADE9to12` columns and `University` from
`COLLEGEENROLL`.
-2. When a configured entry omits `person`, labels or land-use column names
- containing `univ` or `college` match `is_university`; other entries match
- `is_student` and exclude university students.
-3. With more than two configured entries, every non-university-defaulting entry
- must provide `person`; otherwise config validation fails.
-4. A `person` mapping combines all supplied conditions with AND. Scalar and
- list values are both accepted for `school_segment`, `SCHG`, and `pstudent`.
-5. If multiple entries match one person, the first configured entry wins.
+2. When an entry omits `person`, prepare examines labels and land-use column
+ names. Names that contain `univ` or `college` match `is_university`. Other
+ entries match `is_student` and exclude university students.
+3. If there are more than two entries, each non-university default entry must
+ provide `person`. If it does not, configuration validation fails.
+4. A `person` mapping combines all conditions with AND. You can use scalar or
+ list values for `school_segment`, `SCHG`, and `pstudent`.
+5. If multiple entries match one person, the visualizer uses the first entry.
For example, three school levels must select their person rows explicitly:
diff --git a/wiki/20-output-processor.md b/wiki/20-output-processor.md
index da5be12..6900d40 100644
--- a/wiki/20-output-processor.md
+++ b/wiki/20-output-processor.md
@@ -1,7 +1,7 @@
# 20 - Output Processor
-The Output Processor turns ActivitySim model outputs into stable data products
-for the dashboard.
+The Output Processor converts ActivitySim model output to stable data for the
+dashboard.
```text
raw ActivitySim tables or prepared table inputs
@@ -17,7 +17,7 @@ orchestration under [`runtime/workflows`](../runtime/workflows).
## Responsibilities
-The processor is responsible for:
+The processor does these tasks:
- reading raw `.csv` or `.parquet` ActivitySim outputs
- normalizing identifiers and column names
@@ -25,12 +25,12 @@ The processor is responsible for:
- applying weights
- adding geography and zone fields
- optionally joining skim values to trips and tours
-- optionally slicing outputs into configured segments
+- optionally dividing output into configured segments
- writing prepared and summary caches
-- recording manifests and diagnostics so stale outputs can be detected
+- recording manifests and diagnostics to identify stale output
-The dashboard should not re-read raw ActivitySim files. It should consume
-summary caches and, only for pages that explicitly ask for them, prepared tables.
+The dashboard must not read raw ActivitySim files again. It must use summary
+caches. It uses prepared tables only for pages that require them.
## Runtime Data Contract
@@ -50,8 +50,8 @@ The key runtime object is `RunData` in
| `skim_matrix` | Optional distance skim support. |
| `skimjoin_artifacts` | Optional skimjoin manifest and QA reports. |
-Summary builders and prepared-data dashboard pages should depend on this
-prepared contract rather than raw model-specific table layouts.
+Summary builders and prepared-data dashboard pages must use this prepared
+contract. They must not use raw, model-specific table layouts.
## Processor Subsystems
@@ -62,34 +62,35 @@ prepared contract rather than raw model-specific table layouts.
| Summaries | [23 - Summary Functions](23-summary-functions.md) | Build dashboard-ready tables. |
| Summary catalog | [24 - Summary Catalog](24-summary-catalog.md) | Inspect registered summary outputs. |
-The former static prepared-cache schema document recorded one
-`estimation-output` dataset, including its row counts and model-specific
-columns. It was not a portable runtime contract and became stale as inputs
-changed. Use [Prepared Table Names and Fields](21-prepared-tables.md) for the
-stable contract and inspect the manifest and table schema of the actual cache
-when exact model-specific columns are needed.
+The former static prepared-cache schema described one `estimation-output` data
+set. It included row counts and model-specific columns. This schema was not a
+portable runtime contract. It became incorrect when the input changed. Use
+[Prepared Table Names and Fields](21-prepared-tables.md) for the stable
+contract. Examine the applicable cache manifest and table schema for exact
+model-specific columns.
## Where Processor Output Goes
-Prepared caches are reusable canonical data. Summary caches are smaller,
-dashboard-ready CSVs. The summary cache is the normal dashboard input.
+Prepared caches contain reusable canonical data. Summary caches contain
+smaller CSV files for the dashboard. The summary cache is the standard
+dashboard input.
-The processor also carries diagnostic state. A table or summary can be:
+The processor also keeps diagnostic status. A table or summary can be:
- available and populated
- available but empty
- unavailable because an optional input is missing
- failed, with a recorded diagnostic
-This is intentional. The dashboard can show partial results instead of failing
-the entire workflow when one optional table or summary is unavailable.
+This behavior lets the dashboard show partial results. One unavailable optional
+table or summary does not stop the complete workflow.
-“Empty” and “unavailable” are different contracts. Empty means the input and
-calculation were valid but produced zero rows. Unavailable means a prerequisite
-table/column was absent or a declared operation could not run. Failed means an
-exception was recorded under the configured failure policy. Preserve the
-availability metadata when copying `RunData`; checking only
-`DataFrame.is_empty()` loses that distinction.
+"Empty" and "unavailable" have different meanings. Empty means that the input
+and calculation were valid, but the result has zero rows. Unavailable means
+that a required table or column was absent. It can also mean that a declared
+operation could not execute. Failed means that the configured failure policy
+recorded an exception. Keep the availability metadata when you copy `RunData`.
+A check of only `DataFrame.is_empty()` removes this information.
### Example: Follow One Metric
@@ -104,20 +105,19 @@ final_trips.csv
-> page reads the table through self.data.summary(...)
```
-Each boundary has one owner. Prepare resolves source filenames and aliases;
-the summary defines the aggregate; the cache validates the persisted contract;
-the page handles presentation. This separation is why a page should not open a
-raw file or reproduce a weighted aggregation. Chapter 44 works through this
-example in code.
+Each boundary has one owner. Prepare resolves source file names and aliases.
+The summary defines the aggregate. The cache validates the stored contract.
+The page controls the presentation. Thus, a page must not open a raw file or
+repeat a weighted aggregation. Chapter 44 gives the code for this example.
## Extension Checklist
-When adding new processor-visible behavior:
+To add processor behavior, do these steps:
-1. Decide whether the new data belongs in prepared tables, skimjoin outputs, or
+1. Decide whether the new data belongs in prepared tables, skimjoin output, or
a summary table.
2. Add or update the smallest processor subsystem that owns that behavior.
-3. Preserve stable output schemas and use typed empty fallbacks where possible.
+3. Keep stable output schemas and use typed empty fallback results when possible.
4. Update dashboard page requirements if a page depends on the new output.
5. Add focused tests for the new behavior.
6. Regenerate wiki catalogs if summary declarations or page definitions changed.
diff --git a/wiki/21-prepared-tables.md b/wiki/21-prepared-tables.md
index e8e70ba..d1803dd 100644
--- a/wiki/21-prepared-tables.md
+++ b/wiki/21-prepared-tables.md
@@ -1,7 +1,7 @@
# 21 - Prepared Tables
-Prepared tables are the processor's canonical form of ActivitySim output. They
-hide raw file naming differences and expose stable fields for summaries and
+Prepared tables are the canonical form of ActivitySim output. They remove
+differences in raw file names. They supply stable fields to summaries and
dashboard pages.
## Prepare Data Flow
@@ -45,7 +45,7 @@ with domain boundaries in `processor/prepare/enrichment/domains.py`.
## Prepared Table Names
-Runtime table names are defined in `processor.models.PreparedTableName`:
+`processor.models.PreparedTableName` defines the runtime table names:
| Config/file table ID | `RunData`/summary-contract name | Meaning |
|---|---|---|
@@ -59,14 +59,15 @@ Runtime table names are defined in `processor.models.PreparedTableName`:
| `land_use` | `land_use` | Land use and geography lookup data. |
| no file-map ID | `skim` | Optional `skim_matrix` support exposed as a special prepared requirement. |
-Use config/file IDs in `files`, `file_map`, and `prepared_table_map`. Use the
-runtime names in `RunData` access and `@summary(required_columns=...)`; for
-example, `run.per` and `required_columns={"per": ("person_type",)}`.
+Use configuration and file IDs in `files`, `file_map`, and
+`prepared_table_map`. Use runtime names to access `RunData` and in
+`@summary(required_columns=...)`. Examples are `run.per` and
+`required_columns={"per": ("person_type",)}`.
## Common Prepared Fields
-The exact schema can differ by model and optional inputs, but summaries commonly
-rely on:
+The exact schema can differ for each model and optional input. Summaries
+frequently use these fields:
- canonical IDs: `household_id`, `person_id`, `tour_id`, `trip_id`
- purpose and mode fields: `tour_purpose`, `trip_purpose`, `tour_mode`, `trip_mode`
@@ -76,48 +77,47 @@ rely on:
- household/person aliases: `HHVEH`, `HHSIZE`, `AUTOSUFF`, `NUMBER_HH`
- aggregation weight: `finalweight`
-Use the prepared field when it exists rather than probing raw names in a summary
+Use the prepared field when it exists. Do not search for raw names in a summary
or page.
-This list is orientation, not a guarantee that every table has every field.
-For a specific summary, the generated catalog in chapter 24 is the authoritative
-list of required prepared columns. At runtime, `@summary` prerequisites and
-prepared-table availability metadata determine whether a calculation can run.
+This list is an introduction. It does not mean that each table has each field.
+For a specified summary, the generated catalog in chapter 24 gives the required
+prepared columns. At runtime, `@summary` requirements and prepared-table
+availability metadata control whether a calculation can execute.
## Inspecting An Exact Prepared Schema
-There is intentionally no repository-wide dump of every column from one sample
-prepared cache. Raw model extensions and optional inputs make such a snapshot
-model-specific and quickly stale.
+The repository does not contain a list of all columns from one sample prepared
+cache. Raw model extensions and optional input make this list model-specific.
+Input changes can also make the list incorrect.
-For the cache you are actually using:
+For the applicable cache, do these steps:
-1. Read the run's `manifest.json` to find the prepared-table files and recorded
- availability state.
-2. Inspect the Parquet or CSV schema for the relevant table.
+1. Read the run's `manifest.json`. Find the prepared-table files and the
+ recorded availability status.
+2. Examine the Parquet or CSV schema for the applicable table.
3. Use `processor.models.RunData` names at runtime and the file/config names in
[Prepared Table Names](#prepared-table-names).
4. Use the generated [Summary Catalog](24-summary-catalog.md) to find the exact
prepared columns required by each registered summary.
-Stable additions belong in the owning prepare enrichment module and should be
-covered by a prepare test. A row count or a column found only in one regional
-model output is evidence about that dataset, not part of the visualizer's
-portable contract.
+Add stable fields to the applicable prepare enrichment module. Add a prepare
+test for each field. A row count or column in only one regional model describes
+that data set. It is not part of the portable visualizer contract.
## Adding A Prepared Column
-For an end-to-end worked example, see
+For a complete example, see
[Add A Column To An Existing Prepared Table](41-data-extension-cookbook.md#worked-example-add-a-column-to-an-existing-prepared-table).
-Use this path when many summaries/pages need the same derived field or when the
-field is part of canonical model-output normalization.
+Use this procedure when many summaries or pages require the same derived field.
+Also use it when the field is part of canonical model-output normalization.
Checklist:
1. Choose the owning enrichment module.
-2. Add the Polars expression or transformation in the appropriate stage.
-3. Keep missing source columns graceful when the input is optional.
+2. Add the Polars expression or transformation in the applicable stage.
+3. If the input is optional, keep the table usable when source columns are missing.
4. Add final type/cast behavior if the field must be stable.
5. Add or update tests that prepare a minimal run and assert the new column.
6. If a summary depends on the column, add it to that summary's contract
@@ -133,12 +133,12 @@ if "source_column" in state.trips.columns:
)
```
-Do not add page-only formatting columns to prepared tables. Prefer page helpers
-or summary output columns for presentation concerns.
+Do not add page formatting columns to prepared tables. Use page helpers or
+summary output columns for presentation.
## Using Prepared Tables As Inputs
-`prepared_table_map` lets a config bypass raw prepare for a run:
+Use `prepared_table_map` to omit raw prepare for a run:
```yaml
runs:
@@ -151,11 +151,11 @@ runs:
land_use: C:\prepared\land_use.parquet
```
-This path assumes the supplied tables already match the prepared contract.
+The supplied tables must agree with the prepared contract.
-Adding a new prepared table type is a larger change covering config, `RunData`,
-reader, availability, cache IO, pruning, and possibly segmentation. Follow the
-[complete worked example](41-data-extension-cookbook.md#worked-example-add-a-prepared-table).
+A new prepared table type changes the configuration, `RunData`, reader,
+availability, cache I/O, and pruning. It can also change segmentation. Follow
+the [complete example](41-data-extension-cookbook.md#worked-example-add-a-prepared-table).
## Related Chapters
diff --git a/wiki/22-skimjoin.md b/wiki/22-skimjoin.md
index bff7d27..67740ea 100644
--- a/wiki/22-skimjoin.md
+++ b/wiki/22-skimjoin.md
@@ -1,26 +1,26 @@
# 22 - Skimjoin
-Skimjoin enriches prepared trips and tours with skim-derived columns. It runs as
-an optional late-prepare step after raw outputs have been normalized.
+Skimjoin adds skim-derived columns to prepared trips and tours. It is an
+optional final part of prepare. It executes after prepare normalizes raw output.
-Use skimjoin when summaries or dashboard pages need values from OMX skims or
-sidecar lookup files, such as time, cost, distance, walk access, or composed
-tour-level attributes.
+Use skimjoin when summaries or dashboard pages require values from OMX skims or
+sidecar lookup files. Examples are time, cost, distance, walk access, and
+combined tour attributes.
-For full field-by-field skimjoin config options, lookup-rule grammar, defaults,
-and examples, see
+For each skimjoin field, lookup rule, default, and example, see
[25 - Skimjoin Config Reference](25-skimjoin-config-reference.md).
-Two YAML files participate in integrated use:
+Integrated skimjoin uses two YAML files:
-- the **main visualizer config** enables the stage with `pipeline.steps` and
- points at files through `skimjoin.defaults` or per-run overrides; and
+- the **main visualizer config** enables the step with `pipeline.steps` and
+ specifies files in `skimjoin.defaults` or run overrides; and
- the **standalone skimjoin config** defines `project`, `activitysim`,
dimensions, mode/component lookup rules, fallbacks, and tour aggregation.
-Paths in the first file resolve from the main config; paths owned by the second
-resolve from the standalone skimjoin config. Supplying a config path alone does
-not enable the stage—`pipeline.steps` must contain `prepare` and `skimjoin`.
+Paths in the first file start from the main configuration file. Paths in the
+second file start from the standalone skimjoin configuration file. A
+configuration path does not enable the step. `pipeline.steps` must contain
+`prepare` and `skimjoin`.
## Runtime Placement
@@ -36,9 +36,9 @@ prepare raw outputs
The runtime adapter is
[`processor/skimjoin/pipeline.py`](../processor/skimjoin/pipeline.py).
-## Config Anatomy
+## Configuration sections
-A skimjoin config describes:
+A skimjoin configuration contains these sections:
| Section | Purpose |
|---|---|
@@ -50,38 +50,38 @@ A skimjoin config describes:
| `modes` | Mode-specific lookup rules. |
| `tour_aggregation` | How trip skim values roll up to tours. |
-Per-run overrides in the main visualizer config can change selected skim files,
-`network_los_file`, or the whole skimjoin config path.
+Run overrides in the main visualizer configuration can change the skim files,
+`network_los_file`, or skimjoin configuration path.
## Adding A Skim Output
Start with the [Basic OD Lookup](25-skimjoin-config-reference.md#basic-od-lookup)
-for a complete mode rule, then add dimensions, fallbacks, or tour aggregation
+for a complete mode rule. Add dimensions, fallback rules, or tour aggregation
only when the new output requires them.
Checklist:
-1. Confirm the prepared trips/tours contain the source columns needed for lookup.
+1. Make sure that prepared trips or tours contain the required lookup columns.
2. Add or update a lookup rule in the skimjoin config.
-3. Choose the output name and keep the `skim_` prefix convention unless there is
- a strong reason not to.
-4. Set missing matrix and missing OD policies deliberately.
-5. Add fallback lookup rules only when a real fallback is meaningful.
+3. Select an output name. Use the `skim_` prefix unless the interface requires a different prefix.
+4. Set the missing-matrix and missing-OD policies.
+5. Add fallback lookup rules only when a valid fallback value is available.
6. If tours need the value, configure tour aggregation or directional outputs.
7. Add/update a summary in `processor/summarize/summaries/skimjoin.py` if the
dashboard needs aggregate reporting.
8. Regenerate wiki catalogs if summary declarations or dashboard requirements
changed.
-Set `skimjoin.create_hypothetical_skim_tables: true` (globally or in a run
-override) when the configured lookups should also produce hypothetical skim
-sidecar tables. This is opt-in because it adds output work and artifacts.
+Set `skimjoin.create_hypothetical_skim_tables: true` globally or in a run
+override to create hypothetical skim sidecar tables. The default is `false`
+because this option creates more output and artifacts.
## Standalone Skimjoin CLI
-The integrated pipeline is the normal visualizer path. A standalone CLI is
-also available for inspecting and validating a skimjoin config or producing
-annotated tables without running the full visualizer:
+The integrated pipeline is the standard visualizer method. You can also use the
+standalone command-line interface. Use it to examine or validate a skimjoin
+configuration. You can also create annotated tables without the full
+visualizer:
```bash
uv run python -m processor.skimjoin.cli COMMAND --config skimjoin.yaml
@@ -93,19 +93,18 @@ uv run python -m processor.skimjoin.cli COMMAND --config skimjoin.yaml
| `validate` | none | Strictly validates config, inventory, and configured ActivitySim tables; writes `config_normalized.yaml` and `validation_report.txt`. Returns exit code 1 and writes a failure report when validation fails. |
| `annotate-trips` | `--out PATH`, `--preview` | Writes annotated trips plus validation, lookup-summary, and missing-lookup artifacts. The default table is `/trips_with_skims.parquet`. |
| `annotate-tours` | `--out PATH`, `--preview` | Writes annotated tours, `tour_aggregation_summary.csv`, and `missing_lookup_report.csv`. The default table is `/tours_with_skims.parquet`. |
-| `run` | `--out-trips PATH`, `--out-tours PATH`, `--preview` | Runs both annotations and writes their validation/QA reports. Defaults to the two filenames above. |
+| `run` | `--out-trips PATH`, `--out-tours PATH`, `--preview` | Executes both annotations and writes their validation and QA reports. Uses the two file names above by default. |
-`--config` is required for every command. Output flags are optional only when
-`project.output_dir` is configured. Standalone table inputs come from
-`activitysim.trips_table` and `activitysim.tours_table`, with the legacy
-`project.trips_table`/`project.tours_table` fallback described in chapter 25.
-Input and output tables must be CSV or Parquet. `--preview` on annotation
-commands adds a compact output-column inventory; it does not limit rows or
-make the command a dry run.
+Each command requires `--config`. Output flags are optional only if you
+configure `project.output_dir`. The standalone input tables come from
+`activitysim.trips_table` and `activitysim.tours_table`. Chapter 25 describes
+the legacy `project.trips_table` and `project.tours_table` fallback. Input and
+output tables must be CSV or Parquet. For annotation commands, `--preview` adds
+a short output-column inventory. It does not limit rows or prevent writes.
## Debugging Skimjoin
-Start with the skimjoin artifacts on the prepared run:
+First, examine these skimjoin artifacts for the prepared run:
- `skim_lookup_summary`
- `missing_lookup_report`
diff --git a/wiki/23-summary-functions.md b/wiki/23-summary-functions.md
index eb68960..2a67d41 100644
--- a/wiki/23-summary-functions.md
+++ b/wiki/23-summary-functions.md
@@ -1,10 +1,10 @@
# 23 - Summary Functions
-Summary functions turn prepared `RunData` into dashboard-ready Polars
-`DataFrame`s. A summary's identity, prerequisites, output schema, cache name,
-and builder are declared together.
+Summary functions convert prepared `RunData` to Polars `DataFrame` objects for
+the dashboard. Declare the summary identity, requirements, output schema, cache
+name, and builder together.
-## Mental Model
+## Data flow
```text
RunData + Config
@@ -15,20 +15,20 @@ RunData + Config
```
Builders live under [`processor/summarize/summaries`](../processor/summarize/summaries).
-`processor.summarize.catalog` explicitly imports those owning modules and
-discovers their declarations. There is no separate summary-spec registry to
-edit.
+`processor.summarize.catalog` imports those modules and finds their
+declarations. Do not edit a separate summary specification registry. It does
+not exist.
## Summary Declaration
Use `@summary(...)` from `processor.summarize`. The declaration provides:
-- the stable summary ID and optional cache filename
+- the stable summary ID and optional cache file name
- an ordered Polars output schema
- required prepared tables and columns
- a typed empty result
- strict result validation
-- whether the summary is built by default
+- default build status
```python
import polars as pl
@@ -68,27 +68,27 @@ def trip_distance_by_mode(run: RunData, config: Config) -> pl.DataFrame:
)
```
-Successful builders must return exactly the declared columns, in the declared
-order and with the declared dtypes. Missing declared inputs are handled before
-the builder runs and produce its typed empty result.
+A successful builder must return the declared columns in the declared order.
+Each column must have the declared data type. The workflow checks for missing
+declared input before it executes the builder. Missing input gives the typed empty
+result.
-Use `required_tables` only when the presence of an entire table or `skim` is
-enough to express the prerequisite. Use `required_columns` for ordinary table
-dependencies; it also implies that the named runtime table must exist. Table
-names here are `RunData` names (`hh`, `per`, `tours`, `trips`,
-`joint_participants`, `land_use`), not config IDs such as `households` or
-`persons`.
+Use `required_tables` only when a complete table or `skim` is enough to state
+the requirement. Use `required_columns` for standard table
+dependencies. It also requires the named runtime table. Use `RunData` table
+names here: `hh`, `per`, `tours`, `trips`, `joint_participants`, and
+`land_use`. Do not use configuration IDs such as `households` or `persons`.
## Weighting
-Builders aggregate `finalweight`; they do not branch on weighting mode. The
-summary workflow supplies the appropriate prepared data for weighted and
+Builders aggregate `finalweight`. They do not select a weighting mode. The
+summary workflow supplies the required prepared data for weighted and
unweighted builds.
## Adding A Summary Function
-For a complete calculation, contract test, catalog, and page-wiring example,
-follow the [Summary Function Cookbook](44-summary-function-cookbook.md).
+For an example with a calculation, contract test, catalog, and page connection,
+use the [Summary Function Cookbook](44-summary-function-cookbook.md).
1. Put the builder in the domain module that owns the calculation.
2. Decorate it with `@summary(...)` and declare identity, ordered schema, and
@@ -100,50 +100,50 @@ follow the [Summary Function Cookbook](44-summary-function-cookbook.md).
declared prerequisites cannot express.
7. Add focused calculation and contract tests.
8. Add the summary ID to a page's required or optional summaries when needed.
-9. Run `uv run python scripts/generate_wiki_catalogs.py`.
+9. Use `uv run python scripts/generate_wiki_catalogs.py`.
-The catalog import rejects duplicate IDs. Ordinary summarize workflows build
-every declaration with `build_by_default=True`; enabled page requirements do
-not narrow or expand that build set. `build_by_default=False` registers a
-contract without adding it to ordinary generated builds. In the current public
-workflow this is the external-table pattern: provide the table through
-`summary_table_map`. Merely listing a non-default ID in a page declaration does
-not cause its builder to run.
+The catalog import rejects duplicate IDs. Standard summarize workflows build
+each declaration that has `build_by_default=True`. Enabled page requirements
+do not change this build set. `build_by_default=False` registers a contract but
+does not add it to standard builds. Use this value for an external table in the
+public workflow. Supply the table through `summary_table_map`. A non-default ID
+in a page declaration does not start its builder.
## Summary CSV Boundary
-Summary caches are the dashboard input and their registered tables are already
-stored as CSV files under each run and weighting mode. Normal summarize runs
-write missing or stale cache tables unless `--skip-summary-cache-write` is used.
+Summary caches are the dashboard input. The visualizer stores their registered
+tables as CSV files for each run and weighting mode. Standard summarize workflows
+write missing or stale cache tables. Use `--skip-summary-cache-write` to prevent
+these writes.
-For a developer diagnostic, this command bypasses reusable summary caches,
-rebuilds the configured summaries, and forces the cache CSVs/manifests to be
-written:
+For a developer diagnostic, use this command to ignore reusable summary caches.
+The command rebuilds configured summaries and writes the cache CSV files and
+manifests:
```bash
uv run activitysim-viz --config local_config.yaml --summarize --write-csvs
```
-It does not create a second export format or a separate calibration directory.
-`processor.summarize.csv_export.write_summary_csvs()` is the shared low-level
-writer used by cache storage. Dashboard pages load registered summaries through
-`self.data`; they do not open those CSVs directly.
+The command does not create a second export format or a separate calibration
+directory. Cache storage uses the shared
+`processor.summarize.csv_export.write_summary_csvs()` writer. Dashboard pages
+load registered summaries through `self.data`. They do not open the CSV files
+directly.
To register a new dashboard-ready table produced outside the visualizer, use
the [outside summary table recipe](41-data-extension-cookbook.md#worked-example-add-an-outside-summary-table).
## Segmentation
-Segmentation runs inside the summarize workflow and builds the same declarations
-for configured slices of the prepared data. Segment sources may be a prepared
-column or a CSV lookup. Dashboard visibility is controlled by
-`segment.dashboard`.
+Segmentation runs in the summarize workflow. It builds the same declarations
+for configured parts of the prepared data. A segment source can be a prepared
+column or a CSV lookup. `segment.dashboard` controls dashboard visibility.
## Summary Catalog
-The generated [24 - Summary Catalog](24-summary-catalog.md) lists every current
-declaration, output filename, builder, schema, and prerequisite. Regenerate it
-after summary declarations change.
+The generated [24 - Summary Catalog](24-summary-catalog.md) lists each current
+declaration, output file name, builder, schema, and requirement. Regenerate the
+catalog after you change a summary declaration.
## Related Chapters
diff --git a/wiki/24-summary-catalog.md b/wiki/24-summary-catalog.md
index 0381f58..4a1290b 100644
--- a/wiki/24-summary-catalog.md
+++ b/wiki/24-summary-catalog.md
@@ -1,53 +1,52 @@
# 24 - Summary Catalog
-This page is the analytical data dictionary for the summary CSV tables exposed
-by the Output Processor. It covers all registered summary tables, explains the
-observation represented by each row, identifies practical travel-analysis uses,
-and defines every output field. The generated developer inventory at the end of
-the page remains the authoritative list of filenames, schemas, builders, and
-input prerequisites.
+This page is the data dictionary for the summary CSV tables from the Output
+Processor. It describes all registered summary tables. For each table, it
+defines a row, gives analysis uses, and defines each output field. The generated
+developer inventory is at the end of the page. Use it as the authoritative list
+of file names, schemas, builders, and input requirements.
## How to Interpret the Tables
- Count, volume, mileage, and boarding fields are numeric measures. In a
- weighted cache they are sums of `finalweight`; in an unweighted cache the
- workflow substitutes unit weights. A `Float64` count can therefore be a
- fractional population estimate, not a literal row count.
-- Rate, percentage, mean, standard-deviation, median, and percentile fields are
- calculated from the weighted observations described for that table.
+ weighted cache, they are sums of `finalweight`. In an unweighted cache, the
+ workflow uses unit weights. Thus, a `Float64` count can be a fractional
+ population estimate. It is not always a row count.
+- The workflow calculates rate, percentage, mean, standard deviation, median,
+ and percentile fields from the weighted observations for that table.
- Values such as `all_geographies`, `all_person_types`, `all_tour_purposes`,
`all_tour_modes`, `All Modes`, `All Auto`, and `Daily` are rollups. Do not add
- a rollup to its component rows; choose either the rollup or the detail level.
+ a rollup to its component rows. Use the rollup or the detail level, but not both.
- `geography_type` names the configured spatial system, such as MAZ, TAZ,
- county, MPO, or a custom geography. `geography_id` is the identifier within
- that system. Home, work, school, destination, and parking geography are
- stated in each table description.
+ county, MPO, or a custom geography. `geography_id` is the identifier in that
+ system. Each table description identifies the home, work, school,
+ destination, or parking geography.
- Distance values use the units of the prepared distance or skim fields,
- normally miles. Integer distance bins use the truncated mile value unless a
- table explicitly says distances are rounded; terminal bins such as `40+`,
- `20+`, or numeric bin `51` include all larger values.
+ usually miles. Integer distance bins use the truncated mile value unless the
+ table specifies rounded values. End bins such as `40+`, `20+`, or numeric bin
+ `51` include all larger values.
- `time_bin` is the prepared ActivitySim period index: 1--24 for hourly inputs
or 1--48 for half-hour-period inputs. Named `time_period` and `count_period`
values come from configured or supplied period labels.
- Category codes and labels come from prepared ActivitySim values and the
- configured category mappings. Analysts should retain the code field for
- joins and use its label field for presentation.
+ configured category mappings. Keep the code field for joins. Use the label
+ field for presentation.
- A valid calculation can produce an empty CSV. That is distinct from a summary
- marked unavailable because an input table or field was absent; consult the
- summary manifest when the distinction matters.
+ marked unavailable because an input table or field was absent. Use the
+ summary manifest to identify the status.
## Build Status
-The normal summarize workflow builds **85** tables. The other **15** registered
-tables have `Default build = no`: the two skim ECDF tables are optional
-on-demand products, while the 13 validation contracts are supplied through
-`summary_table_map` rather than calculated from `RunData`. All 100 contracts are
-documented below and appear in the generated inventory.
+The standard summarize workflow builds **85** tables. The other **15** tables
+have `Default build = no`. The two skim ECDF tables are optional products. An
+external process supplies the 13 validation contracts through
+`summary_table_map`. The visualizer does not calculate them from `RunData`.
+This page and the generated inventory describe all 100 contracts.
## Analytical Table Reference
-The fields listed for a table are the complete persisted output schema. See the
-generated inventory for physical data types and mechanical input requirements.
+The fields for a table are its complete stored output schema. See the generated
+inventory for data types and input requirements.
### Population and Demographics
@@ -97,9 +96,9 @@ generated inventory for physical data types and mechanical input requirements.
### Long-Term Location Distance
-These three tables contain a dense 0--51 distribution for each geography; bin
-51 contains distances of 51 or more. Missing distance values are treated as
-zero by these distribution builders.
+These three tables contain a complete 0--51 distribution for each geography.
+Bin 51 contains distances of 51 or more. These distribution builders use zero
+for a missing distance value.
| Summary table | Information and analytical use | Fields |
|---|---|---|
@@ -112,31 +111,31 @@ zero by these distribution builders.
| Summary table | Information and analytical use | Fields |
|---|---|---|
| `daily_activity_pattern_by_person_type` | Daily activity pattern alternatives by person type, including an all-person-types rollup. Use it to compare mandatory, nonmandatory, and home-stay behavior. | `person_type`: person-type code or rollup.
`daily_activity_pattern`: prepared CDAP/activity-pattern category.
`person_count`: weighted people in the pattern. |
-| `mandatory_tour_frequency_by_person_type` | Positive mandatory-tour-frequency choice by person type, plus an all-person-types rollup. Use it to analyze how many mandatory tours travelers make; people with a choice of zero are excluded. | `person_type`: person-type code or rollup.
`mandatory_tour_frequency`: prepared positive mandatory-tour frequency alternative.
`person_count`: weighted people choosing that frequency. |
+| `mandatory_tour_frequency_by_person_type` | Positive mandatory-tour-frequency choice by person type, plus an all-person-types rollup. Use it to analyze how many mandatory tours travelers make. The table excludes people with a choice of zero. | `person_type`: person-type code or rollup.
`mandatory_tour_frequency`: prepared positive mandatory-tour frequency alternative.
`person_count`: weighted people choosing that frequency. |
| `nonmandatory_tour_frequency_by_person_type` | Count of individual nonmandatory tours plus joint-tour participation per person, grouped as 0, 1, 2, or 3+, by person type and for all types. Use it to compare discretionary travel propensity. | `person_type`: person-type code or rollup.
`nonmandatory_tour_frequency`: combined nonmandatory-tour category `0`, `1`, `2`, or `3+`.
`person_count`: weighted people in the category. |
| `tour_rates_by_person_type_and_tour_purpose` | Tours per weighted person-day by person type and tour purpose, plus all-person-types rates. Use it to compare tour-generation rates while controlling for population composition. | `person_type`: person-type code or rollup.
`tour_purpose`: prepared tour-purpose category.
`tour_rate`: weighted tours divided by weighted persons for the applicable person type. |
| `trip_rates_by_person_type_and_trip_purpose` | Trips per weighted person by person type and trip purpose, plus all-person-types rates. Use it to compare trip-generation rates across demographic markets. | `person_type`: person-type code or rollup.
`trip_purpose`: destination purpose of the trip.
`trip_rate`: weighted trips divided by weighted persons for the applicable person type. |
### School Escorting
-`direction` values distinguish outbound and inbound tour halves. Some tables
-also include `both`, which counts tours or households escorted in both halves,
-or `all_directions`, which sums directional escort incidences and can count the
-same tour twice. These values are not interchangeable.
+`direction` values identify outbound and inbound tour halves. Some tables also
+include `both`. This value counts tours or households with escorts in both
+halves. The `all_directions` value sums escort incidences by direction. It can
+count the same tour two times. Do not use these values as equivalents.
| Summary table | Information and analytical use | Fields |
|---|---|---|
| `escorted_tour_totals` | One run-level total of adult-side tours with an outbound or inbound school-escort condition. Use it as the top-level escorted-tour control total. | `tour_count`: weighted distinct eligible tours with at least one escorted direction. |
| `school_escorted_tours_by_escort_type_and_direction` | Adult-side escorted tours by escort arrangement and direction, with an `all_directions` incidence rollup. Use it to compare ride-share and pure-escort patterns. | `escort_type`: prepared escort arrangement label.
`direction`: `outbound`, `inbound`, or `all_directions`.
`tour_count`: weighted escorted-tour incidences. |
-| `adult_escorted_tour_purposes_by_direction` | Purposes of the adult tours that perform school escorting, by direction and with an all-directions incidence rollup. Use it to see how escorting is linked with work or other adult activities. | `tour_purpose`: adult tour's primary purpose.
`direction`: escorted half or `all_directions`.
`tour_count`: weighted escorted-tour incidences. |
-| `adult_escorted_tours_by_person_type_and_direction` | Adult-side escorted tours by the adult traveler’s person type and escorted direction. Use it to identify who performs school escorting. | `person_type`: adult traveler person-type code.
`direction`: `outbound`, `inbound`, or `both`.
`tour_count`: weighted tours meeting that directional condition. |
+| `adult_escorted_tour_purposes_by_direction` | Purposes of the adult tours that do school escorting, by direction and with an all-directions incidence rollup. Use it to see how escorting connects with work or other adult activities. | `tour_purpose`: adult tour's primary purpose.
`direction`: escorted half or `all_directions`.
`tour_count`: weighted escorted-tour incidences. |
+| `adult_escorted_tours_by_person_type_and_direction` | Adult-side escorted tours by the adult traveler's person type and escorted direction. Use it to identify who performs school escorting. | `person_type`: adult traveler person-type code.
`direction`: `outbound`, `inbound`, or `both`.
`tour_count`: weighted tours meeting that directional condition. |
| `student_school_escort_status_by_direction` | Student school tours classified by normalized escort type for each direction and for tours escorted both ways. Use it to measure the student-side escort experience. | `direction`: `outbound`, `inbound`, or `both`.
`escort_type`: normalized escort arrangement, including unescorted alternatives where present.
`tour_count`: weighted student school tours in the group. |
| `student_households_by_student_count` | Households by the number of school-age/student household members recognized by the escort logic. Use it as a denominator for household escort participation. | `student_count`: students in the household.
`household_count`: weighted households with that count. |
| `households_with_school_escorting_by_student_count_and_direction` | Unique households with at least one escorted student school tour, by number of students and directional condition. Use it to calculate escort-participation rates by household composition. | `student_count`: students in the household.
`direction`: `outbound`, `inbound`, or `both`.
`household_count`: weighted unique households meeting the condition. |
-| `schoolkids_per_escorted_tour_by_student_count_and_direction` | Average number of escorted children on adult-side escorted tours by household student count and direction. Use it to analyze escorting efficiency and child grouping. | `student_count`: students in the adult traveler’s household.
`direction`: `outbound`, `inbound`, or `both`.
`avg_schoolkids_per_tour`: weighted mean number of escortees per eligible tour.
`tour_count`: weighted eligible tours used as the mean denominator. |
+| `schoolkids_per_escorted_tour_by_student_count_and_direction` | Average number of escorted children on adult-side escorted tours by household student count and direction. Use it to analyze escorting efficiency and child grouping. | `student_count`: students in the adult traveler's household.
`direction`: `outbound`, `inbound`, or `both`.
`avg_schoolkids_per_tour`: weighted mean number of escortees per eligible tour.
`tour_count`: weighted eligible tours used as the mean denominator. |
| `adult_escorted_tour_distance_distribution_by_direction` | Adult-side escorted tours by rounded tour distance and directional escort condition. Use it to compare the length of outbound-only, inbound-only, and both-way escort tours. | `distance_bin`: rounded tour-distance label from `0` to `39` or `40+`.
`direction`: `outbound`, `inbound`, or `both`.
`tour_count`: weighted eligible tours in the bin. |
-| `adult_escorted_trip_distance_distribution_by_direction` | Trips belonging to explicitly escorted adult tours, filtered to the corresponding outbound or inbound half, by rounded trip distance. Use it to examine the trip-leg burden of escorting. | `distance_bin`: rounded trip-distance label from `0` to `39` or `40+`.
`direction`: `outbound`, `inbound`, or `both` condition.
`trip_count`: weighted eligible trips in the bin. |
-| `adult_escort_event_stop_distribution` | Number of intermediate stops before and after the school drop-off or pickup event on explicitly escorted adult tours. Use it to analyze chaining around escort events. | `segment`: one of `outbound_before_dropoff`, `outbound_after_dropoff`, `inbound_before_pickup`, or `inbound_after_pickup`.
`stop_count`: prepared count of stops in that segment.
`tour_count`: weighted escort-event records with that stop count. |
+| `adult_escorted_trip_distance_distribution_by_direction` | Trips on adult tours marked as escorted, by outbound or inbound half and rounded trip distance. Use it to examine the trip-leg distance for escorting. | `distance_bin`: rounded trip-distance label from `0` to `39` or `40+`.
`direction`: `outbound`, `inbound`, or `both` condition.
`trip_count`: weighted eligible trips in the bin. |
+| `adult_escort_event_stop_distribution` | Intermediate stops before and after school drop-off or pickup on adult tours marked as escorted. Use it to analyze trip chains around escort events. | `segment`: one of `outbound_before_dropoff`, `outbound_after_dropoff`, `inbound_before_pickup`, or `inbound_after_pickup`.
`stop_count`: prepared count of stops in that segment.
`tour_count`: weighted escort-event records with that stop count. |
| `adult_escort_trip_stop_frequency` | Adult-side escorted tours jointly classified by purpose and outbound, inbound, and total stop counts. Use it to compare stop-making complexity on escort tours. | `tour_purpose`: adult tour purpose.
`outbound_stop_count`: outbound stops capped at 3.
`inbound_stop_count`: inbound stops capped at 3.
`total_stop_count`: total stops capped at 6.
`tour_count`: weighted escorted tours in the combination. |
### Joint Travel
@@ -148,7 +147,7 @@ same tour twice. These values are not interchangeable.
| `joint_tour_party_size_distribution` | Joint tours by number of household participants, with parties of five or more stored in bin 5. Use it to assess joint-tour occupancy. | `party_size`: household participants; value 5 represents `5+`.
`joint_tour_count`: weighted joint tours in the party-size bin. |
| `joint_tour_composition_distribution` | Joint tours by prepared party-composition category. Use it to compare adult-only, child-inclusive, and other modeled compositions. | `tour_composition`: prepared joint-party composition.
`joint_tour_count`: weighted joint tours in the category. |
| `joint_tour_composition_by_party_size` | Joint tours jointly classified by party composition and exact participant count. Use it to study how household makeup and group size interact. | `tour_composition`: prepared party-composition category.
`party_size`: number of tour participants.
`joint_tour_count`: weighted joint tours in the combination. |
-| `person_jtp_by_household_size` | All people and people participating in one or more joint tours by household size. Use the two counts to calculate person-level participation rates. | `household_size`: size of the person’s household.
`joint_tour_person_count`: weighted people with `num_joint_tours > 0`.
`total_person_count`: weighted people in households of that size. |
+| `person_jtp_by_household_size` | All people and people participating in one or more joint tours by household size. Use the two counts to calculate person-level participation rates. | `household_size`: size of the person's household.
`joint_tour_person_count`: weighted people with `num_joint_tours > 0`.
`total_person_count`: weighted people in households of that size. |
| `household_jtp_by_household_size_and_jtf` | For households of size two or more, percentage distribution across 0, 1, and 2+ joint tours within each household size. Use it to compare joint-tour propensity independent of household-size totals. | `jtf`: joint-tour count category `0`, `1`, or `2+`.
`household_size`: household size as a category.
`household_percent`: percent of households of that size in the JTF category. |
### Basic Tour Distributions
@@ -160,9 +159,9 @@ same tour twice. These values are not interchangeable.
### Vehicles Allocated to Tours
-These tables decode the vehicle-type strings allocated under occupancy
-conditions 1, 2, and 3+. They describe modeled allocation incidences, not the
-unique household vehicle inventory.
+These tables decode vehicle-type strings for occupancy conditions 1, 2, and 3+.
+They describe modeled allocation incidences. They do not describe the unique
+household vehicle inventory.
| Summary table | Information and analytical use | Fields |
|---|---|---|
@@ -174,7 +173,7 @@ unique household vehicle inventory.
| Summary table | Information and analytical use | Fields |
|---|---|---|
-| `tour_mode_by_tour_purpose_and_auto_sufficiency` | Tour-mode counts by purpose and household auto sufficiency, with all-purpose rows. Joint tours are expanded by household participants for this summary. Use it to compare mode choice across vehicle-availability markets. | `tour_mode`: prepared tour mode.
`tour_purpose`: tour purpose or `all_tour_purposes`.
`tour_count_zero_auto`: weighted tours from zero-auto households.
`tour_count_auto_deficient`: weighted tours from households with fewer autos than workers.
`tour_count_auto_sufficient`: weighted tours from auto-sufficient households.
`tour_count_all_households`: sum of the three auto-sufficiency counts. |
+| `tour_mode_by_tour_purpose_and_auto_sufficiency` | Tour-mode counts by purpose and household auto sufficiency, with all-purpose rows. The calculation expands joint tours by household participants. Use it to compare mode choice across vehicle-availability markets. | `tour_mode`: prepared tour mode.
`tour_purpose`: tour purpose or `all_tour_purposes`.
`tour_count_zero_auto`: weighted tours from zero-auto households.
`tour_count_auto_deficient`: weighted tours from households with fewer autos than workers.
`tour_count_auto_sufficient`: weighted tours from auto-sufficient households.
`tour_count_all_households`: sum of the three auto-sufficiency counts. |
| `tour_stop_frequency_by_tour_purpose` | Tours jointly classified by purpose and outbound, inbound, and total intermediate-stop counts. Use it to measure tour complexity and stop-generation patterns. | `tour_purpose`: canonical tour purpose.
`outbound_stop_count`: outbound stops capped at 3.
`inbound_stop_count`: inbound stops capped at 3.
`total_stop_count`: total stops capped at 6.
`tour_count`: weighted tours in the combination. |
| `atwork_subtour_frequency_distribution` | Mandatory work tours by their at-work-subtour-frequency alternative. Use it to validate subtour generation from the workplace. | `atwork_subtour_frequency_category`: prepared at-work subtour-frequency category.
`atwork_subtour_count`: weighted parent work tours choosing the category. |
| `tour_time_of_day_by_tour_purpose` | Dense departure, arrival, and duration profiles by tour purpose plus all-purpose totals. Joint tours are participant-expanded. Use it to compare scheduling and duration distributions. | `time_bin`: ActivitySim period index.
`tour_purpose`: tour purpose or `all_tour_purposes`.
`departure_tour_count`: weighted tours starting in the bin.
`arrival_tour_count`: weighted tours ending in the bin.
`duration_tour_count`: weighted tours whose prepared duration falls in the bin. |
@@ -208,9 +207,9 @@ unique household vehicle inventory.
### Skimjoin Diagnostics
-`skim_scenario` distinguishes values for the chosen mode from hypothetical
-mode/scenario sidecars; `all_records` is used for hypothetical values evaluated
-over all applicable records. Each table also includes an all-modes group.
+`skim_scenario` identifies values for the selected mode or a hypothetical mode
+in a sidecar table. `all_records` identifies hypothetical values for all
+applicable records. Each table also includes an all-modes group.
| Summary table | Information and analytical use | Fields |
|---|---|---|
@@ -221,12 +220,12 @@ over all applicable records. Each table also includes an all-modes group.
### Processor-Built Validation Summaries
-Several assignment-based tables accept optional tables attached to `RunData`.
-They remain valid but empty when those optional assignment inputs are absent.
+Some assignment tables accept optional tables attached to `RunData`. The result
+is valid but empty when the optional assignment input is absent.
| Summary table | Information and analytical use | Fields |
|---|---|---|
-| `traffic_count_comparisons` | Observed and modeled traffic counts matched at count-location, direction, and period level. Use it for count scatterplots, percent differences, RMSE, and facility calibration. Only keys present in both sources are retained. | `count_location_id`: traffic-count station/location identifier.
`direction`: observed/modeled direction label.
`count_period`: count time-period label.
`observed_volume`: summed observed count for the key.
`modeled_volume`: summed assigned volume for the matching key. |
+| `traffic_count_comparisons` | Observed and modeled traffic counts that agree at count-location, direction, and period level. Use it for count scatterplots, percent differences, RMSE, and facility calibration. The table keeps only keys in both sources. | `count_location_id`: traffic-count station/location identifier.
`direction`: observed/modeled direction label.
`count_period`: count time-period label.
`observed_volume`: summed observed count for the key.
`modeled_volume`: summed assigned volume for the matching key. |
| `screenline_flow_comparisons` | Observed and modeled screenline flows matched by screenline, direction, and period, with a representative facility type. Use it for corridor-level flow validation and regression analysis. | `screenline_id`: screenline/cutline identifier.
`direction`: flow direction.
`count_period`: comparison period.
`facility_type`: supplied facility class, or `All` if absent.
`observed_volume`: summed observed flow.
`modeled_volume`: summed modeled flow for the matching key. |
| `transit_boardings_by_operator_and_technology` | Assigned transit boardings summed by operator and transit technology. Use it to compare ridership scale across agencies and modes. | `operator`: supplied transit operator identifier or name.
`technology`: supplied transit mode/technology category.
`boardings`: total assigned unlinked passenger boardings. |
| `transit_transfer_rate` | Assigned boardings divided by linked transit trips by operator, technology, and access mode. The value is boardings per linked trip, so values above one indicate transfers; subtract one if a transfers-per-trip measure is needed. | `operator`: transit operator.
`technology`: transit technology/mode.
`access_mode`: mode used to access transit.
`transfer_rate`: assigned boardings divided by linked trips; null for a zero linked-trip denominator. |
@@ -238,11 +237,10 @@ They remain valid but empty when those optional assignment inputs are absent.
### Externally Supplied Validation Contracts
-The following 13 tables are registered so externally prepared CSVs can be
-loaded consistently. Their no-op builders do not calculate values. The stated
-meaning is therefore the contract expected by the dashboard; the supplying
-workflow is responsible for units, period definitions, and internal
-consistency.
+The visualizer registers the following 13 tables for external CSV input. Their
+builders do not calculate values. The table descriptions define the dashboard
+contract. The external workflow must supply consistent units, period
+definitions, and values.
| Summary table | Information and analytical use | Fields |
|---|---|---|
@@ -262,7 +260,7 @@ consistency.
## Generated Developer Inventory
-Regenerate it with:
+Use this command to regenerate the inventory:
```bash
uv run python scripts/generate_wiki_catalogs.py
diff --git a/wiki/25-skimjoin-config-reference.md b/wiki/25-skimjoin-config-reference.md
index 2d4ae47..8aa9ced 100644
--- a/wiki/25-skimjoin-config-reference.md
+++ b/wiki/25-skimjoin-config-reference.md
@@ -1,17 +1,17 @@
# 25 - Skimjoin Config Reference
-This page is the field-by-field reference for the standalone skimjoin config
-file used by the main visualizer `skimjoin` step. For the workflow overview,
-start with [22 - Skimjoin](22-skimjoin.md). The canonical example is
+This page describes each field in the standalone skimjoin configuration file.
+The main visualizer `skimjoin` step uses this file. For a workflow introduction,
+read [22 - Skimjoin](22-skimjoin.md). See this canonical example:
[`example_skimjoin_config.yaml`](../example_skimjoin_config.yaml).
-Skimjoin config answers four questions:
+The skimjoin configuration answers four questions:
-1. Which skim files and optional `network_los.yaml` should be used?
-2. Which prepared trip and tour columns provide modes, ids, dimensions, and OD
+1. Which skim files and optional `network_los.yaml` must skimjoin use?
+2. Which prepared trip and tour columns supply modes, IDs, dimensions, and OD
lookup columns?
-3. Which matrix or sidecar table should be read for each mode/component?
-4. What should happen when matrices, OD pairs, or dimension values are missing?
+3. Which matrix or sidecar table must skimjoin read for each mode and component?
+4. Which policy applies when matrices, OD pairs, or dimension values are missing?
## Common Recipes
@@ -43,7 +43,7 @@ modes:
distance: SOV_DIST
```
-`time: SOV_TIME` is shorthand for:
+`time: SOV_TIME` has the same result as:
```yaml
time:
@@ -122,8 +122,9 @@ modes:
- matrix: SOV_TIME__MD
```
-Fallbacks run after the primary lookup for rows where the earlier step did not
-produce a valid value. Fallback steps share the same final output column.
+Fallbacks execute after the primary lookup. They apply to rows for which the earlier
+step did not supply a valid value. Fallback steps use the same final output
+column.
### Tour Aggregation
@@ -138,8 +139,8 @@ tour_aggregation:
skim_auto_time: true
```
-Tour lookups are also generated directly from mode rules. For tour lookup rules,
-outputs receive `_outbound` and `_inbound` suffixes.
+Mode rules also create tour lookups directly. Tour lookup output gets an
+`_outbound` or `_inbound` suffix.
## Top-Level Sections
@@ -155,7 +156,7 @@ outputs receive `_outbound` and `_inbound` suffixes.
| `modes` | mapping | required | Mode-specific lookup rules. |
| `tour_aggregation` | mapping | `aggregate_trips` with no configured aggregations | Trip-to-tour aggregation settings. |
-Unknown keys are rejected by the Pydantic schema for typed sections.
+The Pydantic schema rejects unknown keys in typed sections.
## `project`
@@ -193,17 +194,17 @@ Column names cannot be blank.
## `defaults`
-Defaults are inherited by every mode, segment, and component unless overridden
-closer to the rule.
+Each mode, segment, and component uses these defaults. A value nearer to the
+rule overrides a default.
| Field | Type | Default | Allowed values | Notes |
|---|---|---|---|---|
| `origin` | string | `origin` | source column | Origin column for OD lookups. |
| `destination` | string | `destination` | source column | Destination column for OD lookups. |
| `output_prefix` | string | `skim_` | any string | Prefix used when a component does not set `output`. |
-| `missing_matrix_policy` | string | `error` | `error`, `warn`, `set_null` | Policy for absent matrices or matrix names that cannot be resolved. |
+| `missing_matrix_policy` | string | `error` | `error`, `warn`, `set_null` | Policy for absent matrices or matrix names that skimjoin cannot resolve. |
| `missing_od_policy` | string | `error` | `error`, `warn`, `set_null` | Policy for missing/out-of-bounds OD values. |
-| `sentinel_values` | list of numbers | `[]` | numeric list | Lookup results equal to these values are treated as missing. |
+| `sentinel_values` | list of numbers | `[]` | numeric list | Skimjoin treats lookup results equal to these values as missing. |
```yaml
defaults:
@@ -217,13 +218,13 @@ defaults:
## Context Inheritance
-These keys may be set at the top `defaults` level, on a mode, inside a mode
-`defaults` block, inside a segment, or inside a component:
+You can set these keys in the top-level `defaults`, a mode, a mode `defaults`
+block, a segment, or a component:
`origin`, `destination`, `output_prefix`, `missing_matrix_policy`,
`missing_od_policy`, `sentinel_values`, `when`, and `dimensions`.
-Closer settings override or merge with parent settings:
+Settings nearer to a rule override or merge with parent settings:
| Key | Merge behavior |
|---|---|
@@ -245,7 +246,7 @@ modes:
## `zone_mapping`
-`zone_mapping` controls OMX lookup-name selection.
+`zone_mapping` controls the selection of an OMX lookup name.
| Field | Type | Default | Allowed values | Notes |
|---|---|---|---|---|
@@ -264,10 +265,10 @@ zone_mapping:
## `dimensions`
-Dimensions provide placeholder values for matrix names such as
+Dimensions supply placeholder values for matrix names such as
`SOV_TIME__{PERIOD}`.
-Each dimension entry has this shape:
+Each dimension entry has these fields:
| Field | Type | Default | Notes |
|---|---|---|---|
@@ -275,11 +276,11 @@ Each dimension entry has this shape:
| `source_columns.outbound_tour_source_column` | string | required | Source column used for outbound tour lookup rules. |
| `source_columns.inbound_tour_source_column` | string | required | Source column used for inbound tour lookup rules. |
| `values_from_network_los` | boolean | `false` | Only supported for `PERIOD`. Requires `project.network_los_file`. |
-| `values` | mapping | `{}` | Raw source value to matrix-name token. Keys and values are normalized to strings. |
+| `values` | mapping | `{}` | Raw source value to matrix-name token. The loader normalizes keys and values to strings. |
-If `values` is empty, the raw source value is converted to a string and inserted
-into the matrix name. If `values` is present, observed values must have a
-mapping.
+If `values` is empty, skimjoin converts the raw source value to a string. It
+then puts the string in the matrix name. If `values` is present, each observed
+value must have a mapping.
```yaml
dimensions:
@@ -303,9 +304,8 @@ dimensions:
## `ignore_modes`
-`ignore_modes` lists trip modes that are allowed to appear in prepared trips
-without a matching `modes` rule. This is useful for modes where skim enrichment
-is intentionally skipped.
+`ignore_modes` lists trip modes that do not require a matching `modes` rule.
+Use this list for modes that do not require skim enrichment.
```yaml
ignore_modes:
@@ -316,8 +316,8 @@ ignore_modes:
## `modes`
-`modes` is the heart of skimjoin. Each key is a prepared trip or tour mode. Each
-mode block may contain context keys plus component lookup rules.
+`modes` contains the main skimjoin rules. Each key is a prepared trip or tour
+mode. A mode block can contain context keys and component lookup rules.
Reserved keys inside a mode block:
@@ -341,7 +341,7 @@ Reserved keys inside a mode block:
| `tour_origin` | Reserved for future/compatibility context. |
| `tour_destination` | Reserved for future/compatibility context. |
-Every non-reserved key in a mode or segment block is treated as a component
+Skimjoin uses each non-reserved key in a mode or segment block as a component
name.
```yaml
@@ -352,11 +352,11 @@ modes:
distance: SR2_DIST
```
-The output names above are `skim_auto_time` and `skim_auto_distance`.
+This example creates `skim_auto_time` and `skim_auto_distance`.
## Component Rules
-A component rule may be a string matrix name or a mapping.
+A component rule can be a matrix-name string or a mapping.
| Field | Type | Default | Allowed values | Notes |
|---|---|---|---|---|
@@ -389,14 +389,14 @@ modes:
matrix: WTW_EGR__{PERIOD}
```
-When multiple rules write the same output on overlapping rows, use
-`combine: sum` on all overlapping rules. Otherwise validation treats the overlap
-as an output collision.
+If multiple rules write the same output for the same rows, set `combine: sum`
+on all applicable rules. Without this setting, validation reports an output
+collision.
## `when` Filters
-`when` narrows a rule to rows that match source column conditions. Conditions
-may be scalar equality or an `in` list.
+`when` applies a rule only to rows that agree with source column conditions. A
+condition can be scalar equality or an `in` list.
```yaml
modes:
@@ -409,14 +409,14 @@ modes:
outbound: true
```
-`when` filters merge through context inheritance. A mode-level filter applies to
-all of its components unless a child filter replaces the same column key.
+`when` filters merge through context inheritance. A mode-level filter applies
+to all its components. A child filter can replace the same column key.
## `segment_on` And `segments`
-Use `segment_on` when one mode needs different lookup rules for different
+Use `segment_on` when one mode requires different lookup rules for different
source values. Each key under `segments` is a value from the `segment_on`
-column. Skimjoin automatically adds a matching `when` filter for each segment.
+column. Skimjoin adds the applicable `when` filter for each segment.
```yaml
modes:
@@ -435,16 +435,15 @@ modes:
destination: DTAZ
```
-Validation checks that observed segment values for a covered mode have
-configured segment blocks.
+Validation makes sure that each observed value for a covered mode has a segment
+block.
## `fallbacks`
-Fallback entries use the same string or mapping shape as primary component
-rules. They are attempted in list order after failed prior steps. A fallback
-inherits the parent component output unless it explicitly sets an output, and
-validation requires all steps in a fallback chain to share the same final
-output.
+Fallback entries use the same string or mapping format as primary component
+rules. Skimjoin tries them in list order after a prior step fails. A fallback
+uses the parent component output unless it sets an output. All steps in a
+fallback chain must use the same final output.
```yaml
modes:
@@ -457,7 +456,7 @@ modes:
missing_matrix_policy: set_null
```
-Fallback reports are written to `fallback_lookup_report`.
+Skimjoin writes fallback reports to `fallback_lookup_report`.
## Lookup Types
@@ -466,9 +465,9 @@ Fallback reports are written to `fallback_lookup_report`.
| `od` | `matrix`, `origin`, `destination` | Reads an OMX OD matrix or CSV OD table by origin and destination. |
| `key` | `matrix`, `key_column` | Reads a keyed sidecar table by one source column. |
-For CSV skim files, inventory code identifies key/value or origin/destination
-columns from the file structure. For OMX, OD lookups use the configured
-`zone_mapping` lookup name.
+For CSV skim files, the inventory code finds key and value columns from the file
+structure. It can also find origin and destination columns. For OMX files, OD
+lookups use the configured `zone_mapping` lookup name.
```yaml
modes:
@@ -482,7 +481,7 @@ modes:
## Trip And Tour Rules
-Every component creates trip and tour lookup rules by default:
+By default, each component creates trip and tour lookup rules:
| Target | Source mode column | Dimension source | Output name |
|---|---|---|---|
@@ -490,12 +489,12 @@ Every component creates trip and tour lookup rules by default:
| Outbound tours | `activitysim.tour_mode_column` | `outbound_tour_source_column` | `output_outbound` |
| Inbound tours | `activitysim.tour_mode_column` | `inbound_tour_source_column` | `output_inbound` |
-Set `apply_to: trips` or `apply_to: tours` when a component should only run on
-one target table.
+Set `apply_to: trips` or `apply_to: tours` to execute a component on only one target
+table.
## `tour_aggregation`
-`tour_aggregation` controls trip-to-tour rollups for skim columns.
+`tour_aggregation` controls trip-to-tour totals for skim columns.
| Field | Type | Default | Allowed values | Notes |
|---|---|---|---|---|
@@ -516,7 +515,7 @@ tour_aggregation:
## Missing Data And Reports
-Skimjoin writes report artifacts during integrated prepare:
+During integrated prepare, skimjoin writes these report artifacts:
| Report | Purpose |
|---|---|
diff --git a/wiki/30-output-visualizer.md b/wiki/30-output-visualizer.md
index 964f751..b679915 100644
--- a/wiki/30-output-visualizer.md
+++ b/wiki/30-output-visualizer.md
@@ -1,7 +1,7 @@
# 30 - Output Visualizer
-The Output Visualizer reads processor outputs and presents them as either a
-live Panel dashboard or a standalone HTML export.
+The Output Visualizer reads processor output. It shows the output in a live
+Panel dashboard or a standalone HTML file.
```text
summary caches + optional prepared tables
@@ -14,7 +14,7 @@ The main code lives under [`dashboard/`](../dashboard).
## Visualizer Responsibilities
-The visualizer is responsible for:
+The visualizer does these tasks:
- loading summary runs
- loading prepared tables only for pages that request them
@@ -23,13 +23,13 @@ The visualizer is responsible for:
- rendering figures, tables, cards, and widgets
- exporting supported page states to standalone HTML
-It should not rebuild summaries. If a summary is missing, run the processor
-workflow first.
+The visualizer must not rebuild summaries. If a summary is missing, execute the
+processor workflow first.
## Live Dashboard
-The live dashboard is assembled in
-[`dashboard/app.py`](../dashboard/app.py). It creates:
+[`dashboard/app.py`](../dashboard/app.py) assembles the live dashboard. It
+creates:
- run colors and run legend
- `DashboardState`
@@ -45,7 +45,7 @@ pipeline:
dashboard_mode: live
```
-Then run the normal config command:
+Then use the standard configuration command:
```bash
uv run activitysim-viz --config local_config.yaml
@@ -53,9 +53,9 @@ uv run activitysim-viz --config local_config.yaml
## HTML Export
-HTML export uses the same page registry, but serializes supported page content
-into one self-contained HTML document. Export only includes states and selector
-variants generated at export time.
+HTML export uses the same page registry. It converts supported page content to
+one self-contained HTML document. The export includes only the states and
+selector variants that exist at export time.
Configure `pipeline.dashboard_mode: export` and an output path:
@@ -69,12 +69,12 @@ dashboard:
output_path: exports/dashboard.html
```
-The same normal config command then writes the export. For details, read
+The standard configuration command then writes the export. For details, read
[34 - HTML Export](34-html-export.md).
## Dashboard State
-`DashboardState` centralizes the global state pages react to:
+`DashboardState` contains the global state that pages use:
- loaded run labels
- selected weighting mode
@@ -82,13 +82,14 @@ The same normal config command then writes the export. For details, read
- optional segmentation type and visibility
- prepared-data provider state
-Pages should read state through the `DashboardPage` helpers instead of
-duplicating cache or run-selection logic.
+Pages must read state through the `DashboardPage` helpers. Do not duplicate
+cache or run-selection logic.
## Extension Path
-The [Dashboard Extension Cookbook](45-dashboard-extension-cookbook.md) shows
-complete page, page-group, selector, widget, table, and figure examples.
+The [Dashboard Extension Cookbook](45-dashboard-extension-cookbook.md) gives
+complete examples for pages, page groups, selectors, widgets, tables, and
+figures.
When adding visual output:
diff --git a/wiki/31-dashboard-pages.md b/wiki/31-dashboard-pages.md
index 5d3a0f7..af7c58e 100644
--- a/wiki/31-dashboard-pages.md
+++ b/wiki/31-dashboard-pages.md
@@ -1,8 +1,8 @@
# 31 - Dashboard Pages
-Dashboard pages are discovered from modules under
-[`dashboard/pages`](../dashboard/pages). Each leaf module contains one
-`DashboardPage` subclass decorated with `@dashboard_page(...)`; page packages
+The visualizer finds dashboard pages in modules under
+[`dashboard/pages`](../dashboard/pages). Each final module contains one
+`DashboardPage` subclass with a `@dashboard_page(...)` decorator. Page packages
export a `DashboardGroupDefinition` as `GROUP`.
## Page Definition Contract
@@ -20,21 +20,21 @@ Important fields:
| `optional_summary_ids` | Independent add-on summaries that may be absent. |
| `required_prepared_tables` | Prepared tables required by the page. |
-These declarations control dashboard cache loading, pruning, availability
-diagnostics, and prepared-table loading. They do **not** select which generated
-summaries the summarize workflow builds; ordinary summarize runs build every
-`build_by_default=True` declaration.
+These declarations control dashboard cache loads, removal of unused data,
+availability diagnostics, and prepared-table loads. They do not select the
+summaries that the summarize workflow builds. Standard summarize runs build
+each declaration that has `build_by_default=True`.
-`required_summary_ids` marks the page's primary data. If no run has a usable
-required table, `self.data.summary(...)` records a required-data warning and the
-page should render a standard unavailable card. `optional_summary_ids` declares
-an independent add-on: its absence should hide or replace only that feature.
-Neither declaration crashes the whole dashboard, and both can be partially
-available when some runs are usable and others are excluded.
+`required_summary_ids` identifies the primary page data. If no run has a usable
+required table, `self.data.summary(...)` records a required-data warning. The
+page must then show a standard unavailable card. `optional_summary_ids`
+identifies an independent feature. If this data is absent, hide or replace only
+that feature. Missing declared data does not stop the complete dashboard. Data
+can also be available for only some runs.
## Enabling Pages
-Live pages are selected in config:
+Select live pages in the configuration:
```yaml
dashboard:
@@ -55,15 +55,15 @@ Group selection modes are:
| `trip_summaries: all` | Every registered child, including children with `default_enabled=False`. |
| `trip_summaries: [trip_mode, trip_stop_distance]` | Exactly the listed children in that order. |
-When `dashboard.live.pages` is omitted, standalone pages and groups must be
-default-enabled, and grouped children must also be default-enabled. A group's
-`default_page_id` selects the initially visible tab/fallback; it does not by
-itself enable every child.
+If you omit `dashboard.live.pages`, the visualizer selects default-enabled
+standalone pages and groups. It also selects default-enabled children in each
+group. A group's `default_page_id` selects the first visible tab or fallback
+tab. It does not enable all children.
-`dashboard.export.pages` modifies matching pages in the resolved live page set;
-it is not an allow-list. Unmentioned live pages keep their default export
-behavior. Use `enabled: false`, `exclude_pages`, or `exclude_groups` to narrow
-the export. Export cannot add a page omitted from `dashboard.live.pages`.
+`dashboard.export.pages` changes matching pages in the resolved live page set.
+It is not an allow-list. Live pages without an entry keep their default export
+behavior. Use `enabled: false`, `exclude_pages`, or `exclude_groups` to remove
+pages. Export cannot add a page that `dashboard.live.pages` omits.
For example, enable only two trip-summary children:
@@ -79,9 +79,9 @@ dashboard:
## Prepared-Data Pages
-Most pages are summary-backed. A prepared-data page declares
-`prepared_data_mode` and `required_prepared_tables`. Use prepared data only when
-the page truly needs disaggregate records.
+Most pages use summary data. A prepared-data page declares `prepared_data_mode`
+and `required_prepared_tables`. Use prepared data only when the page requires
+disaggregate records.
Current runtime behavior is:
@@ -89,43 +89,43 @@ Current runtime behavior is:
|---|---|
| `none` | Prepared caches are not requested for the page. `required_prepared_tables` must be empty. |
| `optional` | Prepared caches are requested, but the page's primary summary-backed workflow should remain useful when they are unavailable. |
-| `required` | Prepared caches are requested and the page should present an unavailable state when they cannot be loaded. |
+| `required` | The runtime requests prepared caches. The page must show an unavailable state if it cannot load them. |
-Both `optional` and `required` trigger loading; the distinction communicates
-feature criticality and contributes to the strongest requirement across enabled
-pages. Page render code remains responsible for the fallback. Standalone HTML
-export does not load prepared tables; see chapter 34 for section-level export
+Both `optional` and `required` cause a data load. The value identifies whether
+the feature requires the data. It also contributes to the strongest requirement
+for all enabled pages. Page render code must supply the fallback. Standalone
+HTML export does not load prepared tables. See chapter 34 for section export
rules.
## Availability And Validation Features
-Page selectors are data-aware. Option providers enumerate values present in
-usable runs, and the page lifecycle repairs a selection when an upstream choice
-makes it invalid. A selector should not offer a value whose dependent section
-would be empty merely because that value exists in a hard-coded domain.
+Page selectors use available data. Option providers list values in usable runs.
+The page lifecycle repairs a selection if an earlier choice makes it invalid.
+Do not show a value only because it occurs in a fixed domain. Show it only when
+its dependent section has data.
-When no usable run remains, pages render the standard data-unavailable card for
-the affected feature. Required data can make the page's primary workflow
-unavailable; missing optional data replaces only its independent feature. Set
-`display.missing_data_display: blank` to suppress these cards globally.
+When no usable run remains, the page shows the standard data-unavailable card
+for the applicable feature. Missing required data can make the primary page
+workflow unavailable. Missing optional data replaces only its independent
+feature. Set `display.missing_data_display: blank` to hide all these cards.
-The validation group currently provides:
+The validation group provides:
| Page | Current behavior |
|---|---|
-| Traffic Validation | Observed-versus-modeled count-location fit, traffic volume summaries, top modeled count locations, link tables, and screenline flow comparison. Count-location diagnostics report location count, RMSE, RMSPE, and R-squared by facility group. Scatterplots include a 1:1 line; fitted equations, R-squared, and sample size appear on fit-line hover. Screenlines can be filtered by time period and facility type before a per-run ordinary-least-squares fit is calculated. RMSPE is blank for a group containing a zero observed count. |
-| Transit Validation | Boardings by operator/technology and transfer rates by operator, technology, and access mode, with calculation notes and unavailable states when the supplied contracts cannot be used. |
+| Traffic Validation | Observed-versus-modeled count-location fit, traffic volume summaries, top modeled count locations, link tables, and screenline flow comparison. Count-location diagnostics report location count, RMSE, RMSPE, and R-squared by facility group. Scatterplots include a 1:1 line. Fit-line hover shows the fitted equation, R-squared, and sample size. Filter screenlines by time period and facility type before the system calculates a fit for each run. RMSPE is blank for a group that contains a zero observed count. |
+| Transit Validation | Boardings by operator and technology.
Transfer rates by operator, technology, and access mode.
The page shows notes and unavailable states if it cannot use the supplied contracts. |
| VMT Validation | Overview comparisons plus selector-driven personal-auto and non-motorized VMT. Optional outside tables add external travel/VMT, commercial travel/VMT, and bicycle facility summaries; each optional feature gets its own unavailable state. |
-| Regional Validation | Optional district or county observed flow matrices, modeled `commuting_flows`, and aligned heatmaps for modeled, observed, difference, percent difference, or absolute percent difference. Totals can be included or excluded. Only flow types backed by available inputs appear in the selector. |
+| Regional Validation | Optional district or county observed flow matrices, modeled `commuting_flows`, and aligned heatmaps. Heatmaps can show modeled, observed, difference, percent difference, or absolute percent difference. You can include or exclude totals. The selector shows only flow types that have available input. |
-Expandable calculation notes beneath these outputs identify source summary IDs,
-filters, formulas, and aggregation details. They are enabled by default and can
-be hidden with `dashboard.include_notes: false`.
+Expandable calculation notes identify source summary IDs, filters, formulas,
+and aggregation details. They occur below the applicable output. They are on by
+default. Set `dashboard.include_notes: false` to hide them.
## Generated Page Catalog
-The catalog below is generated from the dashboard page registry. Regenerate it
-with:
+The dashboard page registry generates the catalog below. Use this command to
+regenerate it:
```bash
uv run python scripts/generate_wiki_catalogs.py
diff --git a/wiki/32-figures-and-widgets.md b/wiki/32-figures-and-widgets.md
index 3e8fb17..04fd9a8 100644
--- a/wiki/32-figures-and-widgets.md
+++ b/wiki/32-figures-and-widgets.md
@@ -1,65 +1,64 @@
# 32 - Figures And Widgets
-Current pages declare data access, selectors, independently refreshable
-sections, and figures through one shared authoring model. Framework code owns
-widget synchronization, query identity, missing-data diagnostics, and export
-metadata.
+Pages use one authoring model for data access, selectors, refreshable sections,
+and figures. The framework controls widget synchronization, query identity,
+missing-data diagnostics, and export metadata.
## Page Lifecycle
-Every page subclasses `DashboardPage` and implements `build_page()`. That method
-declares selectors and sections once and returns a stable Panel layout.
+Each page subclasses `DashboardPage` and implements `build_page()`. This method
+declares selectors and sections one time. It returns a stable Panel layout.
-`DashboardPage.__init__()` calls `build_page()` after it creates `self.data`,
-page state, and the component registries. Ordinary pages should therefore not
-define their own `__init__`. If specialized initialization is unavoidable, it
-must call `super().__init__(state, config)`, and attributes used by
-`build_page()` must exist before that call. In practice, put declarations in
-`build_page()` and keep implementation mixins free of `__init__` methods.
+`DashboardPage.__init__()` creates `self.data`, page state, and the component
+registries. It then calls `build_page()`. Thus, a standard page must not define
+its own `__init__`. If a page requires special initialization, call
+`super().__init__(state, config)`. Create attributes for `build_page()` before
+that call. Put declarations in `build_page()`. Do not put an `__init__` method
+in an implementation mixin.
The main author-facing objects are:
- `self.data` for summary and prepared `RunTables`
-- `self.select(...)` for ordinary dropdowns, including dynamic options
+- `self.select(...)` for standard selection lists, including dynamic options
- `self.selector(...)` only for custom widgets
- `self.section(...)` for refreshable visible regions
- `self.feature(...)` for a namespaced group of selectors and sections
- `self.query(...)` for repeated or expensive transformations
- `self.plot` for figures and tables
-Do not add routine `sync_controls()` or page-authored cache keys. Option
-providers and section dependencies give the framework enough information to do
-that work.
+Do not add routine `sync_controls()` methods or page cache keys. Option
+providers and section dependencies supply the required information to the
+framework.
## Data And Figures
-For end-to-end examples of an ordinary chart, a Plotly customization, a new
-shared figure type, a custom widget, and a table, use the
+For complete examples of a standard chart, a Plotly customization, a new shared
+figure type, a custom widget, and a table, use the
[Dashboard Extension Cookbook](45-dashboard-extension-cookbook.md).
For the complete chart-method, count/share, table, and figure-testing API, see
the [Plotting Reference](35-plotting-reference.md).
-Load the narrowest useful data selection through `self.data.summary(...)` or
-`self.data.summaries(...)`. `RunTables` applies the same Polars operation across
-runs while retaining labels and availability issues; it supports operations
+Load only the required data through `self.data.summary(...)` or
+`self.data.summaries(...)`. `RunTables` applies the same Polars operation to
+each run. It keeps labels and availability issues. It supports operations
such as `where`, `with_columns`, `group`, `select`, `sort`, `join`, `map`,
`requiring`, and `drop_empty`.
-A `RunTables` value is truthy when at least one run has a non-empty compatible
-table. Runs with a missing table, schema mismatch, failure, or empty input are
-excluded from iteration and described in `data.issues`; consequently,
-`data.partial` means there are both usable and excluded runs. Fluent operations
-preserve those issues. Filtering can make a frame empty without removing it, so
-call `.drop_empty()` when downstream code should ignore those runs.
+A `RunTables` value is true when at least one run has a nonempty compatible
+table. Iteration excludes runs with a missing table, schema mismatch, failure,
+or empty input. `data.issues` describes these runs. `data.partial` means that
+there are usable and excluded runs. Query operations keep these issues. A
+filter can make a frame empty without removal. Call `.drop_empty()` when later
+code must ignore these runs.
The `columns=` argument to `summary()` and `prepared()` is a compatibility
-check: a run missing any named column is excluded with a schema diagnostic. It
-does **not** project the returned frames. Use `.select(...)` when a transform
-needs a narrower schema.
+check. The system excludes a run that is missing a named column. It also adds a
+schema diagnostic. The argument does not select columns in the returned frames.
+Use `.select(...)` to select a smaller schema.
-Pass `RunTables` to `self.plot` methods where possible. Shared rendering lives
-under `dashboard/rendering/`, including figures, tables, layout, and plotter
-logic. Cross-page domain helpers live under `dashboard/helpers/`.
+Pass `RunTables` to `self.plot` methods when possible. Figures, tables, layout,
+and plotter logic are in `dashboard/rendering/`. Shared page helpers are in
+`dashboard/helpers/`.
```python
def render_mode_chart(self):
@@ -84,7 +83,7 @@ The page-facing data API is:
| `self.data.summary(id, weighting=None, columns=(), required=None)` | One summary across usable runs. `columns` performs a schema compatibility check. |
| `self.data.summaries(*ids, columns=None, required=None)` | A dictionary of summary ID to `RunTables`. |
| `self.data.prepared(table, columns=(), weighting_mode=None)` | One declared prepared table across loaded runs. |
-| `self.data.prepared_runs(weighting_mode=None)` | Specialized `RunData` escape hatch for features that require matrices or other non-table state. |
+| `self.data.prepared_runs(weighting_mode=None)` | Direct `RunData` access for features that require matrices or other non-table state. |
| `self.data.summary_series(id, weighting=None)` | Specialized skim-summary view that retains summary-series metadata. |
`RunTables` is iterable and indexable as `(run_label, DataFrame)` pairs. Its
@@ -106,19 +105,19 @@ public fluent/query surface is:
## Calculation Notes
-Calculation notes are expandable, dependency-free HTML details displayed
-beneath annotated charts and tables. Users can hide all notes with:
+Calculation notes are expandable HTML details below annotated charts and
+tables. They have no external dependencies. To hide all notes, use:
```yaml
dashboard:
include_notes: false
```
-Content lives in `dashboard/calculation_notes.yaml`. The top-level `methods`
-mapping contains reusable method explanations; `notes` contains stable note
-IDs. Every note requires `summary`, `method`, and a non-empty `sources` list,
-and may add `label`, `method_text`, `formula`, `source_filters`, and grouped
-`details`. Loading validates unknown fields and method references.
+`dashboard/calculation_notes.yaml` contains the content. The top-level `methods`
+mapping contains reusable method explanations. `notes` contains stable note
+IDs. Each note requires `summary`, `method`, and a nonempty `sources` list. A
+note can also contain `label`, `method_text`, `formula`, `source_filters`, and
+grouped `details`. The loader validates unknown fields and method references.
Page authors attach a note to a registered selector-driven section with:
@@ -131,14 +130,14 @@ body = self.section(
return self.noted_section("traffic.observed_model_fit", body)
```
-Use `self.noted_view(note_id, view)` for an individual plot or table that is
-not itself the registered section container. `self.section_note(...)` is the
-lower-level helper and rejects unregistered sections. Notes use the same page
-layout in live mode and HTML export.
+Use `self.noted_view(note_id, view)` for an individual plot or table outside the
+registered section container. `self.section_note(...)` is the low-level helper.
+It rejects unregistered sections. Notes use the same page layout in live mode
+and HTML export.
## Selectors
-Declare a normal dropdown with its option domain in one place:
+Declare a standard selection list and its option domain in one place:
```python
self.purpose = self.select(
@@ -149,10 +148,10 @@ self.purpose = self.select(
)
```
-An option provider is called before dependent sections render. The framework
-repairs stale values. `default` may be `"first"`, `"last"`, or a callable.
-Use `self.selector(...)` only when wrapping a custom checkbox, numeric input, or
-another widget that `select(...)` cannot express.
+The framework calls an option provider before it renders dependent sections.
+It repairs stale values. `default` can be `"first"`, `"last"`, or a callable.
+Use `self.selector(...)` only for a custom checkbox, numeric input, or other
+widget that `select(...)` cannot define.
## Sections And Features
@@ -166,20 +165,20 @@ chart = self.section(
)
```
-A section renderer may return one Panel `Viewable`, or a list/tuple of
-`Viewable` objects. It should not mutate the stable section container itself;
-the lifecycle replaces that container's contents after each render.
+A section renderer can return one Panel `Viewable` or a list or tuple of
+`Viewable` objects. It must not change the stable section container. The
+lifecycle replaces the container content after each render.
-For a large page, use `self.feature("comparison")` to namespace a coherent
+For a large page, use `self.feature("comparison")` to give a name to one
workflow. Feature component IDs become `comparison.metric`, `comparison.body`,
-and so on. Features participate in the same lifecycle and export behavior as
-the parent page.
+and similar names. Features use the same lifecycle and export behavior as the
+parent page.
-Large controllers may also use private implementation mixins under a
-`_/` package. Mixins organize source responsibilities; `PageFeature`
-organizes live components. A refactored page commonly uses both. Keep mixins
-focused, do not give them `__init__` methods, keep pure transforms as functions,
-and preserve page/component IDs during source-only refactors.
+Large controllers can also use private implementation mixins in a `_/`
+package. Mixins organize source responsibilities. `PageFeature` organizes live
+components. A page can use both. Give each mixin one purpose. Do not give it an
+`__init__` method. Keep pure transforms as functions. Do not change page or
+component IDs during a source-only refactor.
### Large-Page Implementation Mixins
@@ -216,9 +215,9 @@ class ExamplePage(
pass
```
-Every mixin method receives the final `ExamplePage` instance. Python resolves
-methods left to right through the declared bases and then `DashboardPage`.
-Mixins are not standalone pages and must not be instantiated.
+Each mixin method receives the final `ExamplePage` instance. Python resolves
+methods from left to right through the declared bases. It resolves
+`DashboardPage` last. Mixins are not standalone pages. Do not instantiate them.
Keep this pattern narrow:
@@ -229,14 +228,13 @@ Keep this pattern narrow:
- keep stateless pure functions outside mixins
- preserve page, selector, section, and export IDs during source-only refactors
-Mixins organize Python source; `PageFeature` organizes registered live
-components. One does not replace the other. Prefer one page class until stable
-composition, domain, transformation, and rendering boundaries make the split
-easier to understand.
+Mixins organize Python source. `PageFeature` organizes registered live
+components. They have different purposes. Use one page class until the
+composition, domain, transformation, and rendering boundaries are stable.
## Shared Helpers
-Check these before adding page-local utilities:
+Before you add page-local utilities, examine these modules:
| Module | Use |
|---|---|
@@ -249,10 +247,10 @@ Check these before adding page-local utilities:
## Export Considerations
-Export behavior derives from the same selectors and sections used live. Keep
-render methods deterministic for each selector state and avoid unregistered
-live-only callbacks. Export can only include selector values generated at
-export time.
+The same selectors and sections control live and export behavior. Make sure
+that render methods give the same result for each selector state. Do not use
+unregistered live-only callbacks. Export can include only selector values that
+exist at export time.
## Related Chapters
diff --git a/wiki/33-dashboard-page-recipes.md b/wiki/33-dashboard-page-recipes.md
index ed8af47..099cfae 100644
--- a/wiki/33-dashboard-page-recipes.md
+++ b/wiki/33-dashboard-page-recipes.md
@@ -1,7 +1,8 @@
# 33 - Dashboard Page Recipes
-Use the smallest page shape that fits the behavior. Each discoverable page
-module contains one class decorated with `@dashboard_page(...)`.
+Use the smallest page structure that supplies the required behavior. Each
+discoverable page module contains one class with a `@dashboard_page(...)`
+decorator.
## Recipe 1: Simple Summary Page
@@ -30,8 +31,8 @@ class MySummaryPage(DashboardPage):
return data_table(data, title="My Summary")
```
-Use `required_summary_ids` for the page's primary workflow and
-`optional_summary_ids` for independent add-on features.
+Use `required_summary_ids` for the primary page workflow. Use
+`optional_summary_ids` for independent optional features.
## Recipe 2: Dynamic Selector
@@ -80,12 +81,12 @@ def render_chart(self):
```
The framework refreshes options and dependent sections. Use
-`self.selector(...)` only for a genuinely custom widget. Keep the label-to-raw
-mapping so display labels do not leak into data filters.
+`self.selector(...)` only for a custom widget. Keep the label-to-raw mapping.
+This mapping prevents display labels from entering data filters.
## Recipe 3: Multi-Workflow Page
-Create one `PageFeature` per coherent user workflow:
+Create one `PageFeature` for each user workflow:
```python
comparison = self.feature("comparison")
@@ -95,10 +96,10 @@ comparison_body = comparison.section(
)
```
-When the Python controller itself becomes difficult to navigate, keep the
-registered page as a small facade and split implementation mixins into a
-private `_/` package. Current examples include tour mode, mandatory
-location choice, escorted tours, VMT, and traffic validation.
+If the Python controller becomes difficult to read, keep the registered page as
+a small facade. Put implementation mixins in a private `_/` package.
+Examples include tour mode, mandatory location choice, escorted tours, VMT,
+and traffic validation.
## Recipe 4: Prepared-Data Page
@@ -137,21 +138,20 @@ class RawTripDemoPage(DashboardPage):
)
```
-Load prepared data through `self.data`, handle an unavailable selection with a
-standard card, and keep disaggregate use limited. Prefer summaries for repeated
-aggregate views. `raw_trip_demo.py`, the skim pages, and parking location show
-the current required/optional patterns.
+Load prepared data through `self.data`. Show a standard card for unavailable
+data. Use disaggregate data only when necessary. Use summaries for repeated
+aggregate views. See `raw_trip_demo.py`, the skim pages, and parking location
+for current required and optional patterns.
-Mark every section that reads prepared data with
+Mark each section that reads prepared data with
`export_data_mode="optional"` or `"required"`. Standalone export does not load
-prepared tables and skips those sections. If the page also has a summary-backed
-view that should export, place it in a separate section whose
-`export_data_mode` remains `"none"`.
+prepared tables. It omits these sections. Put an exportable summary view in a
+separate section. Keep `export_data_mode="none"` for that section.
## Adding A New Page Group
-For a complete file layout, config example, discovery explanation, and tests,
-see [Add A New Page Group](45-dashboard-extension-cookbook.md#add-a-new-page-group).
+For a complete file layout, configuration example, discovery description, and
+tests, see [Add A New Page Group](45-dashboard-extension-cookbook.md#add-a-new-page-group).
Create a package under `dashboard/pages/` and define `GROUP` in `__init__.py`:
@@ -166,9 +166,8 @@ GROUP = DashboardGroupDefinition(
)
```
-Every child decorator sets `group_id="my_group"`. Discovery rejects duplicate
-IDs, missing definitions, unknown groups, and invalid summary or prepared-table
-requirements.
+Set `group_id="my_group"` in each child decorator. Discovery rejects duplicate
+IDs, missing definitions, unknown groups, and invalid data requirements.
## Page Review Checklist
diff --git a/wiki/34-html-export.md b/wiki/34-html-export.md
index c98a828..41d972a 100644
--- a/wiki/34-html-export.md
+++ b/wiki/34-html-export.md
@@ -1,7 +1,7 @@
# 34 - HTML Export
-HTML export writes a standalone dashboard file that can be opened without a
-Python server.
+HTML export writes a standalone dashboard file. You can open this file without
+a Python server.
```text
registered dashboard pages
@@ -13,18 +13,18 @@ registered dashboard pages
## When To Use Export
-Use export when you need:
+Use export for these requirements:
- an offline deliverable
-- a dashboard that can be emailed or archived
-- a frozen set of run comparisons
+- a dashboard that you can send or archive
+- a fixed set of run comparisons
- no Python server dependency for viewers
-Use live mode when you need:
+Use live mode for these requirements:
- full Python-backed interactivity
- exploratory pages that are not export-ready
-- development/debugging feedback
+- development or debug feedback
## Export Configuration
@@ -42,35 +42,35 @@ dashboard:
output_path: exports/dashboard.html
```
-Run the same command used for every configured workflow:
+Use the standard command for a configured workflow:
```bash
uv run activitysim-viz --config local_config.yaml
```
-This writes `artifacts/exports/dashboard.html`. Relative export paths resolve
-below `root`; an absolute path writes elsewhere. Change
-`pipeline.dashboard_mode` back to `live` when the same config should serve the
-dashboard instead.
+This command writes `artifacts/exports/dashboard.html`. Relative export paths
+start from `root`. An absolute path specifies a different location. Set
+`pipeline.dashboard_mode` to `live` to start the dashboard from the same
+configuration.
-It also writes `artifacts/exports/dashboard.diagnostics.json`. The sidecar
-records export warnings and size/state analysis for developers; the HTML does
-not depend on the sidecar when it is opened or shared.
+The command also writes `artifacts/exports/dashboard.diagnostics.json`. This
+sidecar file records export warnings and size or state analysis. The HTML file
+does not require the sidecar file.
-For a one-off override, use `--export-html [PATH]`. With no path, the CLI uses
-the configured output path and then falls back to
-`/exported_dashboard.html`. The dashboard step must still be selected;
-add `--dashboard` when it is absent from `pipeline.steps`.
+For one override, use `--export-html [PATH]`. If you do not give a path, the
+command uses the configured output path. If that path is absent, it uses
+`/exported_dashboard.html`. You must also select the dashboard step. Add
+`--dashboard` if `pipeline.steps` does not contain it.
-Export begins with the pages resolved by `dashboard.live.pages`. The
-`dashboard.export.pages` mapping modifies matching page selectors and parts; it
-does not select the included page set. Use a page override with `enabled: false`,
-`exclude_pages`, or `exclude_groups` to narrow the live set. Export cannot add a
-page that live configuration omitted.
+Export starts with the pages from `dashboard.live.pages`. The
+`dashboard.export.pages` mapping changes matching page selectors and parts. It
+does not select the page set. Use `enabled: false`, `exclude_pages`, or
+`exclude_groups` to remove pages. Export cannot add a page that the live
+configuration omits.
## Supported Runtime Behavior
-The export runtime supports a deliberately small set of rendered objects:
+The export runtime supports these rendered objects:
- containers
- cards
@@ -81,57 +81,53 @@ The export runtime supports a deliberately small set of rendered objects:
- registered regions
- registered selector widgets
-Viewers can collapse and restore the export sidebar with the header button;
-Plotly charts resize after the layout changes. Long run names use compact,
-unique tab and legend labels while their full text remains available in tab
-tooltips and chart hovers.
+Use the header button to close or open the export sidebar. Plotly charts change
+size after the layout changes. Long run names use short, unique tab and legend
+labels. Tab tooltips and chart hover text show the full names.
The Python-to-JavaScript contract lives in `dashboard/export/types.py`, and the
browser runtime lives under `dashboard/export/js_runtime/`.
## Selector Variants
-Page-local export interactivity is pre-rendered. During export, the runtime
-walks configured selector values, renders page regions, serializes them, and
-stores them as variants.
+The exporter creates page interactivity before it writes the file. It processes
+configured selector values and renders page regions. It converts the regions
+to export data and stores them as variants.
-That means:
+These rules apply:
- exported selectors can only switch among values generated at export time
- large selector domains can make export files large
- pages must register selectors and sections through the page API
- live-only callbacks do not automatically work in export
-Selector and part names are author-defined IDs, not widget labels or section
-titles. Find selector IDs in a page's `self.select(...)` and
-`self.selector(...)` calls, and part IDs in `self.section(...)` calls. Feature
-IDs prefix their components (for example, `comparison.metric` and
-`comparison.body`). The page/group IDs are listed in the generated catalog in
-chapter 31, and chapter 13 contains a complete override example. Invalid page,
-selector, part, or selector-value entries fail or produce a targeted warning
-rather than being silently guessed.
+Selector and part names are IDs from the author. They are not widget labels or
+section titles. Find selector IDs in `self.select(...)` and
+`self.selector(...)` calls. Find part IDs in `self.section(...)` calls. Feature
+IDs are prefixes for their components. Examples are `comparison.metric` and
+`comparison.body`. The generated catalog in chapter 31 lists page and group
+IDs. Chapter 13 contains a complete override example. An invalid page,
+selector, part, or selector value causes an error or a specific warning.
For a concrete selector/section declaration that works in both modes, see the
[Dashboard Extension Cookbook](45-dashboard-extension-cookbook.md#add-a-dynamic-selector).
## Page Authoring Contract
-Export metadata comes from the same page registration graph used by the live
-dashboard:
+Live mode and export use the same page registration graph for metadata:
- `@dashboard_page(...)` owns page identity, grouping, order, and data
requirements.
- `build_page()` creates stable widgets, sections, features, and layout.
-- `self.select(...)` registers ordinary dropdowns and their option/default
+- `self.select(...)` registers standard selection lists and their option/default
policy.
- `self.selector(...)` registers custom widgets.
- `self.section(...)` defines refresh and export-region boundaries.
-Keep section renderers deterministic for a given selector state. Set
-`export=False` on a section that should remain static in the exported shell,
-and `exportable=False` on a selector that should remain live-only. Do not add a
-second export-only registry or duplicate selector metadata on the page
-definition.
+Make sure that a section renderer gives the same result for a specified
+selector state. Set `export=False` on a section that must stay static in the
+exported shell. Set `exportable=False` on a live-only selector. Do not add an
+export-only registry. Do not copy selector metadata to the page definition.
Grouped export configuration addresses children by their leaf `page_id`:
@@ -145,13 +141,13 @@ dashboard:
tour_purpose: all
```
-Validation rejects unknown page, group, selector, and part IDs against this
-shared runtime graph.
+Validation compares page, group, selector, and part IDs with this shared runtime
+graph. It rejects unknown IDs.
## Prepared Data Is A Live-Only Boundary
-The export workflow loads summary caches but does not load prepared runs. A
-section that reads prepared data must declare that boundary:
+The export workflow loads summary caches. It does not load prepared runs. A
+section that reads prepared data must declare this limit:
```python
trip_table = self.section(
@@ -161,12 +157,11 @@ trip_table = self.section(
)
```
-During HTML export, any section whose `export_data_mode` is `optional` or
-`required` is skipped. The distinction still documents whether the feature is
-optional or essential in live mode. Summary-only sections use the default
-`export_data_mode="none"` and remain eligible for export. Split mixed pages
-into separate prepared-backed and summary-backed sections so the latter can be
-exported safely.
+HTML export omits a section if its `export_data_mode` is `optional` or
+`required`. These values identify whether the feature is optional or required
+in live mode. Summary-only sections use the default `export_data_mode="none"`.
+Export can include these sections. On a mixed page, put prepared data and
+summary data in separate sections. Export can then include the summary section.
## Important Files
@@ -190,15 +185,15 @@ exported safely.
| API | Behavior |
|---|---|
| `build_export_html_document(runs, config, summary_runs=None) -> str` | Build, serialize, and validate a complete HTML document in memory. Useful for tests and callers that need the string. |
-| `write_export_html_document(output_path, runs, config, summary_runs=None) -> Path` | Build the payload, stream JSON into a temporary HTML file, write the diagnostics sidecar through a temporary file, and replace each destination only after that file is complete. This is the normal workflow path. |
-
-Payload construction sanitizes NumPy/Pandas values before JSON encoding;
-non-finite numeric values become JSON `null`, timestamps become ISO strings,
-and closing script tags are escaped. The writer streams the JSON rather than
-materializing a second payload string or final HTML string, which keeps peak
-memory lower for large selector-state exports. A serialization, shell, write,
-or finalization failure raises an `ExportBuildError` naming the failed phase
-and cleans up temporary files.
+| `write_export_html_document(output_path, runs, config, summary_runs=None) -> Path` | Build the payload and stream JSON into a temporary HTML file.
Write the diagnostics sidecar through a temporary file.
Replace each destination only after the temporary file is complete.
This is the standard workflow method. |
+
+Payload construction cleans NumPy and Pandas values before JSON encoding.
+Nonfinite numeric values become JSON `null`. Timestamps become ISO strings.
+The exporter escapes closing script tags. The writer streams the JSON and does
+not create a second payload string or final HTML string. This decreases peak
+memory use for exports with many selector states. A conversion, shell, write,
+or finalization failure raises an `ExportBuildError`. The error identifies the
+failed phase, and the writer removes temporary files.
## Changing Export Runtime Behavior
diff --git a/wiki/35-plotting-reference.md b/wiki/35-plotting-reference.md
index 0ea9392..6e9c2d0 100644
--- a/wiki/35-plotting-reference.md
+++ b/wiki/35-plotting-reference.md
@@ -1,10 +1,10 @@
# 35 - Plotting Reference
-Dashboard pages use one plotting surface: `self.plot`. It accepts the same
-`RunTables` object returned by `self.data`, applies the session's run colors and
-count/share mode, and returns a Panel view ready for a section.
+Dashboard pages use one plotting interface: `self.plot`. It accepts the
+`RunTables` object that `self.data` returns. It applies the session run colors
+and count or share mode. It returns a Panel view for a section.
-## The normal path
+## Standard method
Fetch, query, and plot without converting the data to tuple lists:
@@ -29,9 +29,9 @@ return self.plot.bar(
)
```
-Every chart argument after `data` is keyword-only. The short names (`x`, `y`,
-`x_title`, and `y_title`) are the complete public vocabulary; the former
-`x_col`, `y_col`, and `xaxis_title` aliases are not supported.
+Each chart argument after `data` is keyword-only. The public names are `x`, `y`,
+`x_title`, and `y_title`. The interface does not support the former `x_col`,
+`y_col`, and `xaxis_title` aliases.
## Chart types
@@ -42,8 +42,8 @@ Use:
- `self.plot.line(...)` for an unfilled profile; and
- `self.plot.scatter(...)` for observed-versus-modeled comparisons.
-All four validate their required columns before calling Plotly. An error names
-the chart type, run, and missing columns.
+All four methods validate their required columns before they call Plotly. An
+error identifies the chart type, run, and missing columns.
```python
return self.plot.density(
@@ -58,8 +58,8 @@ return self.plot.density(
```
Sort ordered data in the query. For categorical bars, pass
-`category_order=[...]` when the configured display order matters or missing
-categories must keep a stable axis position.
+`category_order=[...]` to use the configured display order. Also use it when
+missing categories must keep a stable axis position.
### Keyword Reference
@@ -73,20 +73,20 @@ All figure builders accept `x`, `y`, `title`, `x_title`, `y_title`, and
| `density` | `value_mode="dashboard"`, `x_range=None`, `category_order=None`, `tick_values=None`, `tick_text=None`, `hover_x_title=None` |
| `scatter` | `drop_zero_y=False`, `fit_overlays=None`, `fit_annotation="annotation"`, `one_to_one=False`, `legend_on_right=False` |
-`self.plot.scatter(...)` additionally accepts `panel_aspect_ratio`; this sizes
-the returned Panel pane and is not passed to the Plotly figure builder.
+`self.plot.scatter(...)` also accepts `panel_aspect_ratio`. This argument sets
+the size of the returned Panel pane. It does not go to the Plotly figure builder.
-For fitted scatterplots, `fit_overlays` is another `RunTables` or iterable of
-run/frame pairs. Each fit frame must contain the same `x` and `y` columns used
-by the scatter and may contain the column named by `fit_annotation`. That text
-is shown when the fitted line is hovered. `one_to_one=True` adds a dashed 1:1
-line, gives both axes the same range, and locks their scale. The validation
-pages use this API for per-run equations, R-squared values, and sample sizes.
+For fitted scatterplots, `fit_overlays` is another `RunTables` object or an
+iterable of run and frame pairs. Each fit frame must contain the scatter `x`
+and `y` columns. It can contain the column specified by `fit_annotation`.
+Hovering on the fitted line shows this text. `one_to_one=True` adds a dashed 1:1
+line. It gives both axes the same range and locks their scale. Validation pages
+use this API for run equations, R-squared values, and sample sizes.
-Run labels are presentation-safe without changing their underlying identity.
-Long labels are shortened to unique legend/tab labels, while Plotly hovers and
-exported tab tooltips retain the full label. Scatter point and fit hovers also
-include the owning run name.
+The interface changes run labels for presentation without changing their
+identity. It shortens long labels to unique legend and tab labels. Plotly hover
+text and exported tab tooltips keep the full label. Scatter point and fit hover
+text also include the run name.
## Count and share behavior
@@ -105,8 +105,8 @@ return self.plot.bar(
)
```
-If a summary already contains a specifically defined share, provide that
-column with `share_y`. The renderer selects it only in share mode:
+If a summary contains a defined share, supply that column with `share_y`. The
+renderer selects it only in share mode:
```python
return self.plot.bar(
@@ -117,15 +117,15 @@ return self.plot.bar(
)
```
-Use `share_y` when the denominator has domain meaning that cannot be recovered
-by summing `y`. Do not select between count and percent columns in the page just
-to follow the global toggle. There are no `as_percent`, `normalize`,
-`percent_y_col`, or `pct_col` plotting arguments.
+Use `share_y` when the denominator has a special meaning that a sum of `y`
+cannot reproduce. Do not select count or percent columns in the page only to
+follow the global control. The plotting interface does not have `as_percent`,
+`normalize`, `percent_y_col`, or `pct_col` arguments.
-## Figure-first escape hatch
+## Direct figure API
-The core builders return `plotly.graph_objects.Figure`, which is useful for
-testing or for adding a genuinely page-specific annotation:
+The core builders return `plotly.graph_objects.Figure`. Use this result for
+tests or a page-specific annotation:
```python
figure = self.plot.figure.scatter(
@@ -138,10 +138,10 @@ figure.add_vline(x=1000, line_dash="dot")
return self.plot.panel(figure)
```
-Prefer the normal `self.plot.*` methods when no figure customization is needed.
-They use the same immutable `RenderContext` as export, so live and exported
-charts receive identical colors, labels, hover policy, and value mode without
-module-global setup.
+Use the standard `self.plot.*` methods when a figure does not require a custom
+change. They use the same fixed `RenderContext` as export. Thus, live and export
+charts get identical colors, labels, hover policy, and value mode. They do not
+require module-global setup.
## Tables and layout
@@ -151,10 +151,10 @@ Display helpers are grouped by responsibility under `dashboard.rendering`:
from dashboard.rendering import data_table, selector_row
```
-`data_table(data, title)` accepts `RunTables` directly. Page KPI values should
-use `self.plot.kpi(...)`, which shares the same run context as charts. Selector
-rows, missing-data cards, legends, and other layout helpers live in
-`dashboard.rendering.layout`; numeric and column formatting lives in
+`data_table(data, title)` accepts `RunTables` directly. Use `self.plot.kpi(...)`
+for page KPI values. It uses the same run context as charts. Selector rows,
+missing-data cards, legends, and layout helpers are in
+`dashboard.rendering.layout`. Numeric and column formatting are in
`dashboard.rendering.tables`.
The `dashboard.rendering` facade exports these non-plot helpers:
@@ -188,7 +188,7 @@ assert figure.data[0].name == "Base"
assert list(figure.data[0].x) == ["Walk", "Bike"]
```
-This keeps plot tests fast and isolates data/query behavior from Panel.
+This method keeps plot tests fast. It separates data and query behavior from Panel.
Use the focused plotting target during development:
@@ -196,8 +196,8 @@ Use the focused plotting target during development:
pytest tests/test_figure_builders.py
```
-Page query behavior belongs in `tests/test_page_authoring.py`; the complete
-HTML export suite is a separate release-boundary check.
+Test page query behavior in `tests/test_page_authoring.py`. Execute the complete
+HTML export suite as a separate release check.
## Related Chapters
diff --git a/wiki/36-html-export-schema.md b/wiki/36-html-export-schema.md
index acb1f9d..b4424af 100644
--- a/wiki/36-html-export-schema.md
+++ b/wiki/36-html-export-schema.md
@@ -1,10 +1,12 @@
# 36 - HTML Export Schema
-This document defines the Python-to-JavaScript contract used by the standalone offline dashboard export.
+This document defines the Python-to-JavaScript contract for the standalone
+offline dashboard export.
The implementation lives under `dashboard/export/`:
-- `dashboard/export/html.py`: entry points that build and write the final self-contained HTML document
+- `dashboard/export/html.py`: entry points that build and write the final
+ self-contained HTML document
- `dashboard/export/payload.py`: dashboard-state and top-level payload composition
- `dashboard/export/traversal.py`: page-tree and export-region resolution
- `dashboard/export/selector_states.py`: selector request and canonical-state enumeration
@@ -14,17 +16,18 @@ The implementation lives under `dashboard/export/`:
- `dashboard/export/types.py`: typed payload and node definitions
- `dashboard/export/js_runtime/`: readable browser-runtime source split into small files
- `dashboard/export/assets/export_runtime.js`: client runtime that validates and renders the payload
-- `dashboard/export/build_export_runtime.py`: concatenates `js_runtime/` into the shipped runtime asset
+- `dashboard/export/build_export_runtime.py`: concatenates `js_runtime/` into
+ the shipped runtime asset
## Top-Level Payload
-The exported HTML embeds one JSON payload inside:
+The exported HTML contains one JSON payload in:
```html
```
-The payload shape is defined in `dashboard/export/types.py` as `ExportPayload`.
+`dashboard/export/types.py` defines the payload as `ExportPayload`.
Top-level fields:
@@ -41,7 +44,7 @@ Top-level fields:
| `page_export_support` | `PageExportSupportPayload` | Metadata about export-enabled page selectors |
| `client_runtime` | `str` | Runtime family identifier for diagnostic/debugging purposes |
-`states` is keyed by the dashboard state key built in `dashboard/export/payload.py`:
+`dashboard/export/payload.py` builds the dashboard state key for `states`:
```text
||
@@ -65,7 +68,9 @@ Each `PageDescriptorPayload` contains:
| `children` | `list[PageDescriptorPayload]` | Child page descriptors when this entry is a grouped top-level page |
| `default_page_id` | `str \| None` | Default leaf page used when a grouped export page first loads |
-The top-level page order is resolved through the shared page registry. Grouped pages keep their child pages nested under a single top-level export tab, while serialized page content in `states` remains keyed by leaf page id.
+The shared page registry sets the top-level page order. Grouped pages keep their
+child pages under one top-level export tab. Content in `states` uses the final
+page ID as its key.
## Selector Metadata
@@ -84,7 +89,7 @@ Each `SelectorMetadataPayload` contains:
| `export_enabled` | `bool` | Whether the selector is interactive in export or rendered as a disabled/static control |
| `parent_selector_id` | `str` (optional) | Parent selector for a dependent option domain |
| `options_by_parent_value` | `dict[str, list[str]]` (optional) | Child options keyed by parent value |
-| `disabled_parent_values` | `list[str]` (optional) | Parent values for which the dependent selector is disabled |
+| `disabled_parent_values` | `list[str]` (optional) | Parent values that disable the dependent selector |
Selector config is driven from:
@@ -96,7 +101,7 @@ dashboard:
: ...
```
-Grouped child pages may also be configured as:
+You can also configure grouped child pages as follows:
```yaml
dashboard:
@@ -108,26 +113,28 @@ dashboard:
: ...
```
-Validation comes from the shared page registry:
+The shared page registry supplies these validation rules:
- unknown page ids fail in `validate_page_export_config()`
- unknown selector ids fail in `validate_page_export_config()`
-- unavailable configured selectors log a warning once and fall back to non-interactive region/static page behavior
+- unavailable configured selectors write one warning to the log and use a static region or page
## Page Content Shape
-`PageContentPayload` is always:
+`PageContentPayload` always has these fields:
| Field | Type | Purpose |
|---|---|---|
| `kind` | `"page"` | Discriminator |
-| `content` | `ExportNode` | Serialized page shell rooted at a normal export node tree |
+| `content` | `ExportNode` | Serialized page shell that starts with a standard export node tree |
-Pages without export-enabled selectors serialize as a normal page shell whose tree contains no `region` nodes. Pages with export-enabled selectors serialize one stable page shell with one or more embedded `region` nodes.
+Pages without export-enabled selectors create a standard page shell. Its tree
+does not contain `region` nodes. Pages with export-enabled selectors create one
+stable page shell with one or more `region` nodes.
## Region Nodes
-`region` is a first-class `ExportNode` kind used for subtree-level switching.
+`region` is an `ExportNode` kind that changes one part of a node tree.
Fields:
@@ -142,7 +149,7 @@ Fields:
| `variants` | `dict[str, ExportNode]` | Mapping from selector-combination key to serialized subtree |
| `variant_aliases` | `dict[str, str]` | Alternate selector keys mapped to a canonical rendered variant |
-Variant keys are JSON strings generated by `dashboard.export.serializer.variant_key()`.
+`dashboard.export.serializer.variant_key()` generates JSON variant-key strings.
Example:
@@ -150,13 +157,14 @@ Example:
["All","DRIVE"]
```
-The order of values in the key must match `selector_ids`.
+The value order in the key must agree with `selector_ids`.
-If a configured selector is unavailable for a region at export time, that region serializes with empty `selector_ids`, `default_content`, and no interactive variants.
+If a configured selector is unavailable at export time, the region has empty
+`selector_ids` and `default_content`. It has no interactive variants.
## Supported Node Kinds
-The browser runtime only supports the node kinds declared in `dashboard/export/types.py`.
+The browser runtime supports only the node kinds in `dashboard/export/types.py`.
| Kind | Produced from | Important fields |
|---|---|---|
@@ -170,15 +178,16 @@ The browser runtime only supports the node kinds declared in `dashboard/export/t
| `html` | `pn.pane.Markdown`, `pn.pane.HTML`, plain strings, unsupported fallback markup | `html` |
| `spacer` | `pn.Spacer` | no extra fields |
-Unsupported objects currently serialize to an `html` node containing a visible fallback panel. The runtime itself treats unknown node kinds as an error and shows an error panel.
+An unsupported object becomes an `html` node with a visible fallback panel. The
+runtime identifies an unknown node kind as an error and shows an error panel.
-Supported widget types are `select`, `radio_button_group`, `float_input`,
+The supported widget types are `select`, `radio_button_group`, `float_input`,
`checkbox`, and `button`. `SelectorMetadataPayload.default_value` and widget
-values are JSON-compatible values and are not limited to strings.
+values can be all JSON-compatible values. They are not limited to strings.
## Runtime Validation Rules
-The embedded runtime validates:
+The embedded runtime validates these items:
- payload presence and JSON parseability
- `schema_version` compatibility
@@ -187,7 +196,7 @@ The embedded runtime validates:
- presence of `states`
- presence of `dashboard_controls`
-At render time it also fails visibly on:
+At render time, it shows an error for these conditions:
- unknown rail sections
- unknown widget types
@@ -196,7 +205,8 @@ At render time it also fails visibly on:
- missing region state for the active selector combination
- Plotly runtime failures
-Failures are shown in the HTML via a visible error panel and also logged to the browser console.
+The HTML shows failures in a visible error panel. The runtime also writes them
+to the browser console.
## Schema Versioning Policy
@@ -204,13 +214,16 @@ Failures are shown in the HTML via a visible error panel and also logged to the
Rules:
-1. Change `schema_version` whenever the browser runtime can no longer safely consume payloads emitted by older Python code.
-2. Keep the runtime check strict. A mismatch should fail loudly instead of rendering incorrect content.
-3. Update this document, `dashboard/export/assets/export_runtime.js`, and the export payload tests in the same change.
+1. Change `schema_version` when the browser runtime cannot safely use payloads
+ from older Python code.
+2. Keep the runtime check strict. A mismatch must show an error and must not
+ render incorrect content.
+3. Update this document, `dashboard/export/assets/export_runtime.js`, and the
+ export payload tests in the same change.
## Checklist for Adding a New Node Kind
-When adding a new serialized node kind:
+To add a serialized node kind, do these steps:
1. Add the new typed shape to `dashboard/export/types.py`.
2. Emit it from `dashboard/export/serializer.py`.
diff --git a/wiki/40-developer-workflows.md b/wiki/40-developer-workflows.md
index a6438a5..e7d728d 100644
--- a/wiki/40-developer-workflows.md
+++ b/wiki/40-developer-workflows.md
@@ -1,6 +1,6 @@
# 40 - Developer Workflows
-This chapter is for contributors changing code or documentation.
+Use this chapter when you change code or documentation.
## Codebase Map
@@ -54,7 +54,7 @@ activitysim_visualizer/
## Testing Guidance
-Use focused tests for the subsystem you changed:
+Execute focused tests for the subsystem that you changed:
- prepare changes: minimal raw/prepared input tests and cache identity tests
- skimjoin changes: config normalization, lookup behavior, reports
@@ -68,13 +68,13 @@ Common command:
uv run --with pytest pytest --basetemp .pytest_tmp
```
-Run narrower tests while iterating when possible.
-The [Testing](46-testing.md) chapter documents the fast/full marker split and
-the required release-boundary commands.
+Execute smaller test groups during development when possible. The
+[Testing](46-testing.md) chapter describes the fast and full markers. It also
+gives the required release test commands.
## Generated Wiki Catalogs
-Regenerate catalogs after changing:
+Regenerate the catalogs after you change:
- `@summary(...)` declarations and contracts
- `processor/summarize/catalog.py`
@@ -87,12 +87,12 @@ Command:
uv run python scripts/generate_wiki_catalogs.py
```
-Generated sections are marked with comments. Do not edit inside generated
-markers by hand.
+Comments identify generated sections. Do not manually edit text between the
+generated markers.
## Documentation Maintenance
-When behavior changes, update docs in the same change:
+When behavior changes, update the documentation in the same change:
| Change | Wiki updates |
|---|---|
@@ -110,8 +110,8 @@ When behavior changes, update docs in the same change:
- The change follows the owning subsystem's existing patterns.
- Config and cache behavior are explicit.
-- Missing optional inputs fail gracefully.
-- Summary/page requirements are declared where the runtime can see them.
+- Missing optional input gives a controlled result.
+- Declare summary and page requirements where the runtime can use them.
- Tests cover the behavior rather than only the implementation detail.
- Generated wiki catalogs are current.
- The fast suite passes, and the `full_export` boundary passes when the change
diff --git a/wiki/41-data-extension-cookbook.md b/wiki/41-data-extension-cookbook.md
index 295247e..73a70c5 100644
--- a/wiki/41-data-extension-cookbook.md
+++ b/wiki/41-data-extension-cookbook.md
@@ -1,7 +1,7 @@
# 41 - Data Extension Cookbook
-This chapter contains end-to-end examples for extending the data that reaches
-the dashboard. Each recipe starts at the narrowest supported boundary.
+This chapter gives complete examples for data extensions. Each procedure starts
+at the smallest supported boundary.
## Choose The Smallest Extension
@@ -9,14 +9,14 @@ the dashboard. Each recipe starts at the narrowest supported boundary.
|---|---|
| Load a dashboard-ready file produced elsewhere | Register an external summary schema and use `summary_table_map`. |
| Reuse one derived value in several summaries | Add a column to an existing prepared table. |
-| Carry a genuinely new row grain through the whole application | Add a prepared table. |
+| Carry a new row type through the complete application | Add a prepared table. |
-Adding a prepared table is much more invasive than adding a column. Prefer a
-column unless the new data has its own stable row grain and lifecycle.
+A prepared table changes more interfaces than a prepared column. Add a column
+unless the new data has its own stable row type and lifecycle.
## Worked Example: Add An Outside Summary Table
-Suppose another process writes `regional_emissions.csv`:
+In this example, a different process writes `regional_emissions.csv`:
```csv
pollutant,tons
@@ -24,10 +24,10 @@ CO2,1250.5
NOX,18.2
```
-The visualizer only accepts registered summary IDs with exact schemas. Register
-the outside table with a no-op builder in an owning summary module. For a group
-of project-supplied tables, a module such as
-`processor/summarize/summaries/external_project.py` is appropriate:
+The visualizer accepts only registered summary IDs with exact schemas. Register
+the external table with a builder that does not calculate values. Put the
+builder in the applicable summary module. For multiple project tables, use a
+module such as `processor/summarize/summaries/external_project.py`:
```python
import polars as pl
@@ -49,10 +49,10 @@ def regional_emissions(run: RunData, config: Config) -> pl.DataFrame:
return regional_emissions.empty()
```
-`build_by_default=False` is important: raw ActivitySim runs cannot build this
-table, but the ID and contract must exist so an outside file can be validated.
+Set `build_by_default=False` because the standard ActivitySim workflow cannot
+build this table. The ID and contract must exist to validate an external file.
-If this is a new module, import it and add it to `SUMMARY_MODULES` in
+If you add a module, import it and add it to `SUMMARY_MODULES` in
`processor/summarize/catalog.py`:
```python
@@ -64,7 +64,7 @@ SUMMARY_MODULES = (
)
```
-Point a run at the file:
+Add the file to a run:
```yaml
runs:
@@ -73,18 +73,18 @@ runs:
regional_emissions: inputs/regional_emissions.csv
```
-Relative paths resolve from the main config file. CSV and Parquet are
-supported. The loader:
+A relative path starts from the main configuration file. The loader supports
+CSV and Parquet. It does these checks and actions:
1. rejects unknown summary IDs;
2. rejects missing or unexpected columns;
3. casts to the declared dtypes and declared column order; and
4. exposes the same outside table under every configured weighting mode.
-The fourth behavior matters: an outside table is assumed to be already
-aggregated. Selecting Weighted or Unweighted does not recalculate it.
+The loader assumes that an external table is already aggregated. The Weighted
+and Unweighted selections do not calculate it again.
-Wire the table to a page as optional data:
+Connect the table to a page as optional data:
```python
@dashboard_page(
@@ -103,10 +103,10 @@ class RegionalValidationPage(DashboardPage):
return self.plot.bar(data, x="pollutant", y="tons")
```
-Use `required_summary_ids` only if the page has no meaningful primary view
-without the table.
+Use `required_summary_ids` only if the table is necessary for the primary page
+view.
-Tests should prove registration, strict schema validation, loading, and page
+Tests must verify registration, strict schema validation, loading, and page
requirements:
```python
@@ -130,7 +130,7 @@ def test_external_emissions_loads(tmp_path, config):
assert run.summaries_by_mode["weighted"]["regional_emissions"].height == 1
```
-Run:
+Use these commands:
```bash
uv run --with pytest pytest --basetemp .pytest_tmp tests/test_summary_declarations.py tests/test_runtime_workflows.py
@@ -139,11 +139,11 @@ uv run python scripts/generate_wiki_catalogs.py
## Worked Example: Add A Column To An Existing Prepared Table
-Suppose several summaries need a canonical household field named
-`area_type`. The raw table already contains enough information to derive it.
+In this example, several summaries require a canonical household field named
+`area_type`. The raw table contains the information to calculate it.
-Put the transformation in the enrichment module that owns the domain. For a
-household field, that is normally
+Put the transformation in the enrichment module for the domain. For a
+household field, this module is usually
`processor/prepare/enrichment/households_persons.py`:
```python
@@ -169,7 +169,7 @@ def enrich_people_and_places_domain(state, config):
return state
```
-Then declare the prepared dependency where it is consumed:
+Then declare the prepared dependency where the summary uses it:
```python
@summary(
@@ -192,18 +192,19 @@ def households_by_area_type(run, config):
)
```
-Add a prepare test with the source column present and another with it absent.
-Optional source data should leave the table usable; the summary contract will
-record the new summary as unavailable when `area_type` is absent.
+Add one prepare test with the source column and one without it. If the optional
+source data is absent, the table must stay usable. The summary contract records
+the new summary as unavailable when `area_type` is absent.
-If config affects the derived value, also add that config value to
-`prepare_signature_payload()` in `runtime/config/signatures.py`. Otherwise a
-prepared cache built with old config could be reused incorrectly.
+If configuration changes the derived value, add that configuration value to
+`prepare_signature_payload()` in `runtime/config/signatures.py`. Without this
+change, the visualizer can incorrectly use a cache from an old configuration.
## Worked Example: Add A Prepared Table
-Assume ActivitySim now emits one row per zone in `final_accessibility.csv`, and
-the table cannot sensibly be represented as columns on `land_use`.
+In this example, ActivitySim writes one row for each zone in
+`final_accessibility.csv`. Columns on `land_use` cannot correctly represent
+this table.
### 1. Define Names And Runtime Storage
@@ -234,9 +235,9 @@ class RunData:
accessibility: pl.DataFrame = field(default_factory=pl.DataFrame)
```
-Also update `PREPARED_TABLE_NAMES`, `prune_prepared_run()`, and every explicit
-`RunData(...)` copy constructor. Copy constructors are intentionally explicit;
-missing one is a common source of a table disappearing between workflows.
+Also update `PREPARED_TABLE_NAMES`, `prune_prepared_run()`, and each explicit
+`RunData(...)` copy constructor. The copy constructors are explicit. If you do
+not update one, a workflow can omit the table.
### 2. Read It And Track Availability
@@ -272,16 +273,16 @@ PREPARED_TABLE_ATTRS = (
)
```
-That one tuple drives prepared filenames, manifest entries, writes, and most
-loads. Because it changes the prepared cache contract, increment
-`SCHEMA_VERSION` and decide whether old schema versions remain readable.
+This tuple controls prepared file names, manifest entries, writes, and most
+loads. It changes the prepared cache contract. Increment `SCHEMA_VERSION` and
+decide whether the reader can read old schema versions.
### 4. Decide Segmentation And Dashboard Behavior
-If segmentation must filter or anchor on the new table, add explicit rules in
-`processor/segmentation.py` and aliases in
-`runtime/config/normalize_segmentation.py`. Do not silently copy the full table
-into every segment unless that is correct for its row grain.
+If segmentation must filter or use the new table as an anchor, add rules in
+`processor/segmentation.py`. Add aliases in
+`runtime/config/normalize_segmentation.py`. Do not copy the complete table to
+each segment unless this is correct for its row type.
Pages can now declare:
@@ -296,7 +297,7 @@ Pages can now declare:
### 5. Test Every Boundary
-At minimum, cover:
+At a minimum, test these items:
- config filename and `prepared_table_map` acceptance;
- raw reader success and optional-file absence;
@@ -319,7 +320,7 @@ uv run --with pytest pytest --basetemp .pytest_tmp tests/test_runtime_workflows.
- IDs are stable across config, runtime, cache, and dashboard declarations.
- Cache identity changes whenever config changes data content.
- Missing optional input produces typed empty/unavailable state, not a crash.
-- External schemas reject extra as well as missing columns.
+- External schemas reject extra and missing columns.
- Generated catalogs have been refreshed.
## Related Chapters
diff --git a/wiki/42-config-column-label-cookbook.md b/wiki/42-config-column-label-cookbook.md
index 02a658a..5a97d0e 100644
--- a/wiki/42-config-column-label-cookbook.md
+++ b/wiki/42-config-column-label-cookbook.md
@@ -1,7 +1,8 @@
# 42 - Config, Columns, And Labels
-This chapter shows how one YAML value travels through validation, typed config,
-cache identity, prepared data, and dashboard presentation.
+This chapter shows the complete path of one YAML value. The path includes
+validation, typed configuration, cache identity, prepared data, and dashboard
+presentation.
## First Decide Which Boundary Owns The Setting
@@ -12,13 +13,13 @@ cache identity, prepared data, and dashboard presentation.
| labels, ordering, colors, or page appearance | `display` or `dashboard` | Presentation |
| which workflow executes | `pipeline` | Runtime plan; include data effects in the owning signature too |
-Do not add a setting only to `Config`. A complete setting has validation,
-normalization, a typed field, cache/signature ownership, a consumer, an example,
-and tests.
+Do not add a setting only to `Config`. Add validation, normalization, a typed
+field, and cache signature ownership. Also add a consumer, an example, and
+tests.
## Worked Example: Add A New Config Item
-Suppose the dashboard needs a presentation-only switch:
+In this example, the dashboard requires a presentation-only control:
```yaml
display:
@@ -62,12 +63,12 @@ class Config:
show_zero_categories: bool
```
-Downstream code should read `config.show_zero_categories`, never the raw YAML
-mapping.
+Downstream code must read `config.show_zero_categories`. It must not read the
+raw YAML mapping.
### 3. Put It In The Correct Signature
-Because this switch changes only rendering, add it to
+This control changes only rendering. Add it to
`presentation_signature_payload()` in `runtime/config/signatures.py`:
```python
@@ -77,32 +78,32 @@ return {
}
```
-Do not add it to the prepare or summary signatures. That would cause expensive
-cache rebuilds for a display-only change.
+Do not add it to the prepare or summary signatures. If you add it, a display
+change causes unnecessary cache rebuilds.
### 4. Consume It At The Presentation Boundary
-For example, a shared category helper can choose whether to complete absent
-categories:
+For example, a shared category helper can add absent categories when the value
+is true:
```python
if config.show_zero_categories:
chart_data = complete_category_rows(chart_data, expected_categories)
```
-Prefer a shared helper if several pages need the setting. Keep one-off behavior
-on the owning page.
+Use a shared helper if several pages require the setting. Keep behavior for one
+page on that page.
### 5. Document And Test It
-Update `config.yaml` and chapter 13. Add tests for the default, explicit value,
-wrong type, signature ownership, and visible consumer behavior:
+Update `config.yaml` and chapter 13. Test the default, an explicit value, and an
+incorrect type. Also test signature ownership and visible consumer behavior.
-The snippets below use illustrative module-local helpers named
-`_write_config()` and `_raw_run()`. They are not repository-wide pytest
-fixtures: define the minimal helper in the owning test module, or adapt that
-module's existing config/run factory. Likewise, `extra_lines` and
-`column_lines` are example helper arguments rather than public config APIs.
+The examples below use module-local helpers named `_write_config()` and
+`_raw_run()`. These helpers are not repository-wide pytest fixtures. Define a
+small helper in the applicable test module, or use its existing configuration
+and run factory. `extra_lines` and `column_lines` are example helper arguments.
+They are not public configuration APIs.
```python
def test_show_zero_categories_is_presentation_only(tmp_path):
@@ -118,8 +119,8 @@ def test_show_zero_categories_is_presentation_only(tmp_path):
## Worked Example: Wire A Configured Column Name Into Prepare
-Suppose different models call household area type `area_type`, `ATYPE`, or
-`area_class`. The prepared contract should expose one stable name:
+In this example, models use `area_type`, `ATYPE`, or `area_class` for household
+area type. The prepared contract must supply one stable name:
`area_type`.
### 1. Add The Alias Setting
@@ -136,7 +137,7 @@ _ALIAS_COLUMN_DEFAULTS = {
}
```
-`CANONICAL_COLUMN_KEYS` is derived from this mapping, so
+The loader gets `CANONICAL_COLUMN_KEYS` from this mapping. Thus,
`columns.area_type` becomes valid automatically. Add the typed field to
`Config`:
@@ -144,14 +145,14 @@ _ALIAS_COLUMN_DEFAULTS = {
col_area_type: list[str]
```
-The user can now override precedence:
+The user can now change the order of preference:
```yaml
columns:
area_type: [area_class, ATYPE]
```
-The first available candidate wins.
+The loader uses the first available candidate.
### 2. Materialize The Canonical Column
@@ -167,13 +168,13 @@ def _canonicalize_households(hh: pl.DataFrame, config: Config) -> pl.DataFrame:
)
```
-Keep the configured source candidates in config and the stable output name in
-prepare. Summary builders should require `hh.area_type`; they should never
-probe `ATYPE` or `area_class`.
+Keep configured source candidates in the configuration. Keep the stable output
+name in prepare. Summary builders must require `hh.area_type`. They must not
+search for `ATYPE` or `area_class`.
-Use `_materialize_preferred_column(...)` only when candidate selection needs
-extra rules, such as rejecting numeric purpose codes. Use `overwrite=True` only
-when prepare intentionally replaces an existing canonical column.
+Use `_materialize_preferred_column(...)` only when candidate selection requires
+more rules. One example is the rejection of numeric purpose codes. Use
+`overwrite=True` only when prepare must replace an existing canonical column.
### 3. Add Cache Identity
@@ -184,8 +185,8 @@ Add the candidate list to the `columns` mapping returned by
"area_type": list(config.col_area_type),
```
-The summary signature currently incorporates the prepared column payload, so
-this also invalidates affected summary caches.
+The summary signature includes the prepared column payload. Thus, this change
+also invalidates applicable summary caches.
### 4. Test Precedence And Materialization
@@ -214,10 +215,10 @@ Also test the default candidate list and missing-source behavior.
## Worked Example: Add A Label Mapping And Use It On A Page
-Label mappings are presentation data. They do not change raw values used for
-filtering or summary grouping.
+Label mappings are presentation data. They do not change raw values for filters
+or summary groups.
-Suppose a summary contains `employment_status` values `0`, `1`, and `2`:
+In this example, a summary contains `employment_status` values `0`, `1`, and `2`:
```yaml
display:
@@ -231,7 +232,7 @@ display:
```
New category IDs do not require a schema change. `normalize_categories()` loads
-arbitrary category IDs into `config.dashboard_labels`.
+all category IDs into `config.dashboard_labels`.
### Selector With Display-To-Raw Mapping
@@ -257,9 +258,8 @@ def selected_employment_status_raw(self):
return self._employment_status_by_label.get(self.employment_status.value)
```
-The widget shows `Full time`; the data filter still uses raw value `2`. This
-avoids corrupting joins, selector state, or summary contracts with display
-text.
+The widget shows `Full time`. The data filter continues to use raw value `2`.
+Thus, display text does not change joins, selector state, or summary contracts.
### Add A Label Column For A Figure
@@ -283,11 +283,10 @@ return self.plot.bar(
)
```
-If many pages use the category, keep mapping mechanics in
+If many pages use the category, put mapping logic in
`dashboard/helpers/category_helpers.py`. If the mapping changes canonical
-summary values rather than appearance, it belongs under
-`summarize.category_normalization` and must be applied by the owning summary
-logic.
+summary values, put it under `summarize.category_normalization`. The applicable
+summary logic must apply it.
### Test Raw And Display Behavior Separately
@@ -298,13 +297,13 @@ assert config.ordered_values(
) == ["0", "1", "2"]
```
-Add a page/helper test proving that selection of `Full time` filters raw `2`.
-This catches the most common label-wiring regression.
+Add a page or helper test. Verify that a `Full time` selection filters raw value
+`2`. This test identifies a common label connection error.
## Completion Checklist
- Unknown keys and wrong types fail near the config boundary.
-- Raw YAML is normalized once and represented by a typed `Config` field.
+- Normalize raw YAML one time and represent it with a typed `Config` field.
- The setting belongs to exactly the cache signatures it can affect.
- Prepared code emits canonical names; summaries do not probe source aliases.
- Dashboard filtering retains raw values and labels only at presentation time.
diff --git a/wiki/43-weighting-hosting-extensions.md b/wiki/43-weighting-hosting-extensions.md
index 41538d6..29c650b 100644
--- a/wiki/43-weighting-hosting-extensions.md
+++ b/wiki/43-weighting-hosting-extensions.md
@@ -1,18 +1,18 @@
# 43 - Weighting And Hosting Extensions
-Weighting and hosting both cross major runtime boundaries. Ordinary alternative
-weights are configuration-driven. A Python registry remains available for
-calculations that cannot be expressed as column selection, while hosting remains
-a deliberately limited extension point.
+Weighting and hosting affect multiple runtime boundaries. The configuration
+controls standard alternative weights. Use the Python registry for calculations
+that column selection cannot define. Hosting is a limited extension point.
## Worked Example: Add A Weighting Mode
-The built-in modes are `weighted` and `unweighted`. A named column mode adds
-another complete set of summary tables, cache entries, dashboard selector state,
-and export states without requiring Python code.
+The built-in modes are `weighted` and `unweighted`. A named column mode adds a
+set of summary tables, cache entries, dashboard selector state, and export
+states. It does not require Python code.
-Suppose ActivitySim writes `calibrated_hh_weight`, `calibrated_person_weight`,
-and `calibrated_trip_weight` alongside its ordinary weights.
+In this example, ActivitySim writes `calibrated_hh_weight`,
+`calibrated_person_weight`, and `calibrated_trip_weight` with its standard
+weights.
### 1. Define The Named Column Mode
@@ -33,60 +33,58 @@ summarize:
weighting_modes: [weighted, unweighted, calibrated]
```
-`label` is optional; an omitted label is generated from the mode ID. At least one
-column must be configured. Supported source tables are `households`, `persons`,
-and `trips`.
+`label` is optional. If you omit it, the loader creates a label from the mode
+ID. You must configure at least one column. The supported source tables are
+`households`, `persons`, and `trips`.
-This differs from the three weight fields on a run. `hh_weight_col`,
-`person_weight_col`, and `trip_weight_col` choose the one primary `weighted`
-definition during prepare. `weighting.modes` preserves that primary definition
-and adds named alternatives that can be compared in one dashboard.
+This function differs from the three weight fields on a run. `hh_weight_col`,
+`person_weight_col`, and `trip_weight_col` select the primary `weighted`
+definition during prepare. `weighting.modes` keeps that primary definition. It
+adds named alternatives for comparison in one dashboard.
### 2. Understand Propagation
-The configured source columns replace `finalweight` on their respective
-prepared tables. Related tables then receive consistent weights:
+The configured source columns replace `finalweight` on their prepared tables.
+The system then supplies consistent weights to related tables:
- a household source propagates to persons, trips, tours, days, and vehicles
- unless a more specific source is configured;
+ unless you configure a more specific source;
- a person source propagates to trips, tours, and days;
- a trip source propagates to tours as the mean selected trip weight for each
`tour_id`; and
- trip and tour hypothetical-skim sidecars inherit the selected trip and tour
weights.
-You can configure only the levels that differ. For example, a mode containing
-only `trips` changes trips and tours while leaving household, person, day, and
-vehicle weights at their primary prepared values.
+Configure only the levels that differ. For example, a mode that contains only
+`trips` changes trips and tours. Household, person, day, and vehicle weights
+keep their primary prepared values.
-Source columns are validated on every prepared run before summaries begin. A
-misspelling therefore produces an error naming the missing table and column
-instead of silently reverting to another weight. Raw ActivitySim columns are
-normally retained by prepare. When using `prepared_table_map`, include the named
-source columns in those prepared files.
+The workflow validates source columns for each prepared run before summaries
+start. An incorrect name causes an error that identifies the missing table and
+column. It does not select a different weight. Prepare usually keeps raw
+ActivitySim columns. If you use `prepared_table_map`, include the named source
+columns in those prepared files.
### 3. Cache, Dashboard, And Outside-Summary Behavior
-The mode ID, selected source columns, and column-mode implementation version are
-part of summary cache identity. Changing a source column invalidates incompatible
-summary caches. The configured label is used by live and exported dashboard
-selectors.
+The summary cache identity includes the mode ID, source columns, and column-mode
+implementation version. A source column change invalidates incompatible summary
+caches. Live and export dashboard selectors use the configured label.
-Declarative column modes reject mode-independent `summary_table_map` inputs.
-An already aggregated outside table does not contain enough information to
-recalculate another weighting mode. Use generated summaries for these modes or
-provide the outside data through a custom workflow that makes its weighting
-semantics explicit.
+Declarative column modes reject mode-independent `summary_table_map` input. An
+aggregated external table does not contain enough information to calculate
+a different weighting mode. Use generated summaries for these modes. Or supply
+external data through a custom workflow that defines its weighting rules.
## Advanced: Custom Weight Calculations
-Use a Python weighting module only when selecting columns is insufficient; for
-example, when weights must be capped, scaled, joined from a control table, or
-calculated from several prepared columns.
+Use a Python weighting module only when column selection is insufficient. For
+example, use one to limit, scale, join, or calculate weights from multiple
+prepared columns.
### 1. Create An Importable Extension Module
-This example adds a capped form of the primary prepared weights. Create
+This example adds a limited form of the primary prepared weights. Create
`my_project/weighting.py` in an installed package or another location on
`PYTHONPATH`:
@@ -130,9 +128,9 @@ def register_weighting_modes(registry: WeightingModeRegistry) -> None:
)
```
-`map_run_data_tables()` copies the complete `RunData`, transforms each DataFrame
-table, and preserves availability metadata, diagnostics, skims, and skimjoin
-artifacts. A transform must return a new `RunData` and must not mutate its input.
+`map_run_data_tables()` copies the complete `RunData` and transforms each data
+frame. It keeps availability metadata, diagnostics, skims, and skimjoin
+artifacts. A transform must return a new `RunData`. It must not change its input.
The registration fields are:
@@ -148,7 +146,7 @@ The registration fields are:
### 2. Load And Configure The Extension
-Use `extensions.modules` for a project-local/importable module and keep plugin
+Use `extensions.modules` for an importable project module. Put extension
settings under `extensions.settings`:
```yaml
@@ -163,9 +161,10 @@ summarize:
weighting_modes: [weighted, unweighted, capped]
```
-Module imports are executable code, so configuration containing extensions is
-trusted configuration. Extension settings and each selected definition's
-version, requirements, and outside-summary policy enter summary cache identity.
+Module imports execute code. Thus, treat a configuration with extensions
+as trusted configuration. Summary cache identity includes extension settings.
+It also includes each selected definition version, requirements, and external
+summary policy.
An installed package can advertise the same registration function with a
Python entry point instead:
@@ -175,14 +174,13 @@ Python entry point instead:
capped = "my_project.weighting:register_weighting_modes"
```
-Use either the installed entry point or `extensions.modules`, not both for the
-same definition. Duplicate IDs and labels fail during config loading.
+For one definition, use the installed entry point or `extensions.modules`. Do
+not use both. Duplicate IDs and labels cause an error during configuration load.
### 3. Runtime Behavior
-The weighting definition contract is the single source for config validation,
-summary transforms, prepared-data transforms, display labels, and cache
-compatibility:
+The weighting definition contract controls configuration validation, summary
+transforms, prepared-data transforms, display labels, and cache compatibility:
- config preserves the requested mode order and rejects unknown IDs;
- the summary workflow applies each registered transform before running builders;
@@ -193,7 +191,7 @@ compatibility:
cache the result for the dashboard session; and
- required source columns fail before a transform can silently fall back.
-Ordinary pages do not branch on particular modes:
+Standard pages do not branch on particular modes:
```python
prepared = self.data.prepared("trips")
@@ -204,17 +202,17 @@ weighted = self.data.prepared("trips", weighting_mode="weighted")
### 4. Outside Summary Tables
-Built-in `weighted` and `unweighted` definitions explicitly use
-`external_summary_policy="copy"`, preserving current behavior. A custom mode
-defaults to `reject`: a run using `summary_table_map` then fails clearly because
-the runtime cannot prove that an already-aggregated file represents that mode.
+Built-in `weighted` and `unweighted` definitions use
+`external_summary_policy="copy"`. A custom mode uses `reject` by default. A run
+with `summary_table_map` then causes an error. The runtime cannot verify that an
+aggregated file represents the custom mode.
-Set the custom definition to `copy` only when the outside table is genuinely
-mode-independent. Per-mode outside file maps are not currently supported.
+Set the custom definition to `copy` only when the external table does not
+depend on the mode. The system does not support file maps for each mode.
### 5. Test The Whole Mode
-At minimum, prove:
+At a minimum, verify these behaviors:
- config accepts, orders, deduplicates, and rejects mode names correctly;
- both module and installed-entry-point discovery use the registration contract;
@@ -235,13 +233,13 @@ uv run --with pytest pytest --basetemp .pytest_tmp tests/test_dashboard_live.py
## Worked Example: Connect A Hosting Script
-The safest first hosting extension is a thin deployment entrypoint that uses
-the existing config, cache loader, page requirements, and `build_dashboard()`.
-It should not duplicate prepare or summarize logic.
+The first hosting extension must be a small deployment entry point. Use the
+existing configuration, cache loader, page requirements, and `build_dashboard()`.
+Do not duplicate prepare or summarize logic.
-The current `pipeline.dashboard_mode: host` is only a placeholder: `run.py`
-logs a warning and falls back to live `pn.serve`. The `dashboard.host` keys are
-validated but are not yet normalized into `Config` or consumed.
+The current `pipeline.dashboard_mode: host` is a placeholder. `run.py` writes a
+warning and uses live `pn.serve`. Validation accepts the `dashboard.host` keys.
+The loader does not put them in `Config`, and the runtime does not use them.
## Option A: Provider Script Without Core Runtime Changes
@@ -290,12 +288,12 @@ dashboard = build_dashboard(
dashboard.servable()
```
-Panel-compatible hosts can launch this module with their normal command. A
-provider SDK can instead receive `dashboard` from the same script. Keep secrets
-and deployment IDs in environment variables or provider configuration, not the
-main visualizer YAML.
+Panel-compatible hosts can start this module with their standard command. A
+provider SDK can receive `dashboard` from the same script. Put secrets and
+deployment IDs in environment variables or provider configuration. Do not put
+them in the main visualizer YAML.
-This approach has useful properties:
+This method has these properties:
- hosting imports a ready-to-serve object instead of calling blocking
`pn.serve()`;
@@ -303,15 +301,13 @@ This approach has useful properties:
- enabled pages determine the data loaded; and
- provider dependencies can live in an optional dependency group.
-For a hosted service, caches must already exist or be available on persistent
-storage. If startup should build them, call the public prepare/summarize
-workflows before `build_dashboard()` and make the cost and write permissions
-explicit.
+For a hosted service, caches must exist in persistent storage. To build caches
+at startup, call the public prepare and summarize workflows before
+`build_dashboard()`. Make the runtime cost and write permissions explicit.
## Option B: Make `dashboard_mode: host` A Core Adapter
-Use this only when the same hosting provider should be a supported runtime
-mode.
+Use this method only when one hosting provider must be a supported runtime mode.
1. Add a typed `HostSettings` model in `runtime/config/models.py`.
2. Normalize `dashboard.host` in a focused parser and pass it into `Config`.
@@ -325,7 +321,7 @@ mode.
5. Let `resolve_dashboard_execution_mode("host")` remain `host` instead of
converting it to `live`.
-6. Reuse the normal workflow loading and `build_dashboard()` path, then call
+6. Reuse the standard workflow loading and `build_dashboard()` path, then call
the adapter instead of `pn.serve()`.
7. Put provider SDKs in a `hosting` optional dependency group in
`pyproject.toml`.
@@ -335,15 +331,14 @@ The boundary should look like:
```text
config + validated caches
- -> normal dashboard data requirements
+ -> standard dashboard data requirements
-> build_dashboard(...)
-> provider adapter
-> hosted application
```
-Avoid putting provider logic in pages, `dashboard/app.py`, or summary
-workflows. Those layers should remain usable locally, in export, and with any
-future host.
+Do not put provider logic in pages, `dashboard/app.py`, or summary workflows.
+These layers must operate locally, in export, and with a future host.
## Hosting Test Matrix
diff --git a/wiki/44-summary-function-cookbook.md b/wiki/44-summary-function-cookbook.md
index 8026eef..0ee46cd 100644
--- a/wiki/44-summary-function-cookbook.md
+++ b/wiki/44-summary-function-cookbook.md
@@ -1,20 +1,20 @@
# 44 - Summary Function Cookbook
-This chapter follows one new summary from a question to a tested dashboard
-dependency. Use it with the shorter contract reference in chapter 23.
+This chapter shows how to make and test one summary for a dashboard. Use it with
+the short contract reference in chapter 23.
## Worked Example: Trips By Mode
-Suppose a page needs total trips by canonical `trip_mode`. The output grain is
-one row per mode, per run, per weighting mode:
+In this example, a page requires total trips by canonical `trip_mode`. The
+output has one row for each mode, run, and weighting mode:
| trip_mode | trip_count |
|---|---:|
| DRIVEALONE | 14230.0 |
| WALK | 3180.0 |
-Write the grain down first. It determines the grouping keys, schema, tests, and
-figure axes.
+First, define what one row represents. This definition controls grouping keys,
+schema, tests, and figure axes.
## 1. Put Pure Calculation Before Registration
@@ -39,8 +39,8 @@ def trips_by_mode_frame(trips: pl.DataFrame) -> pl.DataFrame:
)
```
-Keeping the transform pure makes the calculation easy to test without cache or
-dashboard setup. Use canonical prepared columns; do not probe raw aliases here.
+A pure transform is easy to test without cache or dashboard setup. Use
+canonical prepared columns. Do not search for raw aliases here.
## 2. Declare The Runtime Contract
@@ -66,21 +66,21 @@ def trips_by_mode(run: RunData, config: Config) -> pl.DataFrame:
return trips_by_mode_frame(run.trips)
```
-The declaration does four jobs:
+The declaration does four tasks:
1. gives the table a stable config/cache ID;
2. prevents the builder from running when inputs are unavailable;
3. supplies a correctly typed empty result; and
4. rejects successful results with wrong columns, order, or dtypes.
-The unused `config` argument is still part of the uniform builder interface. If
-config changes the calculation, use it here and ensure the setting belongs to
-the summary signature.
+The uniform builder interface includes the unused `config` argument. If
+configuration changes the calculation, use the argument here. Make sure that
+the setting is in the summary signature.
## 3. Let The Workflow Handle Weighting
-Always aggregate `finalweight`. The workflow supplies ordinary weights for the
-weighted build and replaces them for the unweighted build. Do not add a
+Always aggregate `finalweight`. The workflow supplies standard weights for the
+weighted build. It replaces them for the unweighted build. Do not add a
`weighted` branch to the builder.
For an average, use a weighted numerator and denominator:
@@ -94,21 +94,21 @@ For an average, use a weighted numerator and denominator:
)
```
-Decide how zero total weight should behave and test it explicitly.
+Define the result for zero total weight and test it.
## 4. Register A New Owning Module Only Once
-Adding a function to an existing module in `SUMMARY_MODULES` needs no catalog
-edit. If you create `processor/summarize/summaries/emissions.py`, import that
-module and add it to `SUMMARY_MODULES` in `processor/summarize/catalog.py`.
+You do not have to change the catalog when you add a function to an existing
+module in `SUMMARY_MODULES`. If you create
+`processor/summarize/summaries/emissions.py`, import it. Then add it to
+`SUMMARY_MODULES` in `processor/summarize/catalog.py`.
-Do not maintain a second list of individual functions. Catalog discovery reads
-decorated functions from the explicitly imported owning modules and rejects
-duplicate IDs.
+Do not keep a second list of functions. Catalog discovery reads decorated
+functions from the imported modules. It rejects duplicate IDs.
## 5. Test Calculation And Contract Separately
-Test the numbers with a tiny frame:
+Test the numbers with a small frame:
```python
def test_trips_by_mode_frame_uses_finalweight():
@@ -131,7 +131,7 @@ def test_trips_by_mode_frame_uses_finalweight():
}
```
-Then test the declaration boundary with a minimal `RunData`:
+Then test the declaration boundary with a small `RunData`:
```python
def test_trips_by_mode_preflights_missing_columns():
@@ -158,9 +158,9 @@ def test_trips_by_mode_preflights_missing_columns():
}
```
-Also add a catalog assertion when a new module is introduced. The shared
-declaration tests already cover generic wrong-schema behavior; domain tests
-should focus on your calculation and prerequisites.
+Also add a catalog assertion when you add a module. The shared declaration
+tests cover general incorrect-schema behavior. Domain tests must test the
+calculation and requirements.
## 6. Wire It To A Page
@@ -177,7 +177,7 @@ class TripModeTotalsPage(DashboardPage):
...
```
-Read the table through page data access and state the columns the view uses:
+Read the table through page data access. Specify the columns that the view uses:
```python
data = self.data.summary(
@@ -196,48 +196,47 @@ return self.plot.bar(
)
```
-The page declaration controls cache pruning and startup requirements. The
-`columns=` check provides a useful page-level diagnostic if an old or external
-cache does not satisfy the view.
+The page declaration controls the removal of unused cache data and startup
+requirements. The `columns=` check gives a page diagnostic if a cache does not
+supply the view. This can occur with an old or external cache.
## 7. Regenerate And Verify
-Run:
+Use these commands:
```bash
uv run python scripts/generate_wiki_catalogs.py
uv run --with pytest pytest --basetemp .pytest_tmp tests/test_summary_declarations.py tests/test_page_registry_contract.py
```
-Confirm the new ID appears in chapter 24 and, once wired to a page, in the page
-catalog in chapter 31.
+Make sure that the new ID occurs in chapter 24. After you connect it to a page,
+make sure that it occurs in the chapter 31 page catalog.
## Variations
### Optional Summary
-Use `optional_summary_ids` when the page retains a meaningful primary view
-without the new table. Render an unavailable card only for the optional
-section.
+Use `optional_summary_ids` when the page has a useful primary view without the
+new table. Show an unavailable card only for the optional section.
### External-Only Summary
-Use `build_by_default=False` and a typed no-op builder for a registered table
-that must come from `summary_table_map`. Follow the outside-table recipe in
-chapter 41.
+Use `build_by_default=False` and a typed builder that does not calculate values.
+Use this configuration for a registered table from `summary_table_map`. Follow
+the external-table procedure in chapter 41.
### Segmented Summary
-Usually no builder change is needed. Segmentation slices prepared `RunData`
-before invoking the same declaration. A summary that depends on a table or
-column removed by segmentation should become unavailable through its declared
-prerequisites, not fail inside the builder.
+Usually, the builder does not require a change. Segmentation divides prepared
+`RunData` before it calls the same declaration. Segmentation can remove a
+required table or column. In this condition, the declared requirements must
+make the summary unavailable. The builder must not fail.
## Review Checklist
-- The row grain and value meaning are written down.
+- The documentation defines the row type and value meaning.
- Grouping uses canonical prepared fields.
-- Counts, totals, and averages apply `finalweight` deliberately.
+- Counts, totals, and averages apply `finalweight` correctly.
- The schema is ordered and explicitly cast.
- Mechanical prerequisites are in the decorator.
- Domain-specific empty conditions return `builder.empty()`.
diff --git a/wiki/45-dashboard-extension-cookbook.md b/wiki/45-dashboard-extension-cookbook.md
index f0f4452..0f78180 100644
--- a/wiki/45-dashboard-extension-cookbook.md
+++ b/wiki/45-dashboard-extension-cookbook.md
@@ -1,20 +1,20 @@
# 45 - Dashboard Extension Cookbook
-This chapter gives worked examples for adding a page, page group, selector,
-custom widget, table, and reusable figure behavior. The examples use the
-current declarative page lifecycle: selectors own option domains, sections own
-refresh dependencies, and pages read data through `self.data`.
+This chapter gives examples for a page, page group, selector, custom widget,
+table, and reusable figure. The examples use the declarative page lifecycle.
+Selectors control option domains. Sections control refresh dependencies. Pages
+read data through `self.data`.
## Worked Example: Add A Page To An Existing Group
-Assume the registered summary `trips_by_mode` has columns `trip_mode` and
-`trip_count`. Create one discoverable leaf module:
+In this example, the registered summary `trips_by_mode` has `trip_mode` and
+`trip_count` columns. Create one discoverable final module:
```text
dashboard/pages/trip_summaries/trip_mode_totals.py
```
-The complete first version can stay small:
+The complete first version can be small:
```python
from __future__ import annotations
@@ -57,7 +57,7 @@ class TripModeTotalsPage(DashboardPage):
```
Discovery imports public child modules automatically. Do not edit a central
-page list. The decorator is the single source for identity and data needs.
+page list. The decorator defines the identity and data requirements.
Enable the page explicitly while developing:
@@ -71,8 +71,8 @@ dashboard:
## Add A Dynamic Selector
-Suppose the summary instead contains `tour_purpose`, `trip_mode`, and
-`trip_count`. Add a purpose dropdown whose options come from the loaded data:
+In this example, the summary contains `tour_purpose`, `trip_mode`, and
+`trip_count`. Add a purpose list with options from the loaded data:
```python
from dashboard.helpers.category_helpers import column_options
@@ -108,11 +108,11 @@ def purpose_options(self):
return options
```
-The option provider runs before a dependent section renders. If available
-options change, the framework repairs a stale selection using the selector's
-`default` policy.
+The option provider executes before the framework renders a dependent section. If
+available options change, the framework uses the `default` policy to repair an
+invalid selection.
-Filter with the raw value, not its display label:
+Use the raw value for the filter. Do not use its display label:
```python
raw_purpose = self._purpose_by_label[self.purpose.value]
@@ -124,8 +124,8 @@ chart_data = self.query(
)
```
-`self.query()` derives its cache identity from global state, active section,
-declared selectors, callable location, and captured values. Do not invent a
+`self.query()` gets its cache identity from global state, the active section,
+declared selectors, callable location, and captured values. Do not create a
page-local cache key.
## Add A Custom Widget
@@ -147,7 +147,7 @@ body = self.section(
)
```
-Then apply its value inside the section query:
+Then use its value in the section query:
```python
if self.hide_auto.value:
@@ -158,13 +158,13 @@ if self.hide_auto.value:
)
```
-Registration is what connects the widget to refresh and HTML export. A widget
-created directly in the layout without `self.select()` or `self.selector()` is
-not part of that lifecycle.
+Registration connects the widget to refresh and HTML export. A widget created
+directly in the layout is not part of this lifecycle. Register it with
+`self.select()` or `self.selector()`.
## Add A Figure With The Existing Plotter
-Pages should normally use `self.plot`:
+Pages must usually use `self.plot`:
```python
chart = self.plot.bar(
@@ -178,11 +178,11 @@ chart = self.plot.bar(
)
```
-This applies run colors, count/share state, layout conventions, and hover
-behavior. Available shared types are `bar`, `line`, `density`, and `scatter`.
+This method applies run colors, count or share state, layout rules, and hover
+behavior. The shared types are `bar`, `line`, `density`, and `scatter`.
-If one page needs a Plotly customization, build the figure through the escape
-hatch, mutate it, and wrap it:
+If one page requires a Plotly customization, build the figure through the
+figure API. Change it and put it in a Panel pane:
```python
figure = self.plot.figure.bar(
@@ -195,13 +195,13 @@ figure.update_layout(legend_title_text="Model Run")
return self.plot.panel(figure)
```
-Keep ordinary titles, axes, modes, category order, and sizing in the shared
-arguments rather than post-processing every page.
+Set standard titles, axes, modes, category order, and size with shared
+arguments. Do not make these changes separately on each page.
## Add A Reusable Figure Type
-When several pages need a genuinely new chart contract, add it to the shared
-renderer instead of copying Plotly construction.
+When several pages require a new chart contract, add it to the shared renderer.
+Do not copy the Plotly construction.
For an area chart:
@@ -213,8 +213,8 @@ For an area chart:
4. validate required columns with the same clear errors as other builders; and
5. test the Plotly figure before testing Panel wrapping.
-Here is a complete minimal builder for `dashboard/rendering/figures.py`. It
-uses the existing internal helpers because it lives beside the other builders:
+This is a complete small builder for `dashboard/rendering/figures.py`. It uses
+the existing internal helpers with the other builders:
```python
def area_figure(
@@ -261,11 +261,10 @@ def area_figure(
return figure
```
-`ChartTables`, `ChartValueMode`, `go`, and `np` are already used by that
-module. The explicit `value_mode` keeps `"dashboard"`, forced count, and forced
-share behavior consistent with the existing figure types. `_require_columns`
-provides a run-specific error, while `RenderContext.color()` preserves the
-configured run-color mapping.
+That module already uses `ChartTables`, `ChartValueMode`, `go`, and `np`. The
+explicit `value_mode` keeps `"dashboard"`, count, and share behavior consistent
+with existing figure types. `_require_columns` gives an error for the applicable
+run. `RenderContext.color()` keeps the configured run-color mapping.
The adapter shape is:
@@ -280,7 +279,7 @@ class Plotter:
return self.panel(self.figure.area(data, **kwargs))
```
-A focused test should inspect traces and layout:
+A focused test must examine traces and layout:
```python
def test_area_figure_uses_run_labels_and_colors():
@@ -326,9 +325,9 @@ return data_table(
)
```
-It produces one run tab per frame and applies shared column titles and numeric
-formatting. Use a page-local `Tabulator` only when the shared table contract
-cannot express the required interaction.
+It creates one run tab for each frame. It applies shared column titles and
+numeric formatting. Use a page-local `Tabulator` only when the shared table
+contract cannot supply the required interaction.
## Add A New Page Group
@@ -356,11 +355,11 @@ GROUP = DashboardGroupDefinition(
)
```
-Every child page declares `group_id="emissions"`. `default_page_id` must name
-one of those children. Private helper packages and modules begin with `_` so
-discovery ignores them.
+Each child page declares `group_id="emissions"`. `default_page_id` must specify
+one of these children. Start private helper package and module names with `_`.
+Discovery ignores these names.
-Users can enable the group's default pages or choose children:
+Users can enable the default group pages or select child pages:
```yaml
dashboard:
@@ -374,8 +373,8 @@ dashboard:
## Test The Extension
-Test pure transforms separately from lifecycle wiring. Then add focused checks
-for declarations:
+Test pure transforms separately from lifecycle connections. Then add focused
+checks for declarations:
```python
def test_trip_mode_page_declares_its_runtime_contract():
@@ -386,13 +385,13 @@ def test_trip_mode_page_declares_its_runtime_contract():
assert definition.required_summary_ids == ("trips_by_mode",)
```
-For selector behavior, instantiate a small test page with `DashboardState`,
-change the option provider's domain, refresh, and assert that stale values are
-repaired. For figures, test `Plotter(RenderContext()).figure` so failures are
-independent of Panel. The full registry suites then prove discovery, unique
-IDs, requirements, and export protocol support.
+For selector behavior, create a small test page with `DashboardState`. Change
+the option provider domain and refresh the page. Verify that the framework
+repairs invalid values. For figures, test `Plotter(RenderContext()).figure`.
+This keeps failures independent of Panel. The full registry tests verify
+discovery, unique IDs, requirements, and export protocol support.
-Run at least:
+Use at least these commands:
```bash
uv run python scripts/generate_wiki_catalogs.py
@@ -407,9 +406,9 @@ uv run --with pytest pytest --basetemp .pytest_tmp tests/test_figure_builders.py
- Required and optional data match the visible workflows.
- Summary reads declare the columns they consume.
- Selectors own options; sections list every selector dependency.
-- Custom widgets are registered rather than inserted raw.
+- Register custom widgets. Do not insert them directly.
- Pure transforms do not depend on Panel state.
-- Existing shared figures and tables are used before adding new renderers.
+- Use existing shared figures and tables before you add renderers.
- Missing data produces a standard diagnostic card.
- Live and export behavior use the same declarations.
- Catalogs and focused tests are current.
diff --git a/wiki/46-testing.md b/wiki/46-testing.md
index 2a89ba7..1d9b656 100644
--- a/wiki/46-testing.md
+++ b/wiki/46-testing.md
@@ -1,40 +1,39 @@
# 46 - Testing
-The default command runs every test, including the exhaustive offline HTML
+The default command executes all tests. It includes the complete offline HTML
export checks:
```powershell
uv run pytest --basetemp .pytest_tmp
```
-For a faster development loop, skip tests marked `full_export`:
+For a faster development test, omit tests marked `full_export`:
```powershell
uv run pytest --basetemp .pytest_tmp -m "not full_export"
```
-Run the exhaustive export boundary on its own before merging export, page,
-plotting, or summary changes:
+Execute the complete export tests before you merge export, page, plotting, or
+summary changes:
```powershell
uv run pytest --basetemp .pytest_tmp -m full_export
```
-The repository uses pytest's built-in `tmp_path` fixture with the workspace-local
-`--basetemp` above. Tests must not create persistent UUID-named directories at
-the repository root.
+The repository uses the built-in pytest `tmp_path` fixture. It uses the
+workspace-local `--basetemp` value above. Tests must not create persistent
+UUID-named directories in the repository root.
-Run the configured correctness lint before pushing:
+Execute this correctness check before you push changes:
```powershell
uv run ruff check .
```
-`full_export` is reserved for behavior that requires every default dashboard
-page and all dashboard states. Tests of writing, validation, individual pages,
-selectors, and diagnostics should configure the smallest page and state set
-that exercises their contract. This keeps those tests focused without reducing
-the end-to-end coverage provided by the full-export tests.
+Use `full_export` only for behavior that requires all default dashboard pages
+and dashboard states. For writes, validation, pages, selectors, and diagnostics,
+configure the smallest applicable page and state set. The full-export tests
+continue to supply complete workflow coverage.
## Which Suite To Run
@@ -45,10 +44,10 @@ the end-to-end coverage provided by the full-export tests.
| Export serializer, payload, runtime, or state behavior | Focused export tests | Fast suite plus `-m full_export` |
| Documentation only | Link/catalog checks and focused documentation tests | Fast suite if CI does not provide a docs-only path |
-The full-export tests are slow because they render every default page and
-dashboard state into a representative standalone HTML document. The shared
-fixture builds that document once per test session, so running the marked group
-together avoids repeating the expensive render.
+The full-export tests render each default page and dashboard state in one
+representative standalone HTML document. Thus, these tests take more time. The
+shared fixture builds the document one time in each test session. Execute the
+marked group together to prevent repeated renders.
## Focused Commands
@@ -58,7 +57,7 @@ uv run pytest --basetemp .pytest_tmp tests/test_figure_builders.py
uv run pytest --basetemp .pytest_tmp tests/test_export_serializer.py tests/test_export_payload.py
```
-Use [Developer Workflows](40-developer-workflows.md) to choose tests by
+Use [Developer Workflows](40-developer-workflows.md) to select tests for a
subsystem.
## Related Chapters
diff --git a/wiki/90-troubleshooting.md b/wiki/90-troubleshooting.md
index 3b134e8..eddba02 100644
--- a/wiki/90-troubleshooting.md
+++ b/wiki/90-troubleshooting.md
@@ -1,17 +1,17 @@
# 90 - Troubleshooting
-Use this chapter when a run, cache, page, or export is not behaving as expected.
+Use this chapter when a run, cache, page, or export does not operate correctly.
-## Fast Triage
+## Initial checks
-1. Confirm the config path you ran.
-2. Check the selected pipeline steps and dashboard mode in logs.
-3. Check whether the issue appears in prepare, summarize, dashboard, or export.
+1. Make sure that you used the correct configuration path.
+2. Find the selected pipeline steps and dashboard mode in the log.
+3. Identify whether the problem occurs in prepare, summarize, dashboard, or export.
4. Inspect `//manifest.json` for summary state and
`//prepared_tables/manifest.json` for final prepared/skimjoin
state (see the [cache layout](12-running-workflows.md#artifact-and-cache-paths)).
-5. Run with `--explain-cache` to inspect reuse and rebuild decisions. If a
- forced rebuild is needed, list only the affected stage in `pipeline.refresh`.
+5. Use `--explain-cache` to examine reuse and rebuild decisions. If you
+ must rebuild, list only the applicable step in `pipeline.refresh`.
## Symptoms
@@ -28,7 +28,7 @@ Use this chapter when a run, cache, page, or export is not behaving as expected.
## Cache Problems
-For a reproducible full rebuild, configure the steps and refresh policy:
+To make a repeatable full rebuild, configure the steps and refresh policy:
```yaml
pipeline:
@@ -37,8 +37,8 @@ pipeline:
refresh: all
```
-Return `refresh` to `[]` after the rebuild. Developers can use targeted
-one-off refresh flags while diagnosing a specific cache layer:
+Set `refresh` to `[]` after the rebuild. During a diagnostic run, developers
+can use a refresh flag for one cache layer:
```bash
uv run activitysim-viz --config local_config.yaml --refresh-prepared-cache
@@ -46,12 +46,12 @@ uv run activitysim-viz --config local_config.yaml --refresh-summary-cache
uv run activitysim-viz --config local_config.yaml --refresh-caches
```
-If only dashboard presentation changed, a refresh usually should not be needed.
-Raw-file, skim-file, and relevant config identities are checked automatically;
-use a manual refresh only when deliberately overriding a valid cache decision.
-Prefer `pipeline.refresh` for reproducible runs. A prepare refresh necessarily
-invalidates skimjoin and summary output; a skimjoin refresh preserves
-`base_prepared_tables`; a summary refresh preserves final prepared data.
+If only dashboard presentation changed, a refresh is usually not necessary.
+The system automatically checks raw-file, skim-file, and applicable
+configuration identities. Use a manual refresh only to override a valid cache
+decision. Use `pipeline.refresh` for repeatable runs. A prepare refresh
+invalidates skimjoin and summary output. A skimjoin refresh keeps
+`base_prepared_tables`. A summary refresh keeps final prepared data.
## Missing Page Data
@@ -60,31 +60,31 @@ Find the page in [31 - Dashboard Pages](31-dashboard-pages.md) and check:
- required summary IDs
- required prepared tables
- prepared-data mode
-- whether the page is enabled in live/export config
+- whether the live or export configuration enables the page
-Then find each summary in [24 - Summary Catalog](24-summary-catalog.md) and
-check the required input tables/columns.
+Then find each summary in [24 - Summary Catalog](24-summary-catalog.md). Check
+the required input tables and columns.
### Worked Triage: A Page Says Data Is Unavailable
-Suppose Trip Mode opens but shows the standard unavailable card:
+Use this procedure if Trip Mode shows the standard unavailable card:
1. Find `trip_mode` in chapter 31. It requires
`trip_mode_by_tour_purpose_and_tour_mode`.
2. Find that ID in chapter 24. Note its required prepared table and columns.
-3. Open `//manifest.json` and inspect the summary entry. If the
- summary is `unavailable`, read its recorded reason before rebuilding
- anything.
+3. Open `//manifest.json` and examine the summary entry. If the
+ summary is `unavailable`, read its recorded reason before a rebuild.
4. If a required prepared column is missing, inspect
`//prepared_tables/manifest.json`, the table schema, and the
canonical column settings in `columns`.
5. If the contract recently changed, rebuild the configured summarize step
with `pipeline.refresh: [summarize]`.
-6. If the summary is present and valid, confirm the page's `columns=` request
- matches the cached schema and that the selected weighting mode exists.
+6. If the summary is valid, make sure that the page's `columns=` request agrees
+ with the cached schema.
+7. Make sure that the selected weighting mode exists.
-This sequence moves backward through the declared contracts. It avoids trying
-random cache refreshes when the real issue is an input or schema mismatch.
+This sequence examines the declared contracts in reverse order. It prevents
+unnecessary cache refreshes when the problem is an input or schema mismatch.
## Skimjoin Problems
@@ -97,7 +97,7 @@ Check the skimjoin reports:
- `tour_aggregation_summary`
- `failure_report`
-Common fixes:
+Common corrections:
- correct skim file globs
- correct `network_los_file`
@@ -108,27 +108,27 @@ Common fixes:
## Export Problems
-If live mode works but export does not:
+If live mode operates correctly but export fails, do these steps:
-1. Confirm the page is included in export page selection.
-2. Confirm ordinary dropdowns use `self.select(...)` and custom widgets use
- `self.selector(...)`.
-3. Confirm affected content is registered with `self.section(...)`.
-4. Check browser console errors.
-5. Inspect the adjacent `.diagnostics.json` sidecar.
-6. Try `?debug_export=1`.
+1. Make sure that export page selection includes the page.
+2. Make sure that standard selection lists use `self.select(...)`.
+3. Make sure that custom widgets use `self.selector(...)`.
+4. Make sure that `self.section(...)` registers the applicable content.
+5. Check browser console errors.
+6. Inspect the adjacent `.diagnostics.json` sidecar.
+7. Try `?debug_export=1`.
-Export cannot reproduce arbitrary Python callbacks. It can only switch among
-serialized states and registered selector variants.
+Export cannot reproduce all Python callbacks. It can change only between stored
+states and registered selector variants.
-## Still Stuck
+## Create a small test case
-Create the smallest reproduction:
+Create the smallest test case:
1. one run
2. one page or one summary
3. one weighting mode
4. fresh cache root
-5. copied log excerpt and manifest diagnostics
+5. A copy of the applicable log text and manifest diagnostics.
-That usually makes the owning subsystem obvious.
+This test case usually identifies the applicable subsystem.
diff --git a/wiki/99-glossary.md b/wiki/99-glossary.md
index aa2b97e..6db4ef5 100644
--- a/wiki/99-glossary.md
+++ b/wiki/99-glossary.md
@@ -6,9 +6,9 @@
| Dashboard page | One registered visualizer page with a stable `page_id`. |
| Dashboard state | Shared visualizer state such as weighting mode, value mode, segmentation, and loaded runs. |
| Export | Standalone HTML dashboard output that does not require a Python server. |
-| `file_map` | Per-run override for raw ActivitySim output filenames. |
+| `file_map` | Run override for raw ActivitySim output file names. |
| `finalweight` | Canonical prepared weight column aggregated by summary builders. |
-| Live mode | Python-backed Panel dashboard served locally. |
+| Live mode | Local Panel dashboard that uses a Python server. |
| MAZ | Micro analysis zone. |
| OMX | Open Matrix file format commonly used for skims. |
| Output Processor | Prepare, skimjoin, segmentation, and summarize workflows. |
@@ -17,22 +17,22 @@
| Prepared table | Normalized table used by summaries and prepared-data pages. |
| `prepared_table_map` | Config mapping that supplies canonical prepared tables directly and skips raw prepare. |
| Run | One ActivitySim scenario/output set shown in the dashboard. |
-| Run key | Cache-directory identifier made by slugifying a run label, such as `Build 2035` to `build-2035`; duplicate normalized labels receive order-dependent `-1`, `-2`, and later suffixes. |
-| Segment | Configured slice of prepared data summarized separately. |
+| Run key | Cache-directory identifier made from a run label. For example, `Build 2035` becomes `build-2035`. Duplicate normalized labels get order-dependent suffixes such as `-1` and `-2`. |
+| Segment | Configured part of prepared data that the workflow summarizes separately. |
| Selector | Registered page-local widget that can refresh sections and participate in export. |
-| Skim | Matrix or lookup data used to attach level-of-service values to trips/tours. |
+| Skim | Matrix or lookup data that supplies level-of-service values to trips or tours. |
| Skimjoin | Optional processor step that joins skim-derived values to prepared trips and tours. |
| Summary builder | Function that converts `RunData` and `Config` into one summary `DataFrame`. |
| Summary cache | Per-run, per-weighting-mode CSV summary tables consumed by dashboard pages. |
| Summary contract | Builder metadata defining output schema and required inputs. |
| TAZ | Traffic analysis zone. |
-| Weighting mode | Versioned registered transform that presents prepared `finalweight` values to summary builders and prepared-data pages under one cache/dashboard mode ID. |
+| Weighting mode | Registered transform with a version. It supplies prepared `finalweight` values under one cache and dashboard mode ID. Summary builders and prepared-data pages use the values. |
## How The Terms Connect
-For a run labeled `Build`, raw `final_trips.csv` is normalized into the
-prepared `trips` table. A summary builder aggregates its canonical
-`finalweight` column and writes a registered summary under the run key's
-weighted and unweighted cache directories. A dashboard page declares that
-summary ID, reads it through `self.data`, and lets registered selectors refresh
-its sections. Export serializes those same declared page states into HTML.
+For a run labeled `Build`, prepare converts raw `final_trips.csv` to the
+prepared `trips` table. A summary builder aggregates the canonical `finalweight`
+column. It writes a registered summary in the weighted and unweighted cache
+directories for the run key. A dashboard page declares the summary ID and reads
+it through `self.data`. Registered selectors refresh its sections. Export
+converts the same declared page states to HTML.
From 0ff64fbb4653570006be4b237a3b920cd683d8e4 Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Wed, 5 Aug 2026 17:28:37 -0400
Subject: [PATCH 15/27] Updated caching fingerprints to not recreate all
summaries when segmentation options are used
---
processor/summarize/builder.py | 17 +-
processor/summarize/cache.py | 75 +++++++-
processor/summarize/cache_storage.py | 174 +++++++++++++++++--
run.py | 18 +-
runtime/config/signatures.py | 92 +++++-----
runtime/workflows/artifacts.py | 7 +-
runtime/workflows/summarize.py | 160 +++++++++++------
tests/test_runtime_workflows.py | 249 +++++++++++++++++++++++++++
tests/test_segmentation_feature.py | 4 +-
wiki/12-running-workflows.md | 15 +-
10 files changed, 698 insertions(+), 113 deletions(-)
diff --git a/processor/summarize/builder.py b/processor/summarize/builder.py
index e4af017..ce6a6ce 100644
--- a/processor/summarize/builder.py
+++ b/processor/summarize/builder.py
@@ -36,11 +36,18 @@ def summary_builder_identity(summary_id: str) -> dict[str, object]:
}
-def summary_digest(summary_id: str, config: Config) -> str:
+def summary_digest(
+ summary_id: str,
+ config: Config,
+ *,
+ analysis_unit_identity: dict[str, object] | None = None,
+) -> str:
payload = {
"summary_config_digest": config.summary_config_digest,
"summary": summary_builder_identity(summary_id),
}
+ if analysis_unit_identity is not None:
+ payload["analysis_unit"] = analysis_unit_identity
return hashlib.sha256(
json.dumps(payload, separators=(",", ":"), ensure_ascii=True).encode("utf-8")
).hexdigest()
@@ -49,9 +56,15 @@ def summary_digest(summary_id: str, config: Config) -> str:
def summary_digests(
config: Config,
summary_ids: list[str] | None = None,
+ *,
+ analysis_unit_identity: dict[str, object] | None = None,
) -> dict[str, str]:
return {
- summary_id: summary_digest(summary_id, config)
+ summary_id: summary_digest(
+ summary_id,
+ config,
+ analysis_unit_identity=analysis_unit_identity,
+ )
for summary_id in (
summary_ids if summary_ids is not None else DEFAULT_SUMMARY_IDS
)
diff --git a/processor/summarize/cache.py b/processor/summarize/cache.py
index 51cafa3..ae5cc57 100644
--- a/processor/summarize/cache.py
+++ b/processor/summarize/cache.py
@@ -4,6 +4,7 @@
from pathlib import Path
+from processor.analysis_units import AnalysisUnit
from processor.cache_identity import build_run_fingerprint, build_run_keys, slugify
from processor.summarize import builder as summary_builder
from processor.summarize.cache_storage import (
@@ -25,6 +26,32 @@
SUMMARY_BY_ID,
)
from runtime.config import Config
+from runtime.config.signatures import segmentation_unit_signature_payload
+
+
+def analysis_unit_key(*, segmentation_type: str, segment_id: str) -> str:
+ return f"{segmentation_type}::{segment_id}"
+
+
+def _analysis_unit_summary_digests(
+ config: Config,
+ *,
+ segmentation_type: str,
+ segment_id: str,
+ summary_ids: list[str],
+) -> dict[str, str]:
+ identity = None
+ if (segmentation_type, segment_id) != ("full", "full"):
+ identity = segmentation_unit_signature_payload(
+ config,
+ segmentation_type=segmentation_type,
+ segment_id=segment_id,
+ )
+ return summary_builder.summary_digests(
+ config,
+ summary_ids,
+ analysis_unit_identity=identity,
+ )
def summary_file_map(summary_ids: list[str]) -> dict[str, str]:
@@ -95,13 +122,25 @@ def write_summary_run_bundle(
) -> Path:
"""Write one run cache directory containing all segment variants."""
requested_ids = list(next(iter(summary_runs[0].summaries_by_mode.values())).keys())
+ summary_digests_by_unit = {
+ analysis_unit_key(
+ segmentation_type=summary_run.segmentation_type,
+ segment_id=summary_run.segment_id,
+ ): _analysis_unit_summary_digests(
+ config,
+ segmentation_type=summary_run.segmentation_type,
+ segment_id=summary_run.segment_id,
+ summary_ids=requested_ids,
+ )
+ for summary_run in summary_runs
+ }
return _write_summary_run_bundle(
summary_runs,
config,
output_root=output_root,
run_fingerprint=run_fingerprint,
prepared_manifest_identity=prepared_manifest_identity,
- summary_digests=summary_builder.summary_digests(config, requested_ids),
+ summary_digests_by_unit=summary_digests_by_unit,
summary_filename_by_id=SUMMARY_FILENAME_BY_ID,
)
@@ -117,7 +156,37 @@ def inspect_summary_run_bundle(
expected_prepared_manifest_identity: dict[str, object] | None = None,
expected_label: str | None = None,
expected_run_key: str | None = None,
+ expected_analysis_units: list[AnalysisUnit] | None = None,
) -> dict[str, object]:
+ requested_ids = list(expected_summary_ids or summary_builder.DEFAULT_SUMMARY_IDS)
+ if expected_analysis_units is not None:
+ unit_keys = [
+ (unit.segmentation_type, unit.segment_id)
+ for unit in expected_analysis_units
+ ]
+ elif config.segmentation.enabled:
+ unit_keys = [
+ ("full", "full"),
+ *[
+ (definition.name, segment.id)
+ for definition in config.segmentation.definitions
+ for segment in definition.segments
+ ],
+ ]
+ else:
+ unit_keys = [("full", "full")]
+ expected_summary_digests_by_unit = {
+ analysis_unit_key(
+ segmentation_type=segmentation_type,
+ segment_id=segment_id,
+ ): _analysis_unit_summary_digests(
+ config,
+ segmentation_type=segmentation_type,
+ segment_id=segment_id,
+ summary_ids=requested_ids,
+ )
+ for segmentation_type, segment_id in unit_keys
+ }
return _inspect_summary_run_bundle(
cache_dir,
config,
@@ -131,6 +200,7 @@ def inspect_summary_run_bundle(
expected_summary_digests=summary_builder.summary_digests(
config, expected_summary_ids
),
+ expected_summary_digests_by_unit=expected_summary_digests_by_unit,
)
@@ -145,6 +215,7 @@ def load_summary_run_bundle(
expected_prepared_manifest_identity: dict[str, object] | None = None,
expected_label: str | None = None,
expected_run_key: str | None = None,
+ expected_summary_ids_by_unit: dict[str, list[str]] | None = None,
) -> list[SummaryRun]:
"""Load one run cache directory and return all segment variants."""
return _load_summary_run_bundle(
@@ -157,6 +228,7 @@ def load_summary_run_bundle(
expected_prepared_manifest_identity=expected_prepared_manifest_identity,
expected_label=expected_label,
expected_run_key=expected_run_key,
+ expected_summary_ids_by_unit=expected_summary_ids_by_unit,
summary_spec_by_id=SUMMARY_BY_ID,
)
@@ -164,6 +236,7 @@ def load_summary_run_bundle(
__all__ = [
"SCHEMA_VERSION",
"SummaryRun",
+ "analysis_unit_key",
"build_run_fingerprint",
"build_run_keys",
"discover_cache_dirs",
diff --git a/processor/summarize/cache_storage.py b/processor/summarize/cache_storage.py
index 09efc94..c27615e 100644
--- a/processor/summarize/cache_storage.py
+++ b/processor/summarize/cache_storage.py
@@ -4,6 +4,7 @@
from datetime import datetime, timezone
from pathlib import Path
+import shutil
import polars as pl
@@ -256,7 +257,7 @@ def write_summary_run_bundle(
output_root: str | Path | None = None,
run_fingerprint: dict[str, object] | None = None,
prepared_manifest_identity: dict[str, object] | None = None,
- summary_digests: dict[str, str] | None = None,
+ summary_digests_by_unit: dict[str, dict[str, str]] | None = None,
summary_filename_by_id: dict[str, str],
) -> Path:
"""Write one run cache directory containing full and segmented summary outputs."""
@@ -283,6 +284,8 @@ def write_summary_run_bundle(
segmentation_type_entries: dict[str, dict[str, object]] = {}
for summary_run in summary_runs:
+ unit_key = f"{summary_run.segmentation_type}::{summary_run.segment_id}"
+ unit_summary_digests = (summary_digests_by_unit or {}).get(unit_key, {})
segment_states: dict[str, dict[str, str]] = {}
segment_diagnostics: dict[str, dict[str, str]] = {}
segment_digests: dict[str, dict[str, str]] = {}
@@ -301,13 +304,13 @@ def write_summary_run_bundle(
failed_summaries[mode] = list(mode_payload["failed_summaries"])
summary_diagnostics[mode] = dict(mode_payload["summary_diagnostics"])
manifest_summary_digests[mode] = {
- summary_id: (summary_digests or {}).get(summary_id, "")
+ summary_id: unit_summary_digests.get(summary_id, "")
for summary_id in summary_ids
}
segment_states[mode] = dict(mode_payload["summary_states"])
segment_diagnostics[mode] = dict(mode_payload["summary_diagnostics"])
segment_digests[mode] = {
- summary_id: (summary_digests or {}).get(summary_id, "")
+ summary_id: unit_summary_digests.get(summary_id, "")
for summary_id in summary_ids
}
mode_dir = (
@@ -348,6 +351,27 @@ def write_summary_run_bundle(
)
)
+ current_segment_keys = {
+ (run.segmentation_type, run.segment_id)
+ for run in summary_runs
+ if not run.is_full_segment
+ }
+ for mode in weighting_modes:
+ segments_root = run_dir / "summary_tables" / mode / "segments"
+ if not segments_root.exists():
+ continue
+ for segmentation_dir in segments_root.iterdir():
+ if not segmentation_dir.is_dir():
+ continue
+ for segment_dir in segmentation_dir.iterdir():
+ if segment_dir.is_dir() and (
+ segmentation_dir.name,
+ segment_dir.name,
+ ) not in current_segment_keys:
+ shutil.rmtree(segment_dir)
+ if not any(segmentation_dir.iterdir()):
+ segmentation_dir.rmdir()
+
manifest = _summary_manifest(
summary_run=full_run,
config=config,
@@ -629,6 +653,54 @@ def _segment_mode_dirs(
return segment_dirs
+def _manifest_unit_metadata(
+ cache_dir: Path,
+ manifest: dict[str, object],
+ expected_modes: list[str],
+) -> dict[str, dict[str, object]]:
+ _, _, _, _, full_digests = _manifest_summary_metadata(manifest)
+ units: dict[str, dict[str, object]] = {
+ "full::full": {
+ "mode_dirs": {
+ mode: (
+ cache_dir / "summary_tables" / mode
+ if (cache_dir / "summary_tables" / mode).exists()
+ else cache_dir / mode
+ )
+ for mode in expected_modes
+ },
+ "summary_digests": full_digests,
+ }
+ }
+ for raw_group in list(manifest.get("segmentation_types", [])):
+ group = dict(raw_group)
+ segmentation_type = str(group.get("segmentation_type", "full"))
+ for raw_segment in list(group.get("segments", [])):
+ segment = dict(raw_segment)
+ segment_id = str(segment.get("segment_id", "full"))
+ summary_roots = {
+ str(mode): str(path)
+ for mode, path in dict(segment.get("summary_roots", {})).items()
+ }
+ units[f"{segmentation_type}::{segment_id}"] = {
+ "mode_dirs": {
+ mode: cache_dir
+ / Path(summary_roots.get(mode, f"summary_tables/{mode}"))
+ for mode in expected_modes
+ },
+ "summary_digests": {
+ str(mode): {
+ str(summary_id): str(digest)
+ for summary_id, digest in dict(mode_digests).items()
+ }
+ for mode, mode_digests in dict(
+ segment.get("summary_digests", {})
+ ).items()
+ },
+ }
+ return units
+
+
def inspect_summary_run_bundle(
cache_dir: str | Path,
config: Config,
@@ -641,6 +713,7 @@ def inspect_summary_run_bundle(
expected_label: str | None = None,
expected_run_key: str | None = None,
expected_summary_digests: dict[str, str] | None = None,
+ expected_summary_digests_by_unit: dict[str, dict[str, str]] | None = None,
) -> dict[str, object]:
cache_dir = Path(cache_dir)
manifest = read_manifest(cache_dir, error_cls=SummaryCacheError)
@@ -675,6 +748,54 @@ def inspect_summary_run_bundle(
manifest_summary_digests,
) = _manifest_summary_metadata(manifest)
expected_summary_digests = dict(expected_summary_digests or {})
+ if expected_summary_digests_by_unit is not None:
+ manifest_units = _manifest_unit_metadata(cache_dir, manifest, expected_modes)
+ reusable_by_unit: dict[str, list[str]] = {}
+ stale_by_unit: dict[str, list[str]] = {}
+ for unit_key, unit_expected_digests in expected_summary_digests_by_unit.items():
+ reusable_by_unit[unit_key] = []
+ stale_by_unit[unit_key] = []
+ unit_metadata = manifest_units.get(unit_key)
+ for summary_id in resolved_summary_ids:
+ if unit_metadata is None:
+ stale_by_unit[unit_key].append(summary_id)
+ continue
+ mode_dirs = dict(unit_metadata["mode_dirs"])
+ unit_manifest_digests = dict(unit_metadata["summary_digests"])
+ filename = summary_files.get(summary_id, f"{summary_id}.csv")
+ is_reusable = all(
+ dict(unit_manifest_digests.get(mode, {})).get(summary_id)
+ == unit_expected_digests.get(summary_id)
+ and (mode_dirs[mode] / filename).exists()
+ for mode in expected_modes
+ )
+ target = reusable_by_unit if is_reusable else stale_by_unit
+ target[unit_key].append(summary_id)
+
+ stale_summary_ids = [
+ summary_id
+ for summary_id in resolved_summary_ids
+ if any(
+ summary_id in unit_stale_ids
+ for unit_stale_ids in stale_by_unit.values()
+ )
+ ]
+ reusable_summary_ids = [
+ summary_id
+ for summary_id in resolved_summary_ids
+ if summary_id not in stale_summary_ids
+ ]
+ return {
+ "manifest": manifest,
+ "reusable_summary_ids": reusable_summary_ids,
+ "stale_summary_ids": stale_summary_ids,
+ "reusable_summary_ids_by_unit": reusable_by_unit,
+ "stale_summary_ids_by_unit": stale_by_unit,
+ "obsolete_unit_keys": sorted(
+ set(manifest_units) - set(expected_summary_digests_by_unit)
+ ),
+ }
+
stale_summary_ids: list[str] = []
reusable_summary_ids: list[str] = []
segment_dirs = _segment_mode_dirs(cache_dir, manifest, expected_modes)
@@ -791,18 +912,26 @@ def load_summary_run_bundle(
expected_prepared_manifest_identity: dict[str, object] | None = None,
expected_label: str | None = None,
expected_run_key: str | None = None,
+ expected_summary_ids_by_unit: dict[str, list[str]] | None = None,
summary_spec_by_id: dict[str, object],
) -> list[SummaryRun]:
"""Load one run cache directory and return all persisted segment variants."""
cache_dir = Path(cache_dir)
manifest = read_manifest(cache_dir, error_cls=SummaryCacheError)
if "segmentation_types" not in manifest and "segments" not in manifest:
+ full_summary_ids = (
+ expected_summary_ids_by_unit.get("full::full", [])
+ if expected_summary_ids_by_unit is not None
+ else expected_summary_ids
+ )
+ if expected_summary_ids_by_unit is not None and not full_summary_ids:
+ return []
return [
load_summary_run_cache(
cache_dir,
config,
expected_modes=expected_modes,
- expected_summary_ids=expected_summary_ids,
+ expected_summary_ids=full_summary_ids,
expected_summary_config_digest=expected_summary_config_digest,
expected_run_fingerprint=expected_run_fingerprint,
expected_prepared_manifest_identity=expected_prepared_manifest_identity,
@@ -827,12 +956,21 @@ def load_summary_run_bundle(
expected_label=expected_label,
expected_run_key=expected_run_key,
)
+ requested_summary_ids = expected_summary_ids
+ if expected_summary_ids_by_unit is not None:
+ requested_summary_ids = list(
+ dict.fromkeys(
+ summary_id
+ for unit_summary_ids in expected_summary_ids_by_unit.values()
+ for summary_id in unit_summary_ids
+ )
+ )
expected_modes, expected_summary_ids = _validated_mode_and_summary_ids(
manifest=manifest,
config=config,
cache_dir=cache_dir,
expected_modes=expected_modes,
- expected_summary_ids=expected_summary_ids,
+ expected_summary_ids=requested_summary_ids,
)
(
summary_files,
@@ -843,16 +981,23 @@ def load_summary_run_bundle(
) = _manifest_summary_metadata(manifest)
loaded_runs: list[SummaryRun] = []
+ full_expected_summary_ids = (
+ expected_summary_ids_by_unit.get("full::full", [])
+ if expected_summary_ids_by_unit is not None
+ else expected_summary_ids
+ )
full_summaries_by_mode: dict[str, dict[str, pl.DataFrame]] = {}
full_summary_metadata_by_mode: dict[str, dict[str, dict[str, object]]] = {}
for mode in expected_modes:
+ if not full_expected_summary_ids:
+ break
full_mode_dir = cache_dir / "summary_tables" / mode
if not full_mode_dir.exists():
full_mode_dir = cache_dir / mode
mode_tables, mode_metadata = _load_mode_tables(
mode_dir=full_mode_dir,
mode=mode,
- expected_summary_ids=expected_summary_ids,
+ expected_summary_ids=full_expected_summary_ids,
summary_files=summary_files,
empty_summaries=empty_summaries,
manifest_summary_states=manifest_summary_states,
@@ -861,8 +1006,9 @@ def load_summary_run_bundle(
)
full_summaries_by_mode[mode] = mode_tables
full_summary_metadata_by_mode[mode] = mode_metadata
- loaded_runs.append(
- SummaryRun(
+ if full_expected_summary_ids:
+ loaded_runs.append(
+ SummaryRun(
label=str(manifest.get("label", cache_dir.name)),
run_key=str(manifest.get("run_key", cache_dir.name)),
summaries_by_mode=full_summaries_by_mode,
@@ -873,8 +1019,8 @@ def load_summary_run_bundle(
is_full_segment=True,
source_run_dir=manifest.get("source_run_dir"),
manifest=manifest,
+ )
)
- )
if "segmentation_types" in manifest:
segment_groups = []
for raw_group in list(manifest.get("segmentation_types", [])):
@@ -888,6 +1034,14 @@ def load_summary_run_bundle(
for raw_segment in list(manifest.get("segments", []))
]
for segmentation_type, segment in segment_groups:
+ unit_key = f"{segmentation_type}::{segment.get('segment_id', 'full')}"
+ unit_expected_summary_ids = (
+ expected_summary_ids_by_unit.get(unit_key, [])
+ if expected_summary_ids_by_unit is not None
+ else expected_summary_ids
+ )
+ if not unit_expected_summary_ids:
+ continue
summary_roots = {
str(mode): str(path)
for mode, path in dict(segment.get("summary_roots", {})).items()
@@ -923,7 +1077,7 @@ def load_summary_run_bundle(
mode_tables, mode_metadata = _load_mode_tables(
mode_dir=cache_dir / mode_root,
mode=mode,
- expected_summary_ids=expected_summary_ids,
+ expected_summary_ids=unit_expected_summary_ids,
summary_files=summary_files,
empty_summaries=empty_summaries,
manifest_summary_states=manifest_summary_states,
diff --git a/run.py b/run.py
index c87b800..66f2812 100644
--- a/run.py
+++ b/run.py
@@ -477,10 +477,22 @@ def decision(action: str, reason: str | None = None) -> str:
expected_label=label,
expected_run_key=run_key,
)
- stale = list(inspection["stale_summary_ids"])
+ stale_count = sum(
+ len(summary_ids)
+ for summary_ids in dict(
+ inspection["stale_summary_ids_by_unit"]
+ ).values()
+ )
+ obsolete_count = len(inspection["obsolete_unit_keys"])
summary_action = (
- decision("REBUILD", f"{len(stale)} summary tables are stale")
- if stale
+ decision(
+ "REBUILD",
+ (
+ f"{stale_count} analysis-unit summary tables are stale; "
+ f"{obsolete_count} analysis units are obsolete"
+ ),
+ )
+ if stale_count or obsolete_count
else "REUSE"
)
except SummaryCacheError as exc:
diff --git a/runtime/config/signatures.py b/runtime/config/signatures.py
index 205ea82..5919a31 100644
--- a/runtime/config/signatures.py
+++ b/runtime/config/signatures.py
@@ -187,46 +187,6 @@ def base_prepare_signature_payload(config: Config) -> dict[str, Any]:
def summary_signature_payload(config: Config) -> dict[str, Any]:
- segmentation_payload: dict[str, Any] = {"enabled": config.segmentation.enabled}
- if config.segmentation.enabled:
- segmentation_payload["definitions"] = [
- {
- "name": definition.name,
- "include_full": definition.include_full,
- "persist_segmented_prepared_tables": definition.persist_segmented_prepared_tables,
- "allow_overlapping": definition.allow_overlapping,
- "on_empty_segment": definition.on_empty_segment,
- "source": (
- {
- "type": "prepared_column",
- "column": definition.source.column,
- "source_table": definition.source.source_table,
- }
- if isinstance(definition.source, PreparedColumnSegmentationSource)
- else {
- "type": "csv_lookup",
- "file": definition.source.file,
- "join_source_table": definition.source.join_source_table,
- "join_source_key_column": definition.source.join_source_key_column,
- "csv_key_column": definition.source.csv_key_column,
- "segment_value_column": definition.source.segment_value_column,
- "lookup_rows": [
- {"key": key, "value": value}
- for key, value in definition.source.lookup_rows
- ],
- }
- ),
- "segments": [
- {
- "id": segment.id,
- "label": segment.label,
- "values": list(segment.values),
- }
- for segment in definition.segments
- ],
- }
- for definition in config.segmentation.definitions
- ]
return {
"weighting_modes": [
definition.signature_payload()
@@ -283,7 +243,57 @@ def summary_signature_payload(config: Config) -> dict[str, Any]:
"prepare": {
"vot_bins": prepare_signature_payload(config)["prepare"]["vot_bins"],
},
- "segmentation": segmentation_payload,
+ }
+
+
+def segmentation_unit_signature_payload(
+ config: Config,
+ *,
+ segmentation_type: str,
+ segment_id: str,
+) -> dict[str, Any]:
+ """Return the cache identity for one full or segmented analysis unit."""
+ if segmentation_type == "full" and segment_id == "full":
+ return {"segmentation_type": "full", "segment_id": "full"}
+
+ definition = config.segmentation.definition_by_name(segmentation_type)
+ if definition is None or definition.source is None:
+ return {"segmentation_type": segmentation_type, "segment_id": segment_id}
+ segment = next(
+ (candidate for candidate in definition.segments if candidate.id == segment_id),
+ None,
+ )
+ if segment is None:
+ return {"segmentation_type": segmentation_type, "segment_id": segment_id}
+
+ source = definition.source
+ if isinstance(source, PreparedColumnSegmentationSource):
+ source_payload: dict[str, Any] = {
+ "type": "prepared_column",
+ "column": source.column,
+ "source_table": source.source_table,
+ }
+ else:
+ segment_values = set(segment.values)
+ source_payload = {
+ "type": "csv_lookup",
+ "file": source.file,
+ "join_source_table": source.join_source_table,
+ "join_source_key_column": source.join_source_key_column,
+ "csv_key_column": source.csv_key_column,
+ "segment_value_column": source.segment_value_column,
+ "lookup_rows": [
+ {"key": key, "value": value}
+ for key, value in source.lookup_rows
+ if value in segment_values
+ ],
+ }
+ return {
+ "segmentation_type": segmentation_type,
+ "segment_id": segment.id,
+ "segment_label": segment.label,
+ "segment_values": list(segment.values),
+ "source": source_payload,
}
diff --git a/runtime/workflows/artifacts.py b/runtime/workflows/artifacts.py
index c59fa8a..ba6d082 100644
--- a/runtime/workflows/artifacts.py
+++ b/runtime/workflows/artifacts.py
@@ -69,5 +69,8 @@ class SummaryCacheInspection:
"""Reusable cached summaries and the table ids that still need rebuilding."""
runs: tuple[Any, ...] = ()
- reusable_summary_ids: tuple[str, ...] = ()
- stale_summary_ids: tuple[str, ...] = ()
+ reusable_summary_ids_by_unit: dict[str, tuple[str, ...]] = field(
+ default_factory=dict
+ )
+ stale_summary_ids_by_unit: dict[str, tuple[str, ...]] = field(default_factory=dict)
+ obsolete_unit_keys: tuple[str, ...] = ()
diff --git a/runtime/workflows/summarize.py b/runtime/workflows/summarize.py
index 1c66b4f..8f966c6 100644
--- a/runtime/workflows/summarize.py
+++ b/runtime/workflows/summarize.py
@@ -59,6 +59,7 @@ def _load_summary_run_from_cache(
run_key: str,
run_fingerprint: dict[str, object],
prepared_manifest_identity: dict[str, object],
+ analysis_units: list[AnalysisUnit] | None = None,
) -> SummaryCacheInspection | None:
"""Load one summary run from cache when valid."""
try:
@@ -72,15 +73,31 @@ def _load_summary_run_from_cache(
expected_prepared_manifest_identity=prepared_manifest_identity,
expected_label=label,
expected_run_key=run_key,
+ expected_analysis_units=analysis_units,
)
- reusable_summary_ids = list(inspection["reusable_summary_ids"])
- stale_summary_ids = list(inspection["stale_summary_ids"])
+ reusable_by_unit = {
+ str(unit_key): tuple(summary_ids)
+ for unit_key, summary_ids in dict(
+ inspection["reusable_summary_ids_by_unit"]
+ ).items()
+ }
+ stale_by_unit = {
+ str(unit_key): tuple(summary_ids)
+ for unit_key, summary_ids in dict(
+ inspection["stale_summary_ids_by_unit"]
+ ).items()
+ }
+ obsolete_unit_keys = tuple(inspection["obsolete_unit_keys"])
cached_runs = (
summary_cache.load_summary_run_bundle(
cache_dir,
config,
expected_modes=config.weighting_modes,
- expected_summary_ids=reusable_summary_ids,
+ expected_summary_ids_by_unit={
+ unit_key: list(summary_ids)
+ for unit_key, summary_ids in reusable_by_unit.items()
+ if summary_ids
+ },
# Per-summary digests were validated by the inspection above.
# Requiring the bundle-wide digest here would discard otherwise
# reusable tables whenever only one builder changed.
@@ -90,23 +107,31 @@ def _load_summary_run_from_cache(
expected_label=label,
expected_run_key=run_key,
)
- if reusable_summary_ids
+ if any(reusable_by_unit.values())
else []
)
+ stale_count = sum(len(summary_ids) for summary_ids in stale_by_unit.values())
+ reusable_count = sum(
+ len(summary_ids) for summary_ids in reusable_by_unit.values()
+ )
LOGGER.info(
"Pipeline decision for %r / summarize: %s (%s)",
label,
- "REUSE" if not stale_summary_ids else "REBUILD",
+ "REUSE" if not stale_count and not obsolete_unit_keys else "REBUILD",
(
- "all summary tables reusable"
- if not stale_summary_ids
- else f"{len(stale_summary_ids)} stale; {len(reusable_summary_ids)} reusable"
+ "all analysis-unit summary tables reusable"
+ if not stale_count and not obsolete_unit_keys
+ else (
+ f"{stale_count} stale; {reusable_count} reusable; "
+ f"{len(obsolete_unit_keys)} obsolete analysis units"
+ )
),
)
return SummaryCacheInspection(
runs=tuple(cached_runs),
- reusable_summary_ids=tuple(reusable_summary_ids),
- stale_summary_ids=tuple(stale_summary_ids),
+ reusable_summary_ids_by_unit=reusable_by_unit,
+ stale_summary_ids_by_unit=stale_by_unit,
+ obsolete_unit_keys=obsolete_unit_keys,
)
except summary_types.SummaryCacheError as exc:
LOGGER.info("Cache miss for %r: %s", label, exc)
@@ -175,9 +200,8 @@ def _merge_summary_runs(
*,
cached_runs: list[Any],
rebuilt_runs: list[Any],
+ analysis_units: list[AnalysisUnit],
) -> list[Any]:
- if not cached_runs:
- return rebuilt_runs
cached_by_segment = {
(run.segmentation_type, run.segment_id): run for run in cached_runs
}
@@ -185,9 +209,14 @@ def _merge_summary_runs(
(run.segmentation_type, run.segment_id): run for run in rebuilt_runs
}
merged: list[Any] = []
- for segment_key in rebuilt_by_segment:
- rebuilt = rebuilt_by_segment[segment_key]
+ for unit in analysis_units:
+ segment_key = (unit.segmentation_type, unit.segment_id)
+ rebuilt = rebuilt_by_segment.get(segment_key)
cached = cached_by_segment.get(segment_key)
+ if rebuilt is None:
+ if cached is not None:
+ merged.append(cached)
+ continue
if cached is None:
merged.append(rebuilt)
continue
@@ -294,6 +323,30 @@ def run_summary_workflow(
run_keys.append(run_key)
run_fingerprints_by_key[run_key] = run_fingerprint
cached_run = None
+ analysis_units: list[AnalysisUnit] | None = None
+ prepared_loaded: tuple[str, RunData] | None = None
+ has_buildable_inputs = bool(entry.get("dir") or entry.get("prepared_table_map"))
+
+ if config.segmentation.enabled and has_buildable_inputs:
+ prepare_artifact = run_prepare_workflow(
+ config=config,
+ prepared_root=prepared_root,
+ run_entries=[entry],
+ prefer_cache=prepared_prefer_cache,
+ write_cache=True,
+ existing=prepare_artifact,
+ plan=plan,
+ )
+ if run_key in prepare_artifact.by_key:
+ prepared_loaded = prepare_artifact.by_key[run_key]
+ existing_prepared_runs_by_key = dict(prepare_artifact.by_key)
+ prepared_runs_by_key[run_key] = prepared_loaded
+ analysis_units = build_analysis_units_for_run(
+ run_key=run_key,
+ run_name=label,
+ prepared_run=prepared_loaded[1],
+ config=config,
+ )
if prefer_cache:
cached_run = _load_summary_run_from_cache(
@@ -303,10 +356,13 @@ def run_summary_workflow(
run_key=run_key,
run_fingerprint=run_fingerprint,
prepared_manifest_identity=prepared_manifest_identity,
+ analysis_units=analysis_units,
)
if cached_run is not None:
- stale_summary_ids = list(cached_run.stale_summary_ids)
- if not stale_summary_ids:
+ if (
+ not any(cached_run.stale_summary_ids_by_unit.values())
+ and not cached_run.obsolete_unit_keys
+ ):
summary_runs.extend(
merge_summary_table_map_run(
list(cached_run.runs),
@@ -324,16 +380,6 @@ def run_summary_workflow(
)
cached_summary_runs = list(cached_run.runs) if cached_run else []
- summary_ids_to_build = list(summary_builder.DEFAULT_SUMMARY_IDS)
- if cached_run is not None:
- summary_ids_to_build = list(cached_run.stale_summary_ids)
- summary_ids_to_build = [
- summary_id
- for summary_id in summary_ids_to_build
- if summary_id not in external_summary_ids
- ]
-
- has_buildable_inputs = bool(entry.get("dir") or entry.get("prepared_table_map"))
if not has_buildable_inputs:
run_summary_runs = merge_summary_table_map_run(
cached_summary_runs,
@@ -359,16 +405,19 @@ def run_summary_workflow(
)
continue
- prepare_artifact = run_prepare_workflow(
- config=config,
- prepared_root=prepared_root,
- run_entries=[entry],
- prefer_cache=prepared_prefer_cache,
- write_cache=True,
- existing=prepare_artifact,
- plan=plan,
- )
- if run_key not in prepare_artifact.by_key:
+ if prepared_loaded is None:
+ prepare_artifact = run_prepare_workflow(
+ config=config,
+ prepared_root=prepared_root,
+ run_entries=[entry],
+ prefer_cache=prepared_prefer_cache,
+ write_cache=True,
+ existing=prepare_artifact,
+ plan=plan,
+ )
+ if run_key in prepare_artifact.by_key:
+ prepared_loaded = prepare_artifact.by_key[run_key]
+ if prepared_loaded is None:
run_summary_runs = merge_summary_table_map_run(
cached_summary_runs,
external_summary_run,
@@ -392,19 +441,35 @@ def run_summary_workflow(
label,
)
continue
- prepared_loaded = prepare_artifact.by_key[run_key]
existing_prepared_runs_by_key = dict(prepare_artifact.by_key)
prepared_runs_by_key[run_key] = prepared_loaded
- analysis_units = build_analysis_units_for_run(
- run_key=run_key,
- run_name=label,
- prepared_run=prepared_loaded[1],
- config=config,
- )
+ if analysis_units is None:
+ analysis_units = build_analysis_units_for_run(
+ run_key=run_key,
+ run_name=label,
+ prepared_run=prepared_loaded[1],
+ config=config,
+ )
run_summary_runs = []
- if summary_ids_to_build:
- for unit in analysis_units:
+ for unit in analysis_units:
+ unit_key = summary_cache.analysis_unit_key(
+ segmentation_type=unit.segmentation_type,
+ segment_id=unit.segment_id,
+ )
+ summary_ids_to_build = (
+ list(summary_builder.DEFAULT_SUMMARY_IDS)
+ if cached_run is None
+ else list(
+ cached_run.stale_summary_ids_by_unit.get(unit_key, ())
+ )
+ )
+ summary_ids_to_build = [
+ summary_id
+ for summary_id in summary_ids_to_build
+ if summary_id not in external_summary_ids
+ ]
+ if summary_ids_to_build:
summaries_by_mode, summary_metadata_by_mode = _build_summary_tables_for_run(
prepared_run=unit.prepared_run,
config=config,
@@ -431,13 +496,12 @@ def run_summary_workflow(
source_run_dir=str(unit.prepared_run.run_dir),
)
)
- if cached_summary_runs and run_summary_runs:
+ if cached_summary_runs or run_summary_runs:
run_summary_runs = _merge_summary_runs(
cached_runs=cached_summary_runs,
rebuilt_runs=run_summary_runs,
+ analysis_units=analysis_units,
)
- elif cached_summary_runs:
- run_summary_runs = cached_summary_runs
run_summary_runs = merge_summary_table_map_run(
run_summary_runs,
external_summary_run,
diff --git a/tests/test_runtime_workflows.py b/tests/test_runtime_workflows.py
index f9ca6c0..7ac1d4e 100644
--- a/tests/test_runtime_workflows.py
+++ b/tests/test_runtime_workflows.py
@@ -1288,6 +1288,60 @@ def _segmented_run_data(label: str, run_dir: str) -> RunData:
skim_zone_map=None,
)
+
+def _market_segmentation_lines(
+ segments: list[tuple[str, str, str]],
+) -> list[str]:
+ lines = [
+ "pipeline:",
+ " steps: [segment, summarize]",
+ "segment:",
+ " definitions:",
+ " market:",
+ " source:",
+ " type: prepared_column",
+ " source_table: hh",
+ " column: market",
+ " segments:",
+ ]
+ for segment_id, label, value in segments:
+ lines.extend(
+ [
+ f" - id: {segment_id}",
+ f" label: {label}",
+ f" values: [{value}]",
+ ]
+ )
+ return lines
+
+
+def _recording_summary_builder(calls: list[tuple[str, ...]]):
+ def build(rd, config, summary_ids=None):
+ markets = (
+ tuple(sorted(rd.hh["market"].to_list()))
+ if "market" in rd.hh.columns
+ else ()
+ )
+ calls.append(markets)
+ requested = list(summary_ids or summary_builder.DEFAULT_SUMMARY_IDS)
+ tables = {
+ mode: {
+ summary_id: pl.DataFrame({"value": [float(rd.hh.height)]})
+ for summary_id in requested
+ }
+ for mode in config.weighting_modes
+ }
+ metadata = {
+ mode: {
+ summary_id: {"state": "available"}
+ for summary_id in requested
+ }
+ for mode in config.weighting_modes
+ }
+ return tables, metadata
+
+ return build
+
def test_run_prepare_workflow_rebuilds_and_writes_prepared_cache_on_cache_miss(
tmp_path: Path,
monkeypatch,
@@ -1540,6 +1594,201 @@ def test_run_summary_workflow_with_segment_step_builds_full_and_segmented_summar
]
+def test_enabling_segmentation_reuses_full_summaries_and_builds_only_segments(
+ tmp_path: Path,
+ monkeypatch,
+) -> None:
+ run_dir = tmp_path / "run_a"
+ runs = [{"dir": str(run_dir), "label": "Run A"}]
+ config = _write_config(tmp_path, runs=runs)
+ read_calls: list[str] = []
+ build_calls: list[tuple[str, ...]] = []
+ monkeypatch.setattr(summary_builder, "DEFAULT_SUMMARY_IDS", ["population_totals"])
+ monkeypatch.setattr(
+ summary_builder,
+ "build_mode_summaries_with_metadata",
+ _recording_summary_builder(build_calls),
+ )
+ _patch_prepare_pipeline(
+ monkeypatch,
+ read_run=lambda run_dir, config, label=None, **kwargs: (
+ read_calls.append(label or Path(run_dir).name),
+ _segmented_run_data(label or Path(run_dir).name, str(run_dir)),
+ )[1],
+ prepare_data=lambda rd, config: rd,
+ )
+
+ runtime_workflows.run_summary_workflow(
+ config=config,
+ cache_root=Path(config.summary_root),
+ run_entries=config.runs,
+ prefer_cache=False,
+ write_cache=True,
+ )
+ build_calls.clear()
+ segmented_config = _write_config(
+ tmp_path,
+ runs=runs,
+ extra_lines=_market_segmentation_lines(
+ [("urban", "Urban", "Urban"), ("rural", "Rural", "Rural")]
+ ),
+ )
+
+ result = runtime_workflows.run_summary_workflow(
+ config=segmented_config,
+ cache_root=Path(segmented_config.summary_root),
+ run_entries=segmented_config.runs,
+ prefer_cache=True,
+ write_cache=True,
+ )
+
+ assert build_calls == [("Urban",), ("Rural",)]
+ assert read_calls == ["Run A"]
+ assert [(run.segmentation_type, run.segment_id) for run in result.runs] == [
+ ("full", "full"),
+ ("market", "urban"),
+ ("market", "rural"),
+ ]
+ build_calls.clear()
+
+ runtime_workflows.run_summary_workflow(
+ config=segmented_config,
+ cache_root=Path(segmented_config.summary_root),
+ run_entries=segmented_config.runs,
+ prefer_cache=False,
+ write_cache=True,
+ )
+
+ assert build_calls == [("Rural", "Urban"), ("Urban",), ("Rural",)]
+ assert read_calls == ["Run A"]
+
+
+def test_changing_one_segment_rebuilds_only_that_segment(
+ tmp_path: Path,
+ monkeypatch,
+) -> None:
+ run_dir = tmp_path / "run_a"
+ runs = [{"dir": str(run_dir), "label": "Run A"}]
+ config = _write_config(
+ tmp_path,
+ runs=runs,
+ extra_lines=_market_segmentation_lines(
+ [("urban", "Urban", "Urban"), ("rural", "Rural", "Rural")]
+ ),
+ )
+ build_calls: list[tuple[str, ...]] = []
+ monkeypatch.setattr(summary_builder, "DEFAULT_SUMMARY_IDS", ["population_totals"])
+ monkeypatch.setattr(
+ summary_builder,
+ "build_mode_summaries_with_metadata",
+ _recording_summary_builder(build_calls),
+ )
+ _patch_prepare_pipeline(
+ monkeypatch,
+ read_run=lambda run_dir, config, label=None, **kwargs: _segmented_run_data(
+ label or Path(run_dir).name, str(run_dir)
+ ),
+ prepare_data=lambda rd, config: rd,
+ )
+ runtime_workflows.run_summary_workflow(
+ config=config,
+ cache_root=Path(config.summary_root),
+ run_entries=config.runs,
+ prefer_cache=False,
+ write_cache=True,
+ )
+ build_calls.clear()
+ changed_config = _write_config(
+ tmp_path,
+ runs=runs,
+ extra_lines=_market_segmentation_lines(
+ [("urban", "Urban households", "Urban"), ("rural", "Rural", "Rural")]
+ ),
+ )
+
+ result = runtime_workflows.run_summary_workflow(
+ config=changed_config,
+ cache_root=Path(changed_config.summary_root),
+ run_entries=changed_config.runs,
+ prefer_cache=True,
+ write_cache=True,
+ )
+
+ assert build_calls == [("Urban",)]
+ assert next(run for run in result.runs if run.segment_id == "urban").segment_label == (
+ "Urban households"
+ )
+
+
+def test_removing_segment_reuses_current_units_and_cleans_obsolete_cache(
+ tmp_path: Path,
+ monkeypatch,
+) -> None:
+ run_dir = tmp_path / "run_a"
+ runs = [{"dir": str(run_dir), "label": "Run A"}]
+ config = _write_config(
+ tmp_path,
+ runs=runs,
+ extra_lines=_market_segmentation_lines(
+ [("urban", "Urban", "Urban"), ("rural", "Rural", "Rural")]
+ ),
+ )
+ monkeypatch.setattr(summary_builder, "DEFAULT_SUMMARY_IDS", ["population_totals"])
+ monkeypatch.setattr(
+ summary_builder,
+ "build_mode_summaries_with_metadata",
+ _recording_summary_builder([]),
+ )
+ _patch_prepare_pipeline(
+ monkeypatch,
+ read_run=lambda run_dir, config, label=None, **kwargs: _segmented_run_data(
+ label or Path(run_dir).name, str(run_dir)
+ ),
+ prepare_data=lambda rd, config: rd,
+ )
+ runtime_workflows.run_summary_workflow(
+ config=config,
+ cache_root=Path(config.summary_root),
+ run_entries=config.runs,
+ prefer_cache=False,
+ write_cache=True,
+ )
+ changed_config = _write_config(
+ tmp_path,
+ runs=runs,
+ extra_lines=_market_segmentation_lines([("urban", "Urban", "Urban")]),
+ )
+ monkeypatch.setattr(
+ summary_builder,
+ "build_mode_summaries_with_metadata",
+ lambda *args, **kwargs: (_ for _ in ()).throw(
+ AssertionError("no current analysis unit should be rebuilt")
+ ),
+ )
+
+ result = runtime_workflows.run_summary_workflow(
+ config=changed_config,
+ cache_root=Path(changed_config.summary_root),
+ run_entries=changed_config.runs,
+ prefer_cache=True,
+ write_cache=True,
+ )
+
+ assert [(run.segmentation_type, run.segment_id) for run in result.runs] == [
+ ("full", "full"),
+ ("market", "urban"),
+ ]
+ assert not (
+ Path(changed_config.summary_root)
+ / "run-a"
+ / "summary_tables"
+ / "weighted"
+ / "segments"
+ / "market"
+ / "rural"
+ ).exists()
+
+
def test_run_prepare_workflow_loads_custom_prepared_tables_without_raw_prepare(
tmp_path: Path,
monkeypatch,
diff --git a/tests/test_segmentation_feature.py b/tests/test_segmentation_feature.py
index 7af02bb..b414e36 100644
--- a/tests/test_segmentation_feature.py
+++ b/tests/test_segmentation_feature.py
@@ -271,7 +271,7 @@ def test_config_requires_dashboard_segmentation_type_to_exist(tmp_path: Path) ->
)
-def test_summary_digest_changes_for_definition_but_not_dashboard_selection(
+def test_summary_digest_excludes_segmentation_definition_and_dashboard_selection(
tmp_path: Path,
) -> None:
base_lines = [
@@ -324,7 +324,7 @@ def test_summary_digest_changes_for_definition_but_not_dashboard_selection(
config_a.presentation_config_digest
!= config_b.presentation_config_digest
)
- assert config_a.summary_config_digest != config_c.summary_config_digest
+ assert config_a.summary_config_digest == config_c.summary_config_digest
def test_build_analysis_units_supports_multiple_segmentation_types(tmp_path: Path) -> None:
diff --git a/wiki/12-running-workflows.md b/wiki/12-running-workflows.md
index 23b3a6c..a6df8d0 100644
--- a/wiki/12-running-workflows.md
+++ b/wiki/12-running-workflows.md
@@ -187,7 +187,13 @@ base/
The visualizer stores segmented summary CSV files in
`summary_tables//segments///`. The
-run-level summary manifest describes these files.
+run-level summary manifest describes these files. With `refresh: []`, summary
+reuse is evaluated separately for the full run and for each configured segment.
+Enabling a new segment therefore reuses compatible full summaries and builds
+only the new segment summaries. Changing one segment rebuilds that segment;
+removing one deletes its obsolete cached summary directory on the next summary
+cache write. The prepared cache is loaded to resolve segment membership, but a
+valid prepared or skimjoin cache is not recomputed.
The run-key directory uses a lowercase, file-system-safe form of the run label.
For example, `Build Scenario` becomes `build-scenario`. Duplicate labels get
@@ -218,7 +224,7 @@ The refresh targets are stage-aware:
|---|---|---|
| `prepare` | nothing upstream of prepare | base prepared data, skimjoin output, summaries |
| `skimjoin` | `base_prepared_tables` | enriched `prepared_tables`, summaries |
-| `summarize` | final `prepared_tables` | stale/default summaries and segmented summaries |
+| `summarize` | final `prepared_tables` | all default summaries for the full run and every configured segment |
Cache reuse also depends on file content information. Prepared manifests record
the path, size, and modification time of each raw input. They also record the
@@ -226,8 +232,9 @@ prepare, skimjoin, and skim input identities. The summary manifest records the
prepared-manifest identity, summary configuration, and declaration digest for
each summary. A changed raw file invalidates prepare and its later output. A
changed skim input can rebuild only skimjoin and its later output. A changed
-summary declaration can rebuild only the applicable summary tables. The
-visualizer keeps compatible tables in the bundle.
+summary declaration can rebuild only the applicable summary table for each
+analysis unit. Segment definitions are tracked independently from full-summary
+configuration, so compatible full and segment tables stay in the bundle.
Use `--explain-cache` to print the cache decision for each run. The command
then exits without table loads, cache deletions, or artifact writes. The report
From 4b1da85ec30350c508aa60fcef655bf1afa0191c Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Wed, 5 Aug 2026 17:39:45 -0400
Subject: [PATCH 16/27] fix bug with scatterplot sizing in exported html
---
dashboard/export/assets/export_runtime.js | 18 +++++++++++++-----
dashboard/export/js_runtime/renderers/plots.js | 18 +++++++++++++-----
dashboard/export/serializer.py | 6 +++++-
dashboard/export/types.py | 7 ++++++-
tests/test_export_runtime_contract.py | 7 +++++++
tests/test_export_serializer.py | 14 ++++++++++++++
6 files changed, 58 insertions(+), 12 deletions(-)
diff --git a/dashboard/export/assets/export_runtime.js b/dashboard/export/assets/export_runtime.js
index fc1dd0c..29b8322 100644
--- a/dashboard/export/assets/export_runtime.js
+++ b/dashboard/export/assets/export_runtime.js
@@ -1441,16 +1441,24 @@
className: "plot-shell",
attrs: { "data-plot-pending": "true" },
});
- if (node.height) {
+ const aspectRatio = Number(node.aspect_ratio);
+ const preserveAspectRatio = Number.isFinite(aspectRatio) && aspectRatio > 0;
+ if (preserveAspectRatio) {
+ plotElement.style.aspectRatio = String(aspectRatio);
+ } else if (node.height) {
plotElement.style.minHeight = String(node.height) + "px";
}
const baseFigure = node.figure || { data: [], layout: {} };
+ const layout = Object.assign({}, baseFigure.layout || {}, {
+ autosize: true,
+ width: null,
+ });
+ if (preserveAspectRatio) {
+ delete layout.height;
+ }
const figure = {
data: baseFigure.data || [],
- layout: Object.assign({}, baseFigure.layout || {}, {
- autosize: true,
- width: null,
- }),
+ layout: layout,
};
context.plotManager.registerPlot(plotElement, figure);
return plotElement;
diff --git a/dashboard/export/js_runtime/renderers/plots.js b/dashboard/export/js_runtime/renderers/plots.js
index 531c0e6..91e5812 100644
--- a/dashboard/export/js_runtime/renderers/plots.js
+++ b/dashboard/export/js_runtime/renderers/plots.js
@@ -7,16 +7,24 @@
className: "plot-shell",
attrs: { "data-plot-pending": "true" },
});
- if (node.height) {
+ const aspectRatio = Number(node.aspect_ratio);
+ const preserveAspectRatio = Number.isFinite(aspectRatio) && aspectRatio > 0;
+ if (preserveAspectRatio) {
+ plotElement.style.aspectRatio = String(aspectRatio);
+ } else if (node.height) {
plotElement.style.minHeight = String(node.height) + "px";
}
const baseFigure = node.figure || { data: [], layout: {} };
+ const layout = Object.assign({}, baseFigure.layout || {}, {
+ autosize: true,
+ width: null,
+ });
+ if (preserveAspectRatio) {
+ delete layout.height;
+ }
const figure = {
data: baseFigure.data || [],
- layout: Object.assign({}, baseFigure.layout || {}, {
- autosize: true,
- width: null,
- }),
+ layout: layout,
};
context.plotManager.registerPlot(plotElement, figure);
return plotElement;
diff --git a/dashboard/export/serializer.py b/dashboard/export/serializer.py
index 149a3fb..398549e 100644
--- a/dashboard/export/serializer.py
+++ b/dashboard/export/serializer.py
@@ -169,7 +169,11 @@ def _container_css_classes(viewable: Any) -> list[str]:
figure = obj.object.to_plotly_json()
layout = figure.get("layout", {}) if isinstance(figure, dict) else {}
height = layout.get("height") if isinstance(layout, dict) else None
- return {"kind": "plotly", "figure": figure, "height": height}
+ node = {"kind": "plotly", "figure": figure, "height": height}
+ aspect_ratio = getattr(obj, "aspect_ratio", None)
+ if isinstance(aspect_ratio, (int, float)) and aspect_ratio > 0:
+ node["aspect_ratio"] = float(aspect_ratio)
+ return node
if isinstance(obj, pn.widgets.Tabulator):
frame = obj.value
title_map = {
diff --git a/dashboard/export/types.py b/dashboard/export/types.py
index 7d2c7e4..950abaa 100644
--- a/dashboard/export/types.py
+++ b/dashboard/export/types.py
@@ -105,7 +105,12 @@ class TabsNode(TypedDict):
tabs: list[TabPayload]
-class PlotlyNode(TypedDict):
+class OptionalPlotlyNode(TypedDict, total=False):
+ height: int | float | None
+ aspect_ratio: float
+
+
+class PlotlyNode(OptionalPlotlyNode):
kind: Literal["plotly"]
figure: dict[str, Any]
diff --git a/tests/test_export_runtime_contract.py b/tests/test_export_runtime_contract.py
index 9998073..9c89032 100644
--- a/tests/test_export_runtime_contract.py
+++ b/tests/test_export_runtime_contract.py
@@ -228,6 +228,13 @@ def test_runtime_asset_contains_explicit_context_action_and_region_helpers() ->
assert "modeBarButtonsToAdd: [makePlotCsvDownloadButton(figure)]" in runtime_js
+def test_runtime_asset_preserves_exported_plot_aspect_ratios() -> None:
+ runtime_js = load_export_runtime_js()
+
+ assert "plotElement.style.aspectRatio = String(aspectRatio);" in runtime_js
+ assert "delete layout.height;" in runtime_js
+
+
def test_runtime_asset_exposes_full_table_and_tab_titles_as_tooltips() -> None:
runtime_js = load_export_runtime_js()
diff --git a/tests/test_export_serializer.py b/tests/test_export_serializer.py
index bb82549..80fbf85 100644
--- a/tests/test_export_serializer.py
+++ b/tests/test_export_serializer.py
@@ -68,6 +68,7 @@ def test_serialize_viewable_supports_plotly_and_table_nodes() -> None:
assert plot_payload["kind"] == "plotly"
assert plot_payload["figure"]["data"][0]["type"] == "bar"
+ assert "aspect_ratio" not in plot_payload
assert table_payload == {
"kind": "table",
"columns": ["Alpha Value", "beta", "Gamma Value"],
@@ -75,6 +76,19 @@ def test_serialize_viewable_supports_plotly_and_table_nodes() -> None:
}
+def test_serialize_viewable_preserves_numeric_plotly_aspect_ratio() -> None:
+ pane = pn.pane.Plotly(
+ go.Figure(data=[go.Scatter(x=[1], y=[1])], layout={"height": 400}),
+ sizing_mode="scale_width",
+ aspect_ratio=1.0,
+ )
+
+ payload = serialize_viewable(pane, disable_widgets=True)
+
+ assert payload["height"] == 400
+ assert payload["aspect_ratio"] == 1.0
+
+
def test_serialize_viewable_supports_widget_nodes_with_export_metadata() -> None:
radio = pn.widgets.RadioButtonGroup(
name="Legacy Mode Name",
From 577956d7704a4aedb44a741c5b600f3bf0607fd7 Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Thu, 6 Aug 2026 10:51:51 -0400
Subject: [PATCH 17/27] Allow full matrix naming in skimjoin
---
processor/skimjoin/annotate/engine.py | 10 ++
.../annotate/trip_lookup_execution.py | 44 ++++++-
processor/skimjoin/config/validation.py | 53 ++++++--
processor/skimjoin/inventory.py | 7 ++
processor/skimjoin/runtime_execution.py | 12 +-
tests/test_skimjoin_integration.py | 119 +++++++++++++++++-
wiki/25-skimjoin-config-reference.md | 22 +++-
7 files changed, 243 insertions(+), 24 deletions(-)
diff --git a/processor/skimjoin/annotate/engine.py b/processor/skimjoin/annotate/engine.py
index db87e76..fdfe293 100644
--- a/processor/skimjoin/annotate/engine.py
+++ b/processor/skimjoin/annotate/engine.py
@@ -277,6 +277,16 @@ def _execute_chain_queue(
queued = step_queue.join(metadata, on="matrix_name", how="left")
+ ambiguous = queued.filter(pl.col("source_kind") == "ambiguous")
+ if not ambiguous.is_empty():
+ row = ambiguous.select(
+ ["matrix_name", "ambiguous_sources"]
+ ).unique(maintain_order=True).row(0, named=True)
+ raise ValueError(
+ f"Ambiguous matrix reference {row['matrix_name']!r}; qualify it with one of: "
+ f"{row['ambiguous_sources']}"
+ )
+
missing_matrix = queued.filter(pl.col("file_path").is_null())
if not missing_matrix.is_empty():
for row in missing_matrix.select(
diff --git a/processor/skimjoin/annotate/trip_lookup_execution.py b/processor/skimjoin/annotate/trip_lookup_execution.py
index 1bcd735..e141843 100644
--- a/processor/skimjoin/annotate/trip_lookup_execution.py
+++ b/processor/skimjoin/annotate/trip_lookup_execution.py
@@ -1,8 +1,11 @@
from __future__ import annotations
+from collections import Counter
+
import polars as pl
from processor.skimjoin.config.schema import NormalizedConfig, NormalizedLookupRule
+from processor.skimjoin.inventory import qualified_matrix_reference
from processor.skimjoin.skimstore.base import SkimStore
from processor.skimjoin.annotate.trip_lookup_reports import _row_trip_id
@@ -23,11 +26,48 @@ def _inventory_metadata_frame(
"destination_column_name",
]
)
- rows = selected.to_dicts()
- for row in rows:
+ inventory_rows = selected.to_dicts()
+ name_counts = Counter(str(row["matrix_name"]) for row in inventory_rows)
+ rows: list[dict[str, object]] = []
+ ambiguous_sources: dict[str, list[str]] = {}
+ for inventory_row in inventory_rows:
+ row = dict(inventory_row)
row["lookup_name"] = normalized.zone_mapping.resolve_lookup_name(
str(row["file_path"])
)
+ row["ambiguous_sources"] = None
+ qualified = dict(row)
+ qualified["matrix_name"] = qualified_matrix_reference(
+ str(row["file_path"]), str(row["matrix_name"])
+ )
+ rows.append(qualified)
+ matrix_name = str(row["matrix_name"])
+ if name_counts[matrix_name] == 1:
+ rows.append(row)
+ else:
+ ambiguous_sources.setdefault(matrix_name, []).append(
+ str(row["file_path"])
+ )
+ for matrix_name, file_paths in ambiguous_sources.items():
+ rows.append(
+ {
+ "matrix_name": matrix_name,
+ "file_path": None,
+ "matrix_path": None,
+ "source_kind": "ambiguous",
+ "key_column_name": None,
+ "value_column_name": None,
+ "origin_column_name": None,
+ "destination_column_name": None,
+ "lookup_name": None,
+ "ambiguous_sources": ", ".join(
+ sorted(
+ qualified_matrix_reference(file_path, matrix_name)
+ for file_path in file_paths
+ )
+ ),
+ }
+ )
return pl.DataFrame(rows, infer_schema_length=None)
diff --git a/processor/skimjoin/config/validation.py b/processor/skimjoin/config/validation.py
index 5444e19..83b13f7 100644
--- a/processor/skimjoin/config/validation.py
+++ b/processor/skimjoin/config/validation.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+from collections import Counter
from dataclasses import dataclass
from typing import Any
@@ -11,6 +12,7 @@
NormalizedConfig,
NormalizedLookupRule,
)
+from processor.skimjoin.inventory import qualified_matrix_reference
class ConfigValidationError(ValueError):
@@ -120,18 +122,32 @@ def _inventory_by_name(
]
).to_dicts()
inventory_by_name: dict[str, dict[str, object]] = {}
- duplicate_names: list[str] = []
+ name_counts = Counter(str(row["matrix_name"]) for row in matrix_rows)
+ qualified_counts = Counter(
+ qualified_matrix_reference(row["file_path"], str(row["matrix_name"]))
+ for row in matrix_rows
+ )
+ ambiguous_sources: dict[str, list[str]] = {}
for row in matrix_rows:
matrix_name = str(row["matrix_name"])
- if matrix_name in inventory_by_name:
- duplicate_names.append(matrix_name)
- continue
- inventory_by_name[matrix_name] = row
+ qualified_name = qualified_matrix_reference(row["file_path"], matrix_name)
+ inventory_by_name.setdefault(qualified_name, row)
+ if name_counts[matrix_name] == 1:
+ inventory_by_name[matrix_name] = row
+ else:
+ ambiguous_sources.setdefault(matrix_name, []).append(qualified_name)
+ for matrix_name, sources in ambiguous_sources.items():
+ inventory_by_name[matrix_name] = {
+ "__ambiguous_sources": sorted(set(sources))
+ }
failures = []
- if duplicate_names:
+ duplicate_qualified_names = sorted(
+ name for name, count in qualified_counts.items() if count > 1
+ )
+ if duplicate_qualified_names:
failures.append(
- "Duplicate matrix names in skim inventory: "
- + ", ".join(sorted(set(duplicate_names)))
+ "Duplicate file-qualified matrix references in skim inventory: "
+ + ", ".join(duplicate_qualified_names)
)
return inventory_by_name, failures
@@ -318,17 +334,25 @@ def _validate_target_table(
failures.extend(f"{rule.name}: {message}" for message in combo_failures)
for combo in combos:
matrix_name = str(combo["matrix_name"])
- if matrix_name not in inventory_by_name:
+ inventory_record = inventory_by_name.get(matrix_name)
+ if inventory_record is None:
failures.append(
f"{rule.name}: referenced matrix {matrix_name!r} was not found in skim inventory"
)
continue
+ ambiguous_sources = inventory_record.get("__ambiguous_sources")
+ if ambiguous_sources:
+ failures.append(
+ f"{rule.name}: ambiguous matrix reference {matrix_name!r}; qualify it with one of: "
+ + ", ".join(str(source) for source in ambiguous_sources)
+ )
+ continue
if (
- str(inventory_by_name[matrix_name]["source_kind"]) == "od_matrix"
+ str(inventory_record["source_kind"]) == "od_matrix"
and rule.lookup == "od"
):
- shape_rows = int(inventory_by_name[matrix_name]["shape_rows"])
- shape_cols = int(inventory_by_name[matrix_name]["shape_cols"])
+ shape_rows = int(inventory_record["shape_rows"])
+ shape_cols = int(inventory_record["shape_cols"])
zone_failures, zone_warnings = _validate_od_bounds(
rule,
combo["rows"],
@@ -641,6 +665,9 @@ def _referenced_matrices(
combos, _ = _rule_matrix_combinations(rule, subset)
for combo in combos:
matrix_name = str(combo["matrix_name"])
- if matrix_name in inventory_by_name:
+ inventory_record = inventory_by_name.get(matrix_name)
+ if inventory_record is not None and not inventory_record.get(
+ "__ambiguous_sources"
+ ):
referenced.add(matrix_name)
return referenced
diff --git a/processor/skimjoin/inventory.py b/processor/skimjoin/inventory.py
index 1c5939d..90ef437 100644
--- a/processor/skimjoin/inventory.py
+++ b/processor/skimjoin/inventory.py
@@ -9,6 +9,9 @@
import polars as pl
+MATRIX_REFERENCE_SEPARATOR = "::"
+
+
@dataclass(frozen=True)
class MatrixRecord:
file_path: str
@@ -24,6 +27,10 @@ class MatrixRecord:
destination_column_name: str | None = None
+def qualified_matrix_reference(file_path: str | Path, matrix_name: str) -> str:
+ return f"{Path(file_path).name}{MATRIX_REFERENCE_SEPARATOR}{matrix_name}"
+
+
def expand_paths(paths: Iterable[str | Path]) -> list[Path]:
expanded: list[Path] = []
for raw_path in paths:
diff --git a/processor/skimjoin/runtime_execution.py b/processor/skimjoin/runtime_execution.py
index c5110b0..dad1f75 100644
--- a/processor/skimjoin/runtime_execution.py
+++ b/processor/skimjoin/runtime_execution.py
@@ -11,7 +11,10 @@
from processor.skimjoin.annotate.tours import annotate_tours
from processor.skimjoin.annotate.trips import annotate_trips
from processor.skimjoin.hypothetical_sidecars import build_hypothetical_sidecars
-from processor.skimjoin.inventory import inventory_skim_files
+from processor.skimjoin.inventory import (
+ inventory_skim_files,
+ qualified_matrix_reference,
+)
from processor.skimjoin.runtime_types import _RuntimeSkimjoinResult
from processor.skimjoin.skimstore.omx import OmxSkimStore
@@ -35,7 +38,8 @@ def _validate_runtime_inventory(inventory: pl.DataFrame) -> None:
)
matrix_names = [
- str(value) for value in inventory.get_column("matrix_name").to_list()
+ qualified_matrix_reference(row["file_path"], row["matrix_name"])
+ for row in inventory.select(["file_path", "matrix_name"]).to_dicts()
]
duplicates = sorted(
matrix_name
@@ -44,8 +48,8 @@ def _validate_runtime_inventory(inventory: pl.DataFrame) -> None:
)
if duplicates:
raise ValueError(
- "Integrated skimjoin requires unique matrix names across skim inputs. "
- + "Duplicate names: "
+ "Integrated skimjoin requires unique file-qualified matrix references. "
+ + "Duplicate references: "
+ ", ".join(repr(name) for name in duplicates)
)
diff --git a/tests/test_skimjoin_integration.py b/tests/test_skimjoin_integration.py
index cd504bf..b50cb55 100644
--- a/tests/test_skimjoin_integration.py
+++ b/tests/test_skimjoin_integration.py
@@ -21,6 +21,7 @@
from processor.skimjoin.config.validation import ConfigValidationError, load_config, validate_config
from processor.skimjoin.inventory import inventory_skim_files
from processor.skimjoin.pipeline import apply_skimjoin
+from processor.skimjoin.runtime_execution import _validate_runtime_inventory
from processor.skimjoin.skimstore.omx import OmxSkimStore
from processor.summarize import cache_types as summary_cache_types
from processor.summarize import builder as summary_builder
@@ -1125,7 +1126,9 @@ def test_config_accepts_mixed_omx_and_csv_skim_inputs(tmp_path: Path) -> None:
assert config.skimjoin.normalized_config is not None
-def test_validate_config_rejects_duplicate_matrix_names_across_sources(tmp_path: Path) -> None:
+def test_validate_config_rejects_ambiguous_unqualified_matrix_reference(
+ tmp_path: Path,
+) -> None:
skim_path = tmp_path / "auto.omx"
csv_path = tmp_path / "auto.csv"
_write_omx(skim_path, matrix_name="auto__time")
@@ -1174,13 +1177,121 @@ def test_validate_config_rejects_duplicate_matrix_names_across_sources(tmp_path:
},
}
inventory = inventory_skim_files([skim_path, csv_path])
- trips = pl.read_parquet(tmp_path / "run" / "final_trips.parquet")
- tours = pl.read_parquet(tmp_path / "run" / "final_tours.parquet")
+ trips = pl.read_parquet(tmp_path / "run" / "final_trips.parquet").with_columns(
+ pl.lit("WALK_TRANSIT").alias("trip_mode")
+ )
+ tours = pl.read_parquet(tmp_path / "run" / "final_tours.parquet").with_columns(
+ pl.lit("WALK_TRANSIT").alias("tour_mode"),
+ pl.lit(101).alias("o_maz"),
+ pl.lit(102).alias("d_maz"),
+ )
- with pytest.raises(ConfigValidationError, match="Duplicate matrix names"):
+ with pytest.raises(ConfigValidationError, match="ambiguous matrix reference 'auto__time'"):
validate_config(config_data, inventory, trips, tours=tours)
+def test_qualified_matrix_references_select_duplicate_names_by_file(
+ tmp_path: Path,
+) -> None:
+ commute_path = tmp_path / "bike_commute.omx"
+ noncommute_path = tmp_path / "bike_noncommute.omx"
+ _write_omx_with_lookup(
+ commute_path,
+ matrix_name="distance",
+ lookup_name="taz",
+ values=np.array([[1.0, 2.0], [3.0, 4.0]]),
+ )
+ _write_omx_with_lookup(
+ noncommute_path,
+ matrix_name="distance",
+ lookup_name="taz",
+ values=np.array([[10.0, 20.0], [30.0, 40.0]]),
+ )
+ _write_skimjoin_config(
+ tmp_path,
+ skim_files=[commute_path, noncommute_path],
+ include_default_mode=False,
+ extra_lines=[
+ "modes:",
+ " BIKE:",
+ " commute_distance:",
+ " output: skim_bike_commute_distance",
+ ' matrix: "bike_commute.omx::distance"',
+ " noncommute_distance:",
+ " output: skim_bike_noncommute_distance",
+ ' matrix: "bike_noncommute.omx::distance"',
+ ],
+ )
+ config = _write_main_config(tmp_path, skimjoin_enabled=True)
+ normalized = config.skimjoin.normalized_config
+ assert normalized is not None
+ inventory = inventory_skim_files(normalized.skim_files)
+ _validate_runtime_inventory(inventory)
+
+ trips = pl.DataFrame(
+ {
+ "trip_id": [1],
+ "trip_mode": ["BIKE"],
+ "OTAZ": [101],
+ "DTAZ": [102],
+ }
+ )
+ annotated, lookup_summary, missing = annotate_trips(
+ trips,
+ normalized,
+ inventory,
+ skim_store=OmxSkimStore(),
+ )
+
+ assert annotated["skim_bike_commute_distance"].to_list() == [2.0]
+ assert annotated["skim_bike_noncommute_distance"].to_list() == [20.0]
+ assert sorted(lookup_summary["matrix_name"].to_list()) == [
+ "bike_commute.omx::distance",
+ "bike_noncommute.omx::distance",
+ ]
+ assert missing.is_empty()
+
+
+def test_annotate_trips_rejects_ambiguous_unqualified_matrix_reference(
+ tmp_path: Path,
+) -> None:
+ first_path = tmp_path / "first.omx"
+ second_path = tmp_path / "second.omx"
+ _write_omx(first_path, matrix_name="distance")
+ _write_omx(second_path, matrix_name="distance")
+ _write_skimjoin_config(
+ tmp_path,
+ skim_files=[first_path, second_path],
+ include_default_mode=False,
+ extra_lines=[
+ "modes:",
+ " BIKE:",
+ " distance:",
+ " matrix: distance",
+ ],
+ )
+ config = _write_main_config(tmp_path, skimjoin_enabled=True)
+ normalized = config.skimjoin.normalized_config
+ assert normalized is not None
+ inventory = inventory_skim_files(normalized.skim_files)
+ trips = pl.DataFrame(
+ {
+ "trip_id": [1],
+ "trip_mode": ["BIKE"],
+ "OTAZ": [101],
+ "DTAZ": [102],
+ }
+ )
+
+ with pytest.raises(ValueError, match="Ambiguous matrix reference 'distance'"):
+ annotate_trips(
+ trips,
+ normalized,
+ inventory,
+ skim_store=OmxSkimStore(),
+ )
+
+
def test_validate_config_allows_summed_output_overlap_but_rejects_replace_overlap(
tmp_path: Path,
) -> None:
diff --git a/wiki/25-skimjoin-config-reference.md b/wiki/25-skimjoin-config-reference.md
index 8aa9ced..1dc150a 100644
--- a/wiki/25-skimjoin-config-reference.md
+++ b/wiki/25-skimjoin-config-reference.md
@@ -50,6 +50,26 @@ time:
matrix: SOV_TIME
```
+When multiple skim files contain the same matrix name, qualify the reference
+with the source filename:
+
+```yaml
+project:
+ skim_files:
+ - C:\skims\bike_commute.omx
+ - C:\skims\bike_noncommute.omx
+
+modes:
+ BIKE:
+ distance:
+ matrix: "bike_commute.omx::distance"
+```
+
+Unqualified references continue to work when the matrix name is unique across
+the configured skim files. An ambiguous unqualified reference fails validation
+and lists the available qualified names. Filename-qualified references may also
+contain dimension placeholders.
+
### Period Dimension Lookup
```yaml
@@ -360,7 +380,7 @@ A component rule can be a matrix-name string or a mapping.
| Field | Type | Default | Allowed values | Notes |
|---|---|---|---|---|
-| `matrix` | string | required | matrix/table value name | Matrix name or matrix-name template using `{DIMENSION}` placeholders. |
+| `matrix` | string | required | matrix/table value name | Matrix name, `filename::matrix` reference, or template using `{DIMENSION}` placeholders. |
| `output` | string | `output_prefix` + component name | output column | Final output column. Tour lookup outputs also receive `_outbound` or `_inbound`. |
| `lookup` | string | `od` | `od`, `key` | Lookup type. |
| `key_column` | string | none | source column | Required when `lookup: key`. |
From 9e2e7bb389e2dc866c7e40ffcf7bcf1f0c14f887 Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Thu, 6 Aug 2026 13:35:41 -0400
Subject: [PATCH 18/27] add ebike rows to skim summary table
---
dashboard/calculation_notes.py | 23 ++++++++++
dashboard/calculation_notes.yaml | 24 +++++++++++
dashboard/pages/skim_summaries/_shared.py | 20 ++++++---
tests/test_calculation_notes.py | 17 ++++++++
tests/test_summary_cache.py | 52 ++++++++++++++++++++++-
5 files changed, 129 insertions(+), 7 deletions(-)
diff --git a/dashboard/calculation_notes.py b/dashboard/calculation_notes.py
index f3fa0a0..167a4c3 100644
--- a/dashboard/calculation_notes.py
+++ b/dashboard/calculation_notes.py
@@ -93,6 +93,7 @@ class CalculationNote:
method_text: str | None = None
sources: tuple[str, ...] = ()
source_filters: tuple[str, ...] = ()
+ column_definitions: tuple[str, ...] = ()
def _nonempty_text(value: object, *, field: str) -> str:
@@ -117,6 +118,7 @@ def _parse_note(
"method_text",
"sources",
"source_filters",
+ "column_definitions",
}
unexpected = sorted(set(raw_note) - allowed_fields)
if unexpected:
@@ -176,6 +178,16 @@ def _parse_note(
for item in raw_source_filters
)
+ raw_column_definitions = raw_note.get("column_definitions", [])
+ if not isinstance(raw_column_definitions, list):
+ raise ValueError(
+ f"Calculation note {note_id!r}.column_definitions must be a list."
+ )
+ column_definitions = tuple(
+ _nonempty_text(item, field=f"{note_id!r}.column_definitions item")
+ for item in raw_column_definitions
+ )
+
raw_details = raw_note.get("details", {})
if not isinstance(raw_details, dict):
raise ValueError(f"Calculation note {note_id!r}.details must be a mapping.")
@@ -205,6 +217,7 @@ def _parse_note(
method_text=method_text,
sources=sources,
source_filters=source_filters,
+ column_definitions=column_definitions,
)
@@ -288,6 +301,16 @@ def render_calculation_note_html(note: CalculationNote) -> str:
f""
""
)
+ if note.column_definitions:
+ rendered_items = "".join(
+ f"{html.escape(item)}" for item in note.column_definitions
+ )
+ sections.append(
+ ""
+ "
Table columns:"
+ f"
"
+ "
{html.escape(source)}" for source in note.sources
diff --git a/dashboard/calculation_notes.yaml b/dashboard/calculation_notes.yaml
index a0c20f6..2e55dbf 100644
--- a/dashboard/calculation_notes.yaml
+++ b/dashboard/calculation_notes.yaml
@@ -561,6 +561,18 @@ notes:
method_text: For each skim component and mode, the summary scans the prepared values once to calculate valid count, missing and zero shares, minimum, maximum, mean, median, and standard deviation.
sources: [skimjoin_trip_component_stats]
summary: The table reports descriptive statistics for the selected trip skim family and scenario.
+ column_definitions:
+ - "Skim Name: Display name of the selected skim component, including its unit when known."
+ - "Trip Mode: Mode represented by the row."
+ - "Total: Total trip weight in weighted mode, or number of trip records in unweighted mode, before missing skim values are removed."
+ - "Valid: Trip weight or record count with a nonmissing skim value."
+ - "Mean: Weighted arithmetic mean of valid values; in unweighted mode, the ordinary arithmetic mean."
+ - "Std Dev: Weighted population standard deviation of valid values; in unweighted mode, the population standard deviation."
+ - "Min / Max: Smallest and largest valid skim values."
+ - "Median: Weighted median of valid values; in unweighted mode, the ordinary median."
+ - "Mode: Skim value with the greatest total weight or record count; ties use the smaller value."
+ - "Zero Share: Share of valid trip weight or records whose skim value equals zero."
+ - "Missing Share: Share of total trip weight or records whose skim value is missing."
details:
Aggregation:
- Prepared skim statistics provide the valid count, missing share, zero share, minimum, maximum, mean, median, and standard deviation for each component and mode.
@@ -585,6 +597,18 @@ notes:
method_text: For each directional skim component and mode, the summary calculates valid count, missing and zero shares, minimum, maximum, mean, median, and standard deviation from prepared tour values.
sources: [skimjoin_tour_component_stats]
summary: The table reports descriptive statistics for the selected tour skim family, direction, and scenario.
+ column_definitions:
+ - "Skim Name: Display name of the selected directional skim component, including its unit when known."
+ - "Tour Mode: Mode represented by the row."
+ - "Total: Total tour weight in weighted mode, or number of tour records in unweighted mode, before missing skim values are removed."
+ - "Valid: Tour weight or record count with a nonmissing skim value."
+ - "Mean: Weighted arithmetic mean of valid values; in unweighted mode, the ordinary arithmetic mean."
+ - "Std Dev: Weighted population standard deviation of valid values; in unweighted mode, the population standard deviation."
+ - "Min / Max: Smallest and largest valid skim values."
+ - "Median: Weighted median of valid values; in unweighted mode, the ordinary median."
+ - "Mode: Skim value with the greatest total weight or record count; ties use the smaller value."
+ - "Zero Share: Share of valid tour weight or records whose skim value equals zero."
+ - "Missing Share: Share of total tour weight or records whose skim value is missing."
details:
Aggregation:
- Prepared skim statistics provide the valid count, missing share, zero share, minimum, maximum, mean, median, and standard deviation for each component and mode.
diff --git a/dashboard/pages/skim_summaries/_shared.py b/dashboard/pages/skim_summaries/_shared.py
index e980e6e..cbf4900 100644
--- a/dashboard/pages/skim_summaries/_shared.py
+++ b/dashboard/pages/skim_summaries/_shared.py
@@ -27,9 +27,9 @@
)
SKIM_FAMILY_MODE_MAP = {
"Auto Skims": ("SOV", "HOV2", "HOV3"),
- "Transit Skims": ("WALK_TRANSIT", "PNR_TRANSIT", "KNR_TRANSIT"),
+ "Transit Skims": ("WALK_TRANSIT", "BIKE_TRANSIT", "PNR_TRANSIT", "KNR_TRANSIT"),
"Walk Skims": ("WALK",),
- "Bike Skims": ("BIKE", "EBIKE", "ESCOOTER", "BIKE_TRANSIT"),
+ "Bike Skims": ("BIKE", "EBIKE", "ESCOOTER"),
}
SUMMARY_METRIC_COLUMNS = [
"n_total",
@@ -463,7 +463,7 @@ def family_stats_table(
direction_suffix = None if direction is None else f"_{direction.lower()}"
target_columns = ["skim_name", mode_column, *SUMMARY_METRIC_COLUMNS]
filtered_list: list[tuple[str, pl.DataFrame]] = []
- for label, _, df in nonempty_series(data_list):
+ for label, series, df in nonempty_series(data_list):
family_definition = family_definitions_by_label.get(label, {}).get(family)
family_modes = family_definition.get("modes", ()) if family_definition else ()
configured_outputs = (
@@ -486,9 +486,19 @@ def family_stats_table(
& (pl.col(mode_column) != ALL_MODES)
)
if configured_outputs:
- filtered = filtered.filter(
- pl.col("component").is_in(list(configured_outputs))
+ outputs_by_mode = _configured_outputs_by_mode(
+ config,
+ series,
+ target_table=target_table,
)
+ mode_output_filters = []
+ for mode in family_modes:
+ mode_outputs = outputs_by_mode.get(mode)
+ mode_filter = pl.col(mode_column) == mode
+ if mode_outputs:
+ mode_filter &= pl.col("component").is_in(sorted(mode_outputs))
+ mode_output_filters.append(mode_filter)
+ filtered = filtered.filter(pl.any_horizontal(mode_output_filters))
if direction_suffix is not None:
filtered = filtered.filter(
pl.col("component").str.ends_with(direction_suffix)
diff --git a/tests/test_calculation_notes.py b/tests/test_calculation_notes.py
index db0f082..0986f52 100644
--- a/tests/test_calculation_notes.py
+++ b/tests/test_calculation_notes.py
@@ -74,6 +74,7 @@ def test_calculation_note_renderer_escapes_configured_text() -> None:
method_text="Join by their shared key.",
sources=("source_one",),
source_filters=("Only A & B.",),
+ column_definitions=("Column : A & B.",),
)
rendered = render_calculation_note_html(note)
@@ -87,6 +88,8 @@ def test_calculation_note_renderer_escapes_configured_text() -> None:
assert "Join <records> by their shared key." in rendered
assert "Generic grouping text" not in rendered
assert "Only A & B." in rendered
+ assert "Table columns:" in rendered
+ assert "Column <A>: A & B." in rendered
assert "source_one" in rendered
assert "Summary Tables Used:" in rendered
assert "Prepared summaries used:" not in rendered
@@ -94,6 +97,20 @@ def test_calculation_note_renderer_escapes_configured_text() -> None:
assert rendered.endswith("")
+def test_skim_summary_notes_render_table_column_definitions() -> None:
+ trip_note = render_calculation_note_html(
+ get_calculation_note("trip_skims.summary_table")
+ )
+ tour_note = render_calculation_note_html(
+ get_calculation_note("tour_skims.summary_table")
+ )
+
+ for rendered in (trip_note, tour_note):
+ assert "Table columns:" in rendered
+ assert "Zero Share:" in rendered
+ assert "Missing Share:" in rendered
+
+
def test_validation_notes_expose_comparison_and_error_formulas() -> None:
regional = get_calculation_note("regional_validation.flows")
facility = get_calculation_note("traffic.facility_summary")
diff --git a/tests/test_summary_cache.py b/tests/test_summary_cache.py
index 5f1eef6..ca28a95 100644
--- a/tests/test_summary_cache.py
+++ b/tests/test_summary_cache.py
@@ -29,8 +29,9 @@
from dashboard.pages.daily_travel.escorted_tours import EscortedToursPage
from dashboard.pages.joint_travel import JointTravelPage
from dashboard.pages.overview import OverviewPage
-from dashboard.pages.skim_summaries.trip_skims import TripSkimsPage
+from dashboard.pages.skim_summaries._shared import family_stats_table, skim_family_for_mode
from dashboard.pages.skim_summaries.tour_skims import TourSkimsPage
+from dashboard.pages.skim_summaries.trip_skims import TripSkimsPage
from dashboard.pages.tour_summaries.tour_mode import (
TourModePage as TourSummariesTourModePage,
)
@@ -59,7 +60,7 @@
from dashboard.pages.validation.transit import TransitValidationPage
from dashboard.pages.validation.regional import RegionalValidationPage
from dashboard.pages.validation.vmt import VMTValidationPage
-from dashboard.data_access import DashboardPreparedRunProvider
+from dashboard.data_access import DashboardPreparedRunProvider, DashboardSummarySeries
from dashboard.state import DashboardState
from dashboard.page_registry import all_page_definitions, page_definitions_for_group
from processor.models import RunData
@@ -2561,6 +2562,53 @@ def test_skim_summaries_group_lists_tour_skims_before_trip_skims() -> None:
]
+def test_bike_transit_uses_transit_skim_family() -> None:
+ assert skim_family_for_mode("BIKE_TRANSIT") == "Transit Skims"
+ assert skim_family_for_mode("EBIKE") == "Bike Skims"
+
+
+def test_skim_family_tables_only_show_outputs_configured_for_each_mode(
+ tmp_path: Path,
+) -> None:
+ config = _write_config(tmp_path)
+ _attach_test_skimjoin_config(config)
+ normalized = config.skimjoin.normalized_config
+ series = DashboardSummarySeries(label="Base", summaries_by_mode={})
+ for target_table, mode_column, direction_suffix in (
+ ("trips", "trip_mode", ""),
+ ("tours", "tour_mode", "_outbound"),
+ ):
+ lookups = getattr(normalized, f"{target_table[:-1]}_lookups")
+ lookups.extend(
+ [
+ SimpleNamespace(mode="BIKE", output=f"skim_bike_distance{direction_suffix}"),
+ SimpleNamespace(mode="BIKE", output=f"skim_bike_logsum{direction_suffix}"),
+ SimpleNamespace(mode="EBIKE", output=f"skim_bike_distance{direction_suffix}"),
+ ]
+ )
+ stats = pl.DataFrame(
+ {
+ "component": [
+ f"skim_bike_distance{direction_suffix}",
+ f"skim_bike_logsum{direction_suffix}",
+ ],
+ mode_column: ["EBIKE", "EBIKE"],
+ "n_valid": [2.0, 0.0],
+ }
+ )
+
+ result = family_stats_table(
+ config,
+ [("Base", series, stats)],
+ family="Bike Skims",
+ mode_column=mode_column,
+ target_table=target_table,
+ direction="outbound" if direction_suffix else None,
+ )[0][1]
+
+ assert result["skim_name"].to_list() == ["TAZ Skim Bike Distance (mi)"]
+
+
def test_trip_skims_page_uses_family_selector_and_two_digit_precision_summary_table(
tmp_path: Path,
) -> None:
From ed0f83afdbe21053b3e826ffc2a094088cf5a155 Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Thu, 6 Aug 2026 14:13:36 -0400
Subject: [PATCH 19/27] Simplify CI tests, update workflow logging
---
.github/workflows/tests.yml | 34 ++++------------------------------
.gitignore | 3 +++
run.py | 5 ++++-
3 files changed, 11 insertions(+), 31 deletions(-)
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 6b7c669..03d6c00 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -7,12 +7,8 @@ on:
workflow_dispatch:
jobs:
- pytest-fast:
+ pytest:
runs-on: windows-latest
- strategy:
- fail-fast: false
- matrix:
- python-version: ["3.10", "3.12"]
steps:
- name: Check out repo
@@ -21,7 +17,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v5
with:
- python-version: ${{ matrix.python-version }}
+ python-version: "3.12"
- name: Install uv
uses: astral-sh/setup-uv@v4
@@ -30,29 +26,7 @@ jobs:
run: uv sync --locked --group dev
- name: Run correctness lint
- if: matrix.python-version == '3.12'
run: uv run ruff check .
- - name: Run fast tests
- run: uv run pytest --basetemp .pytest_tmp -m "not full_export"
-
- pytest-full-export:
- runs-on: windows-latest
-
- steps:
- - name: Check out repo
- uses: actions/checkout@v4
-
- - name: Set up Python
- uses: actions/setup-python@v5
- with:
- python-version: "3.12"
-
- - name: Install uv
- uses: astral-sh/setup-uv@v4
-
- - name: Sync locked dependencies
- run: uv sync --locked --group dev
-
- - name: Run exhaustive export tests
- run: uv run pytest --basetemp .pytest_tmp -m full_export
+ - name: Run tests
+ run: uv run pytest --basetemp .pytest_tmp
diff --git a/.gitignore b/.gitignore
index 9debff0..6185712 100644
--- a/.gitignore
+++ b/.gitignore
@@ -244,3 +244,6 @@ plans/
/prepared_table_filter_example.ipynb
/scratch.ipynb
/test
+/simor_project_outputs/
+
+AGENTS.md
\ No newline at end of file
diff --git a/run.py b/run.py
index 66f2812..72b84b5 100644
--- a/run.py
+++ b/run.py
@@ -527,7 +527,10 @@ def main() -> None:
try:
plan = resolve_effective_plan(args, config)
steps = list(plan.runtime_steps)
- LOGGER.info("Requested workflow steps: %s", ", ".join(steps) if steps else "(none)")
+ LOGGER.info(
+ "Requested workflow steps: %s",
+ ", ".join(plan.logical_steps) if plan.logical_steps else "(none)",
+ )
LOGGER.info("Effective dashboard mode: %s", plan.dashboard_mode)
cache_root = runtime_workflows.summary_cache_root(
config, create="summarize" in steps and not args.explain_cache
From e1b6302dd55afe45fb01f69d0ad474a8eecc9a02 Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Thu, 6 Aug 2026 16:34:55 -0400
Subject: [PATCH 20/27] Update language in wiki; fill out metro configs
---
README.md | 40 +++++------
simor_configs/metro_configs/metro_config.yaml | 56 ++++++++-------
.../metro_configs/metro_skimjoin_config.yaml | 60 ++++++++++++----
...etro_skimjoin_config_alternate_id_col.yaml | 8 +--
wiki/00-home.md | 22 +++---
wiki/01-architecture.md | 50 ++++++-------
wiki/10-getting-started.md | 33 +++++----
wiki/11-configuring-your-data.md | 41 ++++++-----
wiki/12-running-workflows.md | 72 +++++++++----------
wiki/13-configuration-reference.md | 45 ++++++------
wiki/20-output-processor.md | 46 ++++++------
wiki/21-prepared-tables.md | 30 ++++----
wiki/22-skimjoin.md | 25 ++++---
wiki/23-summary-functions.md | 46 ++++++------
wiki/24-summary-catalog.md | 49 +++++++------
wiki/25-skimjoin-config-reference.md | 47 ++++++------
wiki/30-output-visualizer.md | 22 +++---
wiki/31-dashboard-pages.md | 40 +++++------
wiki/32-figures-and-widgets.md | 42 +++++------
wiki/33-dashboard-page-recipes.md | 16 ++---
wiki/34-html-export.md | 48 ++++++-------
wiki/35-plotting-reference.md | 30 ++++----
wiki/36-html-export-schema.md | 18 ++---
wiki/40-developer-workflows.md | 14 ++--
wiki/41-data-extension-cookbook.md | 28 ++++----
wiki/42-config-column-label-cookbook.md | 33 +++++----
wiki/43-weighting-hosting-extensions.md | 35 +++++----
wiki/44-summary-function-cookbook.md | 29 ++++----
wiki/45-dashboard-extension-cookbook.md | 27 ++++---
wiki/46-testing.md | 24 +++----
wiki/90-troubleshooting.md | 38 +++++-----
wiki/99-glossary.md | 10 +--
32 files changed, 573 insertions(+), 551 deletions(-)
diff --git a/README.md b/README.md
index dd49dbc..e523250 100644
--- a/README.md
+++ b/README.md
@@ -1,22 +1,22 @@
# ActivitySim Visualizer
-Use ActivitySim Visualizer to examine [ActivitySim](https://activitysim.github.io/)
-output in an interactive dashboard. You can examine one model run, compare
-multiple runs, or compare model output with survey results.
+ActivitySim Visualizer turns [ActivitySim](https://activitysim.github.io/)
+output into an interactive dashboard. Use it to examine one model run, compare
+multiple runs, or compare model results with survey data.
ActivitySim Visualizer can:
- prepare and summarize household, person, tour, and trip output;
- compare travel patterns, model choices, and validation measures for multiple runs;
-- use valid cached results to decrease the start time; and
-- start a local dashboard or create a standalone HTML file.
+- reuse valid cached results for faster startup; and
+- launch a local dashboard or create a standalone HTML file.
## Quick Start
### 1. Install the project
-In the repository root, use `uv` to create the environment. This command also
-installs the locked dependencies:
+From the repository root, use `uv` to create the environment and install the
+locked dependencies:
```bash
uv sync --locked
@@ -30,8 +30,8 @@ uv sync --locked --link-mode=copy
### 2. Create a configuration
-Copy `config.yaml` to `local_config.yaml`. In the new file, set each `runs.dir`
-value to an ActivitySim output directory:
+Copy `config.yaml` to `local_config.yaml`, then set each `runs.dir` value to an
+ActivitySim output directory:
```yaml
runs:
@@ -55,11 +55,12 @@ zones, see [Getting Started](wiki/10-getting-started.md) and
uv run activitysim-viz --config local_config.yaml
```
-The first execution prepares the input and builds the required summary tables. It
-then starts a local server at [http://localhost:5006](http://localhost:5006).
-Later executions use valid caches. To stop the server, press `Ctrl+C`.
+On the first run, the visualizer prepares the input, builds the required summary
+tables, and starts a local server at
+[http://localhost:5006](http://localhost:5006). Later runs reuse valid caches.
+Press `Ctrl+C` to stop the server.
-If data is missing or the first execution fails, use
+If data is missing or the first run fails, see
[Troubleshooting](wiki/90-troubleshooting.md).
## How It Works
@@ -71,9 +72,8 @@ ActivitySim outputs
-> display a live dashboard or export standalone HTML
```
-The configuration selects the input, workflow steps, output location, and
-dashboard mode. Use the same start command for each workflow. Change the YAML
-configuration to change the workflow.
+The YAML configuration selects the input, workflow steps, output location, and
+dashboard mode. The start command stays the same for every workflow.
| Goal | Where to learn more |
|---|---|
@@ -90,7 +90,7 @@ configuration to change the workflow.
The [wiki home](wiki/00-home.md) is the main documentation index.
-For standard use, read these chapters in sequence:
+For a standard setup, read these chapters in order:
1. [Getting Started](wiki/10-getting-started.md)
2. [Configuring Your Data](wiki/11-configuring-your-data.md)
@@ -108,8 +108,8 @@ Other user references:
## For Contributors
Start with [Architecture](wiki/01-architecture.md) and
-[Developer Workflows](wiki/40-developer-workflows.md). Use these task-specific
-guides:
+[Developer Workflows](wiki/40-developer-workflows.md), then use the guide for
+your task:
- [extending prepared data](wiki/41-data-extension-cookbook.md);
- [adding a summary function](wiki/44-summary-function-cookbook.md);
@@ -118,7 +118,7 @@ guides:
- [skim enrichment](wiki/22-skimjoin.md); and
- [testing](wiki/46-testing.md).
-Execute focused tests during development. To execute all tests, use this command:
+Run focused tests during development. To run the full test suite, use:
```bash
uv run pytest --basetemp .pytest_tmp
diff --git a/simor_configs/metro_configs/metro_config.yaml b/simor_configs/metro_configs/metro_config.yaml
index 883281e..1a68936 100644
--- a/simor_configs/metro_configs/metro_config.yaml
+++ b/simor_configs/metro_configs/metro_config.yaml
@@ -11,8 +11,8 @@ pipeline:
- prepare
- skimjoin
# - segment
- - summarize # automatically rebuild stale summaries
- # - dashboard
+ - summarize # will automatically overwrite summaries with a stale cache
+ - dashboard
dashboard_mode: live # live | export | host
refresh: [] # list stages here only when a forced rebuild is required
@@ -36,8 +36,8 @@ files:
# Optional shared fallback files for optional inputs that may be missing in some
# run folders. These must be explicit .csv or .parquet paths.
-# fallback_files:
-# land_use: C:\Users\wesley.darling\Downloads\dataset_2026-05-21\unfiltered\land_use.csv
+fallback_files:
+ land_use: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\land_use.csv
# ---------------------------------------------------------------------------
# Runs to compare
@@ -123,21 +123,25 @@ runs:
# - C:\Users\wesley.darling\project_data\odot_skims\maz_maz_walk.csv
# network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
- - dir: C:\Users\wesley.darling\Downloads\dataset_2026-05-21\output
- label: Estimation Output
- summary_table_map:
- link_validation_summary: ../outside_summary_tables/estimated_fixtures/estimation-output/link_validation_summary.csv
- count_location_counts_validation_summary: ../outside_summary_tables/countLocCounts.csv
- count_location_volumes_validation_summary: ../outside_summary_tables/estimated_fixtures/estimation-output/count_location_volumes_validation_summary.csv
- screenline_flow_comparisons: ../outside_summary_tables/estimated_fixtures/estimation-output/screenline_flow_comparisons.csv
- commuting_flows: ../outside_summary_tables/estimated_fixtures/estimation-output/commuting_flows.csv
- transit_boardings_by_operator_and_technology: ../outside_summary_tables/estimated_fixtures/estimation-output/transit_boardings_by_operator_and_technology.csv
- transit_transfer_rate: ../outside_summary_tables/estimated_fixtures/estimation-output/transit_transfer_rate.csv
- bicycle_vmt_by_facility_type: ../outside_summary_tables/estimated_fixtures/estimation-output/bicycle_vmt_by_facility_type.csv
- commercial_vehicle_validation_summary: ../outside_summary_tables/estimated_fixtures/estimation-output/commercial_vehicle_validation_summary.csv
- commercial_vehicle_vmt_validation_summary: ../outside_summary_tables/estimated_fixtures/estimation-output/commercial_vehicle_vmt_validation_summary.csv
- external_trip_validation_summary: ../outside_summary_tables/estimated_fixtures/estimation-output/external_trip_validation_summary.csv
- external_vmt_validation_summary: ../outside_summary_tables/estimated_fixtures/estimation-output/external_vmt_validation_summary.csv
+ - dir: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\survey_data
+ label: Base
+ hh_weight_col: hh_weight
+ person_weight_col: person_weight
+ trip_weight_col: linked_trip_weight
+ skimjoin:
+ config_path: metro_skimjoin_config_alternate_id_col.yaml
+
+ - dir: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data
+ label: Override
+ hh_weight_col: hh_weight
+ person_weight_col: person_weight
+ trip_weight_col: linked_trip_weight
+ file_map:
+ households: override_households
+ persons: override_persons
+ tours: override_tours
+ trips: override_trips
+ joint_tour_participants: override_joint_tour_participants
# # skimjoin:
# config_path: will_skimjoin_config.yaml
# skim_files:
@@ -224,10 +228,10 @@ prepare:
validation:
relationship_checks: warn
distance_skim:
- file: C:\Users\wesley.darling\project_data\new_odot_skims\Metro_skims\autoSkims__MD.omx # path relative to run directory, or absolute
+ file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\autoSkims__MD.omx # path relative to run directory, or absolute
matrix: SOV_H_DIST__MD
non_motorized_distance_skim:
- file: C:\Users\wesley.darling\project_data\odot_skims\maz_maz_walk.csv
+ file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\maz_maz_walk.csv
matrix: null
time_periods:
network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
@@ -271,11 +275,11 @@ skimjoin:
# Create optional long-form trip/tour tables with skim values for alternate modes.
create_hypothetical_skim_tables: true
defaults:
- config_path: will_skimjoin_config.yaml
+ config_path: metro_skimjoin_config.yaml
skim_files:
- - C:\Users\wesley.darling\project_data\new_odot_skims\Metro_skims\*.omx
- - C:\Users\wesley.darling\project_data\odot_skims\maz_stop_walk.csv
- - C:\Users\wesley.darling\project_data\odot_skims\maz_maz_walk.csv
+ - 'C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\*.omx'
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\maz_stop_walk.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\maz_maz_walk.csv
network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
segment:
@@ -338,7 +342,7 @@ summarize:
aggregations:
school_district:
source_zone_system: maz
- file: C:\Users\wesley.darling\Downloads\viz\viz\new_output\land_use.csv
+ file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\land_use.csv
zone_id_col: zone_id
geography_col: DIST_9to12
diff --git a/simor_configs/metro_configs/metro_skimjoin_config.yaml b/simor_configs/metro_configs/metro_skimjoin_config.yaml
index 79ffea1..651c2e9 100644
--- a/simor_configs/metro_configs/metro_skimjoin_config.yaml
+++ b/simor_configs/metro_configs/metro_skimjoin_config.yaml
@@ -1,8 +1,8 @@
# project:
# skim_files:
-# - C:\Users\wesley.darling\project_data\odot_skims\*.omx
-# - C:\Users\wesley.darling\project_data\odot_skims\maz_stop_walk.csv
-# - C:\Users\wesley.darling\project_data\odot_skims\maz_maz_walk.csv
+# - 'C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\*.omx'
+# - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\maz_stop_walk.csv
+# - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\maz_maz_walk.csv
# network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
@@ -13,7 +13,7 @@
# trips_table: C:\Users\wesley.darling\projects\activitysim_visualizer\artifacts\prepared_cache\override\trips.parquet
# tours_table: C:\Users\wesley.darling\Downloads\viz\viz\output\tour.csv
# tours_table: C:\Users\wesley.darling\projects\activitysim_visualizer\artifacts\prepared_cache\override\tours.parquet
- # output_dir: will_output
+ # output_dir: metro_output
activitysim:
@@ -36,6 +36,7 @@ zone_mapping:
lookup_name: taz
file_lookup_names:
fares.omx: zone_number
+ 'bike_taz_logsums_*.omx': TAZ
missing_zone_policy: error
dimensions:
@@ -53,11 +54,29 @@ dimensions:
L: L
M: M
H: H
+ BIKE_PURPOSE:
+ source_columns:
+ trip_source_column: tour_purpose
+ outbound_tour_source_column: tour_purpose
+ inbound_tour_source_column: tour_purpose
+ values:
+ work: commute
+ school: commute
+ univ: commute
+ atwork: noncommute
+ business: noncommute
+ eat: noncommute
+ eatout: noncommute
+ escort: noncommute
+ loop: noncommute
+ maint: noncommute
+ othdiscr: noncommute
+ othmaint: noncommute
+ shopping: noncommute
+ social: noncommute
ignore_modes:
- ESCOOTER
- - EBIKE
- - BIKE_TRANSIT
- MISSING
- OTHER
@@ -405,11 +424,28 @@ modes:
matrix: maz_stop_walk__walk_dist_premium_transit
BIKE:
output_prefix: skim_bike_
- distance: WLK_DIST
- maz_bike_distance:
- output: skim_bike_maz_distance
- origin: o_maz
- destination: d_maz
- matrix: maz_maz_walk__DISTWALK
+ distance: "bike_taz_logsums_{BIKE_PURPOSE}.omx::distance_{BIKE_PURPOSE}"
+ logsum: "bike_taz_logsums_{BIKE_PURPOSE}.omx::logsum_{BIKE_PURPOSE}"
+ EBIKE:
+ output_prefix: skim_bike_
+ distance: "bike_taz_logsums_{BIKE_PURPOSE}.omx::distance_{BIKE_PURPOSE}"
+ BIKE_TRANSIT:
+ output_prefix: skim_
+ bike_distance:
+ output: skim_bike_distance
+ matrix: "bike_taz_logsums_{BIKE_PURPOSE}.omx::distance_{BIKE_PURPOSE}"
+ bike_logsum:
+ output: skim_bike_logsum
+ matrix: "bike_taz_logsums_{BIKE_PURPOSE}.omx::logsum_{BIKE_PURPOSE}"
+ transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
+ transit_brt_ivtt: "WTW_BRT__{PERIOD}"
+ transit_bus_ivtt: "WTW_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "WTW_CRT__{PERIOD}"
+ transit_first_wait_time: "WTW_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "WTW_LRT__{PERIOD}"
+ transit_tiv: "WTW_TIV__{PERIOD}"
+ transit_num_transfers: "WTW_XFR__{PERIOD}"
+ transit_transfer_wait_time: "WTW_XWT__{PERIOD}"
+ transit_fare: "fare__{PERIOD}"
diff --git a/simor_configs/metro_configs/metro_skimjoin_config_alternate_id_col.yaml b/simor_configs/metro_configs/metro_skimjoin_config_alternate_id_col.yaml
index fcd58c2..78146c7 100644
--- a/simor_configs/metro_configs/metro_skimjoin_config_alternate_id_col.yaml
+++ b/simor_configs/metro_configs/metro_skimjoin_config_alternate_id_col.yaml
@@ -1,8 +1,8 @@
project:
skim_files:
- - C:\Users\wesley.darling\project_data\new_odot_skims\Metro_skims\*.omx
- - C:\Users\wesley.darling\project_data\odot_skims\maz_stop_walk.csv
- - C:\Users\wesley.darling\project_data\odot_skims\maz_maz_walk.csv
+ - 'C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\*.omx'
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\maz_stop_walk.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\maz_maz_walk.csv
network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
# project.trips_table / project.tours_table / project.output_dir are used by the
# standalone skimjoin CLI only. Integrated runtime skimjoin reads prepared
@@ -11,7 +11,7 @@ project:
# trips_table: C:\Users\wesley.darling\projects\activitysim_visualizer\artifacts\prepared_cache\override\trips.parquet
# tours_table: C:\Users\wesley.darling\Downloads\viz\viz\output\tour.csv
# tours_table: C:\Users\wesley.darling\projects\activitysim_visualizer\artifacts\prepared_cache\override\tours.parquet
- # output_dir: will_output
+ # output_dir: metro_output
activitysim:
diff --git a/wiki/00-home.md b/wiki/00-home.md
index 555ac2c..4e8b6d5 100644
--- a/wiki/00-home.md
+++ b/wiki/00-home.md
@@ -1,10 +1,10 @@
# ActivitySim Visualizer Wiki
-This wiki contains the main documentation for ActivitySim Visualizer. Use it
-for these tasks:
+This wiki is the main documentation for ActivitySim Visualizer. It covers two
+common tasks:
-- run the visualizer with ActivitySim output
-- extend the processor, summaries, skimjoin, or dashboard
+- running the visualizer with ActivitySim output
+- extending the processor, summaries, skimjoin, or dashboard
The main data flow is:
@@ -16,20 +16,20 @@ ActivitySim outputs
-> live dashboard or standalone HTML export
```
-For the subsystem boundaries and the complete repository map, see
+For subsystem boundaries and a complete repository map, see
[01 - Architecture](01-architecture.md).
## I Am Using The Visualizer
-For standard use, read these three chapters:
+For a standard setup, read these three chapters in order:
1. [Get a dashboard running](10-getting-started.md).
2. [Choose raw, prepared, or summary inputs](11-configuring-your-data.md).
3. [Configure a live, export, or processor workflow](12-running-workflows.md).
-Use [Troubleshooting](90-troubleshooting.md) when data is missing. Use the
-[Configuration Reference](13-configuration-reference.md) to find a field or a
-default value. You do not have to read the complete reference.
+If data is missing, see [Troubleshooting](90-troubleshooting.md). Use the
+[Configuration Reference](13-configuration-reference.md) to look up a field or
+default value; you do not need to read it from beginning to end.
## I Am Extending The Visualizer
@@ -91,8 +91,8 @@ default value. You do not have to read the complete reference.
## Generated Pages
-The project generates some wiki sections from code. This process keeps the
-reference material consistent with the code:
+The project generates some wiki sections directly from the code so that the
+reference material stays accurate:
- [24 - Summary Catalog](24-summary-catalog.md)
- the generated page catalog in [31 - Dashboard Pages](31-dashboard-pages.md)
diff --git a/wiki/01-architecture.md b/wiki/01-architecture.md
index 38b6fc3..33acc9b 100644
--- a/wiki/01-architecture.md
+++ b/wiki/01-architecture.md
@@ -6,7 +6,7 @@
2. Build and cache summary tables.
3. Render those summaries in a live Panel dashboard or a standalone HTML export.
-The codebase has a separate subsystem for each job.
+Each job has its own subsystem.
The configuration has top-level sections such as `pipeline`, `dashboard`,
`display`, `summarize`, `segment`, and `skimjoin`.
@@ -54,9 +54,9 @@ run.py
```
The runtime passes one resolved `WorkflowPlan` to these operations.
-`run_prepare_workflow()` returns `PreparedRunsArtifact`.
-`run_summary_workflow()` returns `SummaryRunsArtifact`. The runtime workflows
-control the cache policy. Processor functions only transform tables.
+`run_prepare_workflow()` returns `PreparedRunsArtifact`, and
+`run_summary_workflow()` returns `SummaryRunsArtifact`. Runtime workflows own
+the cache policy; processor functions only transform tables.
## Core Runtime Contracts
@@ -82,7 +82,7 @@ accepts `account`, `app_id`, `title`, and `verify`. The runtime does not store
or use these values.
If a feature adds a configuration key or changes configuration behavior,
-update the README and the applicable wiki chapters in the same change.
+update the README and the related wiki chapters in the same change.
`Config.pipeline` is the canonical home for workflow defaults. Today the
logical step names are:
@@ -99,11 +99,11 @@ resolves `segment` in the summarize workflow.
### `RunData`
-`processor.models.RunData` is the prepared-data contract. Summary builders and
-prepared-data dashboard pages use this contract. Summary code must use
-canonical prepared columns. It must not estimate the names of raw ActivitySim
-columns. The `processor/prepare/` subsystem creates the canonical fields and
-contains the prepared-table cache helpers.
+`processor.models.RunData` is the prepared-data contract used by summary
+builders and prepared-data dashboard pages. Summary code uses canonical
+prepared columns and never guesses the names of raw ActivitySim columns. The
+`processor/prepare/` subsystem creates these canonical fields and contains the
+prepared-table cache helpers.
### `@summary` and the summary catalog
@@ -115,7 +115,7 @@ declaration defines:
- its ordered output schema and prepared-input prerequisites
- default build status
-`processor.summarize.catalog` imports the applicable domain modules. It
+`processor.summarize.catalog` imports the relevant domain modules. It
collects the declarations in a repeatable order and rejects duplicate IDs. The
system validates the columns, column order, and data types of each successful
builder result. The `summarize.failure_policy` setting controls unexpected
@@ -158,8 +158,8 @@ The framework now owns:
- export selector metadata
- export region metadata
-The same selector and section registration graph controls live refresh and
-export behavior. Separate page metadata does not control these behaviors.
+The same selector and section registration graph controls both live refresh and
+export behavior; separate page metadata does not.
The page authoring model includes the shared helpers in `dashboard/helpers/`:
@@ -170,7 +170,7 @@ The page authoring model includes the shared helpers in `dashboard/helpers/`:
- `comparison_helpers.py` centralizes percent-error formatting and base-run comparisons
For page-local table changes, `dashboard.data_access.RunTables` applies one
-query to every run and keeps the run labels. Pages must use its
+query to every run while preserving the run labels. Pages should use its
`where`, `with_columns`, `group`, `select`, `sort`, `join`, `requiring`,
`drop_empty`, and `map` operations when possible. Do not write equivalent loops
through run and data frame pairs.
@@ -181,9 +181,9 @@ that one page family shares. Put more general logic in `dashboard/helpers/`.
## Public Python APIs
-Import these facades when you extend or embed the visualizer. A file that a
-facade does not export is an implementation detail. A cookbook can identify an
-exception as an extension point.
+Use these facades when you extend or embed the visualizer. Files they do not
+export are implementation details unless a cookbook identifies a specific
+extension point.
| Import surface | Public contract |
|---|---|
@@ -201,8 +201,8 @@ The normalized config value objects exported alongside `Config` are
`ExportHTMLSettings`, `ExportSelectorRequest`,
`PrepareNonMotorizedDistanceSkimSettings`, `SegmentationDefinition`,
`PreparedColumnSegmentationSource`, `CsvLookupSegmentationSource`, and
-`StudentTypeConfig`. They are read-only runtime contracts. YAML normalization
-supplies user input. Do not assemble a `Config` manually.
+`StudentTypeConfig`. These are read-only runtime contracts populated through
+YAML normalization; do not assemble a `Config` manually.
The workflow facade also exports `effective_processor_config()`,
`run_entries_with_keys()`, `prepared_cache_root()`, `summary_cache_root()`,
@@ -213,12 +213,12 @@ dashboard facade exports `DashboardPreparedRunProvider` and
`RegisteredPageSelector`, `RegisteredPageSection`, and `SectionContent`
declaration records.
-Chapter 32 describes the page-facing `PageData` and `RunTables` API. Chapter 35
-describes chart keywords. Chapter 23 describes the `@summary` contract. The
-subsystem sections above describe workflow arguments and artifacts. Public code must pass
-an explicit `WorkflowPlan` when the required behavior differs from the loaded
-configuration. The plan records logical steps, runtime boundaries, dashboard
-mode, and refresh targets.
+Chapter 32 describes the page-facing `PageData` and `RunTables` API, chapter 35
+covers chart keywords, and chapter 23 defines the `@summary` contract. The
+subsystem sections above describe workflow arguments and artifacts. When public
+code needs behavior that differs from the loaded configuration, it must pass an
+explicit `WorkflowPlan` that records logical steps, runtime boundaries,
+dashboard mode, and refresh targets.
## Repository Map
diff --git a/wiki/10-getting-started.md b/wiki/10-getting-started.md
index 5775013..7fda740 100644
--- a/wiki/10-getting-started.md
+++ b/wiki/10-getting-started.md
@@ -1,6 +1,6 @@
# 10 - Getting Started
-Use this procedure to start a local dashboard from a repository clone.
+Follow these steps to start a local dashboard from a repository clone.
## 1. Install
@@ -18,8 +18,7 @@ uv sync --locked --link-mode=copy
## 2. Create A Small Config
-Create `local_config.yaml`. This file defines the input and the required
-output:
+Create `local_config.yaml` to define the input and output:
```yaml
root: artifacts
@@ -54,10 +53,10 @@ file can be CSV or Parquet.
If your files have different names, read
[File Names](11-configuring-your-data.md#raw-activitysim-output).
-`root` is the artifact directory. The visualizer writes summary caches in this
-directory. It also resolves relative export paths from this directory. Keep
-the export path in the configuration for a live workflow. You can then create
-HTML by changing only `pipeline.dashboard_mode` from `live` to `export`.
+`root` is the artifact directory, where the visualizer writes summary caches
+and resolves relative export paths. Keep the export path in the configuration
+for a live workflow. To create HTML later, you only need to change
+`pipeline.dashboard_mode` from `live` to `export`.
## 3. Run The Config
@@ -65,25 +64,25 @@ HTML by changing only `pipeline.dashboard_mode` from `live` to `export`.
uv run activitysim-viz --config local_config.yaml
```
-The first execution prepares data and builds summaries. It then starts the dashboard
-at [http://localhost:5006](http://localhost:5006). Later executions use valid caches.
+On the first run, the visualizer prepares the data, builds the summaries, and
+starts the dashboard at [http://localhost:5006](http://localhost:5006). Later
+runs reuse valid caches.
To stop the server, press `Ctrl+C`.
-Use this command for live dashboards, HTML exports, and processor-only
-workflows. Change the `pipeline` and `dashboard` sections to select the
-workflow.
+The command is the same for live dashboards, HTML exports, and processor-only
+workflows. Select the workflow in the `pipeline` and `dashboard` sections.
## If the first execution fails
-Do these checks:
+Check the following:
-1. Make sure that each `runs[*].dir` exists.
-2. Make sure that each required table is a `.csv` or `.parquet` file.
-3. Make sure that `zones.use_maz`, `maz_col`, and `taz_col` agree with the model.
+1. Make sure each `runs[*].dir` exists.
+2. Make sure each required table is a `.csv` or `.parquet` file.
+3. Make sure `zones.use_maz`, `maz_col`, and `taz_col` agree with the model.
4. Find the missing file or column in the log.
-Then use [Troubleshooting](90-troubleshooting.md).
+For more help, see [Troubleshooting](90-troubleshooting.md).
## Next
diff --git a/wiki/11-configuring-your-data.md b/wiki/11-configuring-your-data.md
index caa7fa5..403febe 100644
--- a/wiki/11-configuring-your-data.md
+++ b/wiki/11-configuring-your-data.md
@@ -1,7 +1,6 @@
# 11 - Configuring Your Data
-Select an input type and give each run a label. Use one of these three
-configurations.
+Choose one of the three input types below, and give each run a label.
## Raw ActivitySim Output
@@ -17,7 +16,7 @@ runs:
label: Build
```
-The dashboard shows this label.
+The dashboard uses the label to identify the run.
### File Names
@@ -59,13 +58,13 @@ columns:
trip_mode: mode
```
-The prepare step copies the selected source to the canonical visualizer
-column. See the [complete column list](13-configuration-reference.md#columns)
-in chapter 13.
+The prepare step copies the first available source into the canonical
+visualizer column. See the
+[complete column list](13-configuration-reference.md#columns) in chapter 13.
## Already-Prepared Tables
-Use `prepared_table_map` for canonical tables from a different process. This
+Use `prepared_table_map` for canonical tables created by another process. That
process can prepare, skimjoin, or filter the tables:
```yaml
@@ -79,14 +78,15 @@ runs:
land_use: prepared/land_use.parquet
```
-Each path must end in `.csv` or `.parquet`. A relative path starts from the
+Each path must end in `.csv` or `.parquet`. Relative paths start from the
configuration file directory. The tables must contain the canonical prepared
-columns that the summaries require. The visualizer does not run raw prepare or
-integrated skimjoin for this run.
+columns required by the summaries. For this type of run, the visualizer skips
+raw preparation and integrated skimjoin.
## Dashboard-Ready Summary Tables
-Use `summary_table_map` for registered summary tables from a different process:
+Use `summary_table_map` for registered summary tables created by another
+process:
```yaml
runs:
@@ -96,10 +96,10 @@ runs:
traffic_count_comparisons: summaries/traffic_counts.parquet
```
-Each key must occur in the [Summary Catalog](24-summary-catalog.md). Each file
-must have the registered columns in the specified order. A run can contain
-only external summaries. External summaries can also replace selected
-summaries from raw or prepared data.
+Each key must appear in the [Summary Catalog](24-summary-catalog.md), and each
+file must have the registered columns in the specified order. A run can contain
+only external summaries, or external summaries can replace selected summaries
+from raw or prepared data.
## Weights
@@ -121,12 +121,11 @@ runs:
trip_weight_col: trip_weight
```
-If you do not set weight columns, prepare uses the configured sample-rate
-column when it is available. If this column is not available, prepare uses
-`1.0`.
+If you do not set weight columns, the prepare step uses the configured
+sample-rate column when available and otherwise uses `1.0`.
-If the output tables contain more weights, add a named column mode. Do not
-duplicate the run or write Python for this configuration:
+If the output tables contain other weights, add a named column mode instead of
+duplicating the run or writing Python:
```yaml
weighting:
@@ -142,7 +141,7 @@ summarize:
weighting_modes: [weighted, unweighted, calibrated]
```
-The visualizer validates the named sources. It copies the weights to applicable
+The visualizer validates the named sources and copies the weights to relevant
tours, days, vehicles, and skimjoin sidecar tables. See
[43 - Weighting And Hosting Extensions](43-weighting-hosting-extensions.md#worked-example-add-a-weighting-mode)
for the rules.
diff --git a/wiki/12-running-workflows.md b/wiki/12-running-workflows.md
index a6df8d0..108e35e 100644
--- a/wiki/12-running-workflows.md
+++ b/wiki/12-running-workflows.md
@@ -1,13 +1,14 @@
# 12 - Running Workflows
-The configuration controls the standard workflow. Use one start command:
+One configuration controls the standard workflow, and every workflow uses the
+same start command:
```bash
uv run activitysim-viz --config local_config.yaml
```
-The configuration selects the work, the artifact location, and the dashboard
-mode. Use command-line flags only for development or one diagnostic execution.
+The configuration selects the work, artifact location, and dashboard mode.
+Reserve command-line flags for development or one-time diagnostics.
## The Three Main Steps
@@ -22,11 +23,10 @@ prepare -> summarize -> dashboard
Skimjoin runs inside prepare when selected. Segmentation runs with summarize.
These steps are workflow boundaries, not independent commands. The `summarize`
-step requires prepared data. It uses a valid prepared cache when one is
-available. If the cache is missing or stale, it builds prepared data from the
-configured raw or prepared input. If you add `prepare`, the runtime completes
-and stores that step first. The summarize step then uses the result in memory
-or in the cache. It does not prepare the data a second time.
+step requires prepared data, so it reuses a valid prepared cache or builds the
+data from the configured input. If the workflow includes `prepare`, the runtime
+completes and stores that step first; `summarize` then uses the result from
+memory or the cache without preparing it again.
| Requested step | What it guarantees | Prerequisites resolved automatically |
|---|---|---|
@@ -55,7 +55,7 @@ dashboard:
- trip_summaries
```
-This workflow builds missing or stale artifacts. It uses valid caches and
+This workflow reuses valid caches, builds any missing or stale artifacts, and
starts the dashboard. `dashboard.live.pages` selects the available page groups.
## Configure An HTML Export
@@ -107,10 +107,9 @@ The logical steps are `prepare`, `skimjoin`, `segment`, `summarize`, and
`dashboard`. Put `dashboard` last. `skimjoin` requires `prepare`. `segment`
requires `summarize`.
-If you omit `pipeline.steps`, the default is `[summarize, dashboard]`. This
-default prepares raw input when a valid prepared cache is not available. Add
-`prepare` when cache creation must be a visible step. Also add `prepare` when
-you enable `skimjoin`.
+If you omit `pipeline.steps`, it defaults to `[summarize, dashboard]` and
+prepares raw input when no valid prepared cache is available. Add `prepare`
+when cache creation must be a visible step or when you enable `skimjoin`.
Dashboard modes:
@@ -156,16 +155,15 @@ regional_comparison/
manifest.json
```
-The run-level `manifest.json` describes the summary bundle. Each prepared cache
-has a manifest in its table directory. Thus, a prepare-only workflow writes
-`prepared_tables/manifest.json`. It does not create the run-level summary
+The run-level `manifest.json` describes the summary bundle, while each prepared
+cache has a manifest in its table directory. A prepare-only workflow therefore
+writes `prepared_tables/manifest.json` but does not create the run-level summary
manifest.
When you enable skimjoin, `base_prepared_tables/` contains a second prepared
-manifest. It also contains the canonical tables before skim enrichment. The
-visualizer stores enriched tables, skimjoin reports, and optional hypothetical
-sidecar tables under `prepared_tables/`. Thus, summary and dashboard consumers
-use the same final path:
+manifest and the canonical tables before skim enrichment. The visualizer stores
+enriched tables, skimjoin reports, and optional hypothetical sidecar tables
+under `prepared_tables/`, giving summary and dashboard consumers one final path:
```text
base/
@@ -195,16 +193,16 @@ removing one deletes its obsolete cached summary directory on the next summary
cache write. The prepared cache is loaded to resolve segment membership, but a
valid prepared or skimjoin cache is not recomputed.
-The run-key directory uses a lowercase, file-system-safe form of the run label.
-For example, `Build Scenario` becomes `build-scenario`. Duplicate labels get
-ordered suffixes such as `build-1` and `build-2`. Do not use duplicate labels.
-If you change their order, you change the suffix for each run.
+The run-key directory uses a lowercase, file-system-safe form of the run label;
+for example, `Build Scenario` becomes `build-scenario`. Avoid duplicate labels,
+which receive ordered suffixes such as `build-1` and `build-2`. Changing their
+order also changes each suffix.
Relative paths in `dashboard.export.output_path` resolve below this directory.
Input paths follow the path rules documented in
[Configuration Reference](13-configuration-reference.md#reading-this-reference).
-The visualizer automatically uses valid caches. To rebuild each stored stage
+The visualizer reuses valid caches automatically. To rebuild every stored stage
for the configured steps, temporarily set:
```yaml
@@ -226,15 +224,15 @@ The refresh targets are stage-aware:
| `skimjoin` | `base_prepared_tables` | enriched `prepared_tables`, summaries |
| `summarize` | final `prepared_tables` | all default summaries for the full run and every configured segment |
-Cache reuse also depends on file content information. Prepared manifests record
-the path, size, and modification time of each raw input. They also record the
-prepare, skimjoin, and skim input identities. The summary manifest records the
-prepared-manifest identity, summary configuration, and declaration digest for
-each summary. A changed raw file invalidates prepare and its later output. A
-changed skim input can rebuild only skimjoin and its later output. A changed
-summary declaration can rebuild only the applicable summary table for each
-analysis unit. Segment definitions are tracked independently from full-summary
-configuration, so compatible full and segment tables stay in the bundle.
+Cache reuse also depends on file metadata. Prepared manifests record the path,
+size, and modification time of each raw input, along with the prepare, skimjoin,
+and skim input identities. The summary manifest records the prepared-manifest
+identity, summary configuration, and declaration digest for each summary. A
+changed raw file invalidates prepare and all later output, while a changed skim
+input can rebuild only skimjoin and its later output. A changed declaration can
+rebuild only the affected summary table for each analysis unit. Segment
+definitions are tracked separately from the full-summary configuration, so
+compatible full and segment tables remain in the bundle.
Use `--explain-cache` to print the cache decision for each run. The command
then exits without table loads, cache deletions, or artifact writes. The report
@@ -243,8 +241,8 @@ shows the cache-validation reason when one is available.
## CLI Overrides
-Command-line flags override the configured workflow for one execution. For standard
-operation, change the YAML so that you can reproduce the workflow.
+Command-line flags override the configured workflow for one run. For normal
+operation, change the YAML so that the workflow remains reproducible.
| Flag | Behavior |
|---|---|
@@ -270,7 +268,7 @@ If you use `--prepare`, `--summarize`, or `--dashboard`, these flags replace
`pipeline.steps` with the selected main boundaries. They do not enable the
`skimjoin` or `segment` steps. Do not combine `--from-csvs` with processor steps
or `--write-csvs`. The `--write-csvs` and `--skip-summary-cache-write` flags
-require summarize. Each refresh flag requires its applicable processor
+require summarize. Each refresh flag requires its corresponding processor
boundary. If the configuration omits dashboard, use `--export-html` with
`--dashboard`.
diff --git a/wiki/13-configuration-reference.md b/wiki/13-configuration-reference.md
index 8879150..e67022b 100644
--- a/wiki/13-configuration-reference.md
+++ b/wiki/13-configuration-reference.md
@@ -1,13 +1,12 @@
# 13 - Configuration Reference
-This page describes each field in the main ActivitySim Visualizer configuration
-file. For an introduction, read
-[11 - Configuring Your Data](11-configuring-your-data.md). See
-[`config.yaml`](../config.yaml) for the canonical example.
+This page is a field-by-field reference for the main ActivitySim Visualizer
+configuration. For an introduction, read
+[11 - Configuring Your Data](11-configuring-your-data.md); for the canonical
+example, see [`config.yaml`](../config.yaml).
-This page describes the current canonical configuration. Unknown and removed
-keys cause a validation error. When possible, the error gives the canonical
-replacement.
+Unknown and removed keys cause a validation error, which gives the canonical
+replacement when one is available.
## Reading This Reference
@@ -134,8 +133,8 @@ runs:
## `weighting`
-`weighting.modes` defines named alternatives by pointing at columns already
-present in prepared household, person, or trip tables.
+`weighting.modes` defines named alternatives that use columns already present
+in prepared household, person, or trip tables.
| Field | Type | Default | Impact | Notes |
|---|---|---|---|---|
@@ -202,12 +201,12 @@ pipeline:
The visualizer stores `segment` output in summary bundles. Use
`refresh: [summarize]` to rebuild segmented output. Dashboard rendering does
-not have a persistent processor cache. Thus, dashboard is not a refresh target.
+not have a persistent processor cache, so dashboard is not a refresh target.
## `runs`
-Each run entry describes one scenario. Always set `label` when possible. This
-value becomes the display name and identifies the run in cache and debug output.
+Each run entry describes one scenario. Set `label` whenever possible; it becomes
+the display name and identifies the run in cache and debug output.
| Field | Type | Default | Impact | Notes |
|---|---|---|---|---|
@@ -264,8 +263,8 @@ runs:
## `files` And `fallback_files`
`files` maps logical table IDs to raw ActivitySim output file names. If a value
-has no extension, the reader first searches for `.parquet`. It then searches
-for `.csv` in each run directory.
+has no extension, the reader searches each run directory for `.parquet` first
+and then `.csv`.
| Table id | Default stem |
|---|---|
@@ -280,7 +279,7 @@ for `.csv` in each run directory.
`fallback_files` supports only these optional table IDs: `day`, `vehicles`,
`joint_tour_participants`, and `land_use`. Each value must be an explicit
-`.parquet` or `.csv` path.
+`.parquet` or `.csv` path. Use `fallback_files` when multiple runs share input files.
```yaml
files:
@@ -311,8 +310,8 @@ zones:
## `columns`
-Most `columns` values can be one string or an ordered list of possible source
-column names. The visualizer uses the first available column. It reads the
+Most `columns` values can be a single string or an ordered list of possible
+source names. The visualizer uses the first available column and reads the
scalar fields at the start of the table as single names.
| Field | Default | Impact | Purpose |
@@ -464,7 +463,7 @@ main visualizer config
project/activitysim/defaults/modes: defines the actual lookup rules
```
-The `skimjoin.defaults.config_path` value does not start skimjoin. The
+Setting `skimjoin.defaults.config_path` does not start skimjoin; the
`pipeline.steps` list must also contain `prepare` and `skimjoin`.
| Field | Type | Default | Impact | Notes |
@@ -623,11 +622,11 @@ Use a page ID or a nested group and page ID as an export page override key.
Selector keys depend on the page. A selector value can be `default`, `all`, one
string, or a list of strings. Set `parts.*.enabled` to hide named export parts.
-Export uses the page set from `dashboard.live.pages`.
-`dashboard.export.pages` is an override mapping, not an allow-list. An entry
-for one page does not remove other pages. To remove pages, set `enabled: false`
-or use `exclude_pages` or `exclude_groups`. Export cannot add a page that the
-live configuration did not select. Find valid IDs in these locations:
+Export starts with the page set from `dashboard.live.pages`.
+`dashboard.export.pages` is an override mapping, not an allow-list, so an entry
+for one page does not remove the others. To remove pages, set `enabled: false`
+or use `exclude_pages` or `exclude_groups`. Export cannot add a page omitted by
+the live configuration. Find valid IDs in these locations:
- page and group IDs: the generated catalog in chapter 31;
- selector IDs: `self.select(...)` and `self.selector(...)` calls on the page;
diff --git a/wiki/20-output-processor.md b/wiki/20-output-processor.md
index 6900d40..55b92d9 100644
--- a/wiki/20-output-processor.md
+++ b/wiki/20-output-processor.md
@@ -17,7 +17,7 @@ orchestration under [`runtime/workflows`](../runtime/workflows).
## Responsibilities
-The processor does these tasks:
+The processor is responsible for:
- reading raw `.csv` or `.parquet` ActivitySim outputs
- normalizing identifiers and column names
@@ -29,8 +29,8 @@ The processor does these tasks:
- writing prepared and summary caches
- recording manifests and diagnostics to identify stale output
-The dashboard must not read raw ActivitySim files again. It must use summary
-caches. It uses prepared tables only for pages that require them.
+The dashboard reads summary caches instead of reopening raw ActivitySim files.
+Only pages that require prepared data read the prepared tables.
## Runtime Data Contract
@@ -63,17 +63,16 @@ contract. They must not use raw, model-specific table layouts.
| Summary catalog | [24 - Summary Catalog](24-summary-catalog.md) | Inspect registered summary outputs. |
The former static prepared-cache schema described one `estimation-output` data
-set. It included row counts and model-specific columns. This schema was not a
-portable runtime contract. It became incorrect when the input changed. Use
-[Prepared Table Names and Fields](21-prepared-tables.md) for the stable
-contract. Examine the applicable cache manifest and table schema for exact
-model-specific columns.
+set, including its row counts and model-specific columns. Because those details
+became incorrect when the input changed, the schema was not a portable runtime
+contract. Use [Prepared Table Names and Fields](21-prepared-tables.md) for the
+stable contract, and inspect the relevant cache manifest and table schema for
+exact model-specific columns.
## Where Processor Output Goes
-Prepared caches contain reusable canonical data. Summary caches contain
-smaller CSV files for the dashboard. The summary cache is the standard
-dashboard input.
+Prepared caches contain reusable canonical data, while summary caches contain
+the smaller CSV files that serve as the standard dashboard input.
The processor also keeps diagnostic status. A table or summary can be:
@@ -82,15 +81,14 @@ The processor also keeps diagnostic status. A table or summary can be:
- unavailable because an optional input is missing
- failed, with a recorded diagnostic
-This behavior lets the dashboard show partial results. One unavailable optional
-table or summary does not stop the complete workflow.
+This status information lets the dashboard show partial results instead of
+stopping the entire workflow when an optional table or summary is unavailable.
-"Empty" and "unavailable" have different meanings. Empty means that the input
-and calculation were valid, but the result has zero rows. Unavailable means
-that a required table or column was absent. It can also mean that a declared
-operation could not execute. Failed means that the configured failure policy
-recorded an exception. Keep the availability metadata when you copy `RunData`.
-A check of only `DataFrame.is_empty()` removes this information.
+"Empty" and "unavailable" have different meanings. An empty result is valid but
+has zero rows. An unavailable result is missing a required table or column, or
+its declared operation could not run. A failed result means that the configured
+failure policy recorded an exception. Preserve the availability metadata when
+you copy `RunData`; checking only `DataFrame.is_empty()` loses this distinction.
### Example: Follow One Metric
@@ -105,14 +103,14 @@ final_trips.csv
-> page reads the table through self.data.summary(...)
```
-Each boundary has one owner. Prepare resolves source file names and aliases.
-The summary defines the aggregate. The cache validates the stored contract.
-The page controls the presentation. Thus, a page must not open a raw file or
-repeat a weighted aggregation. Chapter 44 gives the code for this example.
+Each boundary has one owner: prepare resolves source file names and aliases,
+the summary defines the aggregate, the cache validates the stored contract, and
+the page controls presentation. A page should therefore neither open a raw file
+nor repeat a weighted aggregation. Chapter 44 gives the code for this example.
## Extension Checklist
-To add processor behavior, do these steps:
+To add processor behavior:
1. Decide whether the new data belongs in prepared tables, skimjoin output, or
a summary table.
diff --git a/wiki/21-prepared-tables.md b/wiki/21-prepared-tables.md
index d1803dd..0c05d5a 100644
--- a/wiki/21-prepared-tables.md
+++ b/wiki/21-prepared-tables.md
@@ -1,7 +1,7 @@
# 21 - Prepared Tables
Prepared tables are the canonical form of ActivitySim output. They remove
-differences in raw file names. They supply stable fields to summaries and
+differences in raw file names and provide stable fields for summaries and
dashboard pages.
## Prepare Data Flow
@@ -80,43 +80,43 @@ frequently use these fields:
Use the prepared field when it exists. Do not search for raw names in a summary
or page.
-This list is an introduction. It does not mean that each table has each field.
-For a specified summary, the generated catalog in chapter 24 gives the required
+This introductory list does not imply that every table has every field. For a
+specific summary, the generated catalog in chapter 24 lists the required
prepared columns. At runtime, `@summary` requirements and prepared-table
-availability metadata control whether a calculation can execute.
+availability metadata determine whether a calculation can run.
## Inspecting An Exact Prepared Schema
-The repository does not contain a list of all columns from one sample prepared
-cache. Raw model extensions and optional input make this list model-specific.
-Input changes can also make the list incorrect.
+The repository does not treat the columns in one sample prepared cache as a
+fixed schema. Raw model extensions, optional inputs, and other input changes
+make that list model-specific.
-For the applicable cache, do these steps:
+To inspect a cache:
1. Read the run's `manifest.json`. Find the prepared-table files and the
recorded availability status.
-2. Examine the Parquet or CSV schema for the applicable table.
+2. Examine the Parquet or CSV schema for the relevant table.
3. Use `processor.models.RunData` names at runtime and the file/config names in
[Prepared Table Names](#prepared-table-names).
4. Use the generated [Summary Catalog](24-summary-catalog.md) to find the exact
prepared columns required by each registered summary.
-Add stable fields to the applicable prepare enrichment module. Add a prepare
-test for each field. A row count or column in only one regional model describes
-that data set. It is not part of the portable visualizer contract.
+Add stable fields to the relevant prepare enrichment module, with a prepare
+test for each field. A row count or column found in only one regional model
+describes that data set, not the portable visualizer contract.
## Adding A Prepared Column
For a complete example, see
[Add A Column To An Existing Prepared Table](41-data-extension-cookbook.md#worked-example-add-a-column-to-an-existing-prepared-table).
-Use this procedure when many summaries or pages require the same derived field.
-Also use it when the field is part of canonical model-output normalization.
+Use this approach when many summaries or pages need the same derived field, or
+when the field is part of canonical model-output normalization.
Checklist:
1. Choose the owning enrichment module.
-2. Add the Polars expression or transformation in the applicable stage.
+2. Add the Polars expression or transformation in the relevant stage.
3. If the input is optional, keep the table usable when source columns are missing.
4. Add final type/cast behavior if the field must be stable.
5. Add or update tests that prepare a minimal run and assert the new column.
diff --git a/wiki/22-skimjoin.md b/wiki/22-skimjoin.md
index 67740ea..6f784dc 100644
--- a/wiki/22-skimjoin.md
+++ b/wiki/22-skimjoin.md
@@ -1,7 +1,7 @@
# 22 - Skimjoin
-Skimjoin adds skim-derived columns to prepared trips and tours. It is an
-optional final part of prepare. It executes after prepare normalizes raw output.
+Skimjoin adds skim-derived columns to prepared trips and tours. This optional
+final part of prepare runs after the raw output has been normalized.
Use skimjoin when summaries or dashboard pages require values from OMX skims or
sidecar lookup files. Examples are time, cost, distance, walk access, and
@@ -17,9 +17,9 @@ Integrated skimjoin uses two YAML files:
- the **standalone skimjoin config** defines `project`, `activitysim`,
dimensions, mode/component lookup rules, fallbacks, and tour aggregation.
-Paths in the first file start from the main configuration file. Paths in the
-second file start from the standalone skimjoin configuration file. A
-configuration path does not enable the step. `pipeline.steps` must contain
+Paths in the first file start from the main configuration file, while paths in
+the second start from the standalone skimjoin configuration file. Providing a
+configuration path does not enable the step; `pipeline.steps` must contain both
`prepare` and `skimjoin`.
## Runtime Placement
@@ -61,7 +61,7 @@ only when the new output requires them.
Checklist:
-1. Make sure that prepared trips or tours contain the required lookup columns.
+1. Make sure prepared trips or tours contain the required lookup columns.
2. Add or update a lookup rule in the skimjoin config.
3. Select an output name. Use the `skim_` prefix unless the interface requires a different prefix.
4. Set the missing-matrix and missing-OD policies.
@@ -78,10 +78,9 @@ because this option creates more output and artifacts.
## Standalone Skimjoin CLI
-The integrated pipeline is the standard visualizer method. You can also use the
-standalone command-line interface. Use it to examine or validate a skimjoin
-configuration. You can also create annotated tables without the full
-visualizer:
+The integrated pipeline is the standard approach. The standalone command-line
+interface is useful for inspecting or validating a skimjoin configuration, or
+for creating annotated tables without the full visualizer:
```bash
uv run python -m processor.skimjoin.cli COMMAND --config skimjoin.yaml
@@ -95,12 +94,12 @@ uv run python -m processor.skimjoin.cli COMMAND --config skimjoin.yaml
| `annotate-tours` | `--out PATH`, `--preview` | Writes annotated tours, `tour_aggregation_summary.csv`, and `missing_lookup_report.csv`. The default table is `/tours_with_skims.parquet`. |
| `run` | `--out-trips PATH`, `--out-tours PATH`, `--preview` | Executes both annotations and writes their validation and QA reports. Uses the two file names above by default. |
-Each command requires `--config`. Output flags are optional only if you
-configure `project.output_dir`. The standalone input tables come from
+Each command requires `--config`; output flags are optional only when
+`project.output_dir` is configured. Standalone input tables come from
`activitysim.trips_table` and `activitysim.tours_table`. Chapter 25 describes
the legacy `project.trips_table` and `project.tours_table` fallback. Input and
output tables must be CSV or Parquet. For annotation commands, `--preview` adds
-a short output-column inventory. It does not limit rows or prevent writes.
+a short output-column inventory but does not limit rows or prevent writes.
## Debugging Skimjoin
diff --git a/wiki/23-summary-functions.md b/wiki/23-summary-functions.md
index 2a67d41..76e0f5f 100644
--- a/wiki/23-summary-functions.md
+++ b/wiki/23-summary-functions.md
@@ -1,8 +1,8 @@
# 23 - Summary Functions
-Summary functions convert prepared `RunData` to Polars `DataFrame` objects for
-the dashboard. Declare the summary identity, requirements, output schema, cache
-name, and builder together.
+Summary functions convert prepared `RunData` into Polars `DataFrame` objects
+for the dashboard. Each function keeps its identity, requirements, output
+schema, cache name, and builder in one declaration.
## Data flow
@@ -15,9 +15,8 @@ RunData + Config
```
Builders live under [`processor/summarize/summaries`](../processor/summarize/summaries).
-`processor.summarize.catalog` imports those modules and finds their
-declarations. Do not edit a separate summary specification registry. It does
-not exist.
+`processor.summarize.catalog` imports those modules and discovers their
+declarations, so there is no separate summary specification registry to edit.
## Summary Declaration
@@ -68,16 +67,15 @@ def trip_distance_by_mode(run: RunData, config: Config) -> pl.DataFrame:
)
```
-A successful builder must return the declared columns in the declared order.
-Each column must have the declared data type. The workflow checks for missing
-declared input before it executes the builder. Missing input gives the typed empty
-result.
+A successful builder must return the declared columns, in order, with the
+declared data types. Before running the builder, the workflow checks for missing
+declared input and returns the typed empty result if any input is unavailable.
Use `required_tables` only when a complete table or `skim` is enough to state
-the requirement. Use `required_columns` for standard table
-dependencies. It also requires the named runtime table. Use `RunData` table
-names here: `hh`, `per`, `tours`, `trips`, `joint_participants`, and
-`land_use`. Do not use configuration IDs such as `households` or `persons`.
+the requirement. For standard table dependencies, use `required_columns`,
+which also requires the named runtime table. Specify `RunData` names such as
+`hh`, `per`, `tours`, `trips`, `joint_participants`, and `land_use`, not
+configuration IDs such as `households` or `persons`.
## Weighting
@@ -103,21 +101,21 @@ use the [Summary Function Cookbook](44-summary-function-cookbook.md).
9. Use `uv run python scripts/generate_wiki_catalogs.py`.
The catalog import rejects duplicate IDs. Standard summarize workflows build
-each declaration that has `build_by_default=True`. Enabled page requirements
-do not change this build set. `build_by_default=False` registers a contract but
-does not add it to standard builds. Use this value for an external table in the
-public workflow. Supply the table through `summary_table_map`. A non-default ID
-in a page declaration does not start its builder.
+every declaration with `build_by_default=True`, regardless of enabled page
+requirements. Setting `build_by_default=False` registers the contract without
+adding it to standard builds. Use this setting for an external table in the
+public workflow and supply the table through `summary_table_map`. Referencing a
+non-default ID in a page declaration does not start its builder.
## Summary CSV Boundary
-Summary caches are the dashboard input. The visualizer stores their registered
-tables as CSV files for each run and weighting mode. Standard summarize workflows
-write missing or stale cache tables. Use `--skip-summary-cache-write` to prevent
+Summary caches are the dashboard input. The visualizer stores registered tables
+as CSV files for each run and weighting mode, and standard summarize workflows
+write any that are missing or stale. Use `--skip-summary-cache-write` to prevent
these writes.
-For a developer diagnostic, use this command to ignore reusable summary caches.
-The command rebuilds configured summaries and writes the cache CSV files and
+For a developer diagnostic, the following command ignores reusable summary
+caches, rebuilds the configured summaries, and writes their CSV files and
manifests:
```bash
diff --git a/wiki/24-summary-catalog.md b/wiki/24-summary-catalog.md
index 4a1290b..288d540 100644
--- a/wiki/24-summary-catalog.md
+++ b/wiki/24-summary-catalog.md
@@ -1,22 +1,21 @@
# 24 - Summary Catalog
-This page is the data dictionary for the summary CSV tables from the Output
-Processor. It describes all registered summary tables. For each table, it
-defines a row, gives analysis uses, and defines each output field. The generated
-developer inventory is at the end of the page. Use it as the authoritative list
-of file names, schemas, builders, and input requirements.
+This page is the data dictionary for summary CSV tables from the Output
+Processor. It explains what one row represents, how each table is used, and
+what each output field means. The generated developer inventory at the end is
+the authoritative list of file names, schemas, builders, and input requirements.
## How to Interpret the Tables
- Count, volume, mileage, and boarding fields are numeric measures. In a
- weighted cache, they are sums of `finalweight`. In an unweighted cache, the
- workflow uses unit weights. Thus, a `Float64` count can be a fractional
- population estimate. It is not always a row count.
+ weighted cache, they are sums of `finalweight`; in an unweighted cache, the
+ workflow uses unit weights. A `Float64` count can therefore be a fractional
+ population estimate rather than a row count.
- The workflow calculates rate, percentage, mean, standard deviation, median,
and percentile fields from the weighted observations for that table.
- Values such as `all_geographies`, `all_person_types`, `all_tour_purposes`,
- `all_tour_modes`, `All Modes`, `All Auto`, and `Daily` are rollups. Do not add
- a rollup to its component rows. Use the rollup or the detail level, but not both.
+ `all_tour_modes`, `All Modes`, `All Auto`, and `Daily` are rollups. Use either
+ the rollup or its component rows, but do not add them together.
- `geography_type` names the configured spatial system, such as MAZ, TAZ,
county, MPO, or a custom geography. `geography_id` is the identifier in that
system. Each table description identifies the home, work, school,
@@ -37,11 +36,11 @@ of file names, schemas, builders, and input requirements.
## Build Status
-The standard summarize workflow builds **85** tables. The other **15** tables
-have `Default build = no`. The two skim ECDF tables are optional products. An
-external process supplies the 13 validation contracts through
-`summary_table_map`. The visualizer does not calculate them from `RunData`.
-This page and the generated inventory describe all 100 contracts.
+The standard summarize workflow builds **85** tables. Of the **15** tables with
+`Default build = no`, two are optional skim ECDF products and 13 are validation
+contracts supplied by an external process through `summary_table_map`. The
+visualizer does not calculate those contracts from `RunData`. This page and the
+generated inventory describe all 100 contracts.
## Analytical Table Reference
@@ -113,15 +112,15 @@ for a missing distance value.
| `daily_activity_pattern_by_person_type` | Daily activity pattern alternatives by person type, including an all-person-types rollup. Use it to compare mandatory, nonmandatory, and home-stay behavior. | `person_type`: person-type code or rollup.
`daily_activity_pattern`: prepared CDAP/activity-pattern category.
`person_count`: weighted people in the pattern. |
| `mandatory_tour_frequency_by_person_type` | Positive mandatory-tour-frequency choice by person type, plus an all-person-types rollup. Use it to analyze how many mandatory tours travelers make. The table excludes people with a choice of zero. | `person_type`: person-type code or rollup.
`mandatory_tour_frequency`: prepared positive mandatory-tour frequency alternative.
`person_count`: weighted people choosing that frequency. |
| `nonmandatory_tour_frequency_by_person_type` | Count of individual nonmandatory tours plus joint-tour participation per person, grouped as 0, 1, 2, or 3+, by person type and for all types. Use it to compare discretionary travel propensity. | `person_type`: person-type code or rollup.
`nonmandatory_tour_frequency`: combined nonmandatory-tour category `0`, `1`, `2`, or `3+`.
`person_count`: weighted people in the category. |
-| `tour_rates_by_person_type_and_tour_purpose` | Tours per weighted person-day by person type and tour purpose, plus all-person-types rates. Use it to compare tour-generation rates while controlling for population composition. | `person_type`: person-type code or rollup.
`tour_purpose`: prepared tour-purpose category.
`tour_rate`: weighted tours divided by weighted persons for the applicable person type. |
-| `trip_rates_by_person_type_and_trip_purpose` | Trips per weighted person by person type and trip purpose, plus all-person-types rates. Use it to compare trip-generation rates across demographic markets. | `person_type`: person-type code or rollup.
`trip_purpose`: destination purpose of the trip.
`trip_rate`: weighted trips divided by weighted persons for the applicable person type. |
+| `tour_rates_by_person_type_and_tour_purpose` | Tours per weighted person-day by person type and tour purpose, plus all-person-types rates. Use it to compare tour-generation rates while controlling for population composition. | `person_type`: person-type code or rollup.
`tour_purpose`: prepared tour-purpose category.
`tour_rate`: weighted tours divided by weighted persons for that person type. |
+| `trip_rates_by_person_type_and_trip_purpose` | Trips per weighted person by person type and trip purpose, plus all-person-types rates. Use it to compare trip-generation rates across demographic markets. | `person_type`: person-type code or rollup.
`trip_purpose`: destination purpose of the trip.
`trip_rate`: weighted trips divided by weighted persons for that person type. |
### School Escorting
-`direction` values identify outbound and inbound tour halves. Some tables also
-include `both`. This value counts tours or households with escorts in both
-halves. The `all_directions` value sums escort incidences by direction. It can
-count the same tour two times. Do not use these values as equivalents.
+`direction` values identify outbound and inbound tour halves. In some tables,
+`both` counts tours or households with escorts in both halves. By contrast,
+`all_directions` sums escort incidences by direction and can count the same tour
+twice. Do not treat these values as equivalent.
| Summary table | Information and analytical use | Fields |
|---|---|---|
@@ -160,8 +159,8 @@ count the same tour two times. Do not use these values as equivalents.
### Vehicles Allocated to Tours
These tables decode vehicle-type strings for occupancy conditions 1, 2, and 3+.
-They describe modeled allocation incidences. They do not describe the unique
-household vehicle inventory.
+They describe modeled allocation incidences, not the unique household vehicle
+inventory.
| Summary table | Information and analytical use | Fields |
|---|---|---|
@@ -220,8 +219,8 @@ applicable records. Each table also includes an all-modes group.
### Processor-Built Validation Summaries
-Some assignment tables accept optional tables attached to `RunData`. The result
-is valid but empty when the optional assignment input is absent.
+Some assignment tables accept optional tables attached to `RunData`. If the
+optional assignment input is absent, the result is valid but empty.
| Summary table | Information and analytical use | Fields |
|---|---|---|
diff --git a/wiki/25-skimjoin-config-reference.md b/wiki/25-skimjoin-config-reference.md
index 1dc150a..f5b1164 100644
--- a/wiki/25-skimjoin-config-reference.md
+++ b/wiki/25-skimjoin-config-reference.md
@@ -1,8 +1,8 @@
# 25 - Skimjoin Config Reference
-This page describes each field in the standalone skimjoin configuration file.
-The main visualizer `skimjoin` step uses this file. For a workflow introduction,
-read [22 - Skimjoin](22-skimjoin.md). See this canonical example:
+This page is a field-by-field reference for the standalone skimjoin
+configuration used by the main visualizer. For a workflow introduction, read
+[22 - Skimjoin](22-skimjoin.md). For a complete example, see
[`example_skimjoin_config.yaml`](../example_skimjoin_config.yaml).
The skimjoin configuration answers four questions:
@@ -43,7 +43,7 @@ modes:
distance: SOV_DIST
```
-`time: SOV_TIME` has the same result as:
+`time: SOV_TIME` is equivalent to:
```yaml
time:
@@ -142,9 +142,8 @@ modes:
- matrix: SOV_TIME__MD
```
-Fallbacks execute after the primary lookup. They apply to rows for which the earlier
-step did not supply a valid value. Fallback steps use the same final output
-column.
+Fallbacks run after the primary lookup and apply only to rows that do not yet
+have a valid value. Every fallback step uses the same final output column.
### Tour Aggregation
@@ -159,7 +158,7 @@ tour_aggregation:
skim_auto_time: true
```
-Mode rules also create tour lookups directly. Tour lookup output gets an
+Mode rules can also create tour lookups directly. Their output receives an
`_outbound` or `_inbound` suffix.
## Top-Level Sections
@@ -214,8 +213,8 @@ Column names cannot be blank.
## `defaults`
-Each mode, segment, and component uses these defaults. A value nearer to the
-rule overrides a default.
+Each mode, segment, and component inherits these defaults. A value closer to
+the rule takes precedence.
| Field | Type | Default | Allowed values | Notes |
|---|---|---|---|---|
@@ -298,9 +297,9 @@ Each dimension entry has these fields:
| `values_from_network_los` | boolean | `false` | Only supported for `PERIOD`. Requires `project.network_los_file`. |
| `values` | mapping | `{}` | Raw source value to matrix-name token. The loader normalizes keys and values to strings. |
-If `values` is empty, skimjoin converts the raw source value to a string. It
-then puts the string in the matrix name. If `values` is present, each observed
-value must have a mapping.
+If `values` is empty, skimjoin converts the raw source value to a string and
+inserts it into the matrix name. If `values` is present, every observed value
+must have a mapping.
```yaml
dimensions:
@@ -324,8 +323,8 @@ dimensions:
## `ignore_modes`
-`ignore_modes` lists trip modes that do not require a matching `modes` rule.
-Use this list for modes that do not require skim enrichment.
+`ignore_modes` lists trip modes that do not need skim enrichment and therefore
+do not require a matching `modes` rule.
```yaml
ignore_modes:
@@ -410,7 +409,7 @@ modes:
```
If multiple rules write the same output for the same rows, set `combine: sum`
-on all applicable rules. Without this setting, validation reports an output
+on all affected rules. Without this setting, validation reports an output
collision.
## `when` Filters
@@ -436,7 +435,7 @@ to all its components. A child filter can replace the same column key.
Use `segment_on` when one mode requires different lookup rules for different
source values. Each key under `segments` is a value from the `segment_on`
-column. Skimjoin adds the applicable `when` filter for each segment.
+column. Skimjoin adds the corresponding `when` filter for each segment.
```yaml
modes:
@@ -455,15 +454,15 @@ modes:
destination: DTAZ
```
-Validation makes sure that each observed value for a covered mode has a segment
+Validation checks that every observed value for a covered mode has a segment
block.
## `fallbacks`
Fallback entries use the same string or mapping format as primary component
-rules. Skimjoin tries them in list order after a prior step fails. A fallback
-uses the parent component output unless it sets an output. All steps in a
-fallback chain must use the same final output.
+rules. Skimjoin tries them in list order after a previous step fails. A fallback
+uses the parent component output unless it sets its own, and all steps in the
+chain must use the same final output.
```yaml
modes:
@@ -486,8 +485,8 @@ Skimjoin writes fallback reports to `fallback_lookup_report`.
| `key` | `matrix`, `key_column` | Reads a keyed sidecar table by one source column. |
For CSV skim files, the inventory code finds key and value columns from the file
-structure. It can also find origin and destination columns. For OMX files, OD
-lookups use the configured `zone_mapping` lookup name.
+structure and can also identify origin and destination columns. For OMX files,
+OD lookups use the configured `zone_mapping` lookup name.
```yaml
modes:
@@ -509,7 +508,7 @@ By default, each component creates trip and tour lookup rules:
| Outbound tours | `activitysim.tour_mode_column` | `outbound_tour_source_column` | `output_outbound` |
| Inbound tours | `activitysim.tour_mode_column` | `inbound_tour_source_column` | `output_inbound` |
-Set `apply_to: trips` or `apply_to: tours` to execute a component on only one target
+Set `apply_to: trips` or `apply_to: tours` to run a component on only one target
table.
## `tour_aggregation`
diff --git a/wiki/30-output-visualizer.md b/wiki/30-output-visualizer.md
index b679915..a20b10d 100644
--- a/wiki/30-output-visualizer.md
+++ b/wiki/30-output-visualizer.md
@@ -1,7 +1,7 @@
# 30 - Output Visualizer
-The Output Visualizer reads processor output. It shows the output in a live
-Panel dashboard or a standalone HTML file.
+The Output Visualizer turns processor output into either a live Panel dashboard
+or a standalone HTML file.
```text
summary caches + optional prepared tables
@@ -14,7 +14,7 @@ The main code lives under [`dashboard/`](../dashboard).
## Visualizer Responsibilities
-The visualizer does these tasks:
+The visualizer is responsible for:
- loading summary runs
- loading prepared tables only for pages that request them
@@ -23,8 +23,8 @@ The visualizer does these tasks:
- rendering figures, tables, cards, and widgets
- exporting supported page states to standalone HTML
-The visualizer must not rebuild summaries. If a summary is missing, execute the
-processor workflow first.
+The visualizer does not rebuild summaries. If one is missing, run the processor
+workflow first.
## Live Dashboard
@@ -53,9 +53,9 @@ uv run activitysim-viz --config local_config.yaml
## HTML Export
-HTML export uses the same page registry. It converts supported page content to
-one self-contained HTML document. The export includes only the states and
-selector variants that exist at export time.
+HTML export uses the same page registry and converts supported content into one
+self-contained document. It includes only the states and selector variants
+available at export time.
Configure `pipeline.dashboard_mode: export` and an output path:
@@ -69,7 +69,7 @@ dashboard:
output_path: exports/dashboard.html
```
-The standard configuration command then writes the export. For details, read
+The standard configuration command then writes the export. For details, see
[34 - HTML Export](34-html-export.md).
## Dashboard State
@@ -82,8 +82,8 @@ The standard configuration command then writes the export. For details, read
- optional segmentation type and visibility
- prepared-data provider state
-Pages must read state through the `DashboardPage` helpers. Do not duplicate
-cache or run-selection logic.
+Pages read state through the `DashboardPage` helpers, which avoids duplicating
+cache and run-selection logic.
## Extension Path
diff --git a/wiki/31-dashboard-pages.md b/wiki/31-dashboard-pages.md
index af7c58e..62817f2 100644
--- a/wiki/31-dashboard-pages.md
+++ b/wiki/31-dashboard-pages.md
@@ -1,6 +1,6 @@
# 31 - Dashboard Pages
-The visualizer finds dashboard pages in modules under
+The visualizer discovers dashboard pages in modules under
[`dashboard/pages`](../dashboard/pages). Each final module contains one
`DashboardPage` subclass with a `@dashboard_page(...)` decorator. Page packages
export a `DashboardGroupDefinition` as `GROUP`.
@@ -21,16 +21,16 @@ Important fields:
| `required_prepared_tables` | Prepared tables required by the page. |
These declarations control dashboard cache loads, removal of unused data,
-availability diagnostics, and prepared-table loads. They do not select the
-summaries that the summarize workflow builds. Standard summarize runs build
-each declaration that has `build_by_default=True`.
+availability diagnostics, and prepared-table loads. They do not select which
+summaries the summarize workflow builds; standard summarize runs build every
+declaration with `build_by_default=True`.
`required_summary_ids` identifies the primary page data. If no run has a usable
-required table, `self.data.summary(...)` records a required-data warning. The
-page must then show a standard unavailable card. `optional_summary_ids`
-identifies an independent feature. If this data is absent, hide or replace only
-that feature. Missing declared data does not stop the complete dashboard. Data
-can also be available for only some runs.
+required table, `self.data.summary(...)` records a required-data warning and the
+page shows a standard unavailable card. `optional_summary_ids` identifies data
+for an independent feature; if that data is absent, hide or replace only the
+feature. Missing declared data does not stop the entire dashboard, and data can
+be available for only some runs.
## Enabling Pages
@@ -56,9 +56,9 @@ Group selection modes are:
| `trip_summaries: [trip_mode, trip_stop_distance]` | Exactly the listed children in that order. |
If you omit `dashboard.live.pages`, the visualizer selects default-enabled
-standalone pages and groups. It also selects default-enabled children in each
-group. A group's `default_page_id` selects the first visible tab or fallback
-tab. It does not enable all children.
+standalone pages and groups, along with each group's default-enabled children.
+A group's `default_page_id` selects the first visible or fallback tab; it does
+not enable every child.
`dashboard.export.pages` changes matching pages in the resolved live page set.
It is not an allow-list. Live pages without an entry keep their default export
@@ -99,13 +99,13 @@ rules.
## Availability And Validation Features
-Page selectors use available data. Option providers list values in usable runs.
-The page lifecycle repairs a selection if an earlier choice makes it invalid.
-Do not show a value only because it occurs in a fixed domain. Show it only when
-its dependent section has data.
+Page selectors reflect the available data, and option providers list values
+from usable runs. If an earlier choice makes a selection invalid, the page
+lifecycle repairs it. A value should appear only when its dependent section has
+data, not simply because it belongs to a fixed domain.
When no usable run remains, the page shows the standard data-unavailable card
-for the applicable feature. Missing required data can make the primary page
+for the affected feature. Missing required data can make the primary page
workflow unavailable. Missing optional data replaces only its independent
feature. Set `display.missing_data_display: blank` to hide all these cards.
@@ -118,9 +118,9 @@ The validation group provides:
| VMT Validation | Overview comparisons plus selector-driven personal-auto and non-motorized VMT. Optional outside tables add external travel/VMT, commercial travel/VMT, and bicycle facility summaries; each optional feature gets its own unavailable state. |
| Regional Validation | Optional district or county observed flow matrices, modeled `commuting_flows`, and aligned heatmaps. Heatmaps can show modeled, observed, difference, percent difference, or absolute percent difference. You can include or exclude totals. The selector shows only flow types that have available input. |
-Expandable calculation notes identify source summary IDs, filters, formulas,
-and aggregation details. They occur below the applicable output. They are on by
-default. Set `dashboard.include_notes: false` to hide them.
+Expandable calculation notes appear below the related output and identify
+source summary IDs, filters, formulas, and aggregation details. They are shown
+by default; set `dashboard.include_notes: false` to hide them.
## Generated Page Catalog
diff --git a/wiki/32-figures-and-widgets.md b/wiki/32-figures-and-widgets.md
index 04fd9a8..640d5f6 100644
--- a/wiki/32-figures-and-widgets.md
+++ b/wiki/32-figures-and-widgets.md
@@ -6,15 +6,15 @@ missing-data diagnostics, and export metadata.
## Page Lifecycle
-Each page subclasses `DashboardPage` and implements `build_page()`. This method
-declares selectors and sections one time. It returns a stable Panel layout.
+Each page subclasses `DashboardPage` and implements `build_page()`, which
+declares selectors and sections once and returns a stable Panel layout.
`DashboardPage.__init__()` creates `self.data`, page state, and the component
-registries. It then calls `build_page()`. Thus, a standard page must not define
-its own `__init__`. If a page requires special initialization, call
-`super().__init__(state, config)`. Create attributes for `build_page()` before
-that call. Put declarations in `build_page()`. Do not put an `__init__` method
-in an implementation mixin.
+registries before calling `build_page()`. A standard page therefore does not
+need its own `__init__`. If special initialization is necessary, create any
+attributes needed by `build_page()` before calling
+`super().__init__(state, config)`. Keep declarations in `build_page()` and never
+put an `__init__` method in an implementation mixin.
The main author-facing objects are:
@@ -87,7 +87,7 @@ The page-facing data API is:
| `self.data.summary_series(id, weighting=None)` | Specialized skim-summary view that retains summary-series metadata. |
`RunTables` is iterable and indexable as `(run_label, DataFrame)` pairs. Its
-public fluent/query surface is:
+public query interface is:
| API | Behavior |
|---|---|
@@ -148,8 +148,8 @@ self.purpose = self.select(
)
```
-The framework calls an option provider before it renders dependent sections.
-It repairs stale values. `default` can be `"first"`, `"last"`, or a callable.
+Before rendering dependent sections, the framework calls an option provider and
+repairs stale values. `default` can be `"first"`, `"last"`, or a callable.
Use `self.selector(...)` only for a custom checkbox, numeric input, or other
widget that `select(...)` cannot define.
@@ -165,9 +165,9 @@ chart = self.section(
)
```
-A section renderer can return one Panel `Viewable` or a list or tuple of
-`Viewable` objects. It must not change the stable section container. The
-lifecycle replaces the container content after each render.
+A section renderer can return one Panel `Viewable`, or a list or tuple of
+`Viewable` objects. The lifecycle replaces the container content after each
+render, so the renderer must not replace the stable section container itself.
For a large page, use `self.feature("comparison")` to give a name to one
workflow. Feature component IDs become `comparison.metric`, `comparison.body`,
@@ -175,9 +175,9 @@ and similar names. Features use the same lifecycle and export behavior as the
parent page.
Large controllers can also use private implementation mixins in a `_/`
-package. Mixins organize source responsibilities. `PageFeature` organizes live
-components. A page can use both. Give each mixin one purpose. Do not give it an
-`__init__` method. Keep pure transforms as functions. Do not change page or
+package. Mixins organize source responsibilities, while `PageFeature` organizes
+live components; a page can use both. Give each mixin one purpose and no
+`__init__` method. Keep pure transforms as functions, and preserve page and
component IDs during a source-only refactor.
### Large-Page Implementation Mixins
@@ -216,8 +216,8 @@ class ExamplePage(
```
Each mixin method receives the final `ExamplePage` instance. Python resolves
-methods from left to right through the declared bases. It resolves
-`DashboardPage` last. Mixins are not standalone pages. Do not instantiate them.
+methods from left to right through the declared bases, with `DashboardPage`
+last. Mixins are not standalone pages and should not be instantiated.
Keep this pattern narrow:
@@ -228,9 +228,9 @@ Keep this pattern narrow:
- keep stateless pure functions outside mixins
- preserve page, selector, section, and export IDs during source-only refactors
-Mixins organize Python source. `PageFeature` organizes registered live
-components. They have different purposes. Use one page class until the
-composition, domain, transformation, and rendering boundaries are stable.
+Mixins organize Python source, while `PageFeature` organizes registered live
+components. Use one page class until the composition, domain, transformation,
+and rendering boundaries are stable.
## Shared Helpers
diff --git a/wiki/33-dashboard-page-recipes.md b/wiki/33-dashboard-page-recipes.md
index 099cfae..9e0aa02 100644
--- a/wiki/33-dashboard-page-recipes.md
+++ b/wiki/33-dashboard-page-recipes.md
@@ -1,6 +1,6 @@
# 33 - Dashboard Page Recipes
-Use the smallest page structure that supplies the required behavior. Each
+Use the smallest page structure that provides the required behavior. Each
discoverable page module contains one class with a `@dashboard_page(...)`
decorator.
@@ -80,9 +80,9 @@ def render_chart(self):
)
```
-The framework refreshes options and dependent sections. Use
-`self.selector(...)` only for a custom widget. Keep the label-to-raw mapping.
-This mapping prevents display labels from entering data filters.
+The framework refreshes both options and dependent sections. Use
+`self.selector(...)` only for a custom widget, and keep the label-to-raw mapping
+so display labels do not enter data filters.
## Recipe 3: Multi-Workflow Page
@@ -138,10 +138,10 @@ class RawTripDemoPage(DashboardPage):
)
```
-Load prepared data through `self.data`. Show a standard card for unavailable
-data. Use disaggregate data only when necessary. Use summaries for repeated
-aggregate views. See `raw_trip_demo.py`, the skim pages, and parking location
-for current required and optional patterns.
+Load prepared data through `self.data`, and show a standard card when it is
+unavailable. Reserve disaggregate data for cases that need it; use summaries
+for repeated aggregate views. See `raw_trip_demo.py`, the skim pages, and
+parking location for current required and optional patterns.
Mark each section that reads prepared data with
`export_data_mode="optional"` or `"required"`. Standalone export does not load
diff --git a/wiki/34-html-export.md b/wiki/34-html-export.md
index 41d972a..6c70ff5 100644
--- a/wiki/34-html-export.md
+++ b/wiki/34-html-export.md
@@ -1,7 +1,6 @@
# 34 - HTML Export
-HTML export writes a standalone dashboard file. You can open this file without
-a Python server.
+HTML export writes a standalone dashboard that opens without a Python server.
```text
registered dashboard pages
@@ -49,13 +48,13 @@ uv run activitysim-viz --config local_config.yaml
```
This command writes `artifacts/exports/dashboard.html`. Relative export paths
-start from `root`. An absolute path specifies a different location. Set
-`pipeline.dashboard_mode` to `live` to start the dashboard from the same
-configuration.
+start from `root`; use an absolute path for a different location. To start the
+live dashboard from the same configuration, set `pipeline.dashboard_mode` to
+`live`.
-The command also writes `artifacts/exports/dashboard.diagnostics.json`. This
-sidecar file records export warnings and size or state analysis. The HTML file
-does not require the sidecar file.
+The command also writes `artifacts/exports/dashboard.diagnostics.json`, a
+sidecar file that records export warnings and size or state analysis. The HTML
+file does not depend on this sidecar.
For one override, use `--export-html [PATH]`. If you do not give a path, the
command uses the configured output path. If that path is absent, it uses
@@ -90,9 +89,8 @@ browser runtime lives under `dashboard/export/js_runtime/`.
## Selector Variants
-The exporter creates page interactivity before it writes the file. It processes
-configured selector values and renders page regions. It converts the regions
-to export data and stores them as variants.
+Before writing the file, the exporter processes configured selector values,
+renders page regions, and stores those regions as export-data variants.
These rules apply:
@@ -124,8 +122,8 @@ Live mode and export use the same page registration graph for metadata:
- `self.selector(...)` registers custom widgets.
- `self.section(...)` defines refresh and export-region boundaries.
-Make sure that a section renderer gives the same result for a specified
-selector state. Set `export=False` on a section that must stay static in the
+Make sure a section renderer gives the same result for a specified selector
+state. Set `export=False` on a section that must stay static in the
exported shell. Set `exportable=False` on a live-only selector. Do not add an
export-only registry. Do not copy selector metadata to the page definition.
@@ -157,11 +155,11 @@ trip_table = self.section(
)
```
-HTML export omits a section if its `export_data_mode` is `optional` or
-`required`. These values identify whether the feature is optional or required
-in live mode. Summary-only sections use the default `export_data_mode="none"`.
-Export can include these sections. On a mixed page, put prepared data and
-summary data in separate sections. Export can then include the summary section.
+HTML export omits sections whose `export_data_mode` is `optional` or `required`;
+these values indicate whether a prepared-data feature is optional or required
+in live mode. Summary-only sections use the default
+`export_data_mode="none"` and can be exported. On a mixed page, separate the
+prepared-data and summary-data sections so the latter can remain in the export.
## Important Files
@@ -187,13 +185,13 @@ summary data in separate sections. Export can then include the summary section.
| `build_export_html_document(runs, config, summary_runs=None) -> str` | Build, serialize, and validate a complete HTML document in memory. Useful for tests and callers that need the string. |
| `write_export_html_document(output_path, runs, config, summary_runs=None) -> Path` | Build the payload and stream JSON into a temporary HTML file.
Write the diagnostics sidecar through a temporary file.
Replace each destination only after the temporary file is complete.
This is the standard workflow method. |
-Payload construction cleans NumPy and Pandas values before JSON encoding.
-Nonfinite numeric values become JSON `null`. Timestamps become ISO strings.
-The exporter escapes closing script tags. The writer streams the JSON and does
-not create a second payload string or final HTML string. This decreases peak
-memory use for exports with many selector states. A conversion, shell, write,
-or finalization failure raises an `ExportBuildError`. The error identifies the
-failed phase, and the writer removes temporary files.
+Payload construction cleans NumPy and Pandas values before JSON encoding:
+nonfinite numbers become JSON `null`, timestamps become ISO strings, and closing
+script tags are escaped. The writer streams the JSON without creating a second
+payload or final HTML string, which reduces peak memory use for exports with
+many selector states. If conversion, shell creation, writing, or finalization
+fails, an `ExportBuildError` identifies the phase and the writer removes its
+temporary files.
## Changing Export Runtime Behavior
diff --git a/wiki/35-plotting-reference.md b/wiki/35-plotting-reference.md
index 6e9c2d0..2bf6fcb 100644
--- a/wiki/35-plotting-reference.md
+++ b/wiki/35-plotting-reference.md
@@ -1,8 +1,8 @@
# 35 - Plotting Reference
-Dashboard pages use one plotting interface: `self.plot`. It accepts the
-`RunTables` object that `self.data` returns. It applies the session run colors
-and count or share mode. It returns a Panel view for a section.
+Dashboard pages use one plotting interface, `self.plot`. It accepts the
+`RunTables` object returned by `self.data`, applies the session run colors and
+count or share mode, and returns a Panel view for a section.
## Standard method
@@ -42,7 +42,7 @@ Use:
- `self.plot.line(...)` for an unfilled profile; and
- `self.plot.scatter(...)` for observed-versus-modeled comparisons.
-All four methods validate their required columns before they call Plotly. An
+Before calling Plotly, all four methods validate their required columns. Any
error identifies the chart type, run, and missing columns.
```python
@@ -83,10 +83,9 @@ Hovering on the fitted line shows this text. `one_to_one=True` adds a dashed 1:1
line. It gives both axes the same range and locks their scale. Validation pages
use this API for run equations, R-squared values, and sample sizes.
-The interface changes run labels for presentation without changing their
-identity. It shortens long labels to unique legend and tab labels. Plotly hover
-text and exported tab tooltips keep the full label. Scatter point and fit hover
-text also include the run name.
+The interface can shorten long run labels for legends and tabs without changing
+their identity. Plotly hover text and exported tab tooltips keep the full label,
+and scatter point and fit hover text also include the run name.
## Count and share behavior
@@ -117,10 +116,10 @@ return self.plot.bar(
)
```
-Use `share_y` when the denominator has a special meaning that a sum of `y`
-cannot reproduce. Do not select count or percent columns in the page only to
-follow the global control. The plotting interface does not have `as_percent`,
-`normalize`, `percent_y_col`, or `pct_col` arguments.
+Use `share_y` when its denominator has a special meaning that a sum of `y`
+cannot reproduce. Do not select count or percentage columns in the page merely
+to follow the global control. The plotting interface does not have
+`as_percent`, `normalize`, `percent_y_col`, or `pct_col` arguments.
## Direct figure API
@@ -139,7 +138,7 @@ return self.plot.panel(figure)
```
Use the standard `self.plot.*` methods when a figure does not require a custom
-change. They use the same fixed `RenderContext` as export. Thus, live and export
+change. They use the same fixed `RenderContext` as export, so live and export
charts get identical colors, labels, hover policy, and value mode. They do not
require module-global setup.
@@ -188,7 +187,8 @@ assert figure.data[0].name == "Base"
assert list(figure.data[0].x) == ["Walk", "Bike"]
```
-This method keeps plot tests fast. It separates data and query behavior from Panel.
+This approach keeps plot tests fast by separating data and query behavior from
+Panel.
Use the focused plotting target during development:
@@ -196,7 +196,7 @@ Use the focused plotting target during development:
pytest tests/test_figure_builders.py
```
-Test page query behavior in `tests/test_page_authoring.py`. Execute the complete
+Test page query behavior in `tests/test_page_authoring.py`. Run the complete
HTML export suite as a separate release check.
## Related Chapters
diff --git a/wiki/36-html-export-schema.md b/wiki/36-html-export-schema.md
index b4424af..7c308c0 100644
--- a/wiki/36-html-export-schema.md
+++ b/wiki/36-html-export-schema.md
@@ -1,7 +1,7 @@
# 36 - HTML Export Schema
This document defines the Python-to-JavaScript contract for the standalone
-offline dashboard export.
+dashboard export.
The implementation lives under `dashboard/export/`:
@@ -128,9 +128,9 @@ The shared page registry supplies these validation rules:
| `kind` | `"page"` | Discriminator |
| `content` | `ExportNode` | Serialized page shell that starts with a standard export node tree |
-Pages without export-enabled selectors create a standard page shell. Its tree
-does not contain `region` nodes. Pages with export-enabled selectors create one
-stable page shell with one or more `region` nodes.
+Pages without export-enabled selectors create a standard page shell whose tree
+has no `region` nodes. Pages with export-enabled selectors create one stable
+page shell with one or more `region` nodes.
## Region Nodes
@@ -178,8 +178,8 @@ The browser runtime supports only the node kinds in `dashboard/export/types.py`.
| `html` | `pn.pane.Markdown`, `pn.pane.HTML`, plain strings, unsupported fallback markup | `html` |
| `spacer` | `pn.Spacer` | no extra fields |
-An unsupported object becomes an `html` node with a visible fallback panel. The
-runtime identifies an unknown node kind as an error and shows an error panel.
+An unsupported object becomes an `html` node with a visible fallback panel. An
+unknown node kind is an error, which the runtime shows in an error panel.
The supported widget types are `select`, `radio_button_group`, `float_input`,
`checkbox`, and `button`. `SelectorMetadataPayload.default_value` and widget
@@ -205,8 +205,8 @@ At render time, it shows an error for these conditions:
- missing region state for the active selector combination
- Plotly runtime failures
-The HTML shows failures in a visible error panel. The runtime also writes them
-to the browser console.
+The HTML shows failures in a visible error panel and also writes them to the
+browser console.
## Schema Versioning Policy
@@ -223,7 +223,7 @@ Rules:
## Checklist for Adding a New Node Kind
-To add a serialized node kind, do these steps:
+To add a serialized node kind:
1. Add the new typed shape to `dashboard/export/types.py`.
2. Emit it from `dashboard/export/serializer.py`.
diff --git a/wiki/40-developer-workflows.md b/wiki/40-developer-workflows.md
index e7d728d..ad1569c 100644
--- a/wiki/40-developer-workflows.md
+++ b/wiki/40-developer-workflows.md
@@ -1,6 +1,6 @@
# 40 - Developer Workflows
-Use this chapter when you change code or documentation.
+Use this chapter to find the right workflow when changing code or documentation.
## Codebase Map
@@ -54,7 +54,7 @@ activitysim_visualizer/
## Testing Guidance
-Execute focused tests for the subsystem that you changed:
+Run focused tests for the subsystem you changed:
- prepare changes: minimal raw/prepared input tests and cache identity tests
- skimjoin changes: config normalization, lookup behavior, reports
@@ -68,9 +68,9 @@ Common command:
uv run --with pytest pytest --basetemp .pytest_tmp
```
-Execute smaller test groups during development when possible. The
-[Testing](46-testing.md) chapter describes the fast and full markers. It also
-gives the required release test commands.
+During development, use the smallest relevant test group. The
+[Testing](46-testing.md) chapter describes the fast and full markers and gives
+the required release test commands.
## Generated Wiki Catalogs
@@ -87,8 +87,8 @@ Command:
uv run python scripts/generate_wiki_catalogs.py
```
-Comments identify generated sections. Do not manually edit text between the
-generated markers.
+Comments identify generated sections. Do not edit the text between those
+markers by hand.
## Documentation Maintenance
diff --git a/wiki/41-data-extension-cookbook.md b/wiki/41-data-extension-cookbook.md
index 73a70c5..c1825c4 100644
--- a/wiki/41-data-extension-cookbook.md
+++ b/wiki/41-data-extension-cookbook.md
@@ -1,7 +1,7 @@
# 41 - Data Extension Cookbook
-This chapter gives complete examples for data extensions. Each procedure starts
-at the smallest supported boundary.
+This chapter gives complete examples of data extensions, starting at the
+smallest supported boundary for each one.
## Choose The Smallest Extension
@@ -25,9 +25,9 @@ NOX,18.2
```
The visualizer accepts only registered summary IDs with exact schemas. Register
-the external table with a builder that does not calculate values. Put the
-builder in the applicable summary module. For multiple project tables, use a
-module such as `processor/summarize/summaries/external_project.py`:
+the external table with a builder that does not calculate values, and place it
+in the relevant summary module. For multiple project tables, use a module such
+as `processor/summarize/summaries/external_project.py`:
```python
import polars as pl
@@ -73,16 +73,16 @@ runs:
regional_emissions: inputs/regional_emissions.csv
```
-A relative path starts from the main configuration file. The loader supports
-CSV and Parquet. It does these checks and actions:
+Relative paths start from the main configuration file. For both CSV and Parquet,
+the loader performs these checks and actions:
1. rejects unknown summary IDs;
2. rejects missing or unexpected columns;
3. casts to the declared dtypes and declared column order; and
4. exposes the same outside table under every configured weighting mode.
-The loader assumes that an external table is already aggregated. The Weighted
-and Unweighted selections do not calculate it again.
+The loader treats an external table as already aggregated, so the Weighted and
+Unweighted selections do not calculate it again.
Connect the table to a page as optional data:
@@ -235,9 +235,9 @@ class RunData:
accessibility: pl.DataFrame = field(default_factory=pl.DataFrame)
```
-Also update `PREPARED_TABLE_NAMES`, `prune_prepared_run()`, and each explicit
-`RunData(...)` copy constructor. The copy constructors are explicit. If you do
-not update one, a workflow can omit the table.
+Also update `PREPARED_TABLE_NAMES`, `prune_prepared_run()`, and every explicit
+`RunData(...)` copy constructor. Because the copy constructors list fields
+explicitly, missing one can cause a workflow to omit the table.
### 2. Read It And Track Availability
@@ -274,8 +274,8 @@ PREPARED_TABLE_ATTRS = (
```
This tuple controls prepared file names, manifest entries, writes, and most
-loads. It changes the prepared cache contract. Increment `SCHEMA_VERSION` and
-decide whether the reader can read old schema versions.
+loads. Because it changes the prepared cache contract, increment
+`SCHEMA_VERSION` and decide whether the reader can read older schema versions.
### 4. Decide Segmentation And Dashboard Behavior
diff --git a/wiki/42-config-column-label-cookbook.md b/wiki/42-config-column-label-cookbook.md
index 5a97d0e..662ba91 100644
--- a/wiki/42-config-column-label-cookbook.md
+++ b/wiki/42-config-column-label-cookbook.md
@@ -1,8 +1,7 @@
# 42 - Config, Columns, And Labels
-This chapter shows the complete path of one YAML value. The path includes
-validation, typed configuration, cache identity, prepared data, and dashboard
-presentation.
+This chapter follows one YAML value through validation, typed configuration,
+cache identity, prepared data, and dashboard presentation.
## First Decide Which Boundary Owns The Setting
@@ -13,9 +12,9 @@ presentation.
| labels, ordering, colors, or page appearance | `display` or `dashboard` | Presentation |
| which workflow executes | `pipeline` | Runtime plan; include data effects in the owning signature too |
-Do not add a setting only to `Config`. Add validation, normalization, a typed
-field, and cache signature ownership. Also add a consumer, an example, and
-tests.
+A new setting needs more than a field on `Config`. Add validation,
+normalization, a typed field, cache-signature ownership, a consumer, an example,
+and tests.
## Worked Example: Add A New Config Item
@@ -63,8 +62,7 @@ class Config:
show_zero_categories: bool
```
-Downstream code must read `config.show_zero_categories`. It must not read the
-raw YAML mapping.
+Downstream code reads `config.show_zero_categories`, not the raw YAML mapping.
### 3. Put It In The Correct Signature
@@ -91,8 +89,8 @@ if config.show_zero_categories:
chart_data = complete_category_rows(chart_data, expected_categories)
```
-Use a shared helper if several pages require the setting. Keep behavior for one
-page on that page.
+Use a shared helper when several pages need the setting; keep page-specific
+behavior on the page itself.
### 5. Document And Test It
@@ -101,7 +99,7 @@ incorrect type. Also test signature ownership and visible consumer behavior.
The examples below use module-local helpers named `_write_config()` and
`_raw_run()`. These helpers are not repository-wide pytest fixtures. Define a
-small helper in the applicable test module, or use its existing configuration
+small helper in the relevant test module, or use its existing configuration
and run factory. `extra_lines` and `column_lines` are example helper arguments.
They are not public configuration APIs.
@@ -137,7 +135,7 @@ _ALIAS_COLUMN_DEFAULTS = {
}
```
-The loader gets `CANONICAL_COLUMN_KEYS` from this mapping. Thus,
+The loader gets `CANONICAL_COLUMN_KEYS` from this mapping, so
`columns.area_type` becomes valid automatically. Add the typed field to
`Config`:
@@ -185,8 +183,8 @@ Add the candidate list to the `columns` mapping returned by
"area_type": list(config.col_area_type),
```
-The summary signature includes the prepared column payload. Thus, this change
-also invalidates applicable summary caches.
+The summary signature includes the prepared column payload, so this change also
+invalidates the affected summary caches.
### 4. Test Precedence And Materialization
@@ -258,8 +256,9 @@ def selected_employment_status_raw(self):
return self._employment_status_by_label.get(self.employment_status.value)
```
-The widget shows `Full time`. The data filter continues to use raw value `2`.
-Thus, display text does not change joins, selector state, or summary contracts.
+The widget shows `Full time`, while the data filter continues to use raw value
+`2`. Display text therefore does not change joins, selector state, or summary
+contracts.
### Add A Label Column For A Figure
@@ -285,7 +284,7 @@ return self.plot.bar(
If many pages use the category, put mapping logic in
`dashboard/helpers/category_helpers.py`. If the mapping changes canonical
-summary values, put it under `summarize.category_normalization`. The applicable
+summary values, put it under `summarize.category_normalization`. The relevant
summary logic must apply it.
### Test Raw And Display Behavior Separately
diff --git a/wiki/43-weighting-hosting-extensions.md b/wiki/43-weighting-hosting-extensions.md
index 29c650b..ca0b730 100644
--- a/wiki/43-weighting-hosting-extensions.md
+++ b/wiki/43-weighting-hosting-extensions.md
@@ -1,8 +1,8 @@
# 43 - Weighting And Hosting Extensions
-Weighting and hosting affect multiple runtime boundaries. The configuration
-controls standard alternative weights. Use the Python registry for calculations
-that column selection cannot define. Hosting is a limited extension point.
+Weighting and hosting cross several runtime boundaries. Configuration handles
+standard alternative weights, while the Python registry supports calculations
+that column selection cannot express. Hosting remains a limited extension point.
## Worked Example: Add A Weighting Mode
@@ -33,9 +33,9 @@ summarize:
weighting_modes: [weighted, unweighted, calibrated]
```
-`label` is optional. If you omit it, the loader creates a label from the mode
-ID. You must configure at least one column. The supported source tables are
-`households`, `persons`, and `trips`.
+`label` is optional; if omitted, the loader creates one from the mode ID. You
+must configure at least one column from a supported source table:
+`households`, `persons`, or `trips`.
This function differs from the three weight fields on a run. `hh_weight_col`,
`person_weight_col`, and `trip_weight_col` select the primary `weighted`
@@ -59,9 +59,9 @@ Configure only the levels that differ. For example, a mode that contains only
`trips` changes trips and tours. Household, person, day, and vehicle weights
keep their primary prepared values.
-The workflow validates source columns for each prepared run before summaries
-start. An incorrect name causes an error that identifies the missing table and
-column. It does not select a different weight. Prepare usually keeps raw
+Before summaries start, the workflow validates source columns for each prepared
+run. An incorrect name produces an error identifying the missing table and
+column; it does not select a different weight. Prepare usually preserves raw
ActivitySim columns. If you use `prepared_table_map`, include the named source
columns in those prepared files.
@@ -129,8 +129,8 @@ def register_weighting_modes(registry: WeightingModeRegistry) -> None:
```
`map_run_data_tables()` copies the complete `RunData` and transforms each data
-frame. It keeps availability metadata, diagnostics, skims, and skimjoin
-artifacts. A transform must return a new `RunData`. It must not change its input.
+frame while preserving availability metadata, diagnostics, skims, and skimjoin
+artifacts. A transform must return a new `RunData` without changing its input.
The registration fields are:
@@ -161,10 +161,9 @@ summarize:
weighting_modes: [weighted, unweighted, capped]
```
-Module imports execute code. Thus, treat a configuration with extensions
-as trusted configuration. Summary cache identity includes extension settings.
-It also includes each selected definition version, requirements, and external
-summary policy.
+Module imports execute code, so treat any configuration with extensions as
+trusted. Summary cache identity includes the extension settings along with each
+selected definition's version, requirements, and external summary policy.
An installed package can advertise the same registration function with a
Python entry point instead:
@@ -233,7 +232,7 @@ uv run --with pytest pytest --basetemp .pytest_tmp tests/test_dashboard_live.py
## Worked Example: Connect A Hosting Script
-The first hosting extension must be a small deployment entry point. Use the
+The first hosting extension should be a small deployment entry point. Use the
existing configuration, cache loader, page requirements, and `build_dashboard()`.
Do not duplicate prepare or summarize logic.
@@ -293,7 +292,7 @@ provider SDK can receive `dashboard` from the same script. Put secrets and
deployment IDs in environment variables or provider configuration. Do not put
them in the main visualizer YAML.
-This method has these properties:
+This approach has the following properties:
- hosting imports a ready-to-serve object instead of calling blocking
`pn.serve()`;
@@ -307,7 +306,7 @@ at startup, call the public prepare and summarize workflows before
## Option B: Make `dashboard_mode: host` A Core Adapter
-Use this method only when one hosting provider must be a supported runtime mode.
+Use this approach only when one hosting provider must be a supported runtime mode.
1. Add a typed `HostSettings` model in `runtime/config/models.py`.
2. Normalize `dashboard.host` in a focused parser and pass it into `Config`.
diff --git a/wiki/44-summary-function-cookbook.md b/wiki/44-summary-function-cookbook.md
index 0ee46cd..ba58d67 100644
--- a/wiki/44-summary-function-cookbook.md
+++ b/wiki/44-summary-function-cookbook.md
@@ -1,7 +1,7 @@
# 44 - Summary Function Cookbook
-This chapter shows how to make and test one summary for a dashboard. Use it with
-the short contract reference in chapter 23.
+This chapter shows how to create and test one dashboard summary. Use it with the
+short contract reference in chapter 23.
## Worked Example: Trips By Mode
@@ -13,8 +13,8 @@ output has one row for each mode, run, and weighting mode:
| DRIVEALONE | 14230.0 |
| WALK | 3180.0 |
-First, define what one row represents. This definition controls grouping keys,
-schema, tests, and figure axes.
+First, define what one row represents because that decision controls grouping
+keys, schema, tests, and figure axes.
## 1. Put Pure Calculation Before Registration
@@ -39,8 +39,8 @@ def trips_by_mode_frame(trips: pl.DataFrame) -> pl.DataFrame:
)
```
-A pure transform is easy to test without cache or dashboard setup. Use
-canonical prepared columns. Do not search for raw aliases here.
+A pure transform is easy to test without setting up caches or a dashboard. Use
+canonical prepared columns rather than searching for raw aliases here.
## 2. Declare The Runtime Contract
@@ -73,9 +73,9 @@ The declaration does four tasks:
3. supplies a correctly typed empty result; and
4. rejects successful results with wrong columns, order, or dtypes.
-The uniform builder interface includes the unused `config` argument. If
-configuration changes the calculation, use the argument here. Make sure that
-the setting is in the summary signature.
+The uniform builder interface includes `config` even when this example does not
+use it. If configuration changes the calculation, use the argument and include
+the setting in the summary signature.
## 3. Let The Workflow Handle Weighting
@@ -98,10 +98,9 @@ Define the result for zero total weight and test it.
## 4. Register A New Owning Module Only Once
-You do not have to change the catalog when you add a function to an existing
-module in `SUMMARY_MODULES`. If you create
-`processor/summarize/summaries/emissions.py`, import it. Then add it to
-`SUMMARY_MODULES` in `processor/summarize/catalog.py`.
+Adding a function to an existing module in `SUMMARY_MODULES` requires no catalog
+change. If you create `processor/summarize/summaries/emissions.py`, import it and
+add it to `SUMMARY_MODULES` in `processor/summarize/catalog.py`.
Do not keep a second list of functions. Catalog discovery reads decorated
functions from the imported modules. It rejects duplicate IDs.
@@ -209,8 +208,8 @@ uv run python scripts/generate_wiki_catalogs.py
uv run --with pytest pytest --basetemp .pytest_tmp tests/test_summary_declarations.py tests/test_page_registry_contract.py
```
-Make sure that the new ID occurs in chapter 24. After you connect it to a page,
-make sure that it occurs in the chapter 31 page catalog.
+Make sure the new ID appears in chapter 24. After connecting it to a page, make
+sure it also appears in the chapter 31 page catalog.
## Variations
diff --git a/wiki/45-dashboard-extension-cookbook.md b/wiki/45-dashboard-extension-cookbook.md
index 0f78180..c7d95b1 100644
--- a/wiki/45-dashboard-extension-cookbook.md
+++ b/wiki/45-dashboard-extension-cookbook.md
@@ -1,9 +1,9 @@
# 45 - Dashboard Extension Cookbook
-This chapter gives examples for a page, page group, selector, custom widget,
-table, and reusable figure. The examples use the declarative page lifecycle.
-Selectors control option domains. Sections control refresh dependencies. Pages
-read data through `self.data`.
+This chapter gives examples of a page, page group, selector, custom widget,
+table, and reusable figure. They use the declarative page lifecycle: selectors
+control option domains, sections control refresh dependencies, and pages read
+data through `self.data`.
## Worked Example: Add A Page To An Existing Group
@@ -56,8 +56,8 @@ class TripModeTotalsPage(DashboardPage):
)
```
-Discovery imports public child modules automatically. Do not edit a central
-page list. The decorator defines the identity and data requirements.
+Discovery imports public child modules automatically, so there is no central
+page list to edit. The decorator defines identity and data requirements.
Enable the page explicitly while developing:
@@ -108,9 +108,8 @@ def purpose_options(self):
return options
```
-The option provider executes before the framework renders a dependent section. If
-available options change, the framework uses the `default` policy to repair an
-invalid selection.
+The option provider runs before the framework renders a dependent section. If
+the available options change, the `default` policy repairs an invalid selection.
Use the raw value for the filter. Do not use its display label:
@@ -159,12 +158,12 @@ if self.hide_auto.value:
```
Registration connects the widget to refresh and HTML export. A widget created
-directly in the layout is not part of this lifecycle. Register it with
+directly in the layout is outside this lifecycle, so register it with
`self.select()` or `self.selector()`.
## Add A Figure With The Existing Plotter
-Pages must usually use `self.plot`:
+Pages should normally use `self.plot`:
```python
chart = self.plot.bar(
@@ -263,7 +262,7 @@ def area_figure(
That module already uses `ChartTables`, `ChartValueMode`, `go`, and `np`. The
explicit `value_mode` keeps `"dashboard"`, count, and share behavior consistent
-with existing figure types. `_require_columns` gives an error for the applicable
+with existing figure types. `_require_columns` gives an error for the affected
run. `RenderContext.color()` keeps the configured run-color mapping.
The adapter shape is:
@@ -325,9 +324,9 @@ return data_table(
)
```
-It creates one run tab for each frame. It applies shared column titles and
+It creates one run tab for each frame and applies shared column titles and
numeric formatting. Use a page-local `Tabulator` only when the shared table
-contract cannot supply the required interaction.
+contract cannot provide the required interaction.
## Add A New Page Group
diff --git a/wiki/46-testing.md b/wiki/46-testing.md
index 1d9b656..9fbd5ea 100644
--- a/wiki/46-testing.md
+++ b/wiki/46-testing.md
@@ -1,7 +1,7 @@
# 46 - Testing
-The default command executes all tests. It includes the complete offline HTML
-export checks:
+The default command runs all tests, including the complete offline HTML export
+checks:
```powershell
uv run pytest --basetemp .pytest_tmp
@@ -13,18 +13,18 @@ For a faster development test, omit tests marked `full_export`:
uv run pytest --basetemp .pytest_tmp -m "not full_export"
```
-Execute the complete export tests before you merge export, page, plotting, or
-summary changes:
+Run the complete export tests before merging export, page, plotting, or summary
+changes:
```powershell
uv run pytest --basetemp .pytest_tmp -m full_export
```
-The repository uses the built-in pytest `tmp_path` fixture. It uses the
+The repository uses pytest's built-in `tmp_path` fixture with the
workspace-local `--basetemp` value above. Tests must not create persistent
UUID-named directories in the repository root.
-Execute this correctness check before you push changes:
+Run this correctness check before pushing changes:
```powershell
uv run ruff check .
@@ -32,8 +32,8 @@ uv run ruff check .
Use `full_export` only for behavior that requires all default dashboard pages
and dashboard states. For writes, validation, pages, selectors, and diagnostics,
-configure the smallest applicable page and state set. The full-export tests
-continue to supply complete workflow coverage.
+configure the smallest relevant page and state set. The full-export tests still
+provide complete workflow coverage.
## Which Suite To Run
@@ -44,10 +44,10 @@ continue to supply complete workflow coverage.
| Export serializer, payload, runtime, or state behavior | Focused export tests | Fast suite plus `-m full_export` |
| Documentation only | Link/catalog checks and focused documentation tests | Fast suite if CI does not provide a docs-only path |
-The full-export tests render each default page and dashboard state in one
-representative standalone HTML document. Thus, these tests take more time. The
-shared fixture builds the document one time in each test session. Execute the
-marked group together to prevent repeated renders.
+The full-export tests take longer because they render every default page and
+dashboard state in one representative standalone HTML document. A shared
+fixture builds the document once per test session, so run the marked group
+together to avoid repeated renders.
## Focused Commands
diff --git a/wiki/90-troubleshooting.md b/wiki/90-troubleshooting.md
index eddba02..4549f45 100644
--- a/wiki/90-troubleshooting.md
+++ b/wiki/90-troubleshooting.md
@@ -1,17 +1,17 @@
# 90 - Troubleshooting
-Use this chapter when a run, cache, page, or export does not operate correctly.
+Use this chapter when a run, cache, page, or export does not behave as expected.
## Initial checks
-1. Make sure that you used the correct configuration path.
+1. Make sure you used the correct configuration path.
2. Find the selected pipeline steps and dashboard mode in the log.
3. Identify whether the problem occurs in prepare, summarize, dashboard, or export.
4. Inspect `//manifest.json` for summary state and
`//prepared_tables/manifest.json` for final prepared/skimjoin
state (see the [cache layout](12-running-workflows.md#artifact-and-cache-paths)).
5. Use `--explain-cache` to examine reuse and rebuild decisions. If you
- must rebuild, list only the applicable step in `pipeline.refresh`.
+ must rebuild, list only the relevant step in `pipeline.refresh`.
## Symptoms
@@ -46,10 +46,10 @@ uv run activitysim-viz --config local_config.yaml --refresh-summary-cache
uv run activitysim-viz --config local_config.yaml --refresh-caches
```
-If only dashboard presentation changed, a refresh is usually not necessary.
-The system automatically checks raw-file, skim-file, and applicable
-configuration identities. Use a manual refresh only to override a valid cache
-decision. Use `pipeline.refresh` for repeatable runs. A prepare refresh
+If only dashboard presentation changed, a refresh is usually unnecessary. The
+system automatically checks raw-file, skim-file, and relevant configuration
+identities. Use a manual refresh only to override a valid cache decision, and
+use `pipeline.refresh` for repeatable runs. A prepare refresh
invalidates skimjoin and summary output. A skimjoin refresh keeps
`base_prepared_tables`. A summary refresh keeps final prepared data.
@@ -67,7 +67,7 @@ the required input tables and columns.
### Worked Triage: A Page Says Data Is Unavailable
-Use this procedure if Trip Mode shows the standard unavailable card:
+If Trip Mode shows the standard unavailable card, follow these steps:
1. Find `trip_mode` in chapter 31. It requires
`trip_mode_by_tour_purpose_and_tour_mode`.
@@ -81,9 +81,9 @@ Use this procedure if Trip Mode shows the standard unavailable card:
with `pipeline.refresh: [summarize]`.
6. If the summary is valid, make sure that the page's `columns=` request agrees
with the cached schema.
-7. Make sure that the selected weighting mode exists.
+7. Make sure the selected weighting mode exists.
-This sequence examines the declared contracts in reverse order. It prevents
+This sequence works backward through the declared contracts and avoids
unnecessary cache refreshes when the problem is an input or schema mismatch.
## Skimjoin Problems
@@ -108,27 +108,27 @@ Common corrections:
## Export Problems
-If live mode operates correctly but export fails, do these steps:
+If live mode works but export fails:
-1. Make sure that export page selection includes the page.
-2. Make sure that standard selection lists use `self.select(...)`.
-3. Make sure that custom widgets use `self.selector(...)`.
-4. Make sure that `self.section(...)` registers the applicable content.
+1. Make sure export page selection includes the page.
+2. Make sure standard selection lists use `self.select(...)`.
+3. Make sure custom widgets use `self.selector(...)`.
+4. Make sure `self.section(...)` registers the relevant content.
5. Check browser console errors.
6. Inspect the adjacent `.diagnostics.json` sidecar.
7. Try `?debug_export=1`.
-Export cannot reproduce all Python callbacks. It can change only between stored
+Export cannot reproduce every Python callback; it can only switch among stored
states and registered selector variants.
## Create a small test case
-Create the smallest test case:
+Reduce the problem to the smallest test case:
1. one run
2. one page or one summary
3. one weighting mode
4. fresh cache root
-5. A copy of the applicable log text and manifest diagnostics.
+5. A copy of the relevant log text and manifest diagnostics.
-This test case usually identifies the applicable subsystem.
+This usually identifies the responsible subsystem.
diff --git a/wiki/99-glossary.md b/wiki/99-glossary.md
index 6db4ef5..e3a8f0e 100644
--- a/wiki/99-glossary.md
+++ b/wiki/99-glossary.md
@@ -30,9 +30,9 @@
## How The Terms Connect
-For a run labeled `Build`, prepare converts raw `final_trips.csv` to the
+For a run labeled `Build`, prepare converts raw `final_trips.csv` into the
prepared `trips` table. A summary builder aggregates the canonical `finalweight`
-column. It writes a registered summary in the weighted and unweighted cache
-directories for the run key. A dashboard page declares the summary ID and reads
-it through `self.data`. Registered selectors refresh its sections. Export
-converts the same declared page states to HTML.
+column and writes a registered summary to the run key's weighted and unweighted
+cache directories. A dashboard page declares that summary ID and reads it
+through `self.data`, while registered selectors refresh its sections. Export
+turns the same declared page states into HTML.
From 2e55060f5e83507996c8409b10ee6e9aea2f0eb0 Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Thu, 6 Aug 2026 18:41:11 -0400
Subject: [PATCH 21/27] Add dashboard page user guide
---
README.md | 14 +-
scripts/generate_wiki_catalogs.py | 4 +-
wiki/00-home.md | 35 +-
wiki/01-architecture.md | 11 +-
wiki/11-configuring-your-data.md | 41 ++-
wiki/12-running-workflows.md | 45 ++-
wiki/13-configuration-reference.md | 124 +++++--
wiki/14-input-data-contract.md | 197 +++++++++++
wiki/15-cache-manifest-reference.md | 236 +++++++++++++
wiki/16-dashboard-user-guide.md | 57 ++++
wiki/20-output-processor.md | 10 +-
wiki/21-prepared-tables.md | 17 +-
wiki/22-skimjoin.md | 213 ++++++++++--
...nce.md => 23-skimjoin-config-reference.md} | 256 ++++++++++----
wiki/24-segmentation.md | 319 ++++++++++++++++++
...y-functions.md => 25-summary-functions.md} | 71 +++-
...mmary-catalog.md => 26-summary-catalog.md} | 2 +-
wiki/27-geography.md | 255 ++++++++++++++
wiki/30-output-visualizer.md | 66 +++-
wiki/31-dashboard-pages.md | 8 +-
wiki/32-figures-and-widgets.md | 62 +++-
wiki/33-dashboard-page-recipes.md | 2 +-
wiki/34-html-export.md | 7 +
wiki/35-plotting-reference.md | 12 +-
wiki/36-html-export-schema.md | 113 ++++++-
wiki/40-developer-workflows.md | 6 +-
wiki/41-data-extension-cookbook.md | 4 +-
wiki/42-config-column-label-cookbook.md | 164 +++++++++
wiki/43-weighting-hosting-extensions.md | 55 ++-
wiki/44-summary-function-cookbook.md | 8 +-
wiki/45-dashboard-extension-cookbook.md | 10 +-
wiki/46-testing.md | 80 +++++
wiki/90-troubleshooting.md | 134 +++++++-
wiki/99-glossary.md | 14 +
34 files changed, 2470 insertions(+), 182 deletions(-)
create mode 100644 wiki/14-input-data-contract.md
create mode 100644 wiki/15-cache-manifest-reference.md
create mode 100644 wiki/16-dashboard-user-guide.md
rename wiki/{25-skimjoin-config-reference.md => 23-skimjoin-config-reference.md} (66%)
create mode 100644 wiki/24-segmentation.md
rename wiki/{23-summary-functions.md => 25-summary-functions.md} (60%)
rename wiki/{24-summary-catalog.md => 26-summary-catalog.md} (99%)
create mode 100644 wiki/27-geography.md
diff --git a/README.md b/README.md
index e523250..5faef3c 100644
--- a/README.md
+++ b/README.md
@@ -83,8 +83,12 @@ dashboard mode. The start command stays the same for every workflow.
| Run only the processor | [Processor-Only Workflow](wiki/12-running-workflows.md#configure-a-processor-only-workflow) |
| Create a standalone HTML dashboard | [HTML Export](wiki/34-html-export.md) |
| Understand caches and workflow steps | [Running Workflows](wiki/12-running-workflows.md) |
+| Build summaries for configured subsets | [Segmentation](wiki/24-segmentation.md) |
+| Add district, county, or other zone groupings | [Geography](wiki/27-geography.md) |
| Find an exact configuration field | [Configuration Reference](wiki/13-configuration-reference.md) |
-| Understand a summary table or field | [Summary Catalog](wiki/24-summary-catalog.md) |
+| Verify raw or prepared table requirements | [Input Data Contract](wiki/14-input-data-contract.md) |
+| Interpret cache manifests and rebuild decisions | [Cache And Manifest Reference](wiki/15-cache-manifest-reference.md) |
+| Understand a summary table or field | [Summary Catalog](wiki/26-summary-catalog.md) |
## Documentation
@@ -99,9 +103,13 @@ For a standard setup, read these chapters in order:
Other user references:
- [Output Visualizer](wiki/30-output-visualizer.md) explains the dashboard.
-- [Dashboard Pages](wiki/31-dashboard-pages.md) lists the available analyses.
+- [Dashboard User Guide](wiki/16-dashboard-user-guide.md) lists the available analyses and explains how to interpret them.
+- [Input Data Contract](wiki/14-input-data-contract.md) defines source and canonical table boundaries.
+- [Cache And Manifest Reference](wiki/15-cache-manifest-reference.md) explains stored identities and diagnostics.
- [HTML Export](wiki/34-html-export.md) explains how to create an offline file.
-- [Summary Catalog](wiki/24-summary-catalog.md) documents every summary table.
+- [Summary Catalog](wiki/26-summary-catalog.md) documents every summary table.
+- [Segmentation](wiki/24-segmentation.md) explains subset summaries and their caches.
+- [Geography](wiki/27-geography.md) explains zone mappings and spatial outputs.
- [Glossary](wiki/99-glossary.md) defines project terminology.
- [Troubleshooting](wiki/90-troubleshooting.md) covers common failures.
diff --git a/scripts/generate_wiki_catalogs.py b/scripts/generate_wiki_catalogs.py
index 385cd12..e602018 100644
--- a/scripts/generate_wiki_catalogs.py
+++ b/scripts/generate_wiki_catalogs.py
@@ -101,7 +101,7 @@ def _validate_summary_reference() -> None:
"""Keep the hand-written analytical reference aligned with declarations."""
from processor.summarize.catalog import SUMMARY_DEFINITIONS
- path = WIKI / "24-summary-catalog.md"
+ path = WIKI / "26-summary-catalog.md"
reference = path.read_text(encoding="utf-8").split(
"",
1,
@@ -204,7 +204,7 @@ def build_dashboard_page_catalog() -> str:
def main() -> None:
_validate_summary_reference()
_replace_generated_section(
- WIKI / "24-summary-catalog.md",
+ WIKI / "26-summary-catalog.md",
marker="SUMMARY-CATALOG",
generated=build_summary_catalog(),
)
diff --git a/wiki/00-home.md b/wiki/00-home.md
index 4e8b6d5..0356eef 100644
--- a/wiki/00-home.md
+++ b/wiki/00-home.md
@@ -27,6 +27,15 @@ For a standard setup, read these three chapters in order:
2. [Choose raw, prepared, or summary inputs](11-configuring-your-data.md).
3. [Configure a live, export, or processor workflow](12-running-workflows.md).
+After the dashboard starts, use the
+[Dashboard User Guide](16-dashboard-user-guide.md) to choose an analysis and
+interpret its controls and results.
+
+Use [14 - Input Data Contract](14-input-data-contract.md) when you need exact
+table, key, relationship, or bypass-prepare rules. Use
+[15 - Cache And Manifest Reference](15-cache-manifest-reference.md) when you
+need to interpret stored identities and diagnostics.
+
If data is missing, see [Troubleshooting](90-troubleshooting.md). Use the
[Configuration Reference](13-configuration-reference.md) to look up a field or
default value; you do not need to read it from beginning to end.
@@ -39,9 +48,11 @@ default value; you do not need to read it from beginning to end.
| Understand the Output Processor | [20 - Output Processor](20-output-processor.md) |
| Add a prepared column | [41 - Data Extension Cookbook](41-data-extension-cookbook.md#worked-example-add-a-column-to-an-existing-prepared-table) |
| Add or debug skimjoin outputs | [22 - Skimjoin](22-skimjoin.md) |
-| Find every skimjoin config field and lookup option | [25 - Skimjoin Config Reference](25-skimjoin-config-reference.md) |
+| Find every skimjoin config field and lookup option | [23 - Skimjoin Config Reference](23-skimjoin-config-reference.md) |
+| Build summaries for configured subsets | [24 - Segmentation](24-segmentation.md) |
+| Add custom zone-based geographies | [27 - Geography](27-geography.md) |
| Add a summary function | [44 - Summary Function Cookbook](44-summary-function-cookbook.md) |
-| Find every registered summary table | [24 - Summary Catalog](24-summary-catalog.md) |
+| Find every registered summary table | [26 - Summary Catalog](26-summary-catalog.md) |
| Understand the Output Visualizer | [30 - Output Visualizer](30-output-visualizer.md) |
| Add a dashboard page or page group | [45 - Dashboard Extension Cookbook](45-dashboard-extension-cookbook.md) |
| Add a figure, table, selector, or widget | [45 - Dashboard Extension Cookbook](45-dashboard-extension-cookbook.md) |
@@ -59,23 +70,30 @@ default value; you do not need to read it from beginning to end.
- [11 - Configuring Your Data](11-configuring-your-data.md)
- [12 - Running Workflows](12-running-workflows.md)
- [13 - Configuration Reference](13-configuration-reference.md)
+- [14 - Input Data Contract](14-input-data-contract.md)
+- [15 - Cache And Manifest Reference](15-cache-manifest-reference.md)
+- [16 - Dashboard User Guide](16-dashboard-user-guide.md)
### Output Processor
- [20 - Output Processor](20-output-processor.md)
- [21 - Prepared Tables](21-prepared-tables.md)
- [22 - Skimjoin](22-skimjoin.md)
-- [23 - Summary Functions](23-summary-functions.md)
-- [24 - Summary Catalog](24-summary-catalog.md)
-- [25 - Skimjoin Config Reference](25-skimjoin-config-reference.md)
+- [23 - Skimjoin Config Reference](23-skimjoin-config-reference.md)
+- [24 - Segmentation](24-segmentation.md)
+- [25 - Summary Functions](25-summary-functions.md)
+- [26 - Summary Catalog](26-summary-catalog.md)
+- [27 - Geography](27-geography.md)
### Output Visualizer
- [30 - Output Visualizer](30-output-visualizer.md)
-- [31 - Dashboard Pages](31-dashboard-pages.md)
+- [31 - Dashboard Page Contract](31-dashboard-pages.md)
- [32 - Figures and Widgets](32-figures-and-widgets.md)
- [33 - Dashboard Page Recipes](33-dashboard-page-recipes.md)
- [34 - HTML Export](34-html-export.md)
+- [35 - Plotting Reference](35-plotting-reference.md)
+- [36 - HTML Export Schema](36-html-export-schema.md)
### Developer Reference
@@ -94,8 +112,9 @@ default value; you do not need to read it from beginning to end.
The project generates some wiki sections directly from the code so that the
reference material stays accurate:
-- [24 - Summary Catalog](24-summary-catalog.md)
-- the generated page catalog in [31 - Dashboard Pages](31-dashboard-pages.md)
+- [26 - Summary Catalog](26-summary-catalog.md)
+- the generated page catalog in
+ [31 - Dashboard Page Contract](31-dashboard-pages.md)
Regenerate these sections after you change a summary declaration, a summary
contract, a dashboard page definition, or a page data requirement:
diff --git a/wiki/01-architecture.md b/wiki/01-architecture.md
index 33acc9b..c150637 100644
--- a/wiki/01-architecture.md
+++ b/wiki/01-architecture.md
@@ -20,6 +20,9 @@ the workflow uses it. Removed keys and unknown keys cause a specific error.
| CLI and workflow orchestration | Parse step selections, choose cache-first vs rebuild flow, and hand off to prepare/summarize/dashboard workflows | `run.py`, `runtime/workflows/` |
| Shared runtime contracts | Normalize YAML config and expose shared cross-cutting contracts used by both processor and dashboard | public surface `runtime.config`, implementation in `runtime/config/` |
| Processor prepare step | Read raw ActivitySim outputs, materialize canonical prepared columns, and manage prepared-table cache I/O | `processor/models.py`, `processor/prepare/*` |
+| Skim enrichment | Resolve per-run lookup rules and add skim-derived prepared fields | `runtime/config/normalize_skimjoin.py`, `processor/skimjoin/` |
+| Segmentation | Slice related prepared tables into analysis units before summary generation | `runtime/config/normalize_segmentation.py`, `processor/segmentation.py` |
+| Geography | Normalize zone lookups and add role-specific spatial fields during prepare | `runtime/config/normalize_geography.py`, `processor/prepare/enrichment/zones.py` |
| Summary generation | Declare builders and compute weighted/unweighted tables | `processor/summarize/contracts.py`, `processor/summarize/catalog.py`, `processor/summarize/summaries/*.py` |
| Summary cache I/O | Inspect, write, and load cache manifests and CSVs | `processor/summarize/cache.py`, `processor/summarize/cache_storage.py` |
| Dashboard page runtime | Discover pages, validate contracts, refresh declared features, and memoize section queries | `dashboard/page_registry.py`, `dashboard/page_definitions.py`, `dashboard/page_lifecycle.py`, `dashboard/page_declarations.py` |
@@ -43,7 +46,9 @@ run.py
B. run_summary_workflow()
-> inspect reusable/stale tables in the summary bundle
-> run_prepare_workflow() on summary-cache miss
+ -> processor.segmentation.build_analysis_units_for_run() when selected
-> processor.summarize.builder.build_mode_summaries_with_metadata()
+ for the full run and each segment analysis unit
-> merge reusable and rebuilt tables
-> processor.summarize.cache.write_summary_run_bundle()
C. load_summary_runs_from_cache() for dashboard-only cache runs
@@ -97,6 +102,10 @@ The runtime executes three main workflow boundaries: `prepare`, `summarize`,
and `dashboard`. The runtime resolves `skimjoin` in the prepare workflow. It
resolves `segment` in the summarize workflow.
+For detailed flows, see [22 - Skimjoin](22-skimjoin.md),
+[24 - Segmentation](24-segmentation.md), and
+[27 - Geography](27-geography.md).
+
### `RunData`
`processor.models.RunData` is the prepared-data contract used by summary
@@ -214,7 +223,7 @@ dashboard facade exports `DashboardPreparedRunProvider` and
declaration records.
Chapter 32 describes the page-facing `PageData` and `RunTables` API, chapter 35
-covers chart keywords, and chapter 23 defines the `@summary` contract. The
+covers chart keywords, and chapter 25 defines the `@summary` contract. The
subsystem sections above describe workflow arguments and artifacts. When public
code needs behavior that differs from the loaded configuration, it must pass an
explicit `WorkflowPlan` that records logical steps, runtime boundaries,
diff --git a/wiki/11-configuring-your-data.md b/wiki/11-configuring-your-data.md
index 403febe..aad9307 100644
--- a/wiki/11-configuring-your-data.md
+++ b/wiki/11-configuring-your-data.md
@@ -1,6 +1,27 @@
# 11 - Configuring Your Data
-Choose one of the three input types below, and give each run a label.
+Choose one of the three input types below, and give each run a label. For exact
+table, key, relationship, and type rules, see
+[14 - Input Data Contract](14-input-data-contract.md).
+
+## Run Input Decision Matrix
+
+The fields on one run can be combined only when their boundaries make sense:
+
+| Run input | Prepare source | Skimjoin | Segmentation | Generated summaries | Mapped-summary behavior |
+|---|---|---|---|---|---|
+| `dir` | Raw CSV/Parquet files | Available when the step and paths are configured | Available | Available | Optional mapped IDs replace generated IDs. |
+| `prepared_table_map` | Supplied canonical tables; raw prepare is skipped | Skipped for this run | Available | Available | Optional mapped IDs replace generated IDs. |
+| `summary_table_map` only | None | Not available | Not available | Not available | Mapped IDs are the run's summaries. |
+| `dir` plus `summary_table_map` | Raw files | Available | Available | Available for unmapped IDs | The same mapped table replaces its ID for full and segmented units. |
+| `prepared_table_map` plus `summary_table_map` | Supplied canonical tables | Skipped | Available | Available for unmapped IDs | The same mapped table replaces its ID for full and segmented units. |
+| `--from-csvs ` | Existing manifested cache bundle | Already complete | Already complete | Not run | Loads the bundle; it is not a loose-file mapping. |
+
+`file_map`, `skim_file`, and the three run weight fields apply to raw `dir`
+input. `file_map` cannot be combined with `prepared_table_map`. A
+`summary_table_map` entry is mode-independent: built-in weighted and unweighted
+modes copy it, while declarative named modes reject it because they cannot
+recalculate an aggregated file.
## Raw ActivitySim Output
@@ -83,6 +104,10 @@ configuration file directory. The tables must contain the canonical prepared
columns required by the summaries. For this type of run, the visualizer skips
raw preparation and integrated skimjoin.
+The visualizer also skips canonicalization, derived columns, standard weight
+creation, and geography mapping. The supplied files must already satisfy those
+parts of the prepared contract.
+
## Dashboard-Ready Summary Tables
Use `summary_table_map` for registered summary tables created by another
@@ -96,11 +121,15 @@ runs:
traffic_count_comparisons: summaries/traffic_counts.parquet
```
-Each key must appear in the [Summary Catalog](24-summary-catalog.md), and each
+Each key must appear in the [Summary Catalog](26-summary-catalog.md), and each
file must have the registered columns in the specified order. A run can contain
only external summaries, or external summaries can replace selected summaries
from raw or prepared data.
+Mapped summary tables cannot be segmented or reweighted from their rows. When
+combined with buildable input, one mapped table is overlaid unchanged on every
+full or segmented analysis unit.
+
## Weights
Configure the standard modes with:
@@ -169,10 +198,10 @@ zones:
## Optional Features
- For skim enrichment, read [Skimjoin](22-skimjoin.md).
-- For custom geography aggregation, read the
- [`summarize.geography` reference](13-configuration-reference.md#summarize).
-- For segmentation, read the
- [`segment` reference](13-configuration-reference.md#segment).
+- To build the same summaries for configured subsets, read
+ [Segmentation](24-segmentation.md).
+- To add district, county, or other zone mappings, read
+ [Geography](27-geography.md).
- For every accepted key and default, use the
[Configuration Reference](13-configuration-reference.md).
diff --git a/wiki/12-running-workflows.md b/wiki/12-running-workflows.md
index 108e35e..09ab363 100644
--- a/wiki/12-running-workflows.md
+++ b/wiki/12-running-workflows.md
@@ -10,17 +10,31 @@ uv run activitysim-viz --config local_config.yaml
The configuration selects the work, artifact location, and dashboard mode.
Reserve command-line flags for development or one-time diagnostics.
-## The Three Main Steps
+## Workflow Order
```text
-prepare -> summarize -> dashboard
+prepare -> optional skimjoin -> optional segmentation -> summarize -> dashboard
```
- **Prepare** reads raw outputs and creates canonical prepared tables.
- **Summarize** creates the smaller tables used by dashboard pages.
- **Dashboard** starts the live application or writes standalone HTML.
-Skimjoin runs inside prepare when selected. Segmentation runs with summarize.
+Skimjoin runs inside the prepare boundary when selected. Segmentation resolves
+and slices prepared data inside the summarize boundary. Geography enrichment
+runs during prepare and geography-aware aggregations run during summarize; it
+is a feature, not a separate pipeline step.
+
+The written order of non-dashboard values in `pipeline.steps` does not control
+runtime order. Those values select logical capabilities, and the runtime
+resolves the fixed dependency order above. `dashboard`, when present, must be
+the last listed value. Use the canonical order in every example and local
+configuration because it makes intent clear:
+
+```yaml
+pipeline:
+ steps: [prepare, skimjoin, segment, summarize, dashboard]
+```
These steps are workflow boundaries, not independent commands. The `summarize`
step requires prepared data, so it reuses a valid prepared cache or builds the
@@ -104,13 +118,27 @@ For dashboard-ready CSV or Parquet files, configure
## Pipeline Rules
The logical steps are `prepare`, `skimjoin`, `segment`, `summarize`, and
-`dashboard`. Put `dashboard` last. `skimjoin` requires `prepare`. `segment`
-requires `summarize`.
+`dashboard`. Values must be lowercase, unique, and valid. Put `dashboard` last.
+`skimjoin` requires `prepare`, and `segment` requires `summarize`. The runtime
+uses step membership, not the listed order, to form the prepare, summarize,
+and dashboard boundaries.
If you omit `pipeline.steps`, it defaults to `[summarize, dashboard]` and
prepares raw input when no valid prepared cache is available. Add `prepare`
when cache creation must be a visible step or when you enable `skimjoin`.
+Add `segment` when the summarize workflow must build configured subsets:
+
+```yaml
+pipeline:
+ steps: [segment, summarize, dashboard]
+ dashboard_mode: live
+ refresh: []
+```
+
+The `segment` step requires `summarize`. Its configuration alone does not
+enable segmentation.
+
Dashboard modes:
- `live`: local Panel server;
@@ -239,6 +267,9 @@ then exits without table loads, cache deletions, or artifact writes. The report
shows `REUSE`, `REBUILD`, `RUN`, or `DISABLED` for each workflow step. It also
shows the cache-validation reason when one is available.
+For annotated manifest examples, every stored field, and cache-recovery rules,
+see [15 - Cache And Manifest Reference](15-cache-manifest-reference.md).
+
## CLI Overrides
Command-line flags override the configured workflow for one run. For normal
@@ -276,4 +307,8 @@ boundary. If the configuration omits dashboard, use `--export-html` with
- [Getting Started](10-getting-started.md)
- [Configuring Your Data](11-configuring-your-data.md)
+- [Input Data Contract](14-input-data-contract.md)
+- [Cache And Manifest Reference](15-cache-manifest-reference.md)
+- [Segmentation](24-segmentation.md)
+- [Geography](27-geography.md)
- [Troubleshooting](90-troubleshooting.md)
diff --git a/wiki/13-configuration-reference.md b/wiki/13-configuration-reference.md
index e67022b..8abbb60 100644
--- a/wiki/13-configuration-reference.md
+++ b/wiki/13-configuration-reference.md
@@ -5,8 +5,13 @@ configuration. For an introduction, read
[11 - Configuring Your Data](11-configuring-your-data.md); for the canonical
example, see [`config.yaml`](../config.yaml).
-Unknown and removed keys cause a validation error, which gives the canonical
-replacement when one is available.
+Unknown and removed keys are rejected at the canonical top level and in the
+typed sections listed in this reference. Validation depth is not uniform:
+intentional free-form mappings such as `extensions.settings` accept arbitrary
+project keys, and some nested implementation mappings validate their values
+rather than every possible key. Use documented fields, load the configuration
+in a focused test, and do not rely on an unreported nested typo being accepted
+or ignored.
## Reading This Reference
@@ -20,7 +25,7 @@ The field type controls the base directory for a relative path:
| `fallback_files.*`, `prepared_table_map.*`, `summary_table_map.*` | main config directory | Values must include `.parquet` or `.csv`. |
| `prepare.distance_skim.file`, `runs[*].skim_file` | the resolved run directory | The loader resolves a relative legacy distance-skim path separately for each run. |
| other main-config enrichment, lookup, and skimjoin paths | main config directory | Includes `prepare.time_periods.network_los_file`, `prepare.non_motorized_distance_skim.file`, segmentation CSVs, and `skimjoin.defaults.*`. |
-| paths inside the standalone skimjoin config | standalone skimjoin config directory | See chapter 25. |
+| paths inside the standalone skimjoin config | standalone skimjoin config directory | See chapter 23. |
| `dashboard.export.output_path` | resolved `root` | Absolute output paths remain absolute. |
The Impact columns use these terms:
@@ -106,8 +111,14 @@ runs:
skimjoin:
skim_files:
- C:\build_skims\*.omx
+ network_los_file: C:\skims\network_los.yaml
```
+Runs without a `runs[*].skimjoin` block use the fully resolved global defaults.
+When a run has its own override block, put every path that varies or is required
+for that run in the block. Omitted skim and network paths fall back to the
+selected standalone skimjoin file, not to the other global path overrides.
+
## Top-Level Fields
| Field | Type | Default | Impact | Purpose |
@@ -188,7 +199,7 @@ See [Advanced: Custom Weight Calculations](43-weighting-hosting-extensions.md#ad
| Field | Type | Default | Allowed values | Impact | Notes |
|---|---|---|---|---|---|
-| `steps` | non-empty list of strings | `[summarize, dashboard]` | `prepare`, `skimjoin`, `segment`, `summarize`, `dashboard` | Runtime | Steps must be lowercase, unique, and valid. `skimjoin` requires `prepare`; `segment` requires `summarize`; `dashboard` must be last when present. |
+| `steps` | non-empty list of strings | `[summarize, dashboard]` | `prepare`, `skimjoin`, `segment`, `summarize`, `dashboard` | Runtime | Steps must be lowercase, unique, and valid. Membership selects fixed runtime boundaries; non-dashboard list order does not change execution order. `skimjoin` requires `prepare`; `segment` requires `summarize`; `dashboard` must be last when present. |
| `dashboard_mode` | string | `live` | `none`, `live`, `export`, `host` | Runtime, Presentation | Controls the dashboard step. `host` writes a warning and uses the standard live server. It does not publish an application. |
| `refresh` | list of strings or `all` | `[]` | `prepare`, `skimjoin`, `summarize`, `all` | Runtime | Forces only the named stored stages to rebuild. An upstream refresh invalidates enabled downstream stages. Leave empty for standard cache-aware operation. |
@@ -310,9 +321,12 @@ zones:
## `columns`
-Most `columns` values can be a single string or an ordered list of possible
-source names. The visualizer uses the first available column and reads the
-scalar fields at the start of the table as single names.
+Alias fields can be a single string or an ordered list of possible source
+names; the visualizer uses the first available column. Exactly these fields are
+scalar-only column names: `ptype`, `hhsize`, `auto_ownership`, `num_workers`,
+`num_adults`, and `sample_rate`. Every other field in the table below is an
+alias field and accepts a string or list. A list supplied to a scalar-only
+field is not an alias search and must not be used.
| Field | Default | Impact | Purpose |
|---|---|---|---|
@@ -451,21 +465,39 @@ lookups.
## `skimjoin`
The `skimjoin` section connects the visualizer runtime to a separate skimjoin
-configuration file. See
-[25 - Skimjoin Config Reference](25-skimjoin-config-reference.md) for the
+rules file. See
+[23 - Skimjoin Config Reference](23-skimjoin-config-reference.md) for the
lookup-rule schema.
```text
main visualizer config
pipeline.steps: enables the integrated skimjoin stage
- skimjoin.defaults: selects the standalone config and optional path overrides
+ skimjoin.defaults: selects the rules file and can supply shared data paths
+ runs[*].skimjoin: can select another rules file or supply run-specific paths
-> standalone skimjoin config
- project/activitysim/defaults/modes: defines the actual lookup rules
+ activitysim/defaults/dimensions/modes: defines lookup behavior
+ project: supplies paths only when the main config does not
```
Setting `skimjoin.defaults.config_path` does not start skimjoin; the
`pipeline.steps` list must also contain `prepare` and `skimjoin`.
+For integrated use, `project` is optional in the standalone skimjoin file. The
+effective configuration needs:
+
+| Requirement | Where to set it |
+|---|---|
+| Skimjoin rules file | `skimjoin.defaults.config_path` or `runs[*].skimjoin.config_path`. |
+| At least one skim file | Main-config `skim_files`, or `project.skim_files` in the selected skimjoin file. |
+| `network_los.yaml` | Only when `dimensions.PERIOD.values_from_network_los` is `true`; set it in the main config or as `project.network_los_file`. |
+| Prepared trip and tour input | Supplied by the integrated prepare workflow. Do not set `project.trips_table`, `project.tours_table`, or `project.output_dir` for integrated use. |
+
+Paths in the main config resolve from the main config directory. Paths inside
+the standalone file resolve from that file's directory. Main-config skim and
+network values replace their `project` counterparts. Avoid the standalone
+top-level `skim_files` form when you need main-config overrides; use
+`project.skim_files` or omit the path from the standalone file.
+
| Field | Type | Default | Impact | Notes |
|---|---|---|---|---|
| `defaults.config_path` | path string | none | Prepare, Summary | Shared skimjoin config path. |
@@ -475,26 +507,38 @@ Setting `skimjoin.defaults.config_path` does not start skimjoin; the
| `create_hypothetical_skim_tables` | boolean | `false` | Prepare | Enables configured hypothetical skim tables. |
Run-level `runs[*].skimjoin` supports `config_path`, `skim_files`,
-`network_los_file`, and `create_hypothetical_skim_tables`. If you omit the last
-field, it uses the global value. To enable skimjoin, add it to `pipeline.steps`.
-Do not use the removed `skimjoin.enabled` or `skimjoin.config_path` keys.
+`network_los_file`, and `create_hypothetical_skim_tables`. The run-level
+`config_path` replaces the global path. A supplied run-level skim or network
+path replaces the corresponding value in the selected standalone file.
+`create_hypothetical_skim_tables` inherits the global value when omitted.
+
+A run with no override block uses the complete global resolution. Once a run
+override requires the rules file to be reloaded, omitted skim and network path
+overrides come from that standalone file. Repeat a required global path in the
+run block if the standalone file does not contain it.
+
+To enable skimjoin, add it to `pipeline.steps`. Do not use the removed
+`skimjoin.enabled` or `skimjoin.config_path` keys.
Integrated skim files must resolve to `.omx`, `.csv`, `.h5`, or `.hdf5`.
## `segment`
Use `segment` as the canonical section in user YAML. The loader converts it to
-the segmentation runtime settings.
+the segmentation runtime settings. The section is active only when
+`pipeline.steps` contains both `segment` and `summarize`. See
+[24 - Segmentation](24-segmentation.md) for the runtime flow, relationship
+slicing, output paths, and dashboard behavior.
| Field | Type | Default | Allowed values | Impact | Notes |
|---|---|---|---|---|---|
| `dashboard.segmentation_type` | string | first configured definition | configured definition name | Presentation | Selected segment type shown in dashboard/export. |
| `dashboard.visibility` | string | `full_and_segments` | `full_only`, `segments_only`, `full_and_segments` | Presentation | Whether the dashboard shows full-run outputs, segmented outputs, or both. |
| `definitions` | mapping | required when you enable the segment step | path-safe lowercase names | Summary | Segment definitions. |
-| `definitions.*.include_full` | boolean | `true` | `true`, `false` | Summary | Also build full-run summaries. |
-| `definitions.*.persist_segmented_prepared_tables` | boolean | `false` | `true`, `false` | Prepare, Summary | Persist segment-specific prepared tables. |
+| `definitions.*.include_full` | boolean | `true` | `true`, `false` | Summary | Accepted setting. The current runtime always builds one full-run analysis unit. |
+| `definitions.*.persist_segmented_prepared_tables` | boolean | `false` | `true`, `false` | Prepare, Summary | Accepted setting. The current runtime keeps slices in memory and writes segmented summaries, not segmented prepared directories. |
| `definitions.*.allow_overlapping` | boolean | `false` | `true`, `false` | Summary | Allows one source value to appear in multiple segments. |
-| `definitions.*.on_empty_segment` | string | `warn` | `error`, `warn`, `skip` | Summary | Behavior when a segment has no rows. |
+| `definitions.*.on_empty_segment` | string | `warn` | `error`, `warn`, `skip` | Summary | `error` stops, `skip` omits the unit, and `warn` keeps an empty unit. |
| `definitions.*.source` | mapping | required | `prepared_column` or `csv_lookup` | Summary | Source of segment values. |
| `definitions.*.segments` | list | required | list of segment mappings | Summary, Presentation | Segment ids, labels, and matched values. |
@@ -520,7 +564,13 @@ segment:
values: [1]
```
-`source_table` may be `hh`, `per`, `tours`, `trips`, or `land_use`.
+Prepared-column source fields:
+
+| Field | Type | Default | Notes |
+|---|---|---|---|
+| `type` | string | `prepared_column` | Must be `prepared_column`. |
+| `source_table` | string | auto-detect | Use `households`, `persons`, `day`, `tours`, `trips`, `vehicles`, `joint_tour_participants`, or `land_use`, or the runtime aliases `hh`, `per`, and `joint_participants`. Auto-detection requires the column to occur in exactly one table. |
+| `column` | string | required | Prepared column containing the values matched by `segments[*].values`. |
CSV lookup source:
@@ -542,6 +592,24 @@ segment:
values: [north]
```
+CSV lookup source fields:
+
+| Field | Type | Notes |
+|---|---|---|
+| `type` | string | Must be `csv_lookup`. |
+| `file` | path string | CSV path relative to the main config directory. |
+| `join.source_table` | string | Prepared anchor table using the names above. |
+| `join.source_key_column` | string | Join key on the prepared table. |
+| `join.csv_key_column` | string | Join key in the CSV. One key cannot map to different segment values. |
+| `segment_value_column` | string | CSV column matched by `segments[*].values`. |
+
+Each segment requires a path-safe lowercase `id`, a nonblank `label`, and one
+or more `values`. A path-safe name matches `[a-z0-9._-]+` and cannot start or
+end with `.`, `_`, or `-`. Prepared-column values must match the prepared
+column's type; CSV lookup segment values must be strings. Definitions and
+segments are cached independently. The full run uses standard summary paths; segment output uses
+`summary_tables//segments///`.
+
## `summarize`
| Field | Type | Default | Allowed values | Impact | Notes |
@@ -553,10 +621,10 @@ segment:
| `group_joint_tour_purposes` | boolean | `true` | `true`, `false` | Summary | Group joint tour purposes in summaries. |
| `group_atwork_tour_purposes` | boolean | `true` | `true`, `false` | Summary | Group at-work tour purposes in summaries. |
| `group_school_tour_purposes` | boolean | `true` | `true`, `false` | Summary | Group school tour purposes in summaries. |
-| `geography.enabled` | boolean | `false` | `true`, `false` | Summary, Presentation | Enables custom geography mapping and aggregations. |
-| `geography.landuse_col` | string | none | land-use column | Summary | Existing land-use geography column used for geography summaries. |
-| `geography.mapping` | mapping | none | raw value to label | Summary, Presentation | Label mapping for geography values. |
-| `geography.aggregations` | mapping | none | aggregation definitions | Summary, Presentation | Additional zone-to-geography lookup definitions. |
+| `geography.enabled` | boolean | `false` | `true`, `false` | Prepare, Summary, Presentation | Enables the legacy mapping and named geography aggregations. Disabled definitions are ignored. |
+| `geography.landuse_col` | string | none | land-use column | Prepare, Summary | Existing land-use column used to create compatibility `HGEO` and `WGEO` fields. |
+| `geography.mapping` | mapping | none | raw value to label | Prepare, Summary, Presentation | Optional normalization for values from `landuse_col`. |
+| `geography.aggregations` | mapping | `{}` | aggregation definitions | Prepare, Summary, Presentation | Named zone-to-geography lookups that create role-specific prepared columns. |
Each `geography.aggregations.*` entry requires these fields:
@@ -568,6 +636,10 @@ Each `geography.aggregations.*` entry requires these fields:
| `zone_id_col` | string | Required with `file`. |
| `geography_col` | string | Required with `file`. |
+For a complete explanation of source zones, generated columns, summary fields,
+dashboard labels, cache behavior, and lookup validation, see
+[27 - Geography](27-geography.md).
+
```yaml
summarize:
weighting_modes: [weighted, unweighted]
@@ -778,5 +850,9 @@ prepare:
- [11 - Configuring Your Data](11-configuring-your-data.md)
- [12 - Running Workflows](12-running-workflows.md)
+- [14 - Input Data Contract](14-input-data-contract.md)
+- [15 - Cache And Manifest Reference](15-cache-manifest-reference.md)
- [21 - Prepared Tables](21-prepared-tables.md)
-- [25 - Skimjoin Config Reference](25-skimjoin-config-reference.md)
+- [23 - Skimjoin Config Reference](23-skimjoin-config-reference.md)
+- [24 - Segmentation](24-segmentation.md)
+- [27 - Geography](27-geography.md)
diff --git a/wiki/14-input-data-contract.md b/wiki/14-input-data-contract.md
new file mode 100644
index 0000000..ce4acac
--- /dev/null
+++ b/wiki/14-input-data-contract.md
@@ -0,0 +1,197 @@
+# 14 - Input Data Contract
+
+This chapter defines the boundary between ActivitySim output, canonical
+prepared tables, summaries, and dashboard pages. Use it when you create a new
+configuration, generate tables outside the visualizer, or diagnose an
+unavailable summary.
+
+The visualizer accepts three input levels:
+
+```text
+raw ActivitySim tables -> prepare -> canonical prepared tables -> summarize
+canonical prepared tables -------------------------------> summarize
+registered summary tables --------------------------------> dashboard
+```
+
+The level you supply determines which checks and transformations the visualizer
+can perform.
+
+## Raw Table Inventory
+
+The raw reader recognizes these logical table IDs. The default file stems are
+defined by `files` and can be changed globally or with `runs[*].file_map`.
+
+| Table ID | Default stem | Coverage role | Main keys and relationships |
+|---|---|---|---|
+| `households` | `final_households` | Core | One row per `household_id`. Supplies household attributes and home zone. |
+| `persons` | `final_persons` | Core | One row per `person_id`; `household_id` refers to households. |
+| `tours` | `final_tours` | Core | One row per `tour_id`; normally contains `person_id` and `household_id`. |
+| `trips` | `final_trips` | Core | One row per `trip_id`; normally contains `tour_id`, `person_id`, and `household_id`. |
+| `day` | `final_day` | Optional | Person-day or household-day rows. Uses `day_id`, `person_id`, and/or `household_id`. |
+| `vehicles` | `final_vehicles` | Optional | Vehicle rows related to households by `household_id`. |
+| `joint_tour_participants` | `final_joint_tour_participants` | Optional | Participation rows related by `tour_id` and `person_id`. |
+| `land_use` | `final_land_use` | Optional | One row per MAZ or TAZ. Supplies employment, enrollment, parking, and geography data. |
+
+“Core” means the table is part of the standard household-person-tour-trip
+model and is needed for broad default-page coverage. The reader does not stop
+when one core file is absent. It marks that table `unavailable`, continues with
+the tables it can load, and lets summary contracts identify the affected
+outputs. A run is skipped only when none of the four core tables is usable.
+
+Optional tables can still be required by individual summaries or pages. For
+example, vehicle-characteristic summaries need `vehicles`, and several
+geography or parking outputs need `land_use`.
+
+## Raw File And Column Rules
+
+For a configured stem without an extension, the reader first looks for
+`.parquet` and then `.csv` in the run directory. An explicit `.csv`
+or `.parquet` name selects only that file. A configured `fallback_files` path
+is tried after the run-local file is absent; fallbacks are supported for
+`day`, `vehicles`, `joint_tour_participants`, and `land_use`.
+
+Raw column names are not a fixed schema. The `columns` and `zones` settings
+select source names and prepare copies the first available alias into a
+canonical column. These are the minimum relationship concepts for a fully
+connected standard run:
+
+| Concept | Canonical name | Expected tables |
+|---|---|---|
+| Household key | `household_id` | households, persons, tours, trips; optional day and vehicles |
+| Person key | `person_id` | persons, tours, trips; optional day and joint participants |
+| Tour key | `tour_id` | tours, trips, joint participants |
+| Trip key | `trip_id` | trips |
+| Home zone | `home_zone_id` | households and/or persons |
+| Work/school zone | `workplace_zone_id`, `school_zone_id` | persons |
+| Origin/destination | `origin`, `destination` | tours and trips |
+| Land-use zone | `MAZ`, `TAZ` | land use |
+
+The [configuration reference](13-configuration-reference.md#columns) lists all
+configurable aliases. A missing concept does not necessarily invalidate its
+whole table. It makes calculations that declare that column unavailable.
+
+## Canonical Prepared Contract
+
+Prepare preserves source columns and adds or normalizes canonical fields. The
+portable contract is therefore a set of stable concepts, not one exhaustive
+column list for every regional model.
+
+| Prepared table | Stable identifiers | Common normalized or derived fields |
+|---|---|---|
+| `hh` | `household_id` (`Int64`) | `home_zone_id`, `HHVEH`, `HHSIZE`, `WORKERS`, `ADULTS`, `AUTOSUFF`, `HGEO`, `finalweight` |
+| `per` | `person_id`, `household_id` (`Int64`) | `person_type`, home/work/school zones and geographies, worker/student fields, work/school distance, `finalweight` |
+| `day` | `day_id`, `person_id`, `household_id` (`Int64` when present) | activity pattern, date/day fields, `finalweight` |
+| `tours` | `tour_id`, `person_id`, `household_id` (`Int64`) | purpose, mode, category, time, stops, zones, distance, geography, `finalweight` |
+| `trips` | `trip_id`, `tour_id`, `person_id`, `household_id` (`Int64`) | purpose, mode, departure, direction, stops, zones, distance, geography, `finalweight` |
+| `vehicles` | `vehicle_id`, `household_id` (`Int64`) | number, type, body, fuel, age, `finalweight` |
+| `joint_participants` | `tour_id`, `person_id` (`Int64`) | participant attributes retained from the source |
+| `land_use` | `MAZ`, `TAZ` (`Int64` when present) | employment/enrollment, parking, and named geography fields |
+
+The table-specific finalizer casts known canonical numeric fields to integer or
+`Float64`, categorical fields to strings, and `finalweight` to `Float64`.
+Columns not owned by the canonical contract keep their source types. See
+[21 - Prepared Tables](21-prepared-tables.md) for the enrichment stages and
+[26 - Summary Catalog](26-summary-catalog.md) for the exact fields required by
+each summary.
+
+## Prepared Relationship Checks
+
+After prepare or `prepared_table_map` loading, the runtime can check these
+foreign-key relationships:
+
+| Source | Source key | Target | Target key |
+|---|---|---|---|
+| persons | `household_id` | households | `household_id` |
+| day | `household_id` | households | `household_id` |
+| day | `person_id` | persons | `person_id` |
+| tours | `household_id` | households | `household_id` |
+| tours | `person_id` | persons | `person_id` |
+| trips | `household_id` | households | `household_id` |
+| trips | `person_id` | persons | `person_id` |
+| trips | `tour_id` | tours | `tour_id` |
+| vehicles | `household_id` | households | `household_id` |
+| joint participants | `person_id` | persons | `person_id` |
+| joint participants | `tour_id` | tours | `tour_id` |
+
+A check is skipped when a table or key column is unavailable. With
+`prepare.validation.relationships: warn`, orphan rows produce warnings. With
+`error`, they stop the workflow. Direct aggregations can still count an orphan
+row, while an aggregation that joins to the parent can drop it. Fixing keys is
+therefore preferable to suppressing the check.
+
+## Using `prepared_table_map`
+
+`prepared_table_map` loads CSV or Parquet files directly into `RunData`. It
+does not run canonicalization, enrichment, weighting, geography mapping, or
+integrated skimjoin. Supply canonical fields and types yourself.
+
+The accepted keys are the eight config/file table IDs in the inventory above.
+Omitted optional tables are marked `unavailable`. Omitted core tables are
+represented as empty tables. Files that do not exist are `unavailable`; files
+that cannot be read are `failed`. These states and their details flow into
+summary and page diagnostics.
+
+Before using custom prepared tables, verify:
+
+1. IDs and foreign keys use compatible types and values.
+2. Every requested summary has its required columns from chapter 26.
+3. Every weighted table has the intended `finalweight`.
+4. Geography and skimjoin columns already exist if the corresponding outputs
+ depend on them.
+5. Named weighting source columns are present when configured.
+
+## Using `summary_table_map`
+
+A mapped summary file is already at the final aggregation boundary. Its key
+must be a registered summary ID, and its columns, order, and Polars-compatible
+types must match that summary's declared schema.
+
+The visualizer cannot derive prepared rows, alternate weights, or segment
+membership from an aggregated summary file. Built-in weighted and unweighted
+modes copy the same mapped table into both modes. Declarative or custom modes
+normally reject mapped summaries unless their registered external-summary
+policy explicitly permits copying.
+
+If a run also has raw or prepared input, mapped summary tables replace the same
+generated IDs and leave other generated summaries unchanged. During
+segmentation, the same mapped table overlays every analysis unit; it does not
+change by segment.
+
+## Availability States
+
+Tables and summaries use four stored states:
+
+| State | Meaning |
+|---|---|
+| `available` | The source loaded or the calculation returned rows. |
+| `empty` | The source or valid result contains no rows. |
+| `unavailable` | A file, table, or declared prerequisite is absent. |
+| `failed` | Reading or calculation raised an error under the recording policy. |
+
+An empty cache file uses an internal `__empty__` sentinel column so CSV and
+Parquet can store an otherwise zero-column frame. The loader converts it back
+to an empty `DataFrame`; user-created input tables should not use this sentinel
+as application data.
+
+## Contract Checklist
+
+When adding or exchanging data across the boundary:
+
+1. Use config table IDs for file mappings and `RunData` names in Python
+ contracts.
+2. Preserve unique IDs and valid relationships.
+3. Materialize canonical fields before bypassing prepare.
+4. Declare exact summary inputs and output schema.
+5. Treat units and weighting as part of the data contract, even when the file
+ format cannot encode them.
+6. Test unavailable, empty, and partial-run behavior, not only the complete
+ case.
+
+## Related Chapters
+
+- [11 - Configuring Your Data](11-configuring-your-data.md)
+- [13 - Configuration Reference](13-configuration-reference.md)
+- [15 - Cache And Manifest Reference](15-cache-manifest-reference.md)
+- [21 - Prepared Tables](21-prepared-tables.md)
+- [26 - Summary Catalog](26-summary-catalog.md)
+- [90 - Troubleshooting](90-troubleshooting.md)
diff --git a/wiki/15-cache-manifest-reference.md b/wiki/15-cache-manifest-reference.md
new file mode 100644
index 0000000..95bd6de
--- /dev/null
+++ b/wiki/15-cache-manifest-reference.md
@@ -0,0 +1,236 @@
+# 15 - Cache And Manifest Reference
+
+Prepared and summary caches are stored contracts, not temporary copies of
+arbitrary files. Each cache directory has a `manifest.json` that records its
+schema, inputs, configuration identity, table inventory, and diagnostics.
+
+Do not edit a manifest by hand. Change the source data or configuration and let
+the workflow rebuild the affected stage.
+
+## Directory Layout
+
+For run key `base`, the standard layout is:
+
+```text
+/base/
+ manifest.json # summary manifest
+ summary_tables/
+ weighted/.csv
+ unweighted/.csv
+ weighted/segments///.csv
+ base_prepared_tables/ # present when skimjoin is enabled
+ manifest.json
+
+ prepared_tables/
+ manifest.json # final prepared/skimjoin manifest
+
+
+ skimjoin/
+```
+
+Without skimjoin, prepare writes only `prepared_tables`. With skimjoin,
+`base_prepared_tables` stores the reusable pre-skim boundary and
+`prepared_tables` stores the enriched result.
+
+## Prepared Manifest
+
+The current prepared schema version is an implementation value. The loader can
+read a limited set of earlier versions, but users should rely on the fields,
+not a hard-coded version number. A shortened example is:
+
+```json
+{
+ "schema_version": 9,
+ "source": "activitysim-visualizer-prepared-cache",
+ "label": "Base",
+ "run_key": "base",
+ "source_run_dir": "C:\\models\\base\\output",
+ "prepare_config_digest": "...",
+ "table_format": "parquet",
+ "table_root": "prepared_tables",
+ "table_files": {
+ "households": "households.parquet",
+ "persons": "persons.parquet",
+ "tours": "tours.parquet",
+ "trips": "trips.parquet"
+ },
+ "table_states": {
+ "households": "available",
+ "day": "unavailable"
+ },
+ "table_diagnostics": {
+ "day": "Cannot find 'final_day.parquet' or 'final_day.csv' ..."
+ },
+ "run_fingerprint": {},
+ "identity": {
+ "raw_inputs": {},
+ "prepare_config": "...",
+ "skimjoin_config": null,
+ "skim_inputs": []
+ }
+}
+```
+
+Important fields:
+
+| Field | Meaning |
+|---|---|
+| `schema_version`, `source`, `generated_at_utc` | Storage format, producer, and write time. |
+| `label`, `run_key`, `source_run_dir` | Display identity, cache identity, and raw source location. |
+| `config_path`, `prepare_config_digest` | Configuration source and normalized prepare identity. |
+| `table_format`, `table_root`, `table_files` | How to find each prepared table. |
+| `sidecar_root`, `sidecar_files` | Optional hypothetical skim sidecar files. |
+| `table_states`, `table_diagnostics` | Per-table `available`, `empty`, `unavailable`, or `failed` state and reason. |
+| `unavailable_tables`, `failed_tables` | Compatibility views of the same table diagnostics. |
+| `source_file_map`, `run_fingerprint` | Resolved raw input mapping, file identities, skims, run weights, and run-level overrides. |
+| `identity` | Compact upstream identity used by later stages. |
+| `hh_weight_col`, `person_weight_col`, `trip_weight_col` | Primary run-level source weight fields. |
+| `prepare_diagnostics` | Recorded preparation warnings and relationship results. |
+| `skimjoin_*` | Enabled/status/config/input identity, applied outputs, skipped rules, warning/fallback counts, failure detail, and sidecar row counts. |
+
+Every configured prepared table has an entry in `table_files`, including empty
+or unavailable tables. Those tables are stored with the internal empty
+sentinel and restored according to `table_states`.
+
+## Summary Manifest
+
+The run-level summary manifest covers the full run and every segment. A
+shortened example is:
+
+```json
+{
+ "schema_version": 15,
+ "source": "activitysim-visualizer-summary-cache",
+ "label": "Base",
+ "run_key": "base",
+ "summary_config_digest": "...",
+ "weighting_modes": ["weighted", "unweighted"],
+ "summary_ids": ["population_totals", "trip_mode_by_tour_purpose_and_tour_mode"],
+ "summary_files": {
+ "population_totals": "population_totals.csv"
+ },
+ "summary_states": {
+ "weighted": {"population_totals": "available"}
+ },
+ "summary_diagnostics": {"weighted": {}},
+ "summary_digests": {
+ "weighted": {"population_totals": "..."}
+ },
+ "prepared_manifest_identity": {},
+ "segmentation_enabled": true,
+ "segmentation_types": []
+}
+```
+
+Important fields:
+
+| Field | Meaning |
+|---|---|
+| `schema_version`, `source`, `generated_at_utc` | Storage format, producer, and write time. |
+| `label`, `run_key`, `source_run_dir` | Run identity and source location. |
+| `summary_config_digest` | Normalized configuration that can affect summaries. |
+| `weighting_modes` | Stored mode IDs in dashboard order. |
+| `summary_ids`, `summary_files` | Registered IDs and their cache filenames. |
+| `empty_summaries`, `summary_states` | Per-mode empty/state inventory. |
+| `unavailable_summaries`, `failed_summaries`, `summary_diagnostics` | Per-mode problem inventory and explanations. |
+| `summary_digests` | Per-mode declaration/implementation identity. It allows one changed builder to rebuild without discarding unrelated tables. |
+| `run_fingerprint` | Run and external-summary input identity. |
+| `prepared_manifest_identity` | Exact prepared source/config identity used by the summaries. |
+| `identity` | Compact upstream-prepared and summary-config identity. |
+| `segmentation_enabled`, `segmentation_types` | Stored analysis-unit definitions and segment metadata. |
+
+Each entry in `segmentation_types` contains the definition name and source,
+plus a `segments` list. Each segment records its ID, label, matched values,
+source columns or CSV join, summary roots, states, diagnostics, and digests.
+
+## What Makes A Cache Stale
+
+The cache identity is intentionally stage-specific:
+
+| Change | Earliest affected stage |
+|---|---|
+| Raw file path, size, or modification time | prepare |
+| Raw file mapping, run weight fields, prepare enrichment settings | prepare |
+| Skimjoin rules or resolved skim inputs | skimjoin |
+| Geography mapping rows | prepare, then summarize |
+| Segmentation definition or values | affected summary analysis units |
+| Weighting definition or summary configuration | summarize |
+| One summary declaration, builder location, schema, or requirements | that summary in each affected analysis unit |
+| Dashboard labels, page selection, or layout | presentation only; no processor rebuild |
+
+File identity uses resolved path, byte size, and nanosecond modification time.
+It does not hash the full file contents. Replacing content while preserving all
+three values can defeat automatic detection; use an explicit refresh in that
+unusual case.
+
+## Read `--explain-cache` Output
+
+Run:
+
+```bash
+uv run activitysim-viz --config local_config.yaml --explain-cache
+```
+
+The command does not load tables, execute builders, create the cache root, or
+start the dashboard. It prints one plan per run:
+
+```text
+Pipeline plan - Base
+ prepare REUSE
+ skimjoin DISABLED
+ summarize REBUILD - 1 analysis-unit summary tables are stale; 0 analysis units are obsolete
+ dashboard RUN
+```
+
+| Action | Meaning |
+|---|---|
+| `REUSE` | The stored manifest agrees with current identity and requirements. |
+| `REBUILD` | The cache is missing, explicitly refreshed, stale, incompatible, or downstream of a rebuilding stage. The rest of the line gives the reason. |
+| `RUN` | The non-persistent dashboard action will execute. |
+| `DISABLED` | The logical step is not selected. |
+
+A summarize `REBUILD` decision does not always mean every summary will run.
+The summarize cache can reuse compatible tables and rebuild only stale summary
+IDs or analysis units.
+
+## Refresh Boundaries
+
+Prefer the narrowest repeatable refresh:
+
+```yaml
+pipeline:
+ refresh: [skimjoin]
+```
+
+| Refresh | Rebuilds | Keeps |
+|---|---|---|
+| `prepare` | raw prepare, enabled skimjoin, and summaries | nothing downstream |
+| `skimjoin` | final skimjoined prepared cache and summaries | `base_prepared_tables` |
+| `summarize` | full and segmented summaries | final prepared cache |
+| `all` | every enabled stored stage | dashboard has no stored stage to refresh |
+
+After a one-time diagnostic refresh, remove it or set `refresh: []` so normal
+reuse resumes.
+
+## Safe Inspection And Recovery
+
+1. Run `--explain-cache` before deleting anything.
+2. Read the manifest state and diagnostic fields.
+3. Confirm the run key; duplicate normalized labels can add `-1`, `-2`, and so
+ on.
+4. Use `pipeline.refresh` when the cache is valid but you intentionally need a
+ rebuild.
+5. If a cache is corrupt, remove only the exact run/stage directory and rerun
+ the selected workflow. A removed cache is recoverable only by rebuilding it.
+
+Do not move one run's manifest into another run directory, copy tables without
+their manifest, or change digests to force reuse. These actions bypass the
+identity checks that protect cross-run comparisons.
+
+## Related Chapters
+
+- [12 - Running Workflows](12-running-workflows.md)
+- [14 - Input Data Contract](14-input-data-contract.md)
+- [21 - Prepared Tables](21-prepared-tables.md)
+- [24 - Segmentation](24-segmentation.md)
+- [90 - Troubleshooting](90-troubleshooting.md)
diff --git a/wiki/16-dashboard-user-guide.md b/wiki/16-dashboard-user-guide.md
new file mode 100644
index 0000000..51f36a7
--- /dev/null
+++ b/wiki/16-dashboard-user-guide.md
@@ -0,0 +1,57 @@
+# 16 - Dashboard User Guide
+
+Use this guide to choose a dashboard page and interpret its controls and
+results. For general instructions about selecting runs, changing global
+controls, reading partial results, and exporting the dashboard, see
+[30 - Output Visualizer](30-output-visualizer.md).
+
+The visualizer currently registers 27 pages. Page-local controls are
+data-driven, so a selector can be absent or have fewer choices when its source
+summary is unavailable. Pages that are disabled by default must be enabled in
+the dashboard configuration before they appear.
+
+## Page-By-Page Guide
+
+The Group column gives the navigation item that contains the page. Open that
+group, then select the page shown in the Page column. A `Standalone` page is a
+top-level navigation item and does not belong to a group.
+
+| Group | Page | What it answers | Main interpretation or controls |
+|---|---|---|---|
+| N/A | Overview | How large is each run, and what is its basic household/person and auto-VMT profile? | Population cards and core distributions. Difference cards use the first run as the base. |
+| Daily Travel | Daily Activity Pattern | What daily activity patterns and mandatory/non-mandatory tour frequencies occur by person type? | Select person type. Tour/trip rates are per person-day measures and do not become shares with the global Values control. |
+| Daily Travel | Escorted Tours | How much school escorting occurs, who chauffeurs, and what are the tour-leg/stop/distance patterns? | Direction, escort category, student-count, and person-type views depend on available required and optional escort summaries. |
+| N/A | Joint Travel | How common are joint-tour patterns, party sizes, compositions, and household participation? | Household size and party size provide different denominators; read each axis and note before comparing percentages. |
+| Long-Term Choices | Individual Choices | How do license holding, bicycle comfort, transit-pass ownership, and transit subsidy vary by person type? | Person-type selectors apply to individual features. Each feature can be unavailable independently. |
+| Long-Term Choices | Vehicle Ownership and Type | How many vehicles do households own, and what are the modeled age, fuel, and body-type distributions? | Household-size and vehicle-characteristic views use different source tables and populations. Allocated tour vehicle outputs are on Tour Mode, not this page. |
+| Long-Term Choices | Mandatory Location Choice | Where are workers and students located, how far do they travel, and how common are work-from-home/telecommute choices? | Geography, location subject, and distance views use home, work, or school roles. “All geographies” and a selected geography can use different display logic. |
+| Long-Term Choices | Employment/Enrollment Match By Geography | How closely do modeled workers/students match employment/enrollment targets? | Choose geography and, for school results, student type. Residual is modeled minus target; percent error needs a nonzero target. |
+| Skim Summaries | Tour Skims | What are the distributions and statistics of observed or hypothetical tour skim components? | Select skim scenario/family, direction, component, and mode from available data. Prepared tour data adds detailed views in live mode; export remains summary-based. |
+| Skim Summaries | Trip Skims | What are the distributions and statistics of observed or hypothetical trip skim components? | Select skim scenario/family, component, and trip mode. Units come from each skim component and are not converted. |
+| Tour Summaries | Tour Purpose | What shares or counts of tours occur by category and purpose? | Category and purpose are separate summaries. Percent mode compares distributions within each plotted total. |
+| Tour Summaries | Tour Mode | How do tour modes vary by purpose and auto sufficiency, and what vehicle characteristics are allocated to auto tours? | Purpose, auto-sufficiency, occupancy, and vehicle-characteristic controls use distinct summary features. |
+| Tour Summaries | Tour Time | When do tours start and end, and how long do they last? | Select tour purpose. Time bins follow prepared/configured time values. |
+| Tour Summaries | Tour Distance | What are tour distance distributions and average distances by purpose and home geography? | Purpose and geography controls affect different views. Existing labels assume prepared distance is in miles. |
+| Tour Summaries | Tour Stop Frequency | How many outbound, inbound, and total stops occur, and how frequent are at-work subtours? | Select purpose where offered; stop-frequency codes and derived counts are different measures. |
+| Tour Summaries | Internal vs. External Tours | How often do non-mandatory tours cross the model boundary, and where are external destinations? | Select home or destination geography from available summary rows. Do not add overlapping geography totals. |
+| Tour Summaries | Park-and-Ride Location | How do modeled PNR tour counts compare with lot capacity? | Select geography. Residuals require valid PNR modes, zones, and capacity data; MAZ output can be hidden by dashboard config. |
+| Trip Summaries | Trip and Stop Purpose | What trip purposes occur, and what purposes occur at intermediate stops within each tour purpose? | Select tour purpose for the stop view. Trips and stops use different count fields and denominators. |
+| Trip Summaries | Trip Mode | How does trip mode vary by tour purpose and tour mode? | Tour-purpose and tour-mode selectors filter the registered three-dimensional summary. |
+| Trip Summaries | Trip and Stop Time | When do trips and stops depart? | Select tour purpose. Departure trip count and departure stop count are separate series. |
+| Trip Summaries | Trip and Stop Distance | What are direct trip distances and stop out-of-direction distances? | Select tour purpose and distance range where available. The two charts use different prepared distance fields. |
+| Trip Summaries | Parking Location | How do trips parked by zone compare with parking capacity? | Disabled by default and requires live prepared `land_use`. Current summary geography is the base parking MAZ/TAZ, not every named aggregation. |
+| Validation Summaries | Traffic Validation | How closely do modeled link/count-location/screenline volumes match observations? | Select period and facility type. RMSE, RMSPE, R-squared, scatter, fit, and screenline outputs have distinct valid-data rules. |
+| Validation Summaries | Transit Validation | How do boardings and transfer rates vary by operator, technology, and access mode? | Uses supplied validation summary contracts. A missing operator/technology field can remove only the affected feature. |
+| Validation Summaries | VMT Validation | How does personal-auto and non-motorized VMT vary by home geography, income, household size, period, and mode? | Many selectors are dependent. Optional outside tables add external, commercial, and bicycle outputs independently. |
+| Validation Summaries | Regional Validation | How do modeled district/county commute flows compare with observed matrices? | Disabled by default. Select flow type and metric: modeled, observed, difference, percent difference, or absolute percent difference. Percent difference needs nonzero observed flow. |
+
+For page IDs, default-enabled status, data prerequisites, and extension
+contracts, see [31 - Dashboard Page Contract](31-dashboard-pages.md).
+
+## Related Chapters
+
+- [10 - Getting Started](10-getting-started.md)
+- [11 - Configuring Your Data](11-configuring-your-data.md)
+- [12 - Running Workflows](12-running-workflows.md)
+- [30 - Output Visualizer](30-output-visualizer.md)
+- [34 - HTML Export](34-html-export.md)
diff --git a/wiki/20-output-processor.md b/wiki/20-output-processor.md
index 55b92d9..378190b 100644
--- a/wiki/20-output-processor.md
+++ b/wiki/20-output-processor.md
@@ -59,8 +59,10 @@ contract. They must not use raw, model-specific table layouts.
|---|---|---|
| Prepare | [21 - Prepared Tables](21-prepared-tables.md) | Normalize raw outputs and add derived fields. |
| Skimjoin | [22 - Skimjoin](22-skimjoin.md) | Add skim-derived trip and tour columns. |
-| Summaries | [23 - Summary Functions](23-summary-functions.md) | Build dashboard-ready tables. |
-| Summary catalog | [24 - Summary Catalog](24-summary-catalog.md) | Inspect registered summary outputs. |
+| Segmentation | [24 - Segmentation](24-segmentation.md) | Slice related prepared tables and repeat summaries for configured subsets. |
+| Summaries | [25 - Summary Functions](25-summary-functions.md) | Build dashboard-ready tables. |
+| Summary catalog | [26 - Summary Catalog](26-summary-catalog.md) | Inspect registered summary outputs. |
+| Geography | [27 - Geography](27-geography.md) | Add consistent MAZ-, TAZ-, and custom geography fields. |
The former static prepared-cache schema described one `estimation-output` data
set, including its row counts and model-specific columns. Because those details
@@ -124,6 +126,8 @@ To add processor behavior:
- [21 - Prepared Tables](21-prepared-tables.md)
- [22 - Skimjoin](22-skimjoin.md)
-- [23 - Summary Functions](23-summary-functions.md)
+- [24 - Segmentation](24-segmentation.md)
+- [25 - Summary Functions](25-summary-functions.md)
+- [27 - Geography](27-geography.md)
- [44 - Summary Function Cookbook](44-summary-function-cookbook.md)
- [40 - Developer Workflows](40-developer-workflows.md)
diff --git a/wiki/21-prepared-tables.md b/wiki/21-prepared-tables.md
index 0c05d5a..558d161 100644
--- a/wiki/21-prepared-tables.md
+++ b/wiki/21-prepared-tables.md
@@ -4,6 +4,11 @@ Prepared tables are the canonical form of ActivitySim output. They remove
differences in raw file names and provide stable fields for summaries and
dashboard pages.
+[14 - Input Data Contract](14-input-data-contract.md) defines the exact input
+inventory, canonical identifiers and types, relationship checks, availability
+states, and requirements for bypassing prepare. This chapter explains how the
+processor creates and extends that contract.
+
## Prepare Data Flow
```text
@@ -81,7 +86,7 @@ Use the prepared field when it exists. Do not search for raw names in a summary
or page.
This introductory list does not imply that every table has every field. For a
-specific summary, the generated catalog in chapter 24 lists the required
+specific summary, the generated catalog in chapter 26 lists the required
prepared columns. At runtime, `@summary` requirements and prepared-table
availability metadata determine whether a calculation can run.
@@ -98,7 +103,7 @@ To inspect a cache:
2. Examine the Parquet or CSV schema for the relevant table.
3. Use `processor.models.RunData` names at runtime and the file/config names in
[Prepared Table Names](#prepared-table-names).
-4. Use the generated [Summary Catalog](24-summary-catalog.md) to find the exact
+4. Use the generated [Summary Catalog](26-summary-catalog.md) to find the exact
prepared columns required by each registered summary.
Add stable fields to the relevant prepare enrichment module, with a prepare
@@ -160,6 +165,10 @@ the [complete example](41-data-extension-cookbook.md#worked-example-add-a-prepar
## Related Chapters
- [11 - Configuring Your Data](11-configuring-your-data.md#already-prepared-tables)
-- [23 - Summary Functions](23-summary-functions.md)
-- [31 - Dashboard Pages](31-dashboard-pages.md)
+- [14 - Input Data Contract](14-input-data-contract.md)
+- [15 - Cache And Manifest Reference](15-cache-manifest-reference.md)
+- [25 - Summary Functions](25-summary-functions.md)
+- [24 - Segmentation](24-segmentation.md)
+- [27 - Geography](27-geography.md)
+- [31 - Dashboard Page Contract](31-dashboard-pages.md)
- [01 - Architecture](01-architecture.md)
diff --git a/wiki/22-skimjoin.md b/wiki/22-skimjoin.md
index 6f784dc..439f71d 100644
--- a/wiki/22-skimjoin.md
+++ b/wiki/22-skimjoin.md
@@ -4,24 +4,123 @@ Skimjoin adds skim-derived columns to prepared trips and tours. This optional
final part of prepare runs after the raw output has been normalized.
Use skimjoin when summaries or dashboard pages require values from OMX skims or
-sidecar lookup files. Examples are time, cost, distance, walk access, and
-combined tour attributes.
+sidecar lookup files. Examples are time, cost, distance, walk access, and direct
+trip or tour attributes.
For each skimjoin field, lookup rule, default, and example, see
-[25 - Skimjoin Config Reference](25-skimjoin-config-reference.md).
+[23 - Skimjoin Config Reference](23-skimjoin-config-reference.md).
Integrated skimjoin uses two YAML files:
- the **main visualizer config** enables the step with `pipeline.steps` and
- specifies files in `skimjoin.defaults` or run overrides; and
-- the **standalone skimjoin config** defines `project`, `activitysim`,
- dimensions, mode/component lookup rules, fallbacks, and tour aggregation.
+ selects the rules file. It can also supply shared or run-specific data paths.
+- the **skimjoin rules file** defines prepared column names, defaults,
+ dimensions, mode/component lookup rules, and fallbacks. Its optional
+ `project` block supplies data paths for the standalone CLI or as integrated
+ defaults.
Paths in the first file start from the main configuration file, while paths in
the second start from the standalone skimjoin configuration file. Providing a
configuration path does not enable the step; `pipeline.steps` must contain both
`prepare` and `skimjoin`.
+## Where Path Settings Belong
+
+For integrated use, the main config can own the skim paths while the skimjoin
+file contains only reusable lookup rules. This is useful when several model
+runs share rules but use different skim files.
+
+Main visualizer config:
+
+```yaml
+pipeline:
+ steps: [prepare, skimjoin, summarize, dashboard]
+
+skimjoin:
+ defaults:
+ config_path: configs\skimjoin_rules.yaml
+ skim_files:
+ - skims\shared\*.omx
+ network_los_file: skims\shared\network_los.yaml
+
+runs:
+ - dir: C:\models\base\output
+ label: Base
+ - dir: C:\models\build\output
+ label: Build
+ skimjoin:
+ skim_files:
+ - skims\build\*.omx
+ network_los_file: skims\build\network_los.yaml
+```
+
+`configs\skimjoin_rules.yaml`:
+
+```yaml
+activitysim:
+ trip_mode_column: trip_mode
+ tour_mode_column: tour_mode
+ trip_id_column: trip_id
+ tour_id_column: tour_id
+ outbound_column: outbound
+
+defaults:
+ origin: OTAZ
+ destination: DTAZ
+ output_prefix: skim_
+
+dimensions:
+ PERIOD:
+ source_columns:
+ trip_source_column: depart
+ outbound_tour_source_column: start
+ inbound_tour_source_column: first_inbound_trip_depart
+ values_from_network_los: true
+
+modes:
+ SOV:
+ time: SOV_TIME__{PERIOD}
+ distance: SOV_DIST__{PERIOD}
+```
+
+In this pattern, `project` is not required. The integrated prepare workflow
+already supplies trip and tour tables, and the main config supplies the skims.
+
+Use `project` when the skimjoin file must be self-contained, especially for the
+standalone CLI:
+
+```yaml
+project:
+ skim_files:
+ - C:\skims\*.omx
+ network_los_file: C:\skims\network_los.yaml
+ trips_table: C:\prepared\trips.parquet
+ tours_table: C:\prepared\tours.parquet
+ output_dir: C:\skimjoin_output
+```
+
+For integrated use, only `project.skim_files` and, when needed,
+`project.network_los_file` are relevant. `project.trips_table`,
+`project.tours_table`, and `project.output_dir` belong to the standalone CLI.
+
+The effective integrated settings follow these rules:
+
+1. `runs[*].skimjoin.config_path` replaces `skimjoin.defaults.config_path` for
+ that run.
+2. Main-config `skim_files` and `network_los_file` replace the corresponding
+ `project` values in the selected rules file.
+3. A run with no override block uses the fully resolved global defaults.
+4. When a run override causes the rules file to be reloaded, any omitted skim
+ or network path comes from that file. Repeat a required global path in the
+ run block if the rules file does not contain it.
+5. `network_los_file` is required only when
+ `dimensions.PERIOD.values_from_network_los` is `true`.
+
+For predictable overrides, keep integrated skim paths either in `project` or
+in the main config, not in the skimjoin file's top-level `skim_files` field.
+The top-level form is supported but takes precedence during config promotion
+and can prevent a main-config skim override from replacing it.
+
## Runtime Placement
```text
@@ -36,6 +135,41 @@ prepare raw outputs
The runtime adapter is
[`processor/skimjoin/pipeline.py`](../processor/skimjoin/pipeline.py).
+## How The Implementation Works
+
+Integrated skimjoin has six main stages:
+
+1. **Resolve one config per run.** The runtime selects the global or run-level
+ rules file, applies path overrides, resolves paths, and validates the typed
+ schema.
+2. **Normalize rules.** Mode, segment, component, dimension, target-table,
+ missing-data, and fallback settings become ordered trip and tour lookup
+ rules. Strict validation can report output collisions and invalid fallback
+ chains before annotation.
+3. **Inventory skim inputs.** The runtime scans OMX, HDF5, and CSV inputs and
+ records matrix names, qualified source names, shapes, lookup types, and key
+ columns. Duplicate file-qualified references are invalid.
+4. **Select rows for each rule.** A rule matches the configured trip or tour
+ mode, then applies `when`, `segment_on`, target, and dimension conditions.
+ Missing source columns can make a rule unusable and appear in diagnostics.
+5. **Resolve and execute lookups.** Dimension values fill matrix-name
+ placeholders. OD rules map origin and destination IDs through the configured
+ OMX lookup; key rules read keyed sidecar values. Sentinel and missing-data
+ policies determine whether invalid values fail, warn, or become null.
+6. **Package enriched data and diagnostics.** Successful output columns replace
+ the prepared `RunData.trips` and `RunData.tours` tables. The runtime stores a
+ manifest, lookup reports, and optional hypothetical sidecars in the final
+ prepared cache.
+
+Fallback rules run in order only for rows that still lack a valid value. Rules
+that write the same output can overlap only when all of them use
+`combine: sum`; otherwise validation reports an output collision.
+
+Tour lookups use two directional contexts derived from each prepared tour. The
+inbound context swaps origin and destination fields, and dimension settings can
+name separate outbound and inbound source columns. This produces direct tour
+lookup outputs with `_outbound` and `_inbound` suffixes.
+
## Configuration sections
A skimjoin configuration contains these sections:
@@ -47,17 +181,17 @@ A skimjoin configuration contains these sections:
| `defaults` | Default origin, destination, output prefix, and missing-data policies. |
| `zone_mapping` | Optional zone lookup behavior. |
| `dimensions` | Time period or other dimensions used to resolve matrix names. |
+| `ignore_modes` | Trip modes allowed to have no lookup rules. |
| `modes` | Mode-specific lookup rules. |
-| `tour_aggregation` | How trip skim values roll up to tours. |
-Run overrides in the main visualizer configuration can change the skim files,
-`network_los_file`, or skimjoin configuration path.
+The optional `project` section contains paths, not lookup behavior. Main-config
+defaults or run overrides can supply the integrated skim paths instead.
## Adding A Skim Output
-Start with the [Basic OD Lookup](25-skimjoin-config-reference.md#basic-od-lookup)
-for a complete mode rule. Add dimensions, fallback rules, or tour aggregation
-only when the new output requires them.
+Start with the [Basic OD Lookup](23-skimjoin-config-reference.md#basic-od-lookup)
+for a complete mode rule. Add dimensions or fallback rules only when the new
+output requires them.
Checklist:
@@ -66,8 +200,8 @@ Checklist:
3. Select an output name. Use the `skim_` prefix unless the interface requires a different prefix.
4. Set the missing-matrix and missing-OD policies.
5. Add fallback lookup rules only when a valid fallback value is available.
-6. If tours need the value, configure tour aggregation or directional outputs.
-7. Add/update a summary in `processor/summarize/summaries/skimjoin.py` if the
+6. Set `apply_to` when the component belongs only on trips or tours.
+7. Add or update a summary in `processor/summarize/summaries/skimjoin.py` if the
dashboard needs aggregate reporting.
8. Regenerate wiki catalogs if summary declarations or dashboard requirements
changed.
@@ -76,6 +210,27 @@ Set `skimjoin.create_hypothetical_skim_tables: true` globally or in a run
override to create hypothetical skim sidecar tables. The default is `false`
because this option creates more output and artifacts.
+Hypothetical sidecars rerun each configured mode's lookup rules against every
+eligible observed row. They do not change the observed trip or tour mode and
+do not replace the annotated prepared tables. They provide long-form values
+for comparisons such as “what would this trip's auto time be under each
+configured mode?”
+
+| Trip sidecar field | Type | Meaning |
+|---|---|---|
+| `trip_id` | `Int64` | Prepared trip identifier. |
+| `observed_mode` | string | Original configured trip mode. |
+| `hypothetical_mode` | string | Mode whose rules produced the value. |
+| `component` | string | Skim output column name. |
+| `value` | `Float64` | Looked-up component value, or null. |
+| `finalweight` | `Float64` | Prepared trip weight. |
+
+The tour sidecar has the same structure with `tour_id` and one additional
+`direction` field. `direction` is `outbound` or `inbound` when the component
+ends with the corresponding suffix; it is null for unsuffixed outputs.
+Sidecars are empty unless the prepared source contains the configured ID and
+mode columns plus `finalweight`.
+
## Standalone Skimjoin CLI
The integrated pipeline is the standard approach. The standalone command-line
@@ -91,12 +246,12 @@ uv run python -m processor.skimjoin.cli COMMAND --config skimjoin.yaml
| `inventory` | `--preview` | Writes `skim_inventory.csv` and `inventory_debug.log` under `project.output_dir`. Preview also writes trip/tour column inventories and ActivitySim value counts when the configured tables are available. |
| `validate` | none | Strictly validates config, inventory, and configured ActivitySim tables; writes `config_normalized.yaml` and `validation_report.txt`. Returns exit code 1 and writes a failure report when validation fails. |
| `annotate-trips` | `--out PATH`, `--preview` | Writes annotated trips plus validation, lookup-summary, and missing-lookup artifacts. The default table is `/trips_with_skims.parquet`. |
-| `annotate-tours` | `--out PATH`, `--preview` | Writes annotated tours, `tour_aggregation_summary.csv`, and `missing_lookup_report.csv`. The default table is `/tours_with_skims.parquet`. |
+| `annotate-tours` | `--out PATH`, `--preview` | Writes annotated tours and lookup diagnostics. The default table is `/tours_with_skims.parquet`. |
| `run` | `--out-trips PATH`, `--out-tours PATH`, `--preview` | Executes both annotations and writes their validation and QA reports. Uses the two file names above by default. |
Each command requires `--config`; output flags are optional only when
`project.output_dir` is configured. Standalone input tables come from
-`activitysim.trips_table` and `activitysim.tours_table`. Chapter 25 describes
+`activitysim.trips_table` and `activitysim.tours_table`. Chapter 23 describes
the legacy `project.trips_table` and `project.tours_table` fallback. Input and
output tables must be CSV or Parquet. For annotation commands, `--preview` adds
a short output-column inventory but does not limit rows or prevent writes.
@@ -109,9 +264,14 @@ First, examine these skimjoin artifacts for the prepared run:
- `missing_lookup_report`
- `fallback_lookup_report`
- `skipped_rule_report`
-- `tour_aggregation_summary`
- `failure_report`
+Integrated artifacts are stored under
+`//prepared_tables/skimjoin/`. The prepared manifest records the
+status, resolved rules/input identity, applied outputs, skipped rules,
+warning/fallback counts, failure detail, and hypothetical sidecar row counts.
+Chapter 23 gives the exact report schemas and concrete skim file layouts.
+
Common causes:
| Symptom | Check |
@@ -120,23 +280,36 @@ Common causes:
| Rule skipped | Source mode, `when` clause, ignored modes, and required dimensions. |
| Missing matrix | Matrix naming pattern, dimensions, network LOS periods, and OMX contents. |
| Missing OD values | Origin/destination columns, zone mapping, sentinel values, and missing OD policy. |
-| Tours missing values | Tour aggregation config and outbound/inbound source columns. |
+| Tours missing values | `apply_to`, tour mode, outbound/inbound source columns, dimensions, and OD columns. |
+
+With `failure_policy: record`, an integrated failure keeps the original
+prepared trips and tours, writes empty skim sidecars, and records a
+`failure_report`. With `failure_policy: error`, the exception stops the run.
+
+The prepared-cache identity includes the normalized skimjoin rules and resolved
+skim inputs. A changed rules file or skim file invalidates skimjoin and later
+summaries without requiring raw preparation to run again. Use
+`refresh: [skimjoin]` to force that boundary while retaining
+`base_prepared_tables`.
## Where To Change Code
| Task | Start here |
|---|---|
| Config shape or validation | `processor/skimjoin/config/schema.py` |
+| Main/run override resolution | `runtime/config/normalize_skimjoin.py` |
| Config normalization | `processor/skimjoin/config/normalize.py` |
+| Skim inventory | `processor/skimjoin/inventory.py` |
| Skim store behavior | `processor/skimjoin/skimstore/` |
| Trip annotation | `processor/skimjoin/annotate/trips.py` |
| Tour annotation | `processor/skimjoin/annotate/tours.py` |
| Runtime reports | `processor/skimjoin/runtime_reports.py` |
+| Integrated execution | `processor/skimjoin/runtime_execution.py` |
| Skim summary tables | `processor/summarize/summaries/skimjoin.py` |
## Related Chapters
- [13 - Configuration Reference](13-configuration-reference.md#skimjoin)
-- [25 - Skimjoin Config Reference](25-skimjoin-config-reference.md)
-- [23 - Summary Functions](23-summary-functions.md)
+- [23 - Skimjoin Config Reference](23-skimjoin-config-reference.md)
+- [25 - Summary Functions](25-summary-functions.md)
- [90 - Troubleshooting](90-troubleshooting.md)
diff --git a/wiki/25-skimjoin-config-reference.md b/wiki/23-skimjoin-config-reference.md
similarity index 66%
rename from wiki/25-skimjoin-config-reference.md
rename to wiki/23-skimjoin-config-reference.md
index f5b1164..76c405f 100644
--- a/wiki/25-skimjoin-config-reference.md
+++ b/wiki/23-skimjoin-config-reference.md
@@ -1,8 +1,9 @@
-# 25 - Skimjoin Config Reference
+# 23 - Skimjoin Config Reference
This page is a field-by-field reference for the standalone skimjoin
configuration used by the main visualizer. For a workflow introduction, read
-[22 - Skimjoin](22-skimjoin.md). For a complete example, see
+[22 - Skimjoin](22-skimjoin.md). For a complete self-contained example that
+also supports the standalone CLI, see
[`example_skimjoin_config.yaml`](../example_skimjoin_config.yaml).
The skimjoin configuration answers four questions:
@@ -13,15 +14,77 @@ The skimjoin configuration answers four questions:
3. Which matrix or sidecar table must skimjoin read for each mode and component?
4. Which policy applies when matrices, OD pairs, or dimension values are missing?
+## Choose Where Paths Live
+
+The standalone skimjoin file always defines lookup behavior. Its `project`
+section is optional for integrated visualizer use because the main config can
+supply the data paths.
+
+| Use case | Put paths here |
+|---|---|
+| Integrated workflow with shared paths | `skimjoin.defaults` in the main visualizer config. |
+| Integrated workflow with paths that differ by run | `runs[*].skimjoin` in the main visualizer config. |
+| Self-contained skimjoin file or standalone CLI | `project` in the skimjoin config. |
+
+For integrated use, the effective config must have at least one skim file. A
+`network_los_file` is required only when
+`dimensions.PERIOD.values_from_network_los` is `true`. The integrated workflow
+supplies prepared trip and tour tables, so it does not need
+`project.trips_table`, `project.tours_table`, or `project.output_dir`.
+
+Main visualizer config with shared paths:
+
+```yaml
+pipeline:
+ steps: [prepare, skimjoin, summarize, dashboard]
+
+skimjoin:
+ defaults:
+ config_path: configs\skimjoin_rules.yaml
+ skim_files:
+ - skims\*.omx
+ network_los_file: skims\network_los.yaml
+```
+
+The referenced rules file can then omit `project`:
+
+```yaml
+activitysim:
+ trip_mode_column: trip_mode
+ tour_mode_column: tour_mode
+ trip_id_column: trip_id
+ tour_id_column: tour_id
+ outbound_column: outbound
+
+defaults:
+ origin: OTAZ
+ destination: DTAZ
+
+modes:
+ SOV:
+ time: SOV_TIME
+ distance: SOV_DIST
+```
+
+Main-config paths resolve from the main config directory. Paths in `project`
+resolve from the skimjoin file's directory.
+
+Main-config skim and network paths replace the corresponding `project` values.
+Avoid the standalone top-level `skim_files` field when you need this override
+behavior. If that top-level field is present, config promotion keeps it instead
+of the injected `project.skim_files` value.
+
+Run overrides have one additional rule: a run with no override block uses the
+complete global resolution, but a run-specific override reloads the selected
+skimjoin file. Any skim or network path omitted from that run block then comes
+from the selected skimjoin file. Repeat a required global path in the run block
+when the file does not contain it.
+
## Common Recipes
### Basic OD Lookup
```yaml
-project:
- skim_files:
- - C:\skims\auto.omx
-
activitysim:
trip_mode_column: trip_mode
tour_mode_column: tour_mode
@@ -51,14 +114,19 @@ time:
```
When multiple skim files contain the same matrix name, qualify the reference
-with the source filename:
+with its source filename. For example, set the files in the main config:
```yaml
-project:
- skim_files:
- - C:\skims\bike_commute.omx
- - C:\skims\bike_noncommute.omx
+skimjoin:
+ defaults:
+ skim_files:
+ - C:\skims\bike_commute.omx
+ - C:\skims\bike_noncommute.omx
+```
+
+In the skimjoin rules file:
+```yaml
modes:
BIKE:
distance:
@@ -72,12 +140,19 @@ contain dimension placeholders.
### Period Dimension Lookup
+The main config can supply both paths:
+
```yaml
-project:
- skim_files:
- - C:\skims\auto.omx
- network_los_file: C:\skims\network_los.yaml
+skimjoin:
+ defaults:
+ skim_files:
+ - C:\skims\auto.omx
+ network_los_file: C:\skims\network_los.yaml
+```
+
+The rules file defines how to use `network_los.yaml`:
+```yaml
activitysim:
trip_mode_column: trip_mode
tour_mode_column: tour_mode
@@ -145,47 +220,33 @@ modes:
Fallbacks run after the primary lookup and apply only to rows that do not yet
have a valid value. Every fallback step uses the same final output column.
-### Tour Aggregation
-
-```yaml
-tour_aggregation:
- method: aggregate_trips
- aggregations:
- skim_auto_time: sum
- skim_auto_distance: sum
- skim_transit_fare: sum
- directional_outputs:
- skim_auto_time: true
-```
-
-Mode rules can also create tour lookups directly. Their output receives an
-`_outbound` or `_inbound` suffix.
-
## Top-Level Sections
| Section | Type | Default | Purpose |
|---|---|---|---|
| `project` | mapping | optional | Skim paths and standalone CLI paths. |
-| `skim_files` | list | promoted from `project.skim_files` | Direct skim file list. Usually set under `project`. |
+| `skim_files` | list | required after path resolution | Compatibility input promoted from `project.skim_files`. Prefer main-config paths for integrated use and `project.skim_files` for standalone use. |
| `activitysim` | mapping | required | Prepared trip/tour source column names. |
| `defaults` | mapping | built-in lookup defaults | Origin, destination, output prefix, missing-data policy, and sentinels. |
| `zone_mapping` | mapping | no mapping name | OMX zone lookup name behavior. |
| `dimensions` | mapping | `{}` | Placeholder definitions for matrix names. |
| `ignore_modes` | list | `[]` | Trip modes allowed to have no lookup rules. |
| `modes` | mapping | required | Mode-specific lookup rules. |
-| `tour_aggregation` | mapping | `aggregate_trips` with no configured aggregations | Trip-to-tour aggregation settings. |
The Pydantic schema rejects unknown keys in typed sections.
## `project`
+`project` is a path container. It is not required when the main visualizer
+config supplies all paths needed by the integrated workflow.
+
| Field | Type | Default | Notes |
|---|---|---|---|
-| `skim_files` | list of path strings | `[]` | OMX, CSV, HDF5, or H5 skim inputs. In integrated visualizer use, main config overrides may replace this list. |
+| `skim_files` | list of path strings | `[]` | OMX, CSV, HDF5, or H5 skim inputs. Required when the main visualizer config does not supply them, and required by the standalone `inventory` command. |
| `network_los_file` | path string | none | ActivitySim `network_los.yaml`, used when `dimensions.PERIOD.values_from_network_los` is true. |
-| `trips_table` | path string | none | Standalone skimjoin CLI input. Not required for integrated visualizer use. |
-| `tours_table` | path string | none | Standalone skimjoin CLI input. Optional. |
-| `output_dir` | path string | none | Standalone skimjoin CLI output directory. |
+| `trips_table` | path string | none | Standalone CLI input fallback. The integrated workflow ignores it. Prefer `activitysim.trips_table` for standalone use. |
+| `tours_table` | path string | none | Optional standalone CLI input fallback. The integrated workflow ignores it. |
+| `output_dir` | path string | none | Required by standalone `inventory`; used as the default output location for other CLI commands. The integrated workflow ignores it. |
```yaml
project:
@@ -193,8 +254,17 @@ project:
- C:\skims\*.omx
- C:\skims\maz_stop_walk.csv
network_los_file: C:\skims\network_los.yaml
+ trips_table: C:\prepared\trips.parquet
+ tours_table: C:\prepared\tours.parquet
+ output_dir: C:\skimjoin_output
```
+For the standalone `validate`, `annotate-trips`, `annotate-tours`, and `run`
+commands, a trips table must resolve through `activitysim.trips_table` or the
+legacy `project.trips_table`. A tours table is optional unless the requested
+operation needs tour input. Output flags can replace `project.output_dir` for
+annotation commands.
+
## `activitysim`
`activitysim` names columns in prepared trip and tour tables.
@@ -294,7 +364,7 @@ Each dimension entry has these fields:
| `source_columns.trip_source_column` | string | required | Source column used for trip lookup rules. |
| `source_columns.outbound_tour_source_column` | string | required | Source column used for outbound tour lookup rules. |
| `source_columns.inbound_tour_source_column` | string | required | Source column used for inbound tour lookup rules. |
-| `values_from_network_los` | boolean | `false` | Only supported for `PERIOD`. Requires `project.network_los_file`. |
+| `values_from_network_los` | boolean | `false` | Only supported for `PERIOD`. Requires an effective `network_los_file` from the main config or `project`. |
| `values` | mapping | `{}` | Raw source value to matrix-name token. The loader normalizes keys and values to strings. |
If `values` is empty, skimjoin converts the raw source value to a string and
@@ -498,6 +568,58 @@ modes:
output: skim_walk_dist
```
+### Concrete CSV Layouts
+
+A keyed CSV uses its first column as the key and every later numeric column as
+a separate inventory value:
+
+```csv
+MAZ,terminal_walk,parking_cost
+101,2.5,4.00
+102,1.8,6.50
+```
+
+If the file is `maz_access.csv`, its inventory names are
+`maz_access__terminal_walk` and `maz_access__parking_cost`. A key rule can use:
+
+```yaml
+terminal_walk:
+ lookup: key
+ key_column: origin
+ matrix: maz_access__terminal_walk
+```
+
+An OD CSV is recognized only when its first two normalized headers form one of
+these pairs: `origin`/`destination`, `otaz`/`dtaz`, `omaz`/`dmaz`,
+`orig`/`dest`, or `from`/`to`. Every later numeric column becomes a separate OD
+table:
+
+```csv
+OTAZ,DTAZ,time,distance
+1,1,0.0,0.0
+1,2,12.5,8.1
+2,1,13.0,8.1
+2,2,0.0,0.0
+```
+
+For `auto_md.csv`, refer to these values as `auto_md__time` and
+`auto_md__distance`. CSV rows need not form a complete square matrix; an
+unlisted pair follows the configured missing-OD policy. Non-numeric columns
+after the key or OD pair are ignored by the inventory.
+
+### OMX, HDF5, And H5 Layouts
+
+Skimjoin inventories every two-dimensional dataset in `.omx`, `.h5`, and
+`.hdf5` files. The inventory records the full dataset path but uses the final
+path component as its unqualified matrix name. For example, dataset
+`/data/SOV_TIME` is referred to as `SOV_TIME` when unique. If more than one
+file exposes that name, use `filename.omx::SOV_TIME`.
+
+OD matrix row and column positions are resolved with the selected OMX mapping.
+Set `zone_mapping.lookup_name`, or use `file_lookup_names` when files use
+different mappings. Matrix dimensions and mapping positions must agree; a
+missing zone follows `zone_mapping.missing_zone_policy`.
+
## Trip And Tour Rules
By default, each component creates trip and tour lookup rules:
@@ -511,27 +633,6 @@ By default, each component creates trip and tour lookup rules:
Set `apply_to: trips` or `apply_to: tours` to run a component on only one target
table.
-## `tour_aggregation`
-
-`tour_aggregation` controls trip-to-tour totals for skim columns.
-
-| Field | Type | Default | Allowed values | Notes |
-|---|---|---|---|---|
-| `method` | string | `aggregate_trips` | `aggregate_trips` | Only supported aggregation method. |
-| `aggregations` | mapping | `{}` | `sum`, `mean`, `min`, `max`, `first`, `last` | Output column to aggregation method. |
-| `directional_outputs` | mapping | `{}` | output column to boolean | Requests directional outbound/inbound tour outputs for selected components. |
-
-```yaml
-tour_aggregation:
- method: aggregate_trips
- aggregations:
- skim_auto_time: sum
- skim_auto_distance: sum
- skim_transit_fare: sum
- directional_outputs:
- skim_auto_time: true
-```
-
## Missing Data And Reports
During integrated prepare, skimjoin writes these report artifacts:
@@ -542,9 +643,44 @@ During integrated prepare, skimjoin writes these report artifacts:
| `missing_lookup_report` | Missing matrix, missing OD, missing dimension, and skipped lookup details. |
| `fallback_lookup_report` | Fallback attempts and outcomes. |
| `skipped_rule_report` | Rules skipped by missing source columns or other selection conditions. |
-| `tour_aggregation_summary` | Tour lookup and aggregation details. |
| `failure_report` | Runtime failure detail when skimjoin cannot complete. |
+The files are under
+`//prepared_tables/skimjoin/`. They are CSV except for the
+resolved `config_normalized.yaml`. Empty reports are still written with their
+declared headers when the integrated run reaches report packaging.
+
+### Report Schemas
+
+| Report | Columns |
+|---|---|
+| `skim_lookup_summary` | `rule_name`, `mode`, `component`, `output`, `matrix_name`, `n_trips`, `origin_column`, `destination_column`, `mean_value`, `min_value`, `max_value`, `n_missing` |
+| `missing_lookup_report` | `rule_name`, `trip_id`, `origin`, `destination`, `matrix_name`, `reason` |
+| `skipped_rule_report` | `rule_name`, `reason`, `n_rows` |
+| `fallback_lookup_report` | `table_name`, `rule_name`, `output`, `logical_id`, `direction`, `primary_matrix_name`, `fallback_matrix_name`, `fallback_step_index`, `fallback_reason`, `fallback_eligible`, `fallback_attempted`, `fallback_succeeded`, `fallback_exhausted` |
+| `failure_report` | `stage`, `error_type`, `detail` |
+
+`n_trips` is the number of lookup rows covered by a rule/matrix combination,
+including invalid results; `n_missing` is the invalid subset. In the fallback
+report, `logical_id` is the trip or tour ID named by `table_name`, and
+`direction` is populated for directional tour work. The `reason` and
+`fallback_reason` strings are diagnostic codes/details; treat them as
+diagnostics rather than a stable category enumeration for downstream data
+exchange.
+
+The final prepared manifest also stores compact run-level fields:
+
+| Manifest field | Meaning |
+|---|---|
+| `skimjoin_status` | Completed, recorded failure, or other packaged execution state. |
+| `skimjoin_config_digest` | Identity of normalized lookup behavior. |
+| `skimjoin_resolved_network_los_file` | Effective network LOS path, if used. |
+| `skimjoin_applied_outputs` | Enriched trip/tour output names. |
+| `skimjoin_skipped_rules` | Compact skipped-rule records. |
+| `skimjoin_warning_count`, `skimjoin_fallback_count` | Aggregate diagnostic counts. |
+| `skimjoin_fallback_outputs` | Outputs that used fallback values. |
+| `skimjoin_failure_detail` | Recorded exception detail under record policy. |
+
Policies:
| Policy | Behavior |
diff --git a/wiki/24-segmentation.md b/wiki/24-segmentation.md
new file mode 100644
index 0000000..3d340a4
--- /dev/null
+++ b/wiki/24-segmentation.md
@@ -0,0 +1,319 @@
+# 24 - Segmentation
+
+Segmentation runs the standard summary catalog for selected subsets of a model
+run. For example, it can produce the same summaries for urban and rural
+households, for income groups, or for people in different survey samples.
+
+Segmentation does not add a grouping column to one summary. It creates a
+related `RunData` slice for each configured segment, then runs every registered
+default summary against that slice.
+
+## Runtime Placement
+
+```text
+raw or prepared input
+ -> prepare
+ -> optional skimjoin
+ -> resolve segment membership
+ -> slice related prepared tables in memory
+ -> summarize the full run and each segment
+ -> write full and segmented summary caches
+ -> show the selected segmentation in the dashboard
+```
+
+Segmentation is part of the summarize workflow. Enable it by adding `segment`
+to `pipeline.steps`; a `segment` configuration block does not enable the step
+by itself. The step also requires `summarize`.
+
+```yaml
+pipeline:
+ steps: [segment, summarize, dashboard]
+ dashboard_mode: live
+ refresh: []
+```
+
+Add `prepare` when you want cache creation to be explicit. If the workflow also
+uses skimjoin, include both `prepare` and `skimjoin` before `segment`.
+
+## Complete Prepared-Column Example
+
+This example divides each run by a canonical person column:
+
+```yaml
+pipeline:
+ steps: [segment, summarize, dashboard]
+ dashboard_mode: live
+ refresh: []
+
+segment:
+ dashboard:
+ segmentation_type: person_sex
+ visibility: full_and_segments
+ definitions:
+ person_sex:
+ source:
+ type: prepared_column
+ source_table: per
+ column: sex
+ allow_overlapping: false
+ on_empty_segment: warn
+ segments:
+ - id: female
+ label: Female
+ values: [2]
+ - id: male
+ label: Male
+ values: [1]
+```
+
+`person_sex` is the segmentation type. Each segment selects source rows whose
+`sex` value appears in its `values` list. The dashboard presents the results as
+series such as `Base (Female)` and `Base (Male)`.
+
+## How Table Slicing Works
+
+The source table is the anchor for membership. After matching its rows, the
+runtime follows canonical IDs to create a consistent set of related tables:
+
+| Source table | Membership starts with | Related data retained |
+|---|---|---|
+| `hh` | matching households | Their people, days, vehicles, tours, trips, and joint tours. |
+| `per` | matching people | Their households, days, tours, trips, vehicles, and joint participation rows. |
+| `day` | matching day rows | Related people or households, then their tours, trips, vehicles, and joint tours. |
+| `tours` | matching tours | Their people, households, trips, participants, days, and vehicles. |
+| `trips` | matching trips | Their tours, people, households, participants, days, and vehicles. |
+| `vehicles` | matching vehicles | Their households and all related household records. |
+| `joint_participants` | matching participation rows | Their joint tours, tour owners and participants, households, trips, days, and vehicles. |
+| `land_use` | matching MAZ or TAZ rows | Households whose home zone matches, then their related records. |
+
+This relationship expansion is important when interpreting totals. A
+trip-based segment contains only matching trips, but its household total counts
+households associated with those trips. It is not a household classification
+unless the source itself is a household field.
+
+The relationship keys must be present. Vehicle sources need `household_id`;
+joint-participant sources need `tour_id` and `person_id`; day sources need
+`person_id` or `household_id`; and land-use sources need a resolved `MAZ` or
+`TAZ` key.
+
+## Source Types
+
+### Prepared Column
+
+Use `prepared_column` when the segment value already exists in a prepared
+table:
+
+```yaml
+source:
+ type: prepared_column
+ source_table: hh
+ column: income_segment
+```
+
+`source_table` accepts `households`, `persons`, `day`, `tours`, `trips`,
+`vehicles`, `joint_tour_participants`, and `land_use`, along with their runtime
+aliases `hh`, `per`, and `joint_participants`.
+
+You can omit `source_table` if the column occurs in exactly one segmentable
+table. Set it explicitly when the name is absent or appears in more than one
+table.
+
+### CSV Lookup
+
+Use `csv_lookup` when membership comes from an external classification:
+
+```yaml
+segment:
+ definitions:
+ district:
+ source:
+ type: csv_lookup
+ file: lookups\household_district.csv
+ join:
+ source_table: hh
+ source_key_column: household_id
+ csv_key_column: household_id
+ segment_value_column: district
+ segments:
+ - id: north
+ label: North
+ values: [North]
+ - id: south
+ label: South
+ values: [South]
+```
+
+Relative lookup paths start from the main configuration directory. The CSV
+must contain the join key and segment-value columns. Keys and values cannot be
+blank, and one CSV key cannot map to multiple segment values. The join must not
+duplicate rows in the anchor table.
+
+## Definition And Segment Settings
+
+| Field | Default | Behavior |
+|---|---|---|
+| `source` | required | Selects a prepared column or CSV lookup and its anchor table. |
+| `segments` | required | Defines the path-safe lowercase `id`, display `label`, and matched `values` for each segment. |
+| `allow_overlapping` | `false` | When `false`, one source value cannot appear in more than one segment in the same definition. When `true`, the same row can contribute to multiple segments. |
+| `on_empty_segment` | `warn` | `error` stops the run, `skip` omits the analysis unit, and `warn` keeps an empty analysis unit so its summaries can report empty or unavailable results. |
+| `include_full` | `true` | Accepted by the schema. The current runtime always builds one full-run analysis unit, regardless of this value. Control dashboard visibility with `segment.dashboard.visibility`. |
+| `persist_segmented_prepared_tables` | `false` | Accepted by the schema. The current runtime keeps segment slices in memory and does not write separate prepared-table directories. |
+
+Segment IDs and definition names become path components, so they must already
+be lowercase and path-safe. The exact accepted form is one or more lowercase
+letters, digits, periods, underscores, or hyphens (`[a-z0-9._-]+`), with no
+leading or trailing period, underscore, or hyphen. Names can start with a
+digit. Spaces, slashes, uppercase letters, and characters outside that set are
+rejected rather than normalized for you.
+
+Value typing is source-specific:
+
+- `prepared_column` compares each YAML value to the prepared column using its
+ existing type. Use numbers for numeric columns, booleans for Boolean columns,
+ and quoted strings when a numeric-looking code is stored as text.
+- `csv_lookup` trims and stores lookup segment values as strings. The
+ corresponding `segments[*].values` should therefore be strings too. For
+ example, use `values: ["1"]`, not `values: [1]`, for CSV value `1`.
+- CSV join keys are trimmed as strings during config normalization, then cast
+ to the prepared anchor key's type for the join. Values that cannot be cast do
+ not match.
+
+A segment can combine several source values:
+
+```yaml
+- id: low_and_medium
+ label: Low and Medium Income
+ values: [low, medium]
+```
+
+Segments do not have to cover every source value. Unmatched rows remain in the
+full-run summaries but do not appear in any configured segment. If overlapping
+is enabled, do not add segment totals together unless double counting is
+intentional.
+
+## Dashboard And Export Settings
+
+`segment.dashboard` selects which stored series the live dashboard shows:
+
+| Field | Default | Behavior |
+|---|---|---|
+| `segmentation_type` | first definition by name | Selects one configured definition for presentation. Other definitions can still exist in the cache. |
+| `visibility` | `full_and_segments` | `full_only`, `segments_only`, or `full_and_segments`. |
+
+HTML export inherits these values. Override them for one export with:
+
+```yaml
+dashboard:
+ export:
+ dashboard:
+ segmentation_type: district
+ segmentation_visibility: segments_only
+```
+
+An export can only use segmentation types and segments already present in the
+summary cache.
+
+## Outputs And Cache Behavior
+
+Full-run summaries keep their standard paths:
+
+```text
+//summary_tables//.csv
+```
+
+Segmented summaries use:
+
+```text
+//summary_tables//segments/
+ //.csv
+```
+
+The run-level summary manifest records each type and segment, including its
+label, source, matched values, summary states, diagnostics, and digests. The
+prepared cache remains at the normal run path.
+
+One shortened manifest entry looks like this:
+
+```json
+{
+ "segmentation_type": "district",
+ "source_type": "csv_lookup",
+ "segment_column": "district",
+ "source_table": "hh",
+ "source_key_column": "household_id",
+ "csv_file": "C:\\lookups\\household_district.csv",
+ "csv_key_column": "household_id",
+ "csv_segment_value_column": "district",
+ "include_full": false,
+ "segments": [
+ {
+ "segmentation_type": "district",
+ "segment_id": "north",
+ "segment_label": "North",
+ "is_full": false,
+ "source_type": "csv_lookup",
+ "segment_column": "district",
+ "segment_values": ["North"],
+ "summary_roots": {
+ "weighted": "summary_tables/weighted/segments/district/north"
+ },
+ "summary_states": {},
+ "summary_diagnostics": {},
+ "summary_digests": {}
+ }
+ ]
+}
+```
+
+The stored `include_full` value on the type entry describes the segmented tree,
+not the accepted config field. The full run remains at the standard summary
+root and is always built by the current runtime.
+
+With `refresh: []`, cache validation is independent for the full run and each
+segment. Adding or changing one segment rebuilds that segment while compatible
+full-run and other segment summaries remain reusable. Removing a segment prunes
+its obsolete summary directory on the next cache write. Use
+`refresh: [summarize]` to rebuild all full and segmented summaries while keeping
+prepared data.
+
+Segmentation requires raw or prepared rows. A run supplied only through
+`summary_table_map` cannot be segmented because its tables are already
+aggregated. If a run combines raw or prepared input with `summary_table_map`,
+the mapped external table is overlaid unchanged on every full and segmented
+analysis unit. Do not use that pattern for a measure that must vary by segment.
+
+## Implementation And Extension Points
+
+| Task | Start here |
+|---|---|
+| Config validation and normalization | `runtime/config/normalize_segmentation.py` |
+| Relationship slicing and analysis units | `processor/segmentation.py` |
+| Segment identity and metadata | `processor/analysis_units.py` |
+| Summary workflow integration | `runtime/workflows/summarize.py` |
+| Cache paths and manifests | `processor/summarize/cache.py` and `cache_storage.py` |
+| Dashboard series selection | `dashboard/state.py` |
+
+Summary builders normally need no segment-specific code. They receive a sliced
+`RunData` object and use the same declaration, schema, and weighting logic as
+the full run.
+
+## Troubleshooting
+
+| Symptom | Check |
+|---|---|
+| No segmented output | Make sure `pipeline.steps` contains both `segment` and `summarize`. |
+| Source column not found | Set the correct runtime `source_table` and inspect the prepared schema. |
+| CSV lookup fails | Check path resolution, required columns, blank values, duplicate keys, and join-key types. |
+| A segment is empty | Compare its `values` with the prepared or lookup values and review `on_empty_segment`. |
+| Totals overlap | Check `allow_overlapping` and whether the selected anchor represents the population being counted. |
+| Dashboard shows only full or only segmented series | Check `segment.dashboard.visibility` and the selected `segmentation_type`. |
+| Old segment remains on disk | Run summarize so the next cache write can prune obsolete units. |
+
+## Related Chapters
+
+- [12 - Running Workflows](12-running-workflows.md)
+- [13 - Configuration Reference](13-configuration-reference.md#segment)
+- [21 - Prepared Tables](21-prepared-tables.md)
+- [25 - Summary Functions](25-summary-functions.md)
+- [42 - Config, Columns, And Labels](42-config-column-label-cookbook.md)
+- [90 - Troubleshooting](90-troubleshooting.md)
diff --git a/wiki/23-summary-functions.md b/wiki/25-summary-functions.md
similarity index 60%
rename from wiki/23-summary-functions.md
rename to wiki/25-summary-functions.md
index 76e0f5f..96d0199 100644
--- a/wiki/23-summary-functions.md
+++ b/wiki/25-summary-functions.md
@@ -1,4 +1,4 @@
-# 23 - Summary Functions
+# 25 - Summary Functions
Summary functions convert prepared `RunData` into Polars `DataFrame` objects
for the dashboard. Each function keeps its identity, requirements, output
@@ -83,6 +83,60 @@ Builders aggregate `finalweight`. They do not select a weighting mode. The
summary workflow supplies the required prepared data for weighted and
unweighted builds.
+### Weight Resolution And Edge Cases
+
+The primary weighted mode is prepared as follows:
+
+1. An explicit run-level household/person/trip weight column is cast to
+ `Float64` on its table.
+2. If no explicit run weight is supplied at any level, a household
+ `sample_rate` produces `1 / sample_rate`.
+3. Otherwise, household weight defaults to `1.0` when no household source was
+ selected. Supplying only a person or trip weight therefore disables
+ household sample-rate expansion.
+4. Missing lower-level sources inherit through household/person/tour
+ relationships. An unmatched inherited row normally falls back to `1.0`.
+5. When an explicit trip weight is used, tour weight is the mean trip weight
+ for that `tour_id`.
+
+The unweighted mode changes existing `finalweight` columns to `1.0`; it does
+not add that column to a custom prepared table that omitted it. Named column
+modes follow the propagation rules in chapter 43.
+
+The runtime casts weights but does not apply a universal quality rule for
+zero, negative, null, infinite, or extreme values. Consequences are
+calculation-specific:
+
+- a null weight is ignored by a Polars sum and can remove that row's
+ contribution;
+- zero weights contribute no count and can create a zero denominator;
+- negative weights subtract from totals;
+- `sample_rate: 0` can produce an infinite expansion weight; and
+- a weighted average with a zero or invalid denominator can return null, NaN,
+ or infinity unless that builder handles the case.
+
+Validate source weights before production use. A practical contract is finite,
+non-null, nonnegative weights and strictly positive sample rates. If zero
+weights are intentional, test every rate and average that consumes them.
+
+### Units
+
+The visualizer does not maintain a separate unit registry or automatically
+convert source values. Units are part of the source/prepared/summary contract:
+
+| Output kind | Unit rule |
+|---|---|
+| Counts and totals | `finalweight` expansion units; unweighted mode is row counts unless a builder applies occupancy/party logic. |
+| Rates and shares | Ratio of the builder's declared numerator and denominator; dimensionless unless the label states a per-person or per-day basis. |
+| Distance and VMT | Uses prepared distance values as supplied. Existing dashboard labels assume miles. Convert upstream or in prepare if the model uses another unit. |
+| Time | Uses prepared time/hour/period fields and configured time-period mapping. Skim time components keep the skim's unit. |
+| Cost and other skim components | Keeps the matrix or sidecar unit; skimjoin does not convert cents, dollars, minutes, seconds, or generalized cost. |
+| Geography IDs and categories | Labels/identifiers, not measured units. |
+
+When you add a summary, state the unit in its column name, page axis/tooltip, or
+calculation note. Do not combine runs whose underlying distance, time, or cost
+units differ without normalizing them first.
+
## Adding A Summary Function
For an example with a calculation, contract test, catalog, and page connection,
@@ -133,13 +187,16 @@ the [outside summary table recipe](41-data-extension-cookbook.md#worked-example-
## Segmentation
-Segmentation runs in the summarize workflow. It builds the same declarations
-for configured parts of the prepared data. A segment source can be a prepared
-column or a CSV lookup. `segment.dashboard` controls dashboard visibility.
+Segmentation runs in the summarize workflow and builds the same declarations
+for related subsets of prepared data. A source can be a prepared column or a
+CSV lookup. The runtime slices `RunData`, applies the normal weighting modes,
+and writes each result below the segment's summary-cache path. See
+[24 - Segmentation](24-segmentation.md) for source-table relationships, settings,
+outputs, cache behavior, and dashboard selection.
## Summary Catalog
-The generated [24 - Summary Catalog](24-summary-catalog.md) lists each current
+The generated [26 - Summary Catalog](26-summary-catalog.md) lists each current
declaration, output file name, builder, schema, and requirement. Regenerate the
catalog after you change a summary declaration.
@@ -147,5 +204,7 @@ catalog after you change a summary declaration.
- [20 - Output Processor](20-output-processor.md)
- [21 - Prepared Tables](21-prepared-tables.md)
-- [31 - Dashboard Pages](31-dashboard-pages.md)
+- [24 - Segmentation](24-segmentation.md)
+- [27 - Geography](27-geography.md)
+- [31 - Dashboard Page Contract](31-dashboard-pages.md)
- [44 - Summary Function Cookbook](44-summary-function-cookbook.md)
diff --git a/wiki/24-summary-catalog.md b/wiki/26-summary-catalog.md
similarity index 99%
rename from wiki/24-summary-catalog.md
rename to wiki/26-summary-catalog.md
index 288d540..702453a 100644
--- a/wiki/24-summary-catalog.md
+++ b/wiki/26-summary-catalog.md
@@ -1,4 +1,4 @@
-# 24 - Summary Catalog
+# 26 - Summary Catalog
This page is the data dictionary for summary CSV tables from the Output
Processor. It explains what one row represents, how each table is used, and
diff --git a/wiki/27-geography.md b/wiki/27-geography.md
new file mode 100644
index 0000000..9868a5f
--- /dev/null
+++ b/wiki/27-geography.md
@@ -0,0 +1,255 @@
+# 27 - Geography
+
+The geography feature adds consistent spatial groupings to prepared tables and
+summary output. Use it to report the same measures by districts, counties,
+subregions, or other zone-based systems without adding regional logic to each
+summary builder.
+
+## Geography Layers
+
+The visualizer can use several kinds of geography at the same time:
+
+1. **Canonical zones** come from `zones`. Prepare creates MAZ and TAZ fields
+ such as `home_zone_id`, `home_taz`, `OTAZ`, and `DTAZ`.
+2. **Native home geographies** such as `home_county` and `home_mpo` are retained
+ when they already exist in model output.
+3. **A legacy land-use geography** can use `summarize.geography.landuse_col` to
+ create the compatibility fields `HGEO` and `WGEO`.
+4. **Named aggregations** under `summarize.geography.aggregations` map MAZ or TAZ
+ IDs to any number of custom geography systems.
+
+Named aggregations are the preferred method for new custom geography work.
+They keep geography IDs and source zone systems explicit and can support
+several systems in one run.
+
+## Data Flow
+
+```text
+inline mapping or CSV zone lookup
+ -> validate one geography label per zone
+ -> prepare role-specific geography columns
+ -> summary builders emit geography_type and geography_id
+ -> dashboard pages expose available geography options
+```
+
+Although the configuration lives under `summarize`, named geography mappings
+also change prepared tables. Their normalized lookup rows are part of the
+prepare and summary cache identities.
+
+## File-Based Example
+
+Given this CSV:
+
+```csv
+MAZ,district
+101,North
+102,North
+201,South
+```
+
+configure a named aggregation as follows:
+
+```yaml
+zones:
+ use_maz: true
+ maz_col: [MAZ, zone_id]
+ taz_col: [TAZ, taz]
+
+summarize:
+ geography:
+ enabled: true
+ aggregations:
+ district:
+ source_zone_system: maz
+ file: lookups\maz_district.csv
+ zone_id_col: MAZ
+ geography_col: district
+```
+
+Relative file paths start from the main configuration directory. The file must
+be CSV and contain both named columns. Zone IDs must be integers, geography
+labels cannot be blank, and one zone cannot map to different labels.
+
+## Inline Example
+
+For a small, stable mapping, list zone IDs directly:
+
+```yaml
+summarize:
+ geography:
+ enabled: true
+ aggregations:
+ market_area:
+ source_zone_system: taz
+ mapping:
+ Core: [1, 2, 3]
+ Suburban: [4, 5, 6]
+ External: [99]
+```
+
+The mapping direction is `geography label -> zone ID or list of zone IDs`.
+Use either `mapping` or `file` for one aggregation, never both.
+
+## Settings
+
+| Field | Default | Behavior |
+|---|---|---|
+| `summarize.geography.enabled` | `false` | Enables the legacy geography and all named aggregations. When `false`, aggregation definitions are ignored. |
+| `summarize.geography.landuse_col` | none | Names one existing land-use column used to create compatibility `HGEO` and `WGEO` fields. |
+| `summarize.geography.mapping` | none | Maps raw values from `landuse_col` to normalized labels for the legacy geography. |
+| `summarize.geography.aggregations` | `{}` | Defines one or more named MAZ- or TAZ-based lookups. |
+| `dashboard.enable_maz_geographies` | `false` | Allows MAZ options on dashboard pages that support them. This is a presentation setting and does not create geography columns. |
+| `display.labels.geography` | none | Changes display labels for geography type IDs, such as displaying `district` as `School District`. It does not remap zone membership. |
+
+Each named aggregation requires `source_zone_system: maz` or `taz` and exactly
+one lookup form:
+
+| Lookup form | Required fields |
+|---|---|
+| Inline | `mapping` |
+| CSV | `file`, `zone_id_col`, `geography_col` |
+
+Use a short, stable aggregation name such as `district` or `county`. That name
+becomes the `geography_type` value in summaries and part of each prepared column
+name.
+
+## Prepared Outputs
+
+For an aggregation named `district`, prepare can create:
+
+| Prepared table | Output columns |
+|---|---|
+| households | `home_geo__district` |
+| persons | `home_geo__district`, `work_geo__district`, `school_geo__district` |
+| tours | `origin_geo__district`, `destination_geo__district` |
+| trips | `origin_geo__district`, `destination_geo__district` |
+| land use | `land_use_geo__district` |
+
+The source columns depend on `source_zone_system`:
+
+| Role | MAZ source | TAZ source |
+|---|---|---|
+| household/person home | `home_zone_id` | `home_taz` |
+| person work | `workplace_zone_id` | `work_taz` |
+| person school | `school_zone_id` | `school_taz` |
+| tour/trip origin | `origin` | `OTAZ` |
+| tour/trip destination | `destination` | `DTAZ` |
+| land use | `MAZ` | `TAZ` |
+
+Prepare first resolves the canonical MAZ and TAZ fields from `zones`. A named
+aggregation can therefore fail to populate if the corresponding zone system is
+missing or misconfigured. Zones absent from the lookup receive null geography
+values; geography-specific summaries generally exclude those null rows.
+
+## Summary And Dashboard Outputs
+
+Summary tables that support geography use a long-form pair:
+
+- `geography_type` identifies the system, such as `maz`, `home_taz`,
+ `home_county`, or `district`.
+- `geography_id` contains the zone or mapped label within that system.
+
+The exact supported roles vary by summary. For example, population summaries
+use home geography, mandatory-location summaries can use work or school
+geography, and destination summaries use destination geography. Check the
+[Summary Catalog](26-summary-catalog.md) for each table's meaning and required
+prepared columns.
+
+Native `home_county` and `home_mpo` columns can appear in supported summaries
+without a named aggregation. `dashboard.enable_maz_geographies` controls only
+whether supporting pages expose MAZ-level choices; it does not affect TAZ,
+native, or named aggregation columns.
+
+To relabel geography type IDs in the dashboard:
+
+```yaml
+display:
+ labels:
+ geography:
+ mapping:
+ district: School District
+ home_county: County
+ home_taz: TAZ
+```
+
+Keep membership changes under `summarize.geography`. Display labels do not
+change joins, cache data, or geography IDs.
+
+## Compatibility Matrix
+
+Configuring a named aggregation creates every role column in the prepared
+output, but a summary uses that aggregation only when its implementation
+requests the corresponding role. The main supported paths are:
+
+| Geography role | Prepared source | Summary families that use it | Dashboard pages |
+|---|---|---|---|
+| Home | household/person home zone and `home_geo__` | worker internal/external status, work/school/university distance, work from home, telecommuting, average tour distance, internal/external non-mandatory tours, personal-auto and non-motorized VMT | Mandatory Location Choice, Tour Distance, Internal vs. External Tours, VMT Validation |
+| Work | person workplace zone and `work_geo__` | workplace/employment comparison, workplace shadow-price residuals, commuting flows, external workplace locations | Mandatory Location Choice, Employment/Enrollment Match, Regional Validation |
+| School | person school zone and `school_geo__` | school/enrollment comparison and school shadow-price residuals | Mandatory Location Choice, Employment/Enrollment Match |
+| Tour/trip origin and destination | `origin_geo__`, `destination_geo__` | commuting flow matrices and external destination summaries | Mandatory Location Choice, Internal vs. External Tours, Regional Validation |
+| Land use | `land_use_geo__` | employment/enrollment targets, shadow-price comparisons, and PNR capacity comparisons when the owning summary joins land use | Employment/Enrollment Match, Park-and-Ride Location |
+| Parking location | `parking_zone` base zone only | `parking_locations` | Parking Location |
+
+Important limits:
+
+- Overview, generic household/person distributions, Tour Purpose/Mode/Time,
+ and most Trip Purpose/Mode/Time summaries do not gain a geography dimension
+ merely because an aggregation is configured.
+- `parking_locations` currently reports its base MAZ or TAZ parking zone. The
+ prepare step does not create `parking_geo__`, so named aggregations do
+ not automatically appear on that page.
+- Regional Validation uses only modeled geography types that agree with the
+ configured outside flow contract, such as `district`/`home_district` or
+ `county`/`home_county`.
+- A page option is present only when at least one usable run contains non-null
+ rows for that geography type. Configuration alone does not force an empty
+ option into the selector.
+
+For an exact output, find its ID in chapter 26 and confirm that the schema has
+`geography_type`/`geography_id` (or origin/destination geography pairs). Then
+inspect the summary's prepared requirements and the relevant page declaration
+in chapter 31.
+
+## Cache And Refresh Behavior
+
+The normalized mapping rows contribute to prepare and summary identity. A
+change to a lookup file, inline mapping, aggregation name, or source zone system
+invalidates incompatible prepared and summary caches automatically. For a
+repeatable manual rebuild, include `prepare` in `pipeline.steps` and use
+`refresh: [prepare]`; this also rebuilds later skimjoin and summary output.
+
+Setting `summarize.geography.enabled: false` disables both the legacy mapping
+and named aggregations. Definitions left below the disabled setting do not
+affect cache identity.
+
+## Implementation And Extension Points
+
+| Task | Start here |
+|---|---|
+| Config and lookup validation | `runtime/config/normalize_geography.py` |
+| Cache identity | `runtime/config/signatures.py` |
+| Zone context and lookup joins | `processor/prepare/enrichment/zones.py` |
+| Household/person role columns | `processor/prepare/enrichment/households_persons.py` |
+| Tour and trip role columns | `processor/prepare/enrichment/tours.py` and `trips.py` |
+| Summary geography helpers | `processor/summarize/summaries/summary_helpers.py` |
+| Dashboard geography options | `dashboard/helpers/geography_helpers.py` |
+
+## Troubleshooting
+
+| Symptom | Check |
+|---|---|
+| No custom geography columns | `summarize.geography.enabled`, prepared cache identity, and the aggregation name. |
+| All mapped values are null | `source_zone_system`, `zones`, the prepared source columns, and lookup zone IDs. |
+| CSV fails during config load | File path, required column names, integer zone IDs, blank labels, and conflicting duplicate zones. |
+| Geography missing from a page | Whether that summary supports the role and whether any usable run has non-null data. |
+| MAZ option missing | `dashboard.enable_maz_geographies` and the page's supported geography levels. |
+| Labels are wrong but membership is correct | `display.labels.geography`, not the aggregation lookup. |
+
+## Related Chapters
+
+- [11 - Configuring Your Data](11-configuring-your-data.md)
+- [13 - Configuration Reference](13-configuration-reference.md#summarize)
+- [21 - Prepared Tables](21-prepared-tables.md)
+- [26 - Summary Catalog](26-summary-catalog.md)
+- [42 - Config, Columns, And Labels](42-config-column-label-cookbook.md)
+- [90 - Troubleshooting](90-troubleshooting.md)
diff --git a/wiki/30-output-visualizer.md b/wiki/30-output-visualizer.md
index a20b10d..3967acc 100644
--- a/wiki/30-output-visualizer.md
+++ b/wiki/30-output-visualizer.md
@@ -26,6 +26,69 @@ The visualizer is responsible for:
The visualizer does not rebuild summaries. If one is missing, run the processor
workflow first.
+## Use The Live Dashboard
+
+After the server starts, open the URL printed in the terminal. The default is
+`http://localhost:5006`. The left rail identifies the loaded runs and provides
+the dashboard-wide controls; the main area contains standalone page tabs and
+group tabs such as Tour Summaries and Validation Summaries.
+
+| Control | Effect |
+|---|---|
+| Runs Loaded | Shows the color and label used for each run. It is a legend, not a run filter. |
+| Weighting | Selects one stored weighting mode. The control is disabled when only one mode is available. |
+| Values: Percent | Shared distribution charts divide each run's values by the relevant plotted total. This supports shape comparison across runs of different sizes. |
+| Values: Count | Shared distribution charts use stored weighted or unweighted values. |
+| Page selectors | Filter or change only the registered sections that depend on them. Options come from usable data and can differ by configuration. |
+| Calculation notes | Expand below supported output to show source summaries, filters, formulas, and aggregation details. |
+
+Some outputs deliberately ignore the Percent/Count switch. Examples include
+rates, averages, validation statistics, tables, and charts whose builder sets a
+fixed value mode. Read the axis label and calculation note; do not assume every
+number on a Percent dashboard is a share.
+
+Configured segmented summaries appear as separate series such as
+`Base (North)`. `segment.dashboard.visibility` determines whether the full run,
+segments, or both are included, and `segmentation_type` selects the displayed
+definition. These are configuration choices, not live sidebar controls.
+
+## Read Comparisons Correctly
+
+Use the following order when interpreting a chart:
+
+1. Confirm the weighting and Values controls.
+2. Read the chart title, axis units, and active page selectors.
+3. Identify each run or segment by its rail color and full hover label.
+4. Check whether the output is a count, share, rate, average, residual, or
+ modeled-versus-observed comparison.
+5. Expand the calculation note when available.
+
+Percent mode normally normalizes each run independently, so it compares
+distributions rather than regional totals. Count mode can compare totals only
+when runs use compatible sample expansion, model coverage, and source units.
+Distance labels in existing pages assume miles; skim component units remain
+the units in their source matrices or sidecars.
+
+The first configured run is the base for outputs that calculate a difference
+or percent difference. Reordering `runs` can therefore change the comparison
+reference as well as duplicate-label run-key suffixes.
+
+## Missing And Partial Data
+
+A page can use some runs while excluding others. A standard unavailable card
+identifies missing files, unavailable summaries, failed calculations, or
+schema mismatches. A partial result means at least one run was usable and at
+least one was excluded; the chart still renders the usable runs. Hover labels
+and the Runs Loaded legend do not prove that every run contributed to every
+visualization.
+
+Set `display.missing_data_display: blank` only when you intentionally want to
+hide diagnostic cards. During setup and extension work, keep the default
+`card` behavior.
+
+For a page-by-page description, use
+[16 - Dashboard User Guide](16-dashboard-user-guide.md).
+
## Live Dashboard
[`dashboard/app.py`](../dashboard/app.py) assembles the live dashboard. It
@@ -104,7 +167,8 @@ When adding visual output:
## Related Chapters
-- [31 - Dashboard Pages](31-dashboard-pages.md)
+- [16 - Dashboard User Guide](16-dashboard-user-guide.md)
+- [31 - Dashboard Page Contract](31-dashboard-pages.md)
- [32 - Figures and Widgets](32-figures-and-widgets.md)
- [33 - Dashboard Page Recipes](33-dashboard-page-recipes.md)
- [34 - HTML Export](34-html-export.md)
diff --git a/wiki/31-dashboard-pages.md b/wiki/31-dashboard-pages.md
index 62817f2..981f6b3 100644
--- a/wiki/31-dashboard-pages.md
+++ b/wiki/31-dashboard-pages.md
@@ -1,10 +1,15 @@
-# 31 - Dashboard Pages
+# 31 - Dashboard Page Contract
The visualizer discovers dashboard pages in modules under
[`dashboard/pages`](../dashboard/pages). Each final module contains one
`DashboardPage` subclass with a `@dashboard_page(...)` decorator. Page packages
export a `DashboardGroupDefinition` as `GROUP`.
+For descriptions of the analyses and advice about interpreting their controls,
+see [16 - Dashboard User Guide](16-dashboard-user-guide.md). This chapter is
+the authoritative reference for page IDs, data prerequisites, and extension
+contracts.
+
## Page Definition Contract
Important fields:
@@ -180,6 +185,7 @@ Total registered pages: **27**
## Related Chapters
+- [16 - Dashboard User Guide](16-dashboard-user-guide.md)
- [30 - Output Visualizer](30-output-visualizer.md)
- [32 - Figures and Widgets](32-figures-and-widgets.md)
- [33 - Dashboard Page Recipes](33-dashboard-page-recipes.md)
diff --git a/wiki/32-figures-and-widgets.md b/wiki/32-figures-and-widgets.md
index 640d5f6..b9f972b 100644
--- a/wiki/32-figures-and-widgets.md
+++ b/wiki/32-figures-and-widgets.md
@@ -26,9 +26,6 @@ The main author-facing objects are:
- `self.query(...)` for repeated or expensive transformations
- `self.plot` for figures and tables
-Do not add routine `sync_controls()` methods or page cache keys. Option
-providers and section dependencies supply the required information to the
-framework.
## Data And Figures
@@ -86,6 +83,19 @@ The page-facing data API is:
| `self.data.prepared_runs(weighting_mode=None)` | Direct `RunData` access for features that require matrices or other non-table state. |
| `self.data.summary_series(id, weighting=None)` | Specialized skim-summary view that retains summary-series metadata. |
+For `summary()` and `summaries()`, `required` has these exact meanings:
+
+| Value | Behavior when no run is usable |
+|---|---|
+| `None` | Required when the ID appears in the page definition's `required_summary_ids`; optional otherwise. |
+| `True` | Record the selection and emit the page's required-summary warning even when the decorator did not declare it. |
+| `False` | Record diagnostics but suppress the required-summary warning. Use this for an independent optional feature. |
+
+`required` does not make the lookup raise and does not render a card by itself.
+The section must still test the returned `RunTables` and choose its standard
+unavailable or optional-feature fallback. `columns=` is evaluated per run, so
+one compatible run can render while other runs appear in `data.issues`.
+
`RunTables` is iterable and indexable as `(run_label, DataFrame)` pairs. Its
public query interface is:
@@ -103,6 +113,50 @@ public query interface is:
| `.to_list()` | Materialize tuples for an external API that cannot consume `RunTables`. |
| `.available`, `.partial`, `.issues`, `.source_ids` | Availability and provenance metadata retained through fluent operations. |
+Each issue contains `label`, `status`, `detail`, `source_kind`, `source_id`,
+`missing_columns`, and available run/cache identity. Plotting a partial
+`RunTables` value renders only its usable runs and keeps the exclusions in page
+and export diagnostics. Do not replace it with `.to_list()` before the normal
+render boundary unless an external API requires tuples; that discards the
+structured availability object from subsequent fluent operations.
+
+## Query Cache Contract
+
+`self.query(factory)` accepts one zero-argument callable and returns the
+callable's result. On a cache miss it executes `factory`; on a hit it returns
+the stored value. Its identity contains:
+
+- page ID;
+- current global dashboard state, including weighting, Values, and segment
+ presentation state;
+- active section ID;
+- current values of selectors declared by that section;
+- callable module, qualified name, file, and first line; and
+- simple closure/default/keyword-default values. Complex captured objects are
+ represented by type, so capture the scalar values that actually change the
+ calculation.
+
+A selector affects query identity only when its ID appears in the active
+section's `selectors=(...)` declaration. Always declare every selector that
+changes the renderer. Global state or a declared selector change creates a new
+identity automatically. Call `self.clear_query_cache()` only after mutable
+external state changes outside those declared inputs; it clears this page's
+memoized queries.
+
+```python
+def render_body(self):
+ purpose = self._purpose_by_label[self.purpose.value]
+ data = self.data.summary(
+ "trip_mode_by_tour_purpose_and_tour_mode",
+ columns=("tour_purpose", "trip_mode", "trip_count"),
+ )
+ return self.query(
+ lambda: data.where(tour_purpose=purpose)
+ .group("trip_mode", pl.col("trip_count").sum())
+ .drop_empty()
+ )
+```
+
## Calculation Notes
Calculation notes are expandable HTML details below annotated charts and
@@ -254,7 +308,7 @@ exist at export time.
## Related Chapters
-- [31 - Dashboard Pages](31-dashboard-pages.md)
+- [31 - Dashboard Page Contract](31-dashboard-pages.md)
- [33 - Dashboard Page Recipes](33-dashboard-page-recipes.md)
- [34 - HTML Export](34-html-export.md)
- [35 - Plotting Reference](35-plotting-reference.md)
diff --git a/wiki/33-dashboard-page-recipes.md b/wiki/33-dashboard-page-recipes.md
index 9e0aa02..c8157cd 100644
--- a/wiki/33-dashboard-page-recipes.md
+++ b/wiki/33-dashboard-page-recipes.md
@@ -183,7 +183,7 @@ IDs, missing definitions, unknown groups, and invalid data requirements.
## Related Chapters
-- [31 - Dashboard Pages](31-dashboard-pages.md)
+- [31 - Dashboard Page Contract](31-dashboard-pages.md)
- [32 - Figures and Widgets](32-figures-and-widgets.md)
- [34 - HTML Export](34-html-export.md)
- [45 - Dashboard Extension Cookbook](45-dashboard-extension-cookbook.md)
diff --git a/wiki/34-html-export.md b/wiki/34-html-export.md
index 6c70ff5..068cbbb 100644
--- a/wiki/34-html-export.md
+++ b/wiki/34-html-export.md
@@ -56,6 +56,13 @@ The command also writes `artifacts/exports/dashboard.diagnostics.json`, a
sidecar file that records export warnings and size or state analysis. The HTML
file does not depend on this sidecar.
+The sidecar distinguishes rendered, partial, and skipped visualization inputs
+for every exported dashboard state and region variant. It also reports raw,
+valid, aliased, and pruned selector combinations plus estimated JSON bytes by
+state, page, and region. See
+[36 - HTML Export Schema](36-html-export-schema.md#diagnostics-sidecar-schema)
+for every field and current warning threshold.
+
For one override, use `--export-html [PATH]`. If you do not give a path, the
command uses the configured output path. If that path is absent, it uses
`/exported_dashboard.html`. You must also select the dashboard step. Add
diff --git a/wiki/35-plotting-reference.md b/wiki/35-plotting-reference.md
index 2bf6fcb..1a25025 100644
--- a/wiki/35-plotting-reference.md
+++ b/wiki/35-plotting-reference.md
@@ -11,17 +11,17 @@ Fetch, query, and plot without converting the data to tuple lists:
```python
data = (
self.data.summary(
- "trip_mode_by_purpose",
- columns=("purpose", "mode", "trip_count"),
+ "trip_mode_by_tour_purpose_and_tour_mode",
+ columns=("tour_purpose", "trip_mode", "trip_count"),
)
- .where(purpose=self.purpose_sel.value)
- .group("mode", pl.col("trip_count").sum())
- .sort("mode")
+ .where(tour_purpose=self.purpose_sel.value)
+ .group("trip_mode", pl.col("trip_count").sum())
+ .sort("trip_mode")
)
return self.plot.bar(
data,
- x="mode",
+ x="trip_mode",
y="trip_count",
title="Trip Mode",
x_title="Mode",
diff --git a/wiki/36-html-export-schema.md b/wiki/36-html-export-schema.md
index 7c308c0..2f0c768 100644
--- a/wiki/36-html-export-schema.md
+++ b/wiki/36-html-export-schema.md
@@ -44,6 +44,28 @@ Top-level fields:
| `page_export_support` | `PageExportSupportPayload` | Metadata about export-enabled page selectors |
| `client_runtime` | `str` | Runtime family identifier for diagnostic/debugging purposes |
+Current protocol identifiers are `schema_version: "2.0"`,
+`client_runtime: "region-swap-v1"`, and
+`page_export_support.client_side_runtime: "dashboard-and-page-selectors"`.
+Treat them as compatibility identifiers, not user-visible labels.
+
+### Dashboard Chrome And State Fields
+
+| Object | Complete fields |
+|---|---|
+| `runs_loaded[*]` | `label`, `color` |
+| `chrome` | `layout`, `rail_sections`, `controls_enabled` |
+| `chrome.controls_enabled` | Boolean `weighting`, Boolean `values` |
+| `dashboard_controls` | `weighting` list and `values` list |
+| `default_state` | `weighting`, `values` |
+| `page_export_support` | `client_side_runtime`, `enabled_page_selectors` |
+| `enabled_page_selectors[*]` | `page_id`, `selector_id` |
+
+The runtime uses the options in `dashboard_controls` to validate
+`default_state` and to form state keys. A control can remain in the payload
+while `controls_enabled` disables switching because only one value was
+exported.
+
`dashboard/export/payload.py` builds the dashboard state key for `states`:
```text
@@ -185,6 +207,82 @@ The supported widget types are `select`, `radio_button_group`, `float_input`,
`checkbox`, and `button`. `SelectorMetadataPayload.default_value` and widget
values can be all JSON-compatible values. They are not limited to strings.
+### Complete Node Fields
+
+Every node has a `kind` discriminator and only the fields for that kind:
+
+| Kind | Required fields | Optional fields |
+|---|---|---|
+| `container` | `layout`, `children`, `child_count`, `styles`, `css_classes` | none |
+| `card` | `title`, `children` | none |
+| `tabs` | `tabs`; each tab has `title`, `content` | tab `full_title` |
+| `plotly` | `figure` | `height`, `aspect_ratio` |
+| `table` | `columns`, `rows` | `column_tooltips` |
+| `widget` | `widget_type`, `name`, `value`, `options`, `step`, `disabled`, `selector_id`, `export_enabled` | `parent_selector_id`, `options_by_parent_value`, `disabled_parent_values` |
+| `html` | `html` | none |
+| `spacer` | none | none |
+| `region` | `region_id`, `selector_ids`, `content_mode`, `default_key`, `default_content`, `variants`, `variant_aliases` | none |
+
+`container.layout` is `row` or `column`. `widget_type` is one of
+`radio_button_group`, `select`, `float_input`, `checkbox`, or `button`.
+`region.content_mode` is currently `snapshot`.
+
+Plotly's `figure` field is its JSON-compatible figure dictionary. Table rows
+are dictionaries keyed by the ordered `columns`. HTML is already serialized
+markup; the browser runtime does not execute Python pane logic.
+
+## Diagnostics Sidecar Schema
+
+`write_export_html_document()` writes `.diagnostics.json` beside the
+HTML. It is build-time diagnostic data and is not required to open the HTML.
+The top-level shape is:
+
+| Field | Type | Meaning |
+|---|---|---|
+| `schema_version` | integer, currently `1` | Diagnostics format version; separate from export payload `2.0`. |
+| `title` | string | Dashboard title. |
+| `states` | mapping | Page and region diagnostics keyed by `||`. |
+| `size_analysis` | mapping | Estimated compact-JSON bytes by state, page, and region. |
+
+For each dashboard state, `states[state_key][page_id]` is a mapping containing:
+
+- `default`: visualization diagnostics for the default page state;
+- `export_region:`: selector enumeration counts; and
+- `region::`: visualization diagnostics captured for
+ one rendered region variant.
+
+A visualization diagnostic has these fields:
+
+| Field | Meaning |
+|---|---|
+| `visualization_id` | Summary or prepared input used as the diagnostic boundary. |
+| `render_state` | `rendered`, `partial`, or `skipped`. |
+| `input_kind` | `summary`, `prepared`, or `mixed`. |
+| `input_ids` | Source summary/prepared IDs. |
+| `usable_run_labels` | Runs included in that output. |
+| `excluded_runs` | Per-run exclusions. |
+
+Each excluded run contains `label`, `status`, `detail`, `source_kind`,
+`source_id`, and `missing_columns`. An export-region enumeration record
+contains `selector_ids`, `selector_counts`, `raw_state_count`,
+`valid_state_count`, `alias_count`, and `pruned_state_count`.
+
+`size_analysis` contains:
+
+| Field | Contents |
+|---|---|
+| `warning_thresholds` | Byte thresholds for total, strong-total, page, static-region, and selector-region warnings. |
+| `total_payload_bytes`, `state_count` | Whole payload estimate and number of dashboard states. |
+| `states` | Per-state `payload_bytes`; each page has `payload_bytes` and region metrics. |
+| `page_peaks` | Largest state for each page. |
+| `region_peaks` | Largest state for each page/region. |
+
+Region size metrics contain `selector_ids`, `variant_count`,
+`default_content_bytes`, `variants_bytes`, and `total_bytes`. Current warning
+thresholds are 100 MiB total, 250 MiB strong total, 10 MiB per page, 5 MiB for
+a static region, and 1 MiB for a selector region. These are warnings, not hard
+limits.
+
## Runtime Validation Rules
The embedded runtime validates these items:
@@ -218,8 +316,9 @@ Rules:
from older Python code.
2. Keep the runtime check strict. A mismatch must show an error and must not
render incorrect content.
-3. Update this document, `dashboard/export/assets/export_runtime.js`, and the
- export payload tests in the same change.
+3. Update this document, the readable files under
+ `dashboard/export/js_runtime/`, rebuild the generated asset, and update the
+ export payload/runtime tests in the same change.
## Checklist for Adding a New Node Kind
@@ -227,10 +326,12 @@ To add a serialized node kind:
1. Add the new typed shape to `dashboard/export/types.py`.
2. Emit it from `dashboard/export/serializer.py`.
-3. Render it in `dashboard/export/assets/export_runtime.js`.
-4. Add serializer coverage in `tests/test_export_serializer.py`.
-5. Add or update payload/smoke assertions if the new node can appear in representative exports.
-6. Update this document.
+3. Render it in `dashboard/export/js_runtime/`.
+4. Run `uv run python dashboard/export/build_export_runtime.py` to rebuild
+ `dashboard/export/assets/export_runtime.js`; do not edit the built asset.
+5. Add serializer coverage in `tests/test_export_serializer.py`.
+6. Add or update payload/smoke assertions if the new node can appear in representative exports.
+7. Update this document.
## Related Chapters
diff --git a/wiki/40-developer-workflows.md b/wiki/40-developer-workflows.md
index ad1569c..f59c4f6 100644
--- a/wiki/40-developer-workflows.md
+++ b/wiki/40-developer-workflows.md
@@ -40,6 +40,8 @@ activitysim_visualizer/
| New raw-output normalization | [21 - Prepared Tables](21-prepared-tables.md) |
| New prepared column | [21 - Prepared Tables](21-prepared-tables.md#adding-a-prepared-column) |
| New skim-derived output | [22 - Skimjoin](22-skimjoin.md#adding-a-skim-output) |
+| New segmentation source or relationship | [24 - Segmentation](24-segmentation.md#implementation-and-extension-points) |
+| New custom geography behavior | [27 - Geography](27-geography.md#implementation-and-extension-points) |
| New generated summary function/table | [44 - Summary Function Cookbook](44-summary-function-cookbook.md) |
| New figure or table on existing page | [32 - Figures and Widgets](32-figures-and-widgets.md) |
| New dashboard page | [33 - Dashboard Page Recipes](33-dashboard-page-recipes.md) |
@@ -99,7 +101,9 @@ When behavior changes, update the documentation in the same change:
| Config behavior | `11-configuring-your-data.md` and `13-configuration-reference.md` |
| Prepare behavior | `21-prepared-tables.md` |
| Skimjoin behavior | `22-skimjoin.md` |
-| Summary contract or registration | `23-summary-functions.md`, then regenerate catalogs |
+| Segmentation behavior | `24-segmentation.md` and `13-configuration-reference.md` |
+| Geography behavior | `27-geography.md` and `13-configuration-reference.md` |
+| Summary contract or registration | `25-summary-functions.md`, then regenerate catalogs |
| Dashboard page API | `31-dashboard-pages.md`, `32-figures-and-widgets.md`, `33-dashboard-page-recipes.md` |
| Export payload/runtime | `34-html-export.md` |
| Export payload schema | `36-html-export-schema.md` |
diff --git a/wiki/41-data-extension-cookbook.md b/wiki/41-data-extension-cookbook.md
index c1825c4..fcfe15f 100644
--- a/wiki/41-data-extension-cookbook.md
+++ b/wiki/41-data-extension-cookbook.md
@@ -326,6 +326,6 @@ uv run --with pytest pytest --basetemp .pytest_tmp tests/test_runtime_workflows.
## Related Chapters
- [21 - Prepared Tables](21-prepared-tables.md)
-- [23 - Summary Functions](23-summary-functions.md)
-- [24 - Summary Catalog](24-summary-catalog.md)
+- [25 - Summary Functions](25-summary-functions.md)
+- [26 - Summary Catalog](26-summary-catalog.md)
- [42 - Config, Columns, and Labels](42-config-column-label-cookbook.md)
diff --git a/wiki/42-config-column-label-cookbook.md b/wiki/42-config-column-label-cookbook.md
index 662ba91..14fb81c 100644
--- a/wiki/42-config-column-label-cookbook.md
+++ b/wiki/42-config-column-label-cookbook.md
@@ -299,6 +299,167 @@ assert config.ordered_values(
Add a page or helper test. Verify that a `Full time` selection filters raw value
`2`. This test identifies a common label connection error.
+## Worked Example: Segment Runs With An External Lookup
+
+Use a CSV-backed segment when membership does not belong in the canonical model
+output. For example, classify households into planning markets without adding a
+regional column to prepare.
+
+Create `lookups/household_market.csv`:
+
+```csv
+household_id,market
+1,Core
+2,Suburban
+3,Rural
+```
+
+Enable the segment step and join the lookup to prepared households:
+
+```yaml
+pipeline:
+ steps: [segment, summarize, dashboard]
+ dashboard_mode: live
+ refresh: []
+
+segment:
+ dashboard:
+ segmentation_type: market
+ visibility: full_and_segments
+ definitions:
+ market:
+ source:
+ type: csv_lookup
+ file: lookups\household_market.csv
+ join:
+ source_table: hh
+ source_key_column: household_id
+ csv_key_column: household_id
+ segment_value_column: market
+ allow_overlapping: false
+ on_empty_segment: error
+ segments:
+ - id: core
+ label: Core
+ values: [Core]
+ - id: suburban
+ label: Suburban
+ values: [Suburban]
+ - id: rural
+ label: Rural
+ values: [Rural]
+```
+
+The household anchor keeps each matched household and its related people,
+tours, trips, days, vehicles, and joint tours. Every default summary then runs
+for each market and weighting mode. Output appears below:
+
+```text
+summary_tables//segments/market//
+```
+
+Before using a large lookup, check that every key and market value is nonblank
+and that one key does not map to different values. Decide whether missing CSV
+keys should remain only in the full run or indicate an incomplete lookup. The
+runtime permits missing keys but rejects joins that duplicate anchor rows.
+
+See [24 - Segmentation](24-segmentation.md) before using a person-, trip-, or
+other lower-level anchor; the selected relationship changes how population
+totals should be interpreted.
+
+## Worked Example: Add A Custom Geography From CSV
+
+Use a named geography aggregation when several summaries or pages need the same
+zone grouping. Suppose `lookups/maz_district.csv` contains:
+
+```csv
+MAZ,district
+101,North
+102,North
+201,South
+```
+
+Configure the lookup once:
+
+```yaml
+zones:
+ use_maz: true
+ maz_col: [MAZ, zone_id]
+ taz_col: [TAZ, taz]
+
+summarize:
+ geography:
+ enabled: true
+ aggregations:
+ district:
+ source_zone_system: maz
+ file: lookups\maz_district.csv
+ zone_id_col: MAZ
+ geography_col: district
+
+display:
+ labels:
+ geography:
+ mapping:
+ district: Planning District
+```
+
+Prepare creates role-specific columns such as `home_geo__district`,
+`work_geo__district`, `origin_geo__district`,
+`destination_geo__district`, and `land_use_geo__district`. Supporting summaries
+emit `geography_type: district` and the mapped district label as
+`geography_id`.
+
+The display mapping changes only the visible name of the geography type. It
+does not change zone membership. Keep the zone join under
+`summarize.geography.aggregations` and presentation text under
+`display.labels.geography`.
+
+Test at least one zone from each district, an unmapped zone, and a conflicting
+duplicate zone. Changing the CSV changes prepare and summary identity, so valid
+old caches are not reused. See [27 - Geography](27-geography.md) for every
+generated column and source-zone rule.
+
+## Worked Example: Share Skimjoin Rules Across Runs
+
+Keep skimjoin lookup behavior in one rules file and put model-specific data
+paths in the main visualizer config. This avoids copying mode and component
+rules for every scenario.
+
+Main config:
+
+```yaml
+pipeline:
+ steps: [prepare, skimjoin, summarize, dashboard]
+
+skimjoin:
+ defaults:
+ config_path: configs\skimjoin_rules.yaml
+ skim_files:
+ - skims\base\*.omx
+ network_los_file: skims\base\network_los.yaml
+
+runs:
+ - dir: outputs\base
+ label: Base
+ - dir: outputs\build
+ label: Build
+ skimjoin:
+ skim_files:
+ - skims\build\*.omx
+ network_los_file: skims\build\network_los.yaml
+```
+
+`configs/skimjoin_rules.yaml` contains `activitysim`, `defaults`, `dimensions`,
+and `modes`, but no `project` block. Integrated prepare supplies its trip and
+tour tables, while the main config supplies the paths.
+
+Repeat all required path overrides in a run-specific block. Once an override
+causes the selected rules file to be reloaded, an omitted skim or network path
+comes from that rules file rather than from the other global path overrides.
+See [22 - Skimjoin](22-skimjoin.md#where-path-settings-belong) for the full
+precedence rules.
+
## Completion Checklist
- Unknown keys and wrong types fail near the config boundary.
@@ -312,6 +473,9 @@ Add a page or helper test. Verify that a `Full time` selection filters raw value
- [13 - Configuration Reference](13-configuration-reference.md)
- [21 - Prepared Tables](21-prepared-tables.md)
+- [22 - Skimjoin](22-skimjoin.md)
+- [24 - Segmentation](24-segmentation.md)
+- [27 - Geography](27-geography.md)
- [32 - Figures And Widgets](32-figures-and-widgets.md)
- [41 - Data Extension Cookbook](41-data-extension-cookbook.md)
- [43 - Weighting And Hosting Extensions](43-weighting-hosting-extensions.md)
diff --git a/wiki/43-weighting-hosting-extensions.md b/wiki/43-weighting-hosting-extensions.md
index ca0b730..27e4647 100644
--- a/wiki/43-weighting-hosting-extensions.md
+++ b/wiki/43-weighting-hosting-extensions.md
@@ -65,6 +65,13 @@ column; it does not select a different weight. Prepare usually preserves raw
ActivitySim columns. If you use `prepared_table_map`, include the named source
columns in those prepared files.
+Column validation checks presence and castability during use; it does not
+enforce finite, non-null, or nonnegative values. Validate those properties in
+the producing workflow. Zero and negative values can produce zero denominators
+or subtract from counts, and null source weights can be omitted by aggregation.
+See [Summary Functions](25-summary-functions.md#weight-resolution-and-edge-cases)
+for primary-mode fallback and sample-rate behavior.
+
### 3. Cache, Dashboard, And Outside-Summary Behavior
The summary cache identity includes the mode ID, source columns, and column-mode
@@ -287,10 +294,30 @@ dashboard = build_dashboard(
dashboard.servable()
```
-Panel-compatible hosts can start this module with their standard command. A
-provider SDK can receive `dashboard` from the same script. Put secrets and
-deployment IDs in environment variables or provider configuration. Do not put
-them in the main visualizer YAML.
+Start it locally from the repository root:
+
+```powershell
+$env:ACTIVITYSIM_VIZ_CONFIG = "C:\deploy\activitysim_viz\config.yaml"
+uv run panel serve scripts/host_dashboard.py --address 127.0.0.1 --port 5006
+```
+
+For a service behind a reverse proxy, bind the process to all container/host
+interfaces and allow the public WebSocket origin:
+
+```powershell
+uv run panel serve scripts/host_dashboard.py --address 0.0.0.0 --port 5006 --allow-websocket-origin dashboard.example.org
+```
+
+Use the scheme/host value expected by the deployed Panel version when the
+public URL is nonstandard, and repeat the origin option if the deployment has
+several valid hosts. The proxy must forward WebSocket upgrade headers as well
+as ordinary HTTP. Terminate TLS and enforce authentication in the proxy or the
+chosen hosting provider unless the deployment deliberately adds those concerns
+to the application.
+
+A provider SDK can instead receive `dashboard` from the same script. Put
+secrets and deployment IDs in environment variables, a secret store, or
+provider configuration. Do not put them in the main visualizer YAML.
This approach has the following properties:
@@ -304,6 +331,26 @@ For a hosted service, caches must exist in persistent storage. To build caches
at startup, call the public prepare and summarize workflows before
`build_dashboard()`. Make the runtime cost and write permissions explicit.
+### Deployment Requirements
+
+Before treating the command as a production service, verify:
+
+| Requirement | Deployment rule |
+|---|---|
+| Code/imports | Install the package or start from a working directory where `dashboard`, `processor`, and `runtime` are importable. Keep the deployed code version aligned with the cache schema. |
+| Configuration | Set `ACTIVITYSIM_VIZ_CONFIG` to an explicit readable file. Resolve relative paths intentionally; absolute cache/input paths are safer in containers. |
+| Summary caches | Mount `` as persistent readable storage. All enabled summary-backed pages need compatible run manifests. |
+| Prepared caches | Mount them when any enabled live page has optional or required prepared data. HTML export alone cannot replace this live requirement. |
+| Permissions | Read-only cache mounts are sufficient when artifacts are built before deployment. Grant writes only when startup intentionally builds or refreshes caches. |
+| Network | Expose the selected port, configure the public WebSocket origin, and preserve WebSocket upgrades through the proxy/load balancer. |
+| Sessions and memory | Panel creates server-side sessions. Size workers for the loaded summary/prepared data and expected concurrent sessions; do not assume a standalone HTML memory profile. |
+| Security | Put TLS, authentication, secrets, and access logs in the provider/proxy boundary unless a reviewed adapter owns them. |
+| Startup failure | Fail the deployment when config or required caches cannot load. Do not serve a process that silently has no configured runs. |
+
+The `panel serve` command above loads caches and serves the app. It does not run
+prepare or summarize. Build and validate artifacts in a separate deployment
+step unless startup generation is an explicit operational choice.
+
## Option B: Make `dashboard_mode: host` A Core Adapter
Use this approach only when one hosting provider must be a supported runtime mode.
diff --git a/wiki/44-summary-function-cookbook.md b/wiki/44-summary-function-cookbook.md
index ba58d67..6b1266b 100644
--- a/wiki/44-summary-function-cookbook.md
+++ b/wiki/44-summary-function-cookbook.md
@@ -1,7 +1,7 @@
# 44 - Summary Function Cookbook
This chapter shows how to create and test one dashboard summary. Use it with the
-short contract reference in chapter 23.
+short contract reference in chapter 25.
## Worked Example: Trips By Mode
@@ -208,7 +208,7 @@ uv run python scripts/generate_wiki_catalogs.py
uv run --with pytest pytest --basetemp .pytest_tmp tests/test_summary_declarations.py tests/test_page_registry_contract.py
```
-Make sure the new ID appears in chapter 24. After connecting it to a page, make
+Make sure the new ID appears in chapter 26. After connecting it to a page, make
sure it also appears in the chapter 31 page catalog.
## Variations
@@ -246,7 +246,7 @@ make the summary unavailable. The builder must not fail.
## Related Chapters
- [21 - Prepared Tables](21-prepared-tables.md)
-- [23 - Summary Functions](23-summary-functions.md)
-- [24 - Summary Catalog](24-summary-catalog.md)
+- [25 - Summary Functions](25-summary-functions.md)
+- [26 - Summary Catalog](26-summary-catalog.md)
- [41 - Data Extension Cookbook](41-data-extension-cookbook.md)
- [45 - Dashboard Extension Cookbook](45-dashboard-extension-cookbook.md)
diff --git a/wiki/45-dashboard-extension-cookbook.md b/wiki/45-dashboard-extension-cookbook.md
index c7d95b1..2421054 100644
--- a/wiki/45-dashboard-extension-cookbook.md
+++ b/wiki/45-dashboard-extension-cookbook.md
@@ -7,7 +7,9 @@ data through `self.data`.
## Worked Example: Add A Page To An Existing Group
-In this example, the registered summary `trips_by_mode` has `trip_mode` and
+In this example, `trips_by_mode` is the worked example summary created in
+[44 - Summary Function Cookbook](44-summary-function-cookbook.md). It is not a
+built-in summary until you add that declaration. It has `trip_mode` and
`trip_count` columns. Create one discoverable final module:
```text
@@ -98,7 +100,7 @@ def build_page(self):
def purpose_options(self):
- data = self.data.summary("trips_by_mode_and_purpose")
+ data = self.data.summary("trip_mode_by_tour_purpose_and_tour_mode")
options, self._purpose_by_label = column_options(
data.to_list(),
"tour_purpose",
@@ -115,7 +117,7 @@ Use the raw value for the filter. Do not use its display label:
```python
raw_purpose = self._purpose_by_label[self.purpose.value]
-data = self.data.summary("trips_by_mode_and_purpose")
+data = self.data.summary("trip_mode_by_tour_purpose_and_tour_mode")
chart_data = self.query(
lambda: data.where(tour_purpose=raw_purpose).select(
"trip_mode", "trip_count"
@@ -414,7 +416,7 @@ uv run --with pytest pytest --basetemp .pytest_tmp tests/test_figure_builders.py
## Related Chapters
-- [31 - Dashboard Pages](31-dashboard-pages.md)
+- [31 - Dashboard Page Contract](31-dashboard-pages.md)
- [32 - Figures And Widgets](32-figures-and-widgets.md)
- [33 - Dashboard Page Recipes](33-dashboard-page-recipes.md)
- [34 - HTML Export](34-html-export.md)
diff --git a/wiki/46-testing.md b/wiki/46-testing.md
index 9fbd5ea..7b9f268 100644
--- a/wiki/46-testing.md
+++ b/wiki/46-testing.md
@@ -1,5 +1,19 @@
# 46 - Testing
+## Install Test Dependencies
+
+The project supports Python 3.10 or later. The checked-in GitHub Actions job
+uses Python 3.12 on Windows. Install the locked runtime and `dev` dependency
+group before running its commands:
+
+```powershell
+uv sync --locked --group dev
+```
+
+The `dev` group contains `pytest` and `ruff`. The separate `notebooks` group is
+not required for the test suite. Do not add an ad hoc `--with` dependency when
+the locked development environment is already installed.
+
The default command runs all tests, including the complete offline HTML export
checks:
@@ -30,6 +44,23 @@ Run this correctness check before pushing changes:
uv run ruff check .
```
+## Match Continuous Integration
+
+`.github/workflows/tests.yml` runs on pushes to `main`, pull requests, and
+manual dispatch. It checks out the repository, selects Python 3.12, installs
+`uv`, syncs the locked `dev` group, runs Ruff, and then runs the full pytest
+command. To reproduce the CI work locally:
+
+```powershell
+uv sync --locked --group dev
+uv run ruff check .
+uv run pytest --basetemp .pytest_tmp
+```
+
+CI currently has no separate docs-link job. Documentation changes that add or
+rename pages must therefore run the repository's catalog generator and a local
+Markdown link/anchor check in addition to the relevant Python tests.
+
Use `full_export` only for behavior that requires all default dashboard pages
and dashboard states. For writes, validation, pages, selectors, and diagnostics,
configure the smallest relevant page and state set. The full-export tests still
@@ -57,6 +88,55 @@ uv run pytest --basetemp .pytest_tmp tests/test_figure_builders.py
uv run pytest --basetemp .pytest_tmp tests/test_export_serializer.py tests/test_export_payload.py
```
+## Generated Runtime And Catalog Checks
+
+The browser export runtime has readable source under
+`dashboard/export/js_runtime/` and a generated artifact at
+`dashboard/export/assets/export_runtime.js`. After changing the readable
+source, rebuild it and run its contract test:
+
+```powershell
+uv run python dashboard/export/build_export_runtime.py
+uv run pytest --basetemp .pytest_tmp tests/test_export_runtime_build.py tests/test_export_runtime_contract.py
+```
+
+The build test checks that the tracked generated asset matches the runtime
+source. If it fails, rebuild the asset and commit the generated change. Do not
+edit the asset directly.
+
+After changing summary declarations, schemas, page definitions, groups, or
+page data requirements, regenerate and test the catalogs:
+
+```powershell
+uv run python scripts/generate_wiki_catalogs.py
+uv run pytest --basetemp .pytest_tmp tests/test_summary_declarations.py tests/test_page_registry_contract.py
+```
+
+Review the generated diff. A catalog change should follow the code contract
+that caused it; do not hand-edit generated blocks.
+
+## Test Data Conventions
+
+- Build the smallest Polars frames that exercise the contract. Include only
+ the IDs, source fields, and `finalweight` needed by the behavior.
+- Use `tmp_path` for files and `tmp_path_factory` only for intentionally shared
+ session fixtures. Never create persistent test output in the repository
+ root.
+- Write CSV, Parquet, OMX, or YAML inputs inside that temporary directory.
+- Give IDs explicit compatible types when a join or cache schema is under test.
+- Test the complete case plus the relevant empty, unavailable, failed, orphan,
+ or partial-run case.
+- Keep pure summary/transform assertions independent of Panel. Add lifecycle or
+ export tests only for behavior at those boundaries.
+- Reuse the session-scoped `representative_full_export_html` fixture when a
+ test genuinely needs the full default export. Do not rebuild it in each
+ test.
+
+When a regression needs a large real data set, reduce it to a small synthetic
+fixture or store only a reviewed stable fixture under `tests`. Tests must not
+depend on a developer's model-output directory, network service, or existing
+cache root.
+
Use [Developer Workflows](40-developer-workflows.md) to select tests for a
subsystem.
diff --git a/wiki/90-troubleshooting.md b/wiki/90-troubleshooting.md
index 4549f45..d63bc5d 100644
--- a/wiki/90-troubleshooting.md
+++ b/wiki/90-troubleshooting.md
@@ -21,11 +21,62 @@ Use this chapter when a run, cache, page, or export does not behave as expected.
| Summary cache rebuilds unexpectedly | Input fingerprint changed, upstream prepared identity changed, summary config changed, summary declaration changed | run-level summary manifest and `--explain-cache` |
| Page says data unavailable | Required summary missing, optional raw input absent, prepared column missing | page catalog and summary catalog |
| Counts look wrong | Weighting mode, sample rate, explicit weight columns | `summarize.weighting_modes`, prepared `finalweight` |
-| Geography options missing | Geography disabled, land-use columns missing, aggregation config wrong | `zones`, `summarize.geography` |
+| Geography options missing | Geography disabled, zone columns missing, lookup config wrong | [Geography](27-geography.md) and `summarize.geography` |
+| Segmented series missing | Segment step disabled, source values do not match, dashboard visibility hides them | [Segmentation](24-segmentation.md), `pipeline.steps`, and `segment.dashboard` |
| Skim pages empty | Skimjoin disabled, no skim outputs, missing lookup rules | skimjoin manifest and reports |
| Export differs from live | Widget/section not registered, selector values omitted, unsupported node | page selector/section registrations |
| Dashboard-only run fails | Summary cache missing or prepared-data page needs prepared cache | `pipeline.steps`, page prepared-data mode |
+## Configuration And Startup Problems
+
+| Symptom | Cause to distinguish | Action |
+|---|---|---|
+| Unknown top-level or section key | Typo, removed field, or field at the wrong nesting level | Use the replacement in the error and compare the field with chapter 13. Do not move it until you confirm its owning section. |
+| Path exists in YAML but file is not found | Relative path uses a different base than expected | Use the path-resolution table in chapter 13. Raw `files` start from each run directory; most other main-config paths start from the config directory. |
+| `skimjoin` or `segment` config appears ignored | The configuration block does not enable its logical step | Add the step and its prerequisite to `pipeline.steps`. Use canonical step order. |
+| Dashboard starts instead of exporting | `dashboard_mode` is live or a CLI override selected live mode | Set `pipeline.dashboard_mode: export`, or use `--dashboard --export-html`. |
+| `dashboard_mode: host` does not publish | Core host mode is a placeholder | Use the explicit Panel hosting script in chapter 43. |
+| Port is already in use | Another server owns the selected port | Stop that process or use `--port `. |
+
+Configuration validation is strict at documented typed boundaries but some
+extension/nested mappings are intentionally free-form. If a nested setting has
+no effect and no error, confirm its exact spelling and add a focused config
+load test rather than assuming it was applied.
+
+## Raw Input And Prepare Problems
+
+| Symptom | Check | Interpretation |
+|---|---|---|
+| Raw table is unavailable | Effective `files` plus `runs[*].file_map`, run directory, extension, fallback path | A stem tries Parquet before CSV. An explicit extension tries only that file. |
+| Entire run is skipped | Availability of households, persons, tours, and trips | The run is skipped when none of these four core tables is usable. One missing core table instead causes partial summary coverage. |
+| Prepared column is missing | Source alias, owning raw table, enrichment prerequisites, prepared manifest | Prepare only materializes a canonical field when it finds the source needed for that field. |
+| CSV reads with an unexpected type | Mixed values or inference | Prefer Parquet for controlled schemas or normalize the raw column before prepare. Prepared finalization casts only known canonical fields. |
+| Relationship warning reports orphans | Source/target key values and types | Direct summaries can count orphan rows while joined summaries can drop them. Fix the relationship instead of comparing those totals as equivalent. |
+| `prepared_table_map` lacks derived fields | External process supplied raw-like rather than canonical tables | That input bypasses all prepare enrichment, weighting, geography, and skimjoin. Materialize the prepared contract upstream. |
+| Optional table has zero columns after cache load | Stored `empty`, `unavailable`, or `failed` state | Read `table_states` and `table_diagnostics`; the cache loader converted its sentinel back to an empty frame. |
+
+Use [14 - Input Data Contract](14-input-data-contract.md) to identify the
+expected keys and relationship checks. Use chapter 26 to work backward from one
+unavailable summary to its exact prepared columns.
+
+## Weighting, Totals, And Units
+
+If counts differ from expectation, inspect the prepared `finalweight` values on
+the table that the summary actually aggregates. Do not infer trip weights from
+household weights without checking propagation.
+
+| Symptom | Likely reason | Check |
+|---|---|---|
+| Weighted equals unweighted | No source weight/sample rate, all source weights are one, or a mapped external summary was copied to both modes | Prepared `finalweight`, run weight fields, `summary_table_map` behavior |
+| Household totals are not sample-expanded | Any explicit run weight field disables automatic household sample-rate expansion | `hh_weight_col`, `person_weight_col`, `trip_weight_col`, `columns.sample_rate` |
+| Some rows disappear from weighted totals | Null source weights or builder filters | Null/nonfinite weight counts and summary requirements |
+| Negative or infinite result | Negative weights, zero sample rate, or zero weighted denominator | Validate finite nonnegative weights and positive sample rates upstream |
+| Distance/time/cost differs by a fixed factor | Runs or skims use different units | Prepared source columns and skim documentation; the visualizer does not convert units |
+| Percent chart does not sum to 100 | Fixed count/rate chart, missing categories, separate traces, or a builder-specific denominator | Axis title, `value_mode` used by the page, and calculation note |
+
+The first run is the comparison base where a page reports differences. Confirm
+run order before treating a changed difference as a processor regression.
+
## Cache Problems
To make a repeatable full rebuild, configure the steps and refresh policy:
@@ -55,14 +106,15 @@ invalidates skimjoin and summary output. A skimjoin refresh keeps
## Missing Page Data
-Find the page in [31 - Dashboard Pages](31-dashboard-pages.md) and check:
+Find the page in
+[31 - Dashboard Page Contract](31-dashboard-pages.md) and check:
- required summary IDs
- required prepared tables
- prepared-data mode
- whether the live or export configuration enables the page
-Then find each summary in [24 - Summary Catalog](24-summary-catalog.md). Check
+Then find each summary in [26 - Summary Catalog](26-summary-catalog.md). Check
the required input tables and columns.
### Worked Triage: A Page Says Data Is Unavailable
@@ -71,7 +123,7 @@ If Trip Mode shows the standard unavailable card, follow these steps:
1. Find `trip_mode` in chapter 31. It requires
`trip_mode_by_tour_purpose_and_tour_mode`.
-2. Find that ID in chapter 24. Note its required prepared table and columns.
+2. Find that ID in chapter 26. Note its required prepared table and columns.
3. Open `//manifest.json` and examine the summary entry. If the
summary is `unavailable`, read its recorded reason before a rebuild.
4. If a required prepared column is missing, inspect
@@ -94,7 +146,6 @@ Check the skimjoin reports:
- `missing_lookup_report`
- `fallback_lookup_report`
- `skipped_rule_report`
-- `tour_aggregation_summary`
- `failure_report`
Common corrections:
@@ -106,6 +157,53 @@ Common corrections:
- change missing matrix/OD policy only after confirming the missing data is
expected
+Also inspect `config_normalized.yaml` to verify the effective rules and paths.
+For CSV skims, confirm whether the file was inventoried as a keyed table or an
+OD table and use the generated `__` matrix name. For
+OMX/HDF5, qualify duplicate matrix names with `filename::matrix` and verify the
+selected zone mapping.
+
+With `failure_policy: record`, a failure is expected to leave the original
+prepared trip/tour tables in place and record a `failure_report`. With `error`,
+the same failure stops the workflow. Do not interpret “run continued” as proof
+that skim values were applied; read `skimjoin_status` and
+`skimjoin_applied_outputs`.
+
+## Segmentation Problems
+
+| Symptom | Check |
+|---|---|
+| Definition or ID rejected | Lowercase path-safe pattern and no leading/trailing punctuation. |
+| CSV-backed segment is empty | Quote numeric-looking `segments[*].values`; CSV segment values are stored as strings. |
+| Prepared-column segment is empty | Match the prepared column's value and type exactly. |
+| Household/person totals look too broad | The anchor may be trip/tour based; relationship expansion retains related parents and children. |
+| Segment totals exceed full total | Values overlap with `allow_overlapping: true`, or the counted population differs from the anchor. |
+| Mapped external summary is identical in every segment | `summary_table_map` is overlaid unchanged because aggregated rows cannot be re-segmented. |
+| Only one segment changed but many tables rebuilt | Read per-unit/per-summary digests and `--explain-cache`; a shared summary/config change can invalidate all units. |
+
+The run manifest's `segmentation_types` list is the final record of source,
+values, stored paths, states, and diagnostics. If a configured segment is not
+there, review `on_empty_segment: skip` and whether summarize completed a cache
+write.
+
+## Geography Problems
+
+Distinguish preparation from presentation:
+
+1. Confirm `summarize.geography.enabled: true`.
+2. Confirm the named aggregation and source zone system in the loaded config.
+3. Inspect the role-specific prepared column, such as
+ `home_geo__district` or `destination_geo__district`.
+4. Confirm non-null mapped values and lookup coverage.
+5. Confirm the target summary supports that role in chapter 27.
+6. Confirm the summary has rows for the geography type.
+7. Only then inspect the dashboard selector and
+ `dashboard.enable_maz_geographies`.
+
+A valid named mapping does not add geography to every summary. Parking Location
+currently uses its base parking zone, and MAZ presentation can be hidden even
+when MAZ summary rows exist.
+
## Export Problems
If live mode works but export fails:
@@ -121,6 +219,32 @@ If live mode works but export fails:
Export cannot reproduce every Python callback; it can only switch among stored
states and registered selector variants.
+Use the diagnostics sidecar to separate three failure classes:
+
+| Evidence | Meaning |
+|---|---|
+| `render_state: skipped` with excluded runs | Data contract or availability problem before serialization. |
+| Large `raw_state_count` or `size_analysis` peak | Selector enumeration made the payload large; export fewer values or disable that part. |
+| Browser `ExportRuntimeError` | Payload/runtime schema, node, state, or rendering problem; note its error code. |
+
+If you changed the browser runtime, edit `dashboard/export/js_runtime/`, rebuild
+the generated asset, and run runtime build/contract tests. Never patch the
+generated asset as the source change.
+
+## Performance And Hosting Problems
+
+| Symptom | First action |
+|---|---|
+| First run is slow | Separate prepare, skimjoin, summarize, and dashboard timings; later valid runs should reuse caches. |
+| Export is very large or slow to open | Inspect `size_analysis.page_peaks` and `region_peaks`; reduce exported weighting/value/selector states. |
+| Live server uses much more memory than export | Check enabled prepared-data pages and concurrent Panel sessions. Prepared runs can be loaded for live-only features. |
+| Hosted page loads but controls disconnect | Verify reverse-proxy WebSocket upgrades and `--allow-websocket-origin`. |
+| Hosted startup has no runs | Use an explicit config path and persistent compatible caches; fail deployment on missing required data. |
+| Permission error during hosting | Use read-only caches for serve-only deployment; grant writes only if startup deliberately builds artifacts. |
+
+For deployment commands and requirements, see
+[43 - Weighting And Hosting Extensions](43-weighting-hosting-extensions.md#worked-example-connect-a-hosting-script).
+
## Create a small test case
Reduce the problem to the smallest test case:
diff --git a/wiki/99-glossary.md b/wiki/99-glossary.md
index e3a8f0e..e8a0a42 100644
--- a/wiki/99-glossary.md
+++ b/wiki/99-glossary.md
@@ -3,28 +3,42 @@
| Term | Meaning |
|---|---|
| ActivitySim output | Raw model output tables such as households, persons, tours, trips, and land use. |
+| Analysis unit | One full run or one related segment slice passed to the standard summary builders. |
+| Availability state | Stored table or summary status: `available`, `empty`, `unavailable`, or `failed`. Dashboard selections add `missing` and `schema_mismatch` when inspecting a requested input. |
+| Cache identity | Normalized input, configuration, upstream-manifest, and implementation information used to decide whether an artifact is reusable. It is recorded in manifests and is not the same as write time. |
| Dashboard page | One registered visualizer page with a stable `page_id`. |
+| Dashboard page group | Registered navigation container with a stable group ID, ordered child pages, a default child, and default-enabled behavior. |
| Dashboard state | Shared visualizer state such as weighting mode, value mode, segmentation, and loaded runs. |
| Export | Standalone HTML dashboard output that does not require a Python server. |
+| Extension | Trusted importable code or external data that adds weighting behavior, summaries, prepared fields/tables, pages, or hosting integration through a documented boundary. |
+| Failure policy | Config choice that either records a stage/builder failure as diagnostics and continues (`record`) or raises it and stops (`error`). Not every subsystem exposes both choices. |
| `file_map` | Run override for raw ActivitySim output file names. |
| `finalweight` | Canonical prepared weight column aggregated by summary builders. |
+| Geography aggregation | Named mapping from MAZ or TAZ IDs to a custom spatial system, such as a district or subregion. |
| Live mode | Local Panel dashboard that uses a Python server. |
| MAZ | Micro analysis zone. |
| OMX | Open Matrix file format commonly used for skims. |
| Output Processor | Prepare, skimjoin, segmentation, and summarize workflows. |
| Output Visualizer | Live dashboard and HTML export workflows. |
+| Page feature | Page-local object that namespaces a related set of selectors and sections under one feature ID. It is composition within a page, not a discoverable page. |
+| Page section | Registered stable page region with a section ID, declared selector dependencies, renderer, and export/data behavior. Selector changes mark only dependent sections stale. |
| Prepared cache | Per-run cache of canonical prepared tables. |
+| Prepared-data mode | Page declaration value `none`, `optional`, or `required` that controls whether live prepared caches are requested and whether the page's feature is expected to need them. |
| Prepared table | Normalized table used by summaries and prepared-data pages. |
| `prepared_table_map` | Config mapping that supplies canonical prepared tables directly and skips raw prepare. |
| Run | One ActivitySim scenario/output set shown in the dashboard. |
+| `RunData` | Processor dataclass containing one run's canonical prepared tables, optional skim state, table availability, prepare diagnostics, and skimjoin artifacts. Summary builders receive it. |
| Run key | Cache-directory identifier made from a run label. For example, `Build 2035` becomes `build-2035`. Duplicate normalized labels get order-dependent suffixes such as `-1` and `-2`. |
+| `RunTables` | Dashboard multi-run table value that keeps usable `(label, DataFrame)` pairs together with exclusions and source IDs while applying fluent Polars operations. |
| Segment | Configured part of prepared data that the workflow summarizes separately. |
+| Segmentation type | Named segment definition containing one source and one or more segment IDs. |
| Selector | Registered page-local widget that can refresh sections and participate in export. |
| Skim | Matrix or lookup data that supplies level-of-service values to trips or tours. |
| Skimjoin | Optional processor step that joins skim-derived values to prepared trips and tours. |
| Summary builder | Function that converts `RunData` and `Config` into one summary `DataFrame`. |
| Summary cache | Per-run, per-weighting-mode CSV summary tables consumed by dashboard pages. |
| Summary contract | Builder metadata defining output schema and required inputs. |
+| `summary_table_map` | Config mapping from registered summary IDs to dashboard-ready CSV/Parquet files. Mapped tables can replace generated IDs but cannot be reweighted or segmented from aggregate rows. |
| TAZ | Traffic analysis zone. |
| Weighting mode | Registered transform with a version. It supplies prepared `finalweight` values under one cache and dashboard mode ID. Summary builders and prepared-data pages use the values. |
From a5d084d3c0ddea1d7068e69ae2704661ea60e0ff Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Fri, 7 Aug 2026 17:51:35 -0400
Subject: [PATCH 22/27] Stand up metro configs; start work on standing up lcog
and skats
---
dashboard/pages/skim_summaries/_shared.py | 7 +
simor_configs/lcog_configs/lcog_config.yaml | 504 +++++++++++++++++
.../lcog_configs/lcog_skimjoin_config.yaml | 109 ++++
...lcog_skimjoin_config_alternate_id_col.yaml | 109 ++++
simor_configs/metro_configs/metro_config.yaml | 182 +-----
.../metro_configs/metro_skimjoin_config.yaml | 50 +-
...etro_skimjoin_config_alternate_id_col.yaml | 76 ++-
simor_configs/skats_configs/skats_config.yaml | 535 ++++++++++++++++++
.../skats_configs/skats_skimjoin_config.yaml | 428 ++++++++++++++
...kats_skimjoin_config_alternate_id_col.yaml | 428 ++++++++++++++
tests/test_dashboard_helpers_phase1.py | 21 +
11 files changed, 2277 insertions(+), 172 deletions(-)
create mode 100644 simor_configs/lcog_configs/lcog_config.yaml
create mode 100644 simor_configs/lcog_configs/lcog_skimjoin_config.yaml
create mode 100644 simor_configs/lcog_configs/lcog_skimjoin_config_alternate_id_col.yaml
create mode 100644 simor_configs/skats_configs/skats_config.yaml
create mode 100644 simor_configs/skats_configs/skats_skimjoin_config.yaml
create mode 100644 simor_configs/skats_configs/skats_skimjoin_config_alternate_id_col.yaml
diff --git a/dashboard/pages/skim_summaries/_shared.py b/dashboard/pages/skim_summaries/_shared.py
index cbf4900..6c5cfb8 100644
--- a/dashboard/pages/skim_summaries/_shared.py
+++ b/dashboard/pages/skim_summaries/_shared.py
@@ -148,6 +148,13 @@ def component_display_name(
"skim_transit_tiv_inbound": "Transit In-Vehicle Time (min)",
"skim_bike_distance": "TAZ Skim Bike Distance (mi)",
"skim_bike_maz_distance": "MAZ Network Bike Distance (mi)",
+ "skim_bike_transit_distance_bus": (
+ "Total Bike Distance - Local Bus (mi) (Estimated from Walk Skims)"
+ ),
+ "skim_bike_transit_distance_premium": (
+ "Total Bike Distance - Premium Transit (mi) "
+ "(Estimated from Walk Skims)"
+ ),
}
if value in special_labels:
return special_labels[value]
diff --git a/simor_configs/lcog_configs/lcog_config.yaml b/simor_configs/lcog_configs/lcog_config.yaml
new file mode 100644
index 0000000..f2cdac3
--- /dev/null
+++ b/simor_configs/lcog_configs/lcog_config.yaml
@@ -0,0 +1,504 @@
+# ActivitySim Visualizer Configuration
+# Adapt this file for each model deployment.
+# Only the sections you need are required — see comments for which are optional.
+
+name: "LCOG Settings"
+root: ../../simor_project_outputs/lcog_outputs # relative to the config's location
+log_level: INFO
+
+pipeline:
+ steps:
+ - prepare
+ - skimjoin
+ # - segment
+ - summarize # will automatically overwrite summaries with a stale cache
+ - dashboard
+ dashboard_mode: live # live | export | host
+ refresh: [] # list stages here only when a forced rebuild is required
+
+# ---------------------------------------------------------------------------
+# ActivitySim output file names
+# Use stems (no extension) for automatic format detection: the reader will try
+# .parquet first, then .csv. You may also specify an explicit extension to
+# force a particular format (e.g. final_households.csv or final_tours.parquet).
+# ---------------------------------------------------------------------------
+files:
+ households: household
+ persons: person
+ day: day
+ tours: tour
+ trips: trip_linked
+ vehicles: vehicles
+ joint_tour_participants: joint_tour_participants
+ land_use: land_use
+
+
+
+# Optional shared fallback files for optional inputs that may be missing in some
+# run folders. These must be explicit .csv or .parquet paths.
+fallback_files:
+ land_use: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\land_use.csv
+
+# ---------------------------------------------------------------------------
+# Runs to compare
+# Each entry specifies a run directory, a display label, and optional overrides.
+#
+# Weight columns (all optional — leave as null or omit to use default rules):
+# hh_weight_col: explicit household weight column in final_households
+# person_weight_col: explicit person weight column in final_persons
+# trip_weight_col: explicit trip weight column in final_trips
+# (tour weight = average of its trips' weights when set)
+#
+# If none of the above are set and sample_rate is not in columns, all weights = 1.
+# ---------------------------------------------------------------------------
+runs:
+ - dir: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\survey_data
+ label: Survey
+ hh_weight_col: hh_weight
+ person_weight_col: person_weight
+ trip_weight_col: linked_trip_weight
+ skimjoin:
+ config_path: lcog_skimjoin_config_alternate_id_col.yaml
+
+ - dir: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data
+ label: Override
+ hh_weight_col: hh_weight
+ person_weight_col: person_weight
+ trip_weight_col: linked_trip_weight
+ file_map:
+ households: override_households
+ persons: override_persons
+ tours: override_tours
+ trips: override_trips
+ joint_tour_participants: override_joint_tour_participants
+
+
+# ---------------------------------------------------------------------------
+# Zone system
+# Set use_maz: false for TAZ-only models (zone IDs in outputs are already TAZs).
+# For 2-zone (MAZ+TAZ) models, specify which columns in final_land_use
+# hold the MAZ and TAZ IDs.
+# ---------------------------------------------------------------------------
+zones:
+ use_maz: true # false = TAZ-only model, no MAZ→TAZ conversion needed
+ maz_col: [MAZ, zone_id] # column in final_land_use containing MAZ IDs
+ taz_col: TAZ # column in final_land_use containing TAZ IDs
+
+
+# ---------------------------------------------------------------------------
+# Column names in the ActivitySim output files
+# Change only if your outputs use non-standard column names.
+# ---------------------------------------------------------------------------
+columns:
+ ptype: ptype # person type column in persons file
+ hhsize: hhsize # HH size column in households file
+ auto_ownership: auto_ownership # vehicle count column in households file
+ num_workers: num_workers # workers per HH in households file
+ num_adults: num_adults # adults per HH in households file
+ # income_segment: [income_segment, income_broad] # OPTIONAL household income segment/level column for trip enrichment
+ # Prefer a readable purpose column per run. Some survey-style outputs store
+ # numeric codes in primary_purpose/tour_purpose and labels in tour_type.
+ tour_purpose: [primary_purpose, tour_type, purpose]
+ # sample_rate: sample_rate # OPTIONAL override; if omitted, code auto-detects
+ # a households column literally named 'sample_rate'.
+ # finalweight = 1/sample_rate (unless explicit run weight columns are provided)
+ # school_esc_outbound: [school_esc_outbound, out_escort_type]
+ # school_esc_inbound: [school_esc_inbound, inb_escort_type]
+
+prepare:
+ output:
+ file_format: parquet
+ validation:
+ relationship_checks: warn
+ non_motorized_distance_skim:
+ file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\maz_maz_walk.csv
+ matrix: null
+ time_periods:
+ network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
+ trip_period_number_column: depart
+ tour_start_period_number_column: start
+ tour_end_period_number_column: end
+ auto_sufficiency_basis: licensed_drivers # licensed_drivers | workers | adults
+ vot_bins:
+ source_column: income_segment
+ output_column: vot_bin
+ fallback_value: M
+ mappings:
+ survey:
+ 1: L
+ 2: L
+ 3: M
+ 4: M
+ 5: H
+ 6: H
+ 999: M
+ override:
+ 1: L
+ 2: L
+ 3: M
+ 4: M
+ 5: H
+ 6: H
+ 999: M
+
+skimjoin:
+ # Create optional long-form trip/tour tables with skim values for alternate modes.
+ create_hypothetical_skim_tables: true
+ defaults:
+ config_path: lcog_skimjoin_config.yaml
+ skim_files:
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\maz_stop_walk.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\maz_maz_walk.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\bike_maz_logsums_commute.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\bike_maz_logsums_noncommute.csv
+ network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
+
+segment:
+ dashboard:
+ segmentation_type: person_sex
+ visibility: segments_only # full_only | segments_only | full_and_segments
+ definitions:
+ signup_platform:
+ include_full: true
+ persist_segmented_prepared_tables: false
+ allow_overlapping: false
+ on_empty_segment: warn
+ source:
+ type: prepared_column
+ source_table: hh
+ column: signup_platform
+ segments:
+ - id: rmove
+ label: RMove
+ values: ["rmove"]
+ - id: browser
+ label: Browser
+ values: ["browser"]
+ - id: call
+ label: Call
+ values: ["call"]
+
+ person_sex:
+ include_full: false
+ persist_segmented_prepared_tables: false
+ allow_overlapping: false
+ on_empty_segment: warn
+ source:
+ type: prepared_column
+ source_table: per
+ column: SEX
+ segments:
+ - id: male
+ label: Male
+ values: [1]
+ - id: female
+ label: Female
+ values: [2]
+
+summarize:
+ weighting_modes: [unweighted, weighted]
+ pnr_tour_modes:
+ - PNR_TRANSIT
+ # OPTIONAL summary-time grouping for outputs with a tour_purpose dimension
+ # group_joint_tour_purposes: true
+ # group_atwork_tour_purposes: true
+ # group_school_tour_purposes: true
+ # Controls additional mapped geography aggregations only. Summaries may still
+ # emit all_geographies totals, and native prepared home geographies such as
+ # home_taz, home_county, and home_mpo can appear when those columns exist.
+ geography:
+ enabled: true
+ # Configured mappings create columns such as home_geo__school_district,
+ # work_geo__county, or land_use_geo__district.
+ aggregations:
+ school_district_k8:
+ source_zone_system: maz
+ file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\land_use.csv
+ zone_id_col: MAZ
+ geography_col: DIST_KTO8
+ school_district_9_12:
+ source_zone_system: maz
+ file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\land_use.csv
+ zone_id_col: MAZ
+ geography_col: DIST_9TO12
+
+dashboard:
+ title: "LCOG Estimation Visualizer"
+ enable_maz_geographies: false
+ live:
+ pages:
+ - overview
+ - long_term_choices
+ - daily_travel
+ - joint_travel
+ - tour_summaries
+ - trip_summaries
+ - skim_summaries
+ # - validation:
+ # - traffic
+ # - transit
+ # - vmt
+ # - regional_validation
+ export:
+ output_path: exports/dashboard.html
+ # summary series are baked into one exported HTML file.
+ # `dashboard.export.pages` modifies matching live pages; it is not an
+ # inclusion list. Omit selector overrides to export all widget states.
+ # Export segmentation defaults to segmentation.dashboard when omitted.
+ # Use full_only, segments_only, or full_and_segments to choose which
+ dashboard:
+ weighting: [unweighted]
+ # segmentation_type: signup_platform
+ # segmentation_visibility: segments_only
+ pages:
+ long_term_choices:
+ mandatory_location_choice:
+ geography_level:
+ - All Geography Types
+ - County
+ - MPO
+ # omit TAZ
+ # parts:
+ # commuting_flows:
+ # enabled: false
+
+ # shadow_pricing:
+ # enabled: true
+ # geography_level: [all]
+ # student_type: [all]
+ # parts:
+ # workplace_table:
+ # enabled: false
+ # school_table:
+ # enabled: false
+ # tour_summaries:
+ # internal_external_tours:
+ # enabled: false
+ validation:
+ vmt:
+ personal_auto_vmt_breakdown: all
+ personal_auto_vmt_geography_type: all
+ personal_auto_vmt_geography: default
+ personal_auto_vmt_time_period: default
+ personal_auto_vmt_mode: default
+ personal_auto_vmt_income_segment: default
+ personal_auto_vmt_household_size: default
+ non_motorized_vmt_breakdown: all
+ non_motorized_vmt_geography_type: all
+ non_motorized_vmt_geography: default
+ non_motorized_vmt_time_period: default
+ non_motorized_vmt_mode: default
+ non_motorized_vmt_income_segment: default
+ non_motorized_vmt_household_size: default
+
+ external_travel_metric: all
+ external_travel_breakdown: all
+ external_travel_trip_purpose: default
+ external_travel_time_period: default
+
+ demo_commercial_metric: all
+ demo_commercial_breakdown: all
+ demo_commercial_vehicle_type: default
+ demo_commercial_time_period: default
+ host:
+ account: my-connect-cloud-account
+ # app_id: 12345
+ # title: Estimation Mode Comparison Visualizer
+ # First hosted publish may open a browser for Posit Connect Cloud login.
+ # Later runs usually reuse saved local rsconnect metadata.
+ verify: true
+
+
+# Summary-affecting regrouping/normalization belongs under summarize.category_normalization.
+# Cosmetic labels and ordering belong under display.labels.
+display:
+ bar_hover_mode: all # closest | all
+ density_hover_mode: all # closest | all
+ labels:
+ person_type:
+ mapping:
+ all_person_types: All Person Types
+ 1: Full-time worker
+ 2: Part-time worker
+ 3: University student
+ 4: Non-worker adult
+ 5: Retired
+ 6: Student
+ 7: Preschool
+
+ transit_subsidy:
+ mapping:
+ 0: No subsidy
+ 1: Discounted
+ 2: Free
+
+ license_holding_status:
+ mapping:
+ has_license: Has License
+ no_license: No License
+
+ transit_pass_ownership_status:
+ mapping:
+ has_transit_pass: Has Transit Pass
+ no_transit_pass: No Transit Pass
+
+ telecommute_frequency:
+ mapping:
+ 1_day_week: 1 Day per Week
+ 2_3_days_week: 2-3 Days per Week
+ 4_days_week: 4+ Days per Week
+ No_Telecommute: No Telecommute
+
+ mode:
+ mapping:
+ SOV: Drive Alone
+ HOV2: Shared Ride 2
+ HOV3: Shared Ride 3+
+ WALK: Walk
+ BIKE: Bike
+ EBIKE: E-Bike
+ ESCOOTER: E-Scooter
+ WALK_TRANSIT: Walk-Transit
+ BIKE_TRANSIT: Bike-Transit
+ PNR_TRANSIT: PNR-Transit
+ KNR_TRANSIT: KNR-Transit
+ TAXI: Taxi
+ TNC_SINGLE: TNC-Single
+ TNC_SHARED: TNC-Pool
+ SCHOOLBUS: School Bus
+
+
+ # escort:
+ # mapping:
+ # not_escorted: No Escort
+ # pure_escort: Pure Escort
+ # ride_share: Ride Share
+ # 0: No Escort
+ # 1: Pure Escort
+ # 2: Ride Share
+ # "": No Escort
+
+ tour_purpose:
+ mapping:
+ all_tour_purposes: All Tour Purposes
+ work: Work
+ school: School
+ joint: Joint
+ escort: Escort
+ shopping: Shopping
+ othmaint: Other Maintenance
+ eatout: Eat Out
+ social: Social
+ othdiscr: Other Discretionary
+ atwork: At-Work
+ loop: Loop
+
+ trip_purpose:
+ mapping:
+ home: Home
+ work: Work
+ school: School
+ escort: Escort
+ shopping: Shopping
+ othmaint: Other Maintenance
+ eatout: Eat Out
+ social: Social
+ othdiscr: Other Discretionary
+ loop: Loop
+
+ stop_purpose:
+ mapping:
+ work: Work
+ school: School
+ escort: Escort
+ shopping: Shopping
+ othmaint: Other Maintenance
+ eatout: Eat Out
+ social: Social
+ othdiscr: Other Discretionary
+
+ tour_composition:
+ mapping:
+ adults: Adults
+ mixed: Mixed
+ children: Children
+
+ tour_category:
+ mapping:
+ mandatory: Mandatory
+ joint: Joint
+ non_mandatory: Non-Mandatory
+ atwork: At-Work
+
+ atwork_subtour_frequency_category:
+ mapping:
+ no_subtours: None
+ eat: 1 Eating Out
+ business1: 1 Business
+ maint: 1 Maintenance
+ business2: 2 Business
+ eat_business: 1 Eating Out + 1 Business
+
+ geography:
+ mapping:
+ all_geographies: All Geographies
+ county: County
+ home_county: County
+ home_mpo: MPO
+ home_taz: TAZ
+ district: District
+ taz: TAZ
+ maz: MAZ
+ school_district: School District
+
+ mandatory_tour_frequency:
+ # mapping:
+ # work1: Work
+ # school1: School
+ # work2: 2 Work
+ # school2: 2 School
+ # work_and_school: 1 Work + 1 School
+ mapping:
+ 1: Work
+ 3: School
+ 2: 2 Work
+ 4: 2 School
+ 5: 1 Work + 1 School
+
+ daily_activity_pattern:
+ mapping:
+ M : Mandatory
+ N : Non-Mandatory
+ H : At-Home
+
+ facility_type:
+ mapping:
+ 1: "Interstate"
+ 3: "Principal Arterial"
+ 4: "Minor Arterial"
+ 5: "Major Collector"
+ 6: "Minor Collector"
+ 7: "Local Road"
+ 30: "Ramp"
+ 998: "Bike/Walk"
+
+ commercial_vehicle_type:
+ mapping:
+ car: Car
+ su: Single-Unit Truck
+ mu: Multi-Unit Truck
+
+ # run_colors:
+ # - "#298c8c" # Teal
+ # - "#a00000" # Red
+ # - "#b8b8b8" # Light gray
+ # - "#384860" # Dark blue-gray
+ # - "#ff7f0e" # Orange
+ # - "#1f77b4"
+ # - "#ff7f0e"
+ # - "#2ca02c"
+ # - "#d62728"
+ # - "#9467bd"
+ # - "#8c564b"
+ # - "#e377c2"
+ # - "#7f7f7f"
diff --git a/simor_configs/lcog_configs/lcog_skimjoin_config.yaml b/simor_configs/lcog_configs/lcog_skimjoin_config.yaml
new file mode 100644
index 0000000..2b676db
--- /dev/null
+++ b/simor_configs/lcog_configs/lcog_skimjoin_config.yaml
@@ -0,0 +1,109 @@
+project:
+ skim_files:
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\maz_stop_walk.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\maz_maz_walk.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\bike_maz_logsums_commute.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\bike_maz_logsums_noncommute.csv
+ network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
+
+activitysim:
+ trip_mode_column: trip_mode
+ trip_id_column: trip_id
+ tour_mode_column: tour_mode
+ tour_id_column: tour_id
+ outbound_column: outbound
+
+defaults:
+ origin: o_maz
+ destination: d_maz
+ output_prefix: skim_
+ sentinel_values:
+ - 9999
+ - 999999
+
+dimensions:
+ BIKE_PURPOSE:
+ source_columns:
+ trip_source_column: tour_purpose
+ outbound_tour_source_column: tour_purpose
+ inbound_tour_source_column: tour_purpose
+ values:
+ work: commute
+ school: commute
+ univ: commute
+ atwork: noncommute
+ business: noncommute
+ eat: noncommute
+ eatout: noncommute
+ escort: noncommute
+ loop: noncommute
+ maint: noncommute
+ othdiscr: noncommute
+ othmaint: noncommute
+ shopping: noncommute
+ social: noncommute
+
+ignore_modes:
+ - ESCOOTER
+ - HOV2
+ - HOV3
+ - KNR_TRANSIT
+ - MISSING
+ - OTHER
+ - PNR_TRANSIT
+ - SCHOOLBUS
+ - SOV
+ - TAXI
+ - TNC_SHARED
+ - TNC_SINGLE
+ - WALK_TRANSIT
+
+modes:
+ WALK:
+ output_prefix: skim_walk_
+ distance: maz_maz_walk__DISTWALK
+ actual: maz_maz_walk__actual
+ BIKE:
+ output_prefix: skim_bike_
+ maz_distance:
+ output: skim_bike_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: "bike_maz_logsums_{BIKE_PURPOSE}__distance_{BIKE_PURPOSE}"
+ logsum:
+ origin: o_maz
+ destination: d_maz
+ matrix: "bike_maz_logsums_{BIKE_PURPOSE}__logsum_{BIKE_PURPOSE}"
+ EBIKE:
+ output_prefix: skim_bike_
+ maz_distance:
+ output: skim_bike_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: "bike_maz_logsums_{BIKE_PURPOSE}__distance_{BIKE_PURPOSE}"
+ BIKE_TRANSIT:
+ output_prefix: skim_
+ bike_o_maz_stop_distance_bus:
+ output: skim_bike_transit_distance_bus
+ combine: sum
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ bike_d_maz_stop_distance_bus:
+ output: skim_bike_transit_distance_bus
+ combine: sum
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ bike_o_maz_stop_distance_premium:
+ output: skim_bike_transit_distance_premium
+ combine: sum
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ bike_d_maz_stop_distance_premium:
+ output: skim_bike_transit_distance_premium
+ combine: sum
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
diff --git a/simor_configs/lcog_configs/lcog_skimjoin_config_alternate_id_col.yaml b/simor_configs/lcog_configs/lcog_skimjoin_config_alternate_id_col.yaml
new file mode 100644
index 0000000..e0e3a11
--- /dev/null
+++ b/simor_configs/lcog_configs/lcog_skimjoin_config_alternate_id_col.yaml
@@ -0,0 +1,109 @@
+project:
+ skim_files:
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\maz_stop_walk.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\maz_maz_walk.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\bike_maz_logsums_commute.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\bike_maz_logsums_noncommute.csv
+ network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
+
+activitysim:
+ trip_mode_column: trip_mode
+ trip_id_column: linked_trip_id
+ tour_mode_column: tour_mode
+ tour_id_column: tour_id
+ outbound_column: outbound
+
+defaults:
+ origin: o_maz
+ destination: d_maz
+ output_prefix: skim_
+ sentinel_values:
+ - 9999
+ - 999999
+
+dimensions:
+ BIKE_PURPOSE:
+ source_columns:
+ trip_source_column: tour_purpose
+ outbound_tour_source_column: tour_purpose
+ inbound_tour_source_column: tour_purpose
+ values:
+ work: commute
+ school: commute
+ univ: commute
+ atwork: noncommute
+ business: noncommute
+ eat: noncommute
+ eatout: noncommute
+ escort: noncommute
+ loop: noncommute
+ maint: noncommute
+ othdiscr: noncommute
+ othmaint: noncommute
+ shopping: noncommute
+ social: noncommute
+
+ignore_modes:
+ - ESCOOTER
+ - HOV2
+ - HOV3
+ - KNR_TRANSIT
+ - MISSING
+ - OTHER
+ - PNR_TRANSIT
+ - SCHOOLBUS
+ - SOV
+ - TAXI
+ - TNC_SHARED
+ - TNC_SINGLE
+ - WALK_TRANSIT
+
+modes:
+ WALK:
+ output_prefix: skim_walk_
+ distance: maz_maz_walk__DISTWALK
+ actual: maz_maz_walk__actual
+ BIKE:
+ output_prefix: skim_bike_
+ maz_distance:
+ output: skim_bike_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: "bike_maz_logsums_{BIKE_PURPOSE}__distance_{BIKE_PURPOSE}"
+ logsum:
+ origin: o_maz
+ destination: d_maz
+ matrix: "bike_maz_logsums_{BIKE_PURPOSE}__logsum_{BIKE_PURPOSE}"
+ EBIKE:
+ output_prefix: skim_bike_
+ maz_distance:
+ output: skim_bike_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: "bike_maz_logsums_{BIKE_PURPOSE}__distance_{BIKE_PURPOSE}"
+ BIKE_TRANSIT:
+ output_prefix: skim_
+ bike_o_maz_stop_distance_bus:
+ output: skim_bike_transit_distance_bus
+ combine: sum
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ bike_d_maz_stop_distance_bus:
+ output: skim_bike_transit_distance_bus
+ combine: sum
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ bike_o_maz_stop_distance_premium:
+ output: skim_bike_transit_distance_premium
+ combine: sum
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ bike_d_maz_stop_distance_premium:
+ output: skim_bike_transit_distance_premium
+ combine: sum
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
diff --git a/simor_configs/metro_configs/metro_config.yaml b/simor_configs/metro_configs/metro_config.yaml
index 1a68936..09429cf 100644
--- a/simor_configs/metro_configs/metro_config.yaml
+++ b/simor_configs/metro_configs/metro_config.yaml
@@ -52,79 +52,8 @@ fallback_files:
# If none of the above are set and sample_rate is not in columns, all weights = 1.
# ---------------------------------------------------------------------------
runs:
- # - dir: C:\Users\wesley.darling\Downloads\dataset_2026-05-21\unfiltered
- # label: Unfiltered
- # skimjoin:
- # config_path: will_skimjoin_config_alternate_id_col.yaml
- # # Synthetic assignment-side summaries exercise validation-page wiring.
- # # Replace these paths with real summaries without changing dashboard code.
- # summary_table_map:
- # link_validation_summary: ../outside_summary_tables/estimated_fixtures/unfiltered/link_validation_summary.csv
- # count_location_counts_validation_summary: ../outside_summary_tables/countLocCounts.csv
- # count_location_volumes_validation_summary: ../outside_summary_tables/estimated_fixtures/unfiltered/count_location_volumes_validation_summary.csv
- # screenline_flow_comparisons: ../outside_summary_tables/estimated_fixtures/unfiltered/screenline_flow_comparisons.csv
- # commuting_flows: ../outside_summary_tables/estimated_fixtures/unfiltered/commuting_flows.csv
- # transit_boardings_by_operator_and_technology: ../outside_summary_tables/estimated_fixtures/unfiltered/transit_boardings_by_operator_and_technology.csv
- # transit_transfer_rate: ../outside_summary_tables/estimated_fixtures/unfiltered/transit_transfer_rate.csv
- # bicycle_vmt_by_facility_type: ../outside_summary_tables/estimated_fixtures/unfiltered/bicycle_vmt_by_facility_type.csv
- # commercial_vehicle_validation_summary: ../outside_summary_tables/estimated_fixtures/unfiltered/commercial_vehicle_validation_summary.csv
- # commercial_vehicle_vmt_validation_summary: ../outside_summary_tables/estimated_fixtures/unfiltered/commercial_vehicle_vmt_validation_summary.csv
- # external_trip_validation_summary: ../outside_summary_tables/estimated_fixtures/unfiltered/external_trip_validation_summary.csv
- # external_vmt_validation_summary: ../outside_summary_tables/estimated_fixtures/unfiltered/external_vmt_validation_summary.csv
- # # skim_files:
- # - C:\Users\wesley.darling\project_data\odot_skims\*.omx
- # - C:\Users\wesley.darling\project_data\odot_skims\maz_stop_walk.csv
- # - C:\Users\wesley.darling\project_data\odot_skims\maz_maz_walk.csv
- # network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
-
- # - dir: C:\Users\wesley.darling\Downloads\dataset_2026-05-21\filtered
- # label: Filtered
- # skimjoin:
- # config_path: will_skimjoin_config_alternate_id_col.yaml
- # summary_table_map:
- # link_validation_summary: ../outside_summary_tables/estimated_fixtures/filtered/link_validation_summary.csv
- # count_location_counts_validation_summary: ../outside_summary_tables/countLocCounts.csv
- # count_location_volumes_validation_summary: ../outside_summary_tables/estimated_fixtures/filtered/count_location_volumes_validation_summary.csv
- # screenline_flow_comparisons: ../outside_summary_tables/estimated_fixtures/filtered/screenline_flow_comparisons.csv
- # commuting_flows: ../outside_summary_tables/estimated_fixtures/filtered/commuting_flows.csv
- # transit_boardings_by_operator_and_technology: ../outside_summary_tables/estimated_fixtures/filtered/transit_boardings_by_operator_and_technology.csv
- # transit_transfer_rate: ../outside_summary_tables/estimated_fixtures/filtered/transit_transfer_rate.csv
- # bicycle_vmt_by_facility_type: ../outside_summary_tables/estimated_fixtures/filtered/bicycle_vmt_by_facility_type.csv
- # commercial_vehicle_validation_summary: ../outside_summary_tables/estimated_fixtures/filtered/commercial_vehicle_validation_summary.csv
- # commercial_vehicle_vmt_validation_summary: ../outside_summary_tables/estimated_fixtures/filtered/commercial_vehicle_vmt_validation_summary.csv
- # external_trip_validation_summary: ../outside_summary_tables/estimated_fixtures/filtered/external_trip_validation_summary.csv
- # external_vmt_validation_summary: ../outside_summary_tables/estimated_fixtures/filtered/external_vmt_validation_summary.csv
- # # skim_files:
- # - C:\Users\wesley.darling\project_data\odot_skims\*.omx
- # - C:\Users\wesley.darling\project_data\odot_skims\maz_stop_walk.csv
- # - C:\Users\wesley.darling\project_data\odot_skims\maz_maz_walk.csv
- # network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
-
- # - dir: C:\Users\wesley.darling\Downloads\dataset_2026-05-21\override
- # label: Override
- # summary_table_map:
- # link_validation_summary: ../outside_summary_tables/estimated_fixtures/override/link_validation_summary.csv
- # count_location_counts_validation_summary: ../outside_summary_tables/countLocCounts.csv
- # count_location_volumes_validation_summary: ../outside_summary_tables/estimated_fixtures/override/count_location_volumes_validation_summary.csv
- # screenline_flow_comparisons: ../outside_summary_tables/estimated_fixtures/override/screenline_flow_comparisons.csv
- # commuting_flows: ../outside_summary_tables/estimated_fixtures/override/commuting_flows.csv
- # transit_boardings_by_operator_and_technology: ../outside_summary_tables/estimated_fixtures/override/transit_boardings_by_operator_and_technology.csv
- # transit_transfer_rate: ../outside_summary_tables/estimated_fixtures/override/transit_transfer_rate.csv
- # bicycle_vmt_by_facility_type: ../outside_summary_tables/estimated_fixtures/override/bicycle_vmt_by_facility_type.csv
- # commercial_vehicle_validation_summary: ../outside_summary_tables/estimated_fixtures/override/commercial_vehicle_validation_summary.csv
- # commercial_vehicle_vmt_validation_summary: ../outside_summary_tables/estimated_fixtures/override/commercial_vehicle_vmt_validation_summary.csv
- # external_trip_validation_summary: ../outside_summary_tables/estimated_fixtures/override/external_trip_validation_summary.csv
- # external_vmt_validation_summary: ../outside_summary_tables/estimated_fixtures/override/external_vmt_validation_summary.csv
- # # skimjoin:
- # config_path: will_skimjoin_config.yaml
- # skim_files:
- # - C:\Users\wesley.darling\project_data\odot_skims\*.omx
- # - C:\Users\wesley.darling\project_data\odot_skims\maz_stop_walk.csv
- # - C:\Users\wesley.darling\project_data\odot_skims\maz_maz_walk.csv
- # network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
-
- dir: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\survey_data
- label: Base
+ label: Survey
hh_weight_col: hh_weight
person_weight_col: person_weight
trip_weight_col: linked_trip_weight
@@ -142,53 +71,6 @@ runs:
tours: override_tours
trips: override_trips
joint_tour_participants: override_joint_tour_participants
- # # skimjoin:
- # config_path: will_skimjoin_config.yaml
- # skim_files:
- # - C:\Users\wesley.darling\project_data\odot_skims\*.omx
- # - C:\Users\wesley.darling\project_data\odot_skims\maz_stop_walk.csv
- # - C:\Users\wesley.darling\project_data\odot_skims\maz_maz_walk.csv
- # network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
-
- # - dir: C:\Users\wesley.darling\Downloads\viz\viz\new_override
- # label: New Override
-
- # - dir: C:\Users\wesley.darling\Downloads\viz\viz\new_estimation_5-20
- # label: New Estimation Output 20
- # file_map:
- # households: final_households
- # persons: final_persons
- # tours: final_tours
- # trips: final_trips
- # joint_tour_participants: final_joint_tour_participants
- # land_use: final_land_use
- # vehicles: final_vehicles
-
- # - label: Will Prepared Tables
- # prepared_table_map:
- # households: C:/Users/wesley.darling/projects/activitysim_visualizer/will_prepared_tables/households.parquet
- # persons: C:/Users/wesley.darling/projects/activitysim_visualizer/will_prepared_tables/persons.parquet
- # tours: C:/Users/wesley.darling/projects/activitysim_visualizer/will_prepared_tables/tours.parquet
- # trips: C:/Users/wesley.darling/projects/activitysim_visualizer/will_prepared_tables/trips.parquet
- # joint_tour_participants: C:/Users/wesley.darling/projects/activitysim_visualizer/will_prepared_tables/joint_tour_participants.parquet
- # land_use: C:/Users/wesley.darling/projects/activitysim_visualizer/will_prepared_tables/land_use.parquet
-
- # - label: Demo Validation Data
- # summary_table_map:
- # link_validation_summary: ../outside_summary_tables/allLinkSummary.csv
- # count_location_counts_validation_summary: ../outside_summary_tables/countLocCounts.csv
- # count_location_volumes_validation_summary: ../outside_summary_tables/countLocVolumes.csv
- # district_commuting_flows_validation_summary: ../outside_summary_tables/countyFlows.csv
- # county_commuting_flows_validation_summary: ../outside_summary_tables/countyFlows_JoJa.csv
- # commercial_vehicle_validation_summary: ../outside_summary_tables/cvm_summary.csv
- # commercial_vehicle_vmt_validation_summary: ../outside_summary_tables/cvm_vmt_summary.csv
- # external_trip_validation_summary: ../outside_summary_tables/ext_summary.csv
- # external_vmt_validation_summary: ../outside_summary_tables/ext_vmt_summary.csv
- # auto_vmt_validation_summary: ../outside_summary_tables/vmtSummary.csv
- # work_from_home_validation_summary: ../outside_summary_tables/wfh_summary.csv
- # transit_boardings_by_operator_and_technology: ../outside_summary_tables/estimated_fixtures/observed/transit_boardings_by_operator_and_technology.csv
- # transit_transfer_rate: ../outside_summary_tables/estimated_fixtures/observed/transit_transfer_rate.csv
- # bicycle_vmt_by_facility_type: ../outside_summary_tables/estimated_fixtures/observed/bicycle_vmt_by_facility_type.csv
# ---------------------------------------------------------------------------
# Zone system
@@ -244,32 +126,22 @@ prepare:
output_column: vot_bin
fallback_value: M
mappings:
- unfiltered:
- 1: L
- 2: L
- 3: M
- 4: M
- 5: H
- 6: H
- new-filtered:
+ survey:
1: L
2: L
3: M
4: M
5: H
6: H
- new-override:
+ 999: M
+ override:
1: L
2: L
3: M
4: M
5: H
6: H
- new-estimation-output:
- 1: L
- 2: M
- 3: M
- 4: H
+ 999: M
skimjoin:
# Create optional long-form trip/tour tables with skim values for alternate modes.
@@ -280,6 +152,8 @@ skimjoin:
- 'C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\*.omx'
- C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\maz_stop_walk.csv
- C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\maz_maz_walk.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\bike_maz_logsums_commute.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\bike_maz_logsums_noncommute.csv
network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
segment:
@@ -307,22 +181,6 @@ segment:
label: Call
values: ["call"]
- person_sex:
- include_full: false
- persist_segmented_prepared_tables: false
- allow_overlapping: false
- on_empty_segment: warn
- source:
- type: prepared_column
- source_table: per
- column: SEX
- segments:
- - id: male
- label: Male
- values: [1]
- - id: female
- label: Female
- values: [2]
summarize:
weighting_modes: [unweighted, weighted]
@@ -336,14 +194,24 @@ summarize:
# emit all_geographies totals, and native prepared home geographies such as
# home_taz, home_county, and home_mpo can appear when those columns exist.
geography:
- enabled: false
+ enabled: true
# Configured mappings create columns such as home_geo__school_district,
# work_geo__county, or land_use_geo__district.
aggregations:
- school_district:
+ regional_district:
+ source_zone_system: maz
+ file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\land_use.csv
+ zone_id_col: MAZ
+ geography_col: DISTRICT9
+ school_district_k8:
+ source_zone_system: maz
+ file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\land_use.csv
+ zone_id_col: MAZ
+ geography_col: DIST_Kto8
+ school_district_9_12:
source_zone_system: maz
file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\land_use.csv
- zone_id_col: zone_id
+ zone_id_col: MAZ
geography_col: DIST_9to12
dashboard:
@@ -358,11 +226,11 @@ dashboard:
- tour_summaries
- trip_summaries
- skim_summaries
- - validation:
- - traffic
- - transit
- - vmt
- - regional_validation
+ # - validation:
+ # - traffic
+ # - transit
+ # - vmt
+ # - regional_validation
export:
output_path: exports/dashboard.html
# summary series are baked into one exported HTML file.
diff --git a/simor_configs/metro_configs/metro_skimjoin_config.yaml b/simor_configs/metro_configs/metro_skimjoin_config.yaml
index 651c2e9..9e4922d 100644
--- a/simor_configs/metro_configs/metro_skimjoin_config.yaml
+++ b/simor_configs/metro_configs/metro_skimjoin_config.yaml
@@ -3,6 +3,8 @@
# - 'C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\*.omx'
# - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\maz_stop_walk.csv
# - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\maz_maz_walk.csv
+# - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\bike_maz_logsums_commute.csv
+# - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\bike_maz_logsums_noncommute.csv
# network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
@@ -36,7 +38,6 @@ zone_mapping:
lookup_name: taz
file_lookup_names:
fares.omx: zone_number
- 'bike_taz_logsums_*.omx': TAZ
missing_zone_policy: error
dimensions:
@@ -424,19 +425,48 @@ modes:
matrix: maz_stop_walk__walk_dist_premium_transit
BIKE:
output_prefix: skim_bike_
- distance: "bike_taz_logsums_{BIKE_PURPOSE}.omx::distance_{BIKE_PURPOSE}"
- logsum: "bike_taz_logsums_{BIKE_PURPOSE}.omx::logsum_{BIKE_PURPOSE}"
+ maz_distance:
+ output: skim_bike_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: "bike_maz_logsums_{BIKE_PURPOSE}__distance_{BIKE_PURPOSE}"
+ logsum:
+ origin: o_maz
+ destination: d_maz
+ matrix: "bike_maz_logsums_{BIKE_PURPOSE}__logsum_{BIKE_PURPOSE}"
EBIKE:
output_prefix: skim_bike_
- distance: "bike_taz_logsums_{BIKE_PURPOSE}.omx::distance_{BIKE_PURPOSE}"
+ maz_distance:
+ output: skim_bike_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: "bike_maz_logsums_{BIKE_PURPOSE}__distance_{BIKE_PURPOSE}"
BIKE_TRANSIT:
output_prefix: skim_
- bike_distance:
- output: skim_bike_distance
- matrix: "bike_taz_logsums_{BIKE_PURPOSE}.omx::distance_{BIKE_PURPOSE}"
- bike_logsum:
- output: skim_bike_logsum
- matrix: "bike_taz_logsums_{BIKE_PURPOSE}.omx::logsum_{BIKE_PURPOSE}"
+ bike_o_maz_stop_distance_bus:
+ output: skim_bike_transit_distance_bus
+ combine: sum
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ bike_d_maz_stop_distance_bus:
+ output: skim_bike_transit_distance_bus
+ combine: sum
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ bike_o_maz_stop_distance_premium:
+ output: skim_bike_transit_distance_premium
+ combine: sum
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ bike_d_maz_stop_distance_premium:
+ output: skim_bike_transit_distance_premium
+ combine: sum
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
transit_brt_ivtt: "WTW_BRT__{PERIOD}"
transit_bus_ivtt: "WTW_BUS__{PERIOD}"
diff --git a/simor_configs/metro_configs/metro_skimjoin_config_alternate_id_col.yaml b/simor_configs/metro_configs/metro_skimjoin_config_alternate_id_col.yaml
index 78146c7..5362370 100644
--- a/simor_configs/metro_configs/metro_skimjoin_config_alternate_id_col.yaml
+++ b/simor_configs/metro_configs/metro_skimjoin_config_alternate_id_col.yaml
@@ -3,6 +3,8 @@ project:
- 'C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\*.omx'
- C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\maz_stop_walk.csv
- C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\maz_maz_walk.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\bike_maz_logsums_commute.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\metro_data\bike_maz_logsums_noncommute.csv
network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
# project.trips_table / project.tours_table / project.output_dir are used by the
# standalone skimjoin CLI only. Integrated runtime skimjoin reads prepared
@@ -53,11 +55,29 @@ dimensions:
L: L
M: M
H: H
+ BIKE_PURPOSE:
+ source_columns:
+ trip_source_column: tour_purpose
+ outbound_tour_source_column: tour_purpose
+ inbound_tour_source_column: tour_purpose
+ values:
+ work: commute
+ school: commute
+ univ: commute
+ atwork: noncommute
+ business: noncommute
+ eat: noncommute
+ eatout: noncommute
+ escort: noncommute
+ loop: noncommute
+ maint: noncommute
+ othdiscr: noncommute
+ othmaint: noncommute
+ shopping: noncommute
+ social: noncommute
ignore_modes:
- ESCOOTER
- - EBIKE
- - BIKE_TRANSIT
- MISSING
- OTHER
@@ -405,11 +425,57 @@ modes:
matrix: maz_stop_walk__walk_dist_premium_transit
BIKE:
output_prefix: skim_bike_
- distance: WLK_DIST
- maz_bike_distance:
+ maz_distance:
output: skim_bike_maz_distance
origin: o_maz
destination: d_maz
- matrix: maz_maz_walk__DISTWALK
+ matrix: "bike_maz_logsums_{BIKE_PURPOSE}__distance_{BIKE_PURPOSE}"
+ logsum:
+ origin: o_maz
+ destination: d_maz
+ matrix: "bike_maz_logsums_{BIKE_PURPOSE}__logsum_{BIKE_PURPOSE}"
+ EBIKE:
+ output_prefix: skim_bike_
+ maz_distance:
+ output: skim_bike_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: "bike_maz_logsums_{BIKE_PURPOSE}__distance_{BIKE_PURPOSE}"
+ BIKE_TRANSIT:
+ output_prefix: skim_
+ bike_o_maz_stop_distance_bus:
+ output: skim_bike_transit_distance_bus
+ combine: sum
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ bike_d_maz_stop_distance_bus:
+ output: skim_bike_transit_distance_bus
+ combine: sum
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ bike_o_maz_stop_distance_premium:
+ output: skim_bike_transit_distance_premium
+ combine: sum
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ bike_d_maz_stop_distance_premium:
+ output: skim_bike_transit_distance_premium
+ combine: sum
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
+ transit_brt_ivtt: "WTW_BRT__{PERIOD}"
+ transit_bus_ivtt: "WTW_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "WTW_CRT__{PERIOD}"
+ transit_first_wait_time: "WTW_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "WTW_LRT__{PERIOD}"
+ transit_tiv: "WTW_TIV__{PERIOD}"
+ transit_num_transfers: "WTW_XFR__{PERIOD}"
+ transit_transfer_wait_time: "WTW_XWT__{PERIOD}"
+ transit_fare: "fare__{PERIOD}"
diff --git a/simor_configs/skats_configs/skats_config.yaml b/simor_configs/skats_configs/skats_config.yaml
new file mode 100644
index 0000000..26312f1
--- /dev/null
+++ b/simor_configs/skats_configs/skats_config.yaml
@@ -0,0 +1,535 @@
+# ActivitySim Visualizer Configuration
+# Adapt this file for each model deployment.
+# Only the sections you need are required — see comments for which are optional.
+
+name: "SKATS Settings"
+root: ../../simor_project_outputs/skats_outputs # relative to the config's location
+log_level: INFO
+
+pipeline:
+ steps:
+ - prepare
+ - skimjoin
+ # - segment
+ - summarize # will automatically overwrite summaries with a stale cache
+ - dashboard
+ dashboard_mode: live # live | export | host
+ refresh: [] # list stages here only when a forced rebuild is required
+
+# ---------------------------------------------------------------------------
+# ActivitySim output file names
+# Use stems (no extension) for automatic format detection: the reader will try
+# .parquet first, then .csv. You may also specify an explicit extension to
+# force a particular format (e.g. final_households.csv or final_tours.parquet).
+# ---------------------------------------------------------------------------
+files:
+ households: household
+ persons: person
+ day: day
+ tours: tour
+ trips: trip_linked
+ vehicles: vehicles
+ joint_tour_participants: joint_tour_participants
+ land_use: land_use
+
+
+
+# Optional shared fallback files for optional inputs that may be missing in some
+# run folders. These must be explicit .csv or .parquet paths.
+fallback_files:
+ land_use: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\land_use.csv
+
+# ---------------------------------------------------------------------------
+# Runs to compare
+# Each entry specifies a run directory, a display label, and optional overrides.
+#
+# Weight columns (all optional — leave as null or omit to use default rules):
+# hh_weight_col: explicit household weight column in final_households
+# person_weight_col: explicit person weight column in final_persons
+# trip_weight_col: explicit trip weight column in final_trips
+# (tour weight = average of its trips' weights when set)
+#
+# If none of the above are set and sample_rate is not in columns, all weights = 1.
+# ---------------------------------------------------------------------------
+runs:
+ - dir: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\survey_data
+ label: Survey
+ hh_weight_col: hh_weight
+ person_weight_col: person_weight
+ trip_weight_col: linked_trip_weight
+ skimjoin:
+ config_path: skats_skimjoin_config_alternate_id_col.yaml
+
+ - dir: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data
+ label: Override
+ hh_weight_col: hh_weight
+ person_weight_col: person_weight
+ trip_weight_col: linked_trip_weight
+ file_map:
+ households: override_households
+ persons: override_persons
+ tours: override_tours
+ trips: override_trips
+ joint_tour_participants: override_joint_tour_participants
+ # # skimjoin:
+ # config_path: will_skimjoin_config.yaml
+ # skim_files:
+ # - C:\Users\wesley.darling\project_data\odot_skims\*.omx
+ # - C:\Users\wesley.darling\project_data\odot_skims\maz_stop_walk.csv
+ # - C:\Users\wesley.darling\project_data\odot_skims\maz_maz_walk.csv
+ # network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
+
+
+# ---------------------------------------------------------------------------
+# Zone system
+# Set use_maz: false for TAZ-only models (zone IDs in outputs are already TAZs).
+# For 2-zone (MAZ+TAZ) models, specify which columns in final_land_use
+# hold the MAZ and TAZ IDs.
+# ---------------------------------------------------------------------------
+zones:
+ use_maz: true # false = TAZ-only model, no MAZ→TAZ conversion needed
+ maz_col: [MAZ, zone_id] # column in final_land_use containing MAZ IDs
+ taz_col: TAZ # column in final_land_use containing TAZ IDs
+
+
+# ---------------------------------------------------------------------------
+# Column names in the ActivitySim output files
+# Change only if your outputs use non-standard column names.
+# ---------------------------------------------------------------------------
+columns:
+ ptype: ptype # person type column in persons file
+ hhsize: hhsize # HH size column in households file
+ auto_ownership: auto_ownership # vehicle count column in households file
+ num_workers: num_workers # workers per HH in households file
+ num_adults: num_adults # adults per HH in households file
+ # income_segment: [income_segment, income_broad] # OPTIONAL household income segment/level column for trip enrichment
+ # Prefer a readable purpose column per run. Some survey-style outputs store
+ # numeric codes in primary_purpose/tour_purpose and labels in tour_type.
+ tour_purpose: [primary_purpose, tour_type, purpose]
+ # sample_rate: sample_rate # OPTIONAL override; if omitted, code auto-detects
+ # a households column literally named 'sample_rate'.
+ # finalweight = 1/sample_rate (unless explicit run weight columns are provided)
+ # school_esc_outbound: [school_esc_outbound, out_escort_type]
+ # school_esc_inbound: [school_esc_inbound, inb_escort_type]
+
+prepare:
+ output:
+ file_format: parquet
+ validation:
+ relationship_checks: warn
+ distance_skim:
+ file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\autoSkims__MD.omx # path relative to run directory, or absolute
+ matrix: SOV_H_DIST__MD
+ non_motorized_distance_skim:
+ file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\maz_maz_walk.csv
+ matrix: null
+ time_periods:
+ network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
+ trip_period_number_column: depart
+ tour_start_period_number_column: start
+ tour_end_period_number_column: end
+ auto_sufficiency_basis: licensed_drivers # licensed_drivers | workers | adults
+ vot_bins:
+ source_column: income_segment
+ output_column: vot_bin
+ fallback_value: M
+ mappings:
+ survey:
+ 1: L
+ 2: L
+ 3: M
+ 4: M
+ 5: H
+ 6: H
+ 999: M
+ override:
+ 1: L
+ 2: L
+ 3: M
+ 4: M
+ 5: H
+ 6: H
+ 999: M
+
+skimjoin:
+ # Create optional long-form trip/tour tables with skim values for alternate modes.
+ create_hypothetical_skim_tables: true
+ defaults:
+ config_path: skats_skimjoin_config.yaml
+ skim_files:
+ - 'C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\*.omx'
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\maz_stop_walk.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\maz_maz_walk.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\bike_maz_logsums_commute.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\bike_maz_logsums_noncommute.csv
+ network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
+
+segment:
+ dashboard:
+ segmentation_type: person_sex
+ visibility: segments_only # full_only | segments_only | full_and_segments
+ definitions:
+ signup_platform:
+ include_full: true
+ persist_segmented_prepared_tables: false
+ allow_overlapping: false
+ on_empty_segment: warn
+ source:
+ type: prepared_column
+ source_table: hh
+ column: signup_platform
+ segments:
+ - id: rmove
+ label: RMove
+ values: ["rmove"]
+ - id: browser
+ label: Browser
+ values: ["browser"]
+ - id: call
+ label: Call
+ values: ["call"]
+
+ person_sex:
+ include_full: false
+ persist_segmented_prepared_tables: false
+ allow_overlapping: false
+ on_empty_segment: warn
+ source:
+ type: prepared_column
+ source_table: per
+ column: SEX
+ segments:
+ - id: male
+ label: Male
+ values: [1]
+ - id: female
+ label: Female
+ values: [2]
+
+summarize:
+ weighting_modes: [unweighted, weighted]
+ pnr_tour_modes:
+ - PNR_TRANSIT
+ # OPTIONAL summary-time grouping for outputs with a tour_purpose dimension
+ # group_joint_tour_purposes: true
+ # group_atwork_tour_purposes: true
+ # group_school_tour_purposes: true
+ # Controls additional mapped geography aggregations only. Summaries may still
+ # emit all_geographies totals, and native prepared home geographies such as
+ # home_taz, home_county, and home_mpo can appear when those columns exist.
+ geography:
+ enabled: true
+ # Configured mappings create columns such as home_geo__school_district,
+ # work_geo__county, or land_use_geo__district.
+ aggregations:
+ county:
+ source_zone_system: maz
+ file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\land_use.csv
+ zone_id_col: MAZ
+ geography_col: COUNTY
+ city_code:
+ source_zone_system: maz
+ file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\land_use.csv
+ zone_id_col: MAZ
+ geography_col: CITY
+ urban_growth_boundary:
+ source_zone_system: maz
+ file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\land_use.csv
+ zone_id_col: MAZ
+ geography_col: UGB
+ regional_district:
+ source_zone_system: maz
+ file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\land_use.csv
+ zone_id_col: MAZ
+ geography_col: DISTRICT06
+ school_district_k8:
+ source_zone_system: maz
+ file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\land_use.csv
+ zone_id_col: MAZ
+ geography_col: DIST_KTO8
+ school_district_9_12:
+ source_zone_system: maz
+ file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\land_use.csv
+ zone_id_col: MAZ
+ geography_col: DIST_9TO12
+
+dashboard:
+ title: "SKATS Estimation Visualizer"
+ enable_maz_geographies: false
+ live:
+ pages:
+ - overview
+ - long_term_choices
+ - daily_travel
+ - joint_travel
+ - tour_summaries
+ - trip_summaries
+ - skim_summaries
+ # - validation:
+ # - traffic
+ # - transit
+ # - vmt
+ # - regional_validation
+ export:
+ output_path: exports/dashboard.html
+ # summary series are baked into one exported HTML file.
+ # `dashboard.export.pages` modifies matching live pages; it is not an
+ # inclusion list. Omit selector overrides to export all widget states.
+ # Export segmentation defaults to segmentation.dashboard when omitted.
+ # Use full_only, segments_only, or full_and_segments to choose which
+ dashboard:
+ weighting: [unweighted]
+ # segmentation_type: signup_platform
+ # segmentation_visibility: segments_only
+ pages:
+ long_term_choices:
+ mandatory_location_choice:
+ geography_level:
+ - All Geography Types
+ - County
+ - MPO
+ # omit TAZ
+ # parts:
+ # commuting_flows:
+ # enabled: false
+
+ # shadow_pricing:
+ # enabled: true
+ # geography_level: [all]
+ # student_type: [all]
+ # parts:
+ # workplace_table:
+ # enabled: false
+ # school_table:
+ # enabled: false
+ # tour_summaries:
+ # internal_external_tours:
+ # enabled: false
+ validation:
+ vmt:
+ personal_auto_vmt_breakdown: all
+ personal_auto_vmt_geography_type: all
+ personal_auto_vmt_geography: default
+ personal_auto_vmt_time_period: default
+ personal_auto_vmt_mode: default
+ personal_auto_vmt_income_segment: default
+ personal_auto_vmt_household_size: default
+ non_motorized_vmt_breakdown: all
+ non_motorized_vmt_geography_type: all
+ non_motorized_vmt_geography: default
+ non_motorized_vmt_time_period: default
+ non_motorized_vmt_mode: default
+ non_motorized_vmt_income_segment: default
+ non_motorized_vmt_household_size: default
+
+ external_travel_metric: all
+ external_travel_breakdown: all
+ external_travel_trip_purpose: default
+ external_travel_time_period: default
+
+ demo_commercial_metric: all
+ demo_commercial_breakdown: all
+ demo_commercial_vehicle_type: default
+ demo_commercial_time_period: default
+ host:
+ account: my-connect-cloud-account
+ # app_id: 12345
+ # title: Estimation Mode Comparison Visualizer
+ # First hosted publish may open a browser for Posit Connect Cloud login.
+ # Later runs usually reuse saved local rsconnect metadata.
+ verify: true
+
+
+# Summary-affecting regrouping/normalization belongs under summarize.category_normalization.
+# Cosmetic labels and ordering belong under display.labels.
+display:
+ bar_hover_mode: all # closest | all
+ density_hover_mode: all # closest | all
+ labels:
+ person_type:
+ mapping:
+ all_person_types: All Person Types
+ 1: Full-time worker
+ 2: Part-time worker
+ 3: University student
+ 4: Non-worker adult
+ 5: Retired
+ 6: Student
+ 7: Preschool
+
+ transit_subsidy:
+ mapping:
+ 0: No subsidy
+ 1: Discounted
+ 2: Free
+
+ license_holding_status:
+ mapping:
+ has_license: Has License
+ no_license: No License
+
+ transit_pass_ownership_status:
+ mapping:
+ has_transit_pass: Has Transit Pass
+ no_transit_pass: No Transit Pass
+
+ telecommute_frequency:
+ mapping:
+ 1_day_week: 1 Day per Week
+ 2_3_days_week: 2-3 Days per Week
+ 4_days_week: 4+ Days per Week
+ No_Telecommute: No Telecommute
+
+ mode:
+ mapping:
+ SOV: Drive Alone
+ HOV2: Shared Ride 2
+ HOV3: Shared Ride 3+
+ WALK: Walk
+ BIKE: Bike
+ EBIKE: E-Bike
+ ESCOOTER: E-Scooter
+ WALK_TRANSIT: Walk-Transit
+ BIKE_TRANSIT: Bike-Transit
+ PNR_TRANSIT: PNR-Transit
+ KNR_TRANSIT: KNR-Transit
+ TAXI: Taxi
+ TNC_SINGLE: TNC-Single
+ TNC_SHARED: TNC-Pool
+ SCHOOLBUS: School Bus
+
+
+ # escort:
+ # mapping:
+ # not_escorted: No Escort
+ # pure_escort: Pure Escort
+ # ride_share: Ride Share
+ # 0: No Escort
+ # 1: Pure Escort
+ # 2: Ride Share
+ # "": No Escort
+
+ tour_purpose:
+ mapping:
+ all_tour_purposes: All Tour Purposes
+ work: Work
+ school: School
+ joint: Joint
+ escort: Escort
+ shopping: Shopping
+ othmaint: Other Maintenance
+ eatout: Eat Out
+ social: Social
+ othdiscr: Other Discretionary
+ atwork: At-Work
+ loop: Loop
+
+ trip_purpose:
+ mapping:
+ home: Home
+ work: Work
+ school: School
+ escort: Escort
+ shopping: Shopping
+ othmaint: Other Maintenance
+ eatout: Eat Out
+ social: Social
+ othdiscr: Other Discretionary
+ loop: Loop
+
+ stop_purpose:
+ mapping:
+ work: Work
+ school: School
+ escort: Escort
+ shopping: Shopping
+ othmaint: Other Maintenance
+ eatout: Eat Out
+ social: Social
+ othdiscr: Other Discretionary
+
+ tour_composition:
+ mapping:
+ adults: Adults
+ mixed: Mixed
+ children: Children
+
+ tour_category:
+ mapping:
+ mandatory: Mandatory
+ joint: Joint
+ non_mandatory: Non-Mandatory
+ atwork: At-Work
+
+ atwork_subtour_frequency_category:
+ mapping:
+ no_subtours: None
+ eat: 1 Eating Out
+ business1: 1 Business
+ maint: 1 Maintenance
+ business2: 2 Business
+ eat_business: 1 Eating Out + 1 Business
+
+ geography:
+ mapping:
+ all_geographies: All Geographies
+ county: County
+ home_county: County
+ home_mpo: MPO
+ home_taz: TAZ
+ district: District
+ taz: TAZ
+ maz: MAZ
+ school_district: School District
+
+ mandatory_tour_frequency:
+ # mapping:
+ # work1: Work
+ # school1: School
+ # work2: 2 Work
+ # school2: 2 School
+ # work_and_school: 1 Work + 1 School
+ mapping:
+ 1: Work
+ 3: School
+ 2: 2 Work
+ 4: 2 School
+ 5: 1 Work + 1 School
+
+ daily_activity_pattern:
+ mapping:
+ M : Mandatory
+ N : Non-Mandatory
+ H : At-Home
+
+ facility_type:
+ mapping:
+ 1: "Interstate"
+ 3: "Principal Arterial"
+ 4: "Minor Arterial"
+ 5: "Major Collector"
+ 6: "Minor Collector"
+ 7: "Local Road"
+ 30: "Ramp"
+ 998: "Bike/Walk"
+
+ commercial_vehicle_type:
+ mapping:
+ car: Car
+ su: Single-Unit Truck
+ mu: Multi-Unit Truck
+
+ # run_colors:
+ # - "#298c8c" # Teal
+ # - "#a00000" # Red
+ # - "#b8b8b8" # Light gray
+ # - "#384860" # Dark blue-gray
+ # - "#ff7f0e" # Orange
+ # - "#1f77b4"
+ # - "#ff7f0e"
+ # - "#2ca02c"
+ # - "#d62728"
+ # - "#9467bd"
+ # - "#8c564b"
+ # - "#e377c2"
+ # - "#7f7f7f"
diff --git a/simor_configs/skats_configs/skats_skimjoin_config.yaml b/simor_configs/skats_configs/skats_skimjoin_config.yaml
new file mode 100644
index 0000000..c5e4ff8
--- /dev/null
+++ b/simor_configs/skats_configs/skats_skimjoin_config.yaml
@@ -0,0 +1,428 @@
+project:
+ skim_files:
+ - 'C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\*.omx'
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\maz_stop_walk.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\maz_maz_walk.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\bike_maz_logsums_commute.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\bike_maz_logsums_noncommute.csv
+ network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
+ # project.trips_table / project.tours_table / project.output_dir are used by the
+ # standalone skimjoin CLI only. Integrated runtime skimjoin reads prepared
+ # in-memory trips/tours from the processor pipeline instead.
+ # trips_table: C:\Users\wesley.darling\Downloads\viz\viz\output\trip_linked.csv
+ # trips_table: C:\Users\wesley.darling\projects\activitysim_visualizer\artifacts\prepared_cache\override\trips.parquet
+ # tours_table: C:\Users\wesley.darling\Downloads\viz\viz\output\tour.csv
+ # tours_table: C:\Users\wesley.darling\projects\activitysim_visualizer\artifacts\prepared_cache\override\tours.parquet
+ # output_dir: skats_output
+
+
+activitysim:
+ # activitysim now only defines structural trip/tour fields used by skimjoin.
+ trip_mode_column: trip_mode
+ trip_id_column: trip_id
+ tour_mode_column: tour_mode
+ tour_id_column: tour_id
+ outbound_column: outbound
+
+defaults:
+ origin: OTAZ
+ destination: DTAZ
+ output_prefix: skim_
+ sentinel_values:
+ - 9999
+ - 999999
+
+zone_mapping:
+ lookup_name: taz
+ file_lookup_names:
+ fares.omx: zone_number
+ missing_zone_policy: error
+
+dimensions:
+ PERIOD:
+ source_columns:
+ trip_source_column: trip_period
+ outbound_tour_source_column: start_period
+ inbound_tour_source_column: first_inbound_trip_period
+ VOT:
+ source_columns:
+ trip_source_column: vot_bin
+ outbound_tour_source_column: vot_bin
+ inbound_tour_source_column: vot_bin
+ values:
+ L: L
+ M: M
+ H: H
+ BIKE_PURPOSE:
+ source_columns:
+ trip_source_column: tour_purpose
+ outbound_tour_source_column: tour_purpose
+ inbound_tour_source_column: tour_purpose
+ values:
+ work: commute
+ school: commute
+ univ: commute
+ atwork: noncommute
+ business: noncommute
+ eat: noncommute
+ eatout: noncommute
+ escort: noncommute
+ loop: noncommute
+ maint: noncommute
+ othdiscr: noncommute
+ othmaint: noncommute
+ shopping: noncommute
+ social: noncommute
+
+ignore_modes:
+ - ESCOOTER
+ - KNR_TRANSIT
+ - MISSING
+ - OTHER
+
+modes:
+ SOV:
+ output_prefix: skim_auto_
+ time: "SOV_{VOT}_TIME__{PERIOD}"
+ cost: "SOV_{VOT}_COST__{PERIOD}"
+ distance: "SOV_{VOT}_DIST__{PERIOD}"
+ HOV2:
+ output_prefix: skim_auto_
+ time: "SR2_{VOT}_TIME__{PERIOD}"
+ cost: "SR2_{VOT}_COST__{PERIOD}"
+ distance: "SR2_{VOT}_DIST__{PERIOD}"
+ HOV3:
+ output_prefix: skim_auto_
+ time: "SR3_{VOT}_TIME__{PERIOD}"
+ cost: "SR3_{VOT}_COST__{PERIOD}"
+ distance: "SR3_{VOT}_DIST__{PERIOD}"
+ PNR_TRANSIT:
+ output_prefix: skim_
+ segment_on: outbound
+ segments:
+ true:
+ auto_time:
+ matrix: "SOV_{VOT}_TIME__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ auto_distance:
+ matrix: "SOV_{VOT}_DIST__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ auto_cost:
+ matrix: "SOV_{VOT}_COST__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_auxiliary_walk_time:
+ matrix: "WTW_AUX__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_brt_ivtt:
+ matrix: "WTW_BRT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_bus_ivtt:
+ matrix: "WTW_BUS__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_commuter_rail_ivtt:
+ matrix: "WTW_CRT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_first_wait_time:
+ matrix: "WTW_FWT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_light_rail_ivtt:
+ matrix: "WTW_LRT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_tiv:
+ matrix: "WTW_TIV__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_num_transfers:
+ matrix: "WTW_XFR__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_transfer_wait_time:
+ matrix: "WTW_XWT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_fare:
+ matrix: "fare__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: pnr_zone_id
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: pnr_zone_id
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ false:
+ auto_time:
+ matrix: "SOV_{VOT}_TIME__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ auto_distance:
+ matrix: "SOV_{VOT}_DIST__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ auto_cost:
+ matrix: "SOV_{VOT}_COST__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_auxiliary_walk_time:
+ matrix: "WTW_AUX__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_brt_ivtt:
+ matrix: "WTW_BRT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_bus_ivtt:
+ matrix: "WTW_BUS__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_commuter_rail_ivtt:
+ matrix: "WTW_CRT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_first_wait_time:
+ matrix: "WTW_FWT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_light_rail_ivtt:
+ matrix: "WTW_LRT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_tiv:
+ matrix: "WTW_TIV__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_num_transfers:
+ matrix: "WTW_XFR__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_transfer_wait_time:
+ matrix: "WTW_XWT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_fare:
+ matrix: "fare__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: pnr_zone_id
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: pnr_zone_id
+ matrix: maz_stop_walk__walk_dist_premium_transit
+
+ SCHOOLBUS:
+ output_prefix: skim_
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
+ transit_bus_ivtt: "WTW_BUS__{PERIOD}"
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ transit_first_wait_time: "WTW_FWT__{PERIOD}"
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ fallbacks:
+ - matrix: "WTW_ACC__{PERIOD}"
+ origin: OTAZ
+ destination: DTAZ
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ fallbacks:
+ - matrix: "WTW_EGR__{PERIOD}"
+ origin: OTAZ
+ destination: DTAZ
+ TAXI:
+ output_prefix: skim_auto_
+ time: "SOV_{VOT}_TIME__{PERIOD}"
+ cost: "SOV_{VOT}_COST__{PERIOD}"
+ distance: "SOV_{VOT}_DIST__{PERIOD}"
+ TNC_SHARED:
+ output_prefix: skim_auto_
+ time: "SR2_{VOT}_TIME__{PERIOD}"
+ cost: "SR2_{VOT}_COST__{PERIOD}"
+ distance: "SR2_{VOT}_DIST__{PERIOD}"
+ TNC_SINGLE:
+ output_prefix: skim_auto_
+ time: "SOV_{VOT}_TIME__{PERIOD}"
+ cost: "SOV_{VOT}_COST__{PERIOD}"
+ distance: "SOV_{VOT}_DIST__{PERIOD}"
+ WALK:
+ output_prefix: skim_walk_
+ distance: WLK_DIST
+ maz_walk_distance:
+ output: skim_walk_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: maz_maz_walk__DISTWALK
+ maz_walk_actual:
+ output: skim_walk_maz_actual
+ origin: o_maz
+ destination: d_maz
+ matrix: maz_maz_walk__actual
+ WALK_TRANSIT:
+ output_prefix: skim_
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
+ transit_brt_ivtt: "WTW_BRT__{PERIOD}"
+ transit_bus_ivtt: "WTW_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "WTW_CRT__{PERIOD}"
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ transit_first_wait_time: "WTW_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "WTW_LRT__{PERIOD}"
+ transit_tiv: "WTW_TIV__{PERIOD}"
+ transit_num_transfers: "WTW_XFR__{PERIOD}"
+ transit_transfer_wait_time: "WTW_XWT__{PERIOD}"
+ transit_fare: "fare__{PERIOD}"
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ BIKE:
+ output_prefix: skim_bike_
+ maz_distance:
+ output: skim_bike_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: "bike_maz_logsums_{BIKE_PURPOSE}__distance_{BIKE_PURPOSE}"
+ logsum:
+ origin: o_maz
+ destination: d_maz
+ matrix: "bike_maz_logsums_{BIKE_PURPOSE}__logsum_{BIKE_PURPOSE}"
+ EBIKE:
+ output_prefix: skim_bike_
+ maz_distance:
+ output: skim_bike_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: "bike_maz_logsums_{BIKE_PURPOSE}__distance_{BIKE_PURPOSE}"
+ BIKE_TRANSIT:
+ output_prefix: skim_
+ bike_o_maz_stop_distance_bus:
+ output: skim_bike_transit_distance_bus
+ combine: sum
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ bike_d_maz_stop_distance_bus:
+ output: skim_bike_transit_distance_bus
+ combine: sum
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ bike_o_maz_stop_distance_premium:
+ output: skim_bike_transit_distance_premium
+ combine: sum
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ bike_d_maz_stop_distance_premium:
+ output: skim_bike_transit_distance_premium
+ combine: sum
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
+ transit_brt_ivtt: "WTW_BRT__{PERIOD}"
+ transit_bus_ivtt: "WTW_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "WTW_CRT__{PERIOD}"
+ transit_first_wait_time: "WTW_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "WTW_LRT__{PERIOD}"
+ transit_tiv: "WTW_TIV__{PERIOD}"
+ transit_num_transfers: "WTW_XFR__{PERIOD}"
+ transit_transfer_wait_time: "WTW_XWT__{PERIOD}"
+ transit_fare: "fare__{PERIOD}"
+
+
diff --git a/simor_configs/skats_configs/skats_skimjoin_config_alternate_id_col.yaml b/simor_configs/skats_configs/skats_skimjoin_config_alternate_id_col.yaml
new file mode 100644
index 0000000..9aa99f0
--- /dev/null
+++ b/simor_configs/skats_configs/skats_skimjoin_config_alternate_id_col.yaml
@@ -0,0 +1,428 @@
+project:
+ skim_files:
+ - 'C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\*.omx'
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\maz_stop_walk.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\maz_maz_walk.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\bike_maz_logsums_commute.csv
+ - C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\skats_data\bike_maz_logsums_noncommute.csv
+ network_los_file: C:\Users\wesley.darling\project_data\odot_skims\network_los.yaml
+ # project.trips_table / project.tours_table / project.output_dir are used by the
+ # standalone skimjoin CLI only. Integrated runtime skimjoin reads prepared
+ # in-memory trips/tours from the processor pipeline instead.
+ # trips_table: C:\Users\wesley.darling\Downloads\viz\viz\output\trip_linked.csv
+ # trips_table: C:\Users\wesley.darling\projects\activitysim_visualizer\artifacts\prepared_cache\override\trips.parquet
+ # tours_table: C:\Users\wesley.darling\Downloads\viz\viz\output\tour.csv
+ # tours_table: C:\Users\wesley.darling\projects\activitysim_visualizer\artifacts\prepared_cache\override\tours.parquet
+ # output_dir: skats_output
+
+
+activitysim:
+ # activitysim now only defines structural trip/tour fields used by skimjoin.
+ trip_mode_column: trip_mode
+ trip_id_column: linked_trip_id
+ tour_mode_column: tour_mode
+ tour_id_column: tour_id
+ outbound_column: outbound
+
+defaults:
+ origin: OTAZ
+ destination: DTAZ
+ output_prefix: skim_
+ sentinel_values:
+ - 9999
+ - 999999
+
+zone_mapping:
+ lookup_name: taz
+ file_lookup_names:
+ fares.omx: zone_number
+ missing_zone_policy: error
+
+dimensions:
+ PERIOD:
+ source_columns:
+ trip_source_column: trip_period
+ outbound_tour_source_column: start_period
+ inbound_tour_source_column: first_inbound_trip_period
+ VOT:
+ source_columns:
+ trip_source_column: vot_bin
+ outbound_tour_source_column: vot_bin
+ inbound_tour_source_column: vot_bin
+ values:
+ L: L
+ M: M
+ H: H
+ BIKE_PURPOSE:
+ source_columns:
+ trip_source_column: tour_purpose
+ outbound_tour_source_column: tour_purpose
+ inbound_tour_source_column: tour_purpose
+ values:
+ work: commute
+ school: commute
+ univ: commute
+ atwork: noncommute
+ business: noncommute
+ eat: noncommute
+ eatout: noncommute
+ escort: noncommute
+ loop: noncommute
+ maint: noncommute
+ othdiscr: noncommute
+ othmaint: noncommute
+ shopping: noncommute
+ social: noncommute
+
+ignore_modes:
+ - ESCOOTER
+ - KNR_TRANSIT
+ - MISSING
+ - OTHER
+
+modes:
+ SOV:
+ output_prefix: skim_auto_
+ time: "SOV_{VOT}_TIME__{PERIOD}"
+ cost: "SOV_{VOT}_COST__{PERIOD}"
+ distance: "SOV_{VOT}_DIST__{PERIOD}"
+ HOV2:
+ output_prefix: skim_auto_
+ time: "SR2_{VOT}_TIME__{PERIOD}"
+ cost: "SR2_{VOT}_COST__{PERIOD}"
+ distance: "SR2_{VOT}_DIST__{PERIOD}"
+ HOV3:
+ output_prefix: skim_auto_
+ time: "SR3_{VOT}_TIME__{PERIOD}"
+ cost: "SR3_{VOT}_COST__{PERIOD}"
+ distance: "SR3_{VOT}_DIST__{PERIOD}"
+ PNR_TRANSIT:
+ output_prefix: skim_
+ segment_on: outbound
+ segments:
+ true:
+ auto_time:
+ matrix: "SOV_{VOT}_TIME__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ auto_distance:
+ matrix: "SOV_{VOT}_DIST__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ auto_cost:
+ matrix: "SOV_{VOT}_COST__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_auxiliary_walk_time:
+ matrix: "WTW_AUX__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_brt_ivtt:
+ matrix: "WTW_BRT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_bus_ivtt:
+ matrix: "WTW_BUS__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_commuter_rail_ivtt:
+ matrix: "WTW_CRT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_first_wait_time:
+ matrix: "WTW_FWT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_light_rail_ivtt:
+ matrix: "WTW_LRT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_tiv:
+ matrix: "WTW_TIV__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_num_transfers:
+ matrix: "WTW_XFR__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_transfer_wait_time:
+ matrix: "WTW_XWT__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_fare:
+ matrix: "fare__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: pnr_zone_id
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: pnr_zone_id
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ false:
+ auto_time:
+ matrix: "SOV_{VOT}_TIME__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ auto_distance:
+ matrix: "SOV_{VOT}_DIST__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ auto_cost:
+ matrix: "SOV_{VOT}_COST__{PERIOD}"
+ origin: pnr_taz
+ destination: DTAZ
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_auxiliary_walk_time:
+ matrix: "WTW_AUX__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_brt_ivtt:
+ matrix: "WTW_BRT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_bus_ivtt:
+ matrix: "WTW_BUS__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_commuter_rail_ivtt:
+ matrix: "WTW_CRT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_first_wait_time:
+ matrix: "WTW_FWT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_light_rail_ivtt:
+ matrix: "WTW_LRT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_tiv:
+ matrix: "WTW_TIV__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_num_transfers:
+ matrix: "WTW_XFR__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_transfer_wait_time:
+ matrix: "WTW_XWT__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_fare:
+ matrix: "fare__{PERIOD}"
+ origin: OTAZ
+ destination: pnr_taz
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: pnr_zone_id
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: pnr_zone_id
+ matrix: maz_stop_walk__walk_dist_premium_transit
+
+ SCHOOLBUS:
+ output_prefix: skim_
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
+ transit_bus_ivtt: "WTW_BUS__{PERIOD}"
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ transit_first_wait_time: "WTW_FWT__{PERIOD}"
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ fallbacks:
+ - matrix: "WTW_ACC__{PERIOD}"
+ origin: OTAZ
+ destination: DTAZ
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ fallbacks:
+ - matrix: "WTW_EGR__{PERIOD}"
+ origin: OTAZ
+ destination: DTAZ
+ TAXI:
+ output_prefix: skim_auto_
+ time: "SOV_{VOT}_TIME__{PERIOD}"
+ cost: "SOV_{VOT}_COST__{PERIOD}"
+ distance: "SOV_{VOT}_DIST__{PERIOD}"
+ TNC_SHARED:
+ output_prefix: skim_auto_
+ time: "SR2_{VOT}_TIME__{PERIOD}"
+ cost: "SR2_{VOT}_COST__{PERIOD}"
+ distance: "SR2_{VOT}_DIST__{PERIOD}"
+ TNC_SINGLE:
+ output_prefix: skim_auto_
+ time: "SOV_{VOT}_TIME__{PERIOD}"
+ cost: "SOV_{VOT}_COST__{PERIOD}"
+ distance: "SOV_{VOT}_DIST__{PERIOD}"
+ WALK:
+ output_prefix: skim_walk_
+ distance: WLK_DIST
+ maz_walk_distance:
+ output: skim_walk_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: maz_maz_walk__DISTWALK
+ maz_walk_actual:
+ output: skim_walk_maz_actual
+ origin: o_maz
+ destination: d_maz
+ matrix: maz_maz_walk__actual
+ WALK_TRANSIT:
+ output_prefix: skim_
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
+ transit_brt_ivtt: "WTW_BRT__{PERIOD}"
+ transit_bus_ivtt: "WTW_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "WTW_CRT__{PERIOD}"
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ transit_first_wait_time: "WTW_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "WTW_LRT__{PERIOD}"
+ transit_tiv: "WTW_TIV__{PERIOD}"
+ transit_num_transfers: "WTW_XFR__{PERIOD}"
+ transit_transfer_wait_time: "WTW_XWT__{PERIOD}"
+ transit_fare: "fare__{PERIOD}"
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ BIKE:
+ output_prefix: skim_bike_
+ maz_distance:
+ output: skim_bike_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: "bike_maz_logsums_{BIKE_PURPOSE}__distance_{BIKE_PURPOSE}"
+ logsum:
+ origin: o_maz
+ destination: d_maz
+ matrix: "bike_maz_logsums_{BIKE_PURPOSE}__logsum_{BIKE_PURPOSE}"
+ EBIKE:
+ output_prefix: skim_bike_
+ maz_distance:
+ output: skim_bike_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: "bike_maz_logsums_{BIKE_PURPOSE}__distance_{BIKE_PURPOSE}"
+ BIKE_TRANSIT:
+ output_prefix: skim_
+ bike_o_maz_stop_distance_bus:
+ output: skim_bike_transit_distance_bus
+ combine: sum
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ bike_d_maz_stop_distance_bus:
+ output: skim_bike_transit_distance_bus
+ combine: sum
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ bike_o_maz_stop_distance_premium:
+ output: skim_bike_transit_distance_premium
+ combine: sum
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ bike_d_maz_stop_distance_premium:
+ output: skim_bike_transit_distance_premium
+ combine: sum
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
+ transit_brt_ivtt: "WTW_BRT__{PERIOD}"
+ transit_bus_ivtt: "WTW_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "WTW_CRT__{PERIOD}"
+ transit_first_wait_time: "WTW_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "WTW_LRT__{PERIOD}"
+ transit_tiv: "WTW_TIV__{PERIOD}"
+ transit_num_transfers: "WTW_XFR__{PERIOD}"
+ transit_transfer_wait_time: "WTW_XWT__{PERIOD}"
+ transit_fare: "fare__{PERIOD}"
+
+
diff --git a/tests/test_dashboard_helpers_phase1.py b/tests/test_dashboard_helpers_phase1.py
index 73acf66..1efccc7 100644
--- a/tests/test_dashboard_helpers_phase1.py
+++ b/tests/test_dashboard_helpers_phase1.py
@@ -55,12 +55,33 @@
timebin_label,
)
from dashboard.page_base import DashboardPage
+from dashboard.pages.skim_summaries._shared import component_display_name
from dashboard.pages.trip_summaries.parking_location import parking_scatter_data
from dashboard.state import DashboardState
from processor.models import RunData
from test_export_html import _full_summary_run, _write_config
+@pytest.mark.parametrize(
+ ("component", "expected"),
+ [
+ (
+ "skim_bike_transit_distance_bus",
+ "Total Bike Distance - Local Bus (mi) (Estimated from Walk Skims)",
+ ),
+ (
+ "skim_bike_transit_distance_premium",
+ "Total Bike Distance - Premium Transit (mi) (Estimated from Walk Skims)",
+ ),
+ ],
+)
+def test_component_display_name_labels_estimated_bike_transit_distances(
+ component: str,
+ expected: str,
+) -> None:
+ assert component_display_name(component) == expected
+
+
def test_run_table_view_filters_transforms_and_joins_by_run_label() -> None:
counts = RunTables.from_runs(
[
From fbdb463b9469daaf996d7dbf8ec0d7c637b50880 Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Mon, 10 Aug 2026 11:08:46 -0400
Subject: [PATCH 23/27] stand up skats and lcog visualizers
---
simor_configs/lcog_configs/lcog_config.yaml | 14 +-
.../lcog_configs/lcog_skimjoin_config.yaml | 255 +++++++++++++++++-
...lcog_skimjoin_config_alternate_id_col.yaml | 255 +++++++++++++++++-
simor_configs/skats_configs/skats_config.yaml | 14 +-
.../skats_configs/skats_skimjoin_config.yaml | 179 +-----------
...kats_skimjoin_config_alternate_id_col.yaml | 179 +-----------
6 files changed, 504 insertions(+), 392 deletions(-)
diff --git a/simor_configs/lcog_configs/lcog_config.yaml b/simor_configs/lcog_configs/lcog_config.yaml
index f2cdac3..63ec886 100644
--- a/simor_configs/lcog_configs/lcog_config.yaml
+++ b/simor_configs/lcog_configs/lcog_config.yaml
@@ -110,6 +110,9 @@ prepare:
file_format: parquet
validation:
relationship_checks: warn
+ distance_skim:
+ file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\autoSkims__MD.omx # path relative to run directory, or absolute
+ matrix: SOV_H_DIST__MD
non_motorized_distance_skim:
file: C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\maz_maz_walk.csv
matrix: null
@@ -147,6 +150,7 @@ skimjoin:
defaults:
config_path: lcog_skimjoin_config.yaml
skim_files:
+ - 'C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\*.omx'
- C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\maz_stop_walk.csv
- C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\maz_maz_walk.csv
- C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\bike_maz_logsums_commute.csv
@@ -488,11 +492,11 @@ display:
su: Single-Unit Truck
mu: Multi-Unit Truck
- # run_colors:
- # - "#298c8c" # Teal
- # - "#a00000" # Red
- # - "#b8b8b8" # Light gray
- # - "#384860" # Dark blue-gray
+ run_colors:
+ - "#298c8c" # Teal
+ - "#a00000" # Red
+ - "#b8b8b8" # Light gray
+ - "#384860" # Dark blue-gray
# - "#ff7f0e" # Orange
# - "#1f77b4"
# - "#ff7f0e"
diff --git a/simor_configs/lcog_configs/lcog_skimjoin_config.yaml b/simor_configs/lcog_configs/lcog_skimjoin_config.yaml
index 2b676db..13711b1 100644
--- a/simor_configs/lcog_configs/lcog_skimjoin_config.yaml
+++ b/simor_configs/lcog_configs/lcog_skimjoin_config.yaml
@@ -1,5 +1,6 @@
project:
skim_files:
+ - 'C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\*.omx'
- C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\maz_stop_walk.csv
- C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\maz_maz_walk.csv
- C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\bike_maz_logsums_commute.csv
@@ -14,14 +15,32 @@ activitysim:
outbound_column: outbound
defaults:
- origin: o_maz
- destination: d_maz
+ origin: OTAZ
+ destination: DTAZ
output_prefix: skim_
sentinel_values:
- 9999
- 999999
+zone_mapping:
+ lookup_name: taz
+ missing_zone_policy: error
+
dimensions:
+ PERIOD:
+ source_columns:
+ trip_source_column: trip_period
+ outbound_tour_source_column: start_period
+ inbound_tour_source_column: first_inbound_trip_period
+ VOT:
+ source_columns:
+ trip_source_column: vot_bin
+ outbound_tour_source_column: vot_bin
+ inbound_tour_source_column: vot_bin
+ values:
+ L: L
+ M: M
+ H: H
BIKE_PURPOSE:
source_columns:
trip_source_column: tour_purpose
@@ -45,24 +64,223 @@ dimensions:
ignore_modes:
- ESCOOTER
- - HOV2
- - HOV3
- - KNR_TRANSIT
- MISSING
- OTHER
- - PNR_TRANSIT
- - SCHOOLBUS
- - SOV
- - TAXI
- - TNC_SHARED
- - TNC_SINGLE
- - WALK_TRANSIT
modes:
+ SOV:
+ output_prefix: skim_auto_
+ time: "SOV_{VOT}_TIME__{PERIOD}"
+ cost: "SOV_{VOT}_COST__{PERIOD}"
+ distance: "SOV_{VOT}_DIST__{PERIOD}"
+ HOV2:
+ output_prefix: skim_auto_
+ time: "SR2_{VOT}_TIME__{PERIOD}"
+ cost: "SR2_{VOT}_COST__{PERIOD}"
+ distance: "SR2_{VOT}_DIST__{PERIOD}"
+ HOV3:
+ output_prefix: skim_auto_
+ time: "SR3_{VOT}_TIME__{PERIOD}"
+ cost: "SR3_{VOT}_COST__{PERIOD}"
+ distance: "SR3_{VOT}_DIST__{PERIOD}"
+ KNR_TRANSIT:
+ output_prefix: skim_
+ segment_on: outbound
+ segments:
+ true:
+ auto_time: "KTW_ACC__{PERIOD}"
+ auto_distance: "KTW_TDD__{PERIOD}"
+ transit_auxiliary_walk_time: "KTW_AUX__{PERIOD}"
+ transit_brt_ivtt: "KTW_BRT__{PERIOD}"
+ transit_bus_ivtt: "KTW_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "KTW_CRT__{PERIOD}"
+ walk_time: "KTW_EGR__{PERIOD}"
+ transit_first_wait_time: "KTW_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "KTW_LRT__{PERIOD}"
+ transit_tiv: "KTW_TIV__{PERIOD}"
+ transit_num_transfers: "KTW_XFR__{PERIOD}"
+ transit_transfer_wait_time: "KTW_XWT__{PERIOD}"
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ false:
+ walk_time: "WTK_ACC__{PERIOD}"
+ transit_auxiliary_walk_time: "WTK_AUX__{PERIOD}"
+ transit_brt_ivtt: "WTK_BRT__{PERIOD}"
+ transit_bus_ivtt: "WTK_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "WTK_CRT__{PERIOD}"
+ auto_time: "WTK_EGR__{PERIOD}"
+ auto_distance: "WTK_TDD__{PERIOD}"
+ transit_first_wait_time: "WTK_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "WTK_LRT__{PERIOD}"
+ transit_tiv: "WTK_TIV__{PERIOD}"
+ transit_num_transfers: "WTK_XFR__{PERIOD}"
+ transit_transfer_wait_time: "WTK_XWT__{PERIOD}"
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ PNR_TRANSIT:
+ output_prefix: skim_
+ segment_on: outbound
+ segments:
+ true:
+ auto_time: "KTW_ACC__{PERIOD}"
+ auto_distance: "KTW_TDD__{PERIOD}"
+ transit_auxiliary_walk_time: "KTW_AUX__{PERIOD}"
+ transit_brt_ivtt: "KTW_BRT__{PERIOD}"
+ transit_bus_ivtt: "KTW_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "KTW_CRT__{PERIOD}"
+ walk_time: "KTW_EGR__{PERIOD}"
+ transit_first_wait_time: "KTW_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "KTW_LRT__{PERIOD}"
+ transit_tiv: "KTW_TIV__{PERIOD}"
+ transit_num_transfers: "KTW_XFR__{PERIOD}"
+ transit_transfer_wait_time: "KTW_XWT__{PERIOD}"
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ false:
+ walk_time: "WTK_ACC__{PERIOD}"
+ transit_auxiliary_walk_time: "WTK_AUX__{PERIOD}"
+ transit_brt_ivtt: "WTK_BRT__{PERIOD}"
+ transit_bus_ivtt: "WTK_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "WTK_CRT__{PERIOD}"
+ auto_time: "WTK_EGR__{PERIOD}"
+ auto_distance: "WTK_TDD__{PERIOD}"
+ transit_first_wait_time: "WTK_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "WTK_LRT__{PERIOD}"
+ transit_tiv: "WTK_TIV__{PERIOD}"
+ transit_num_transfers: "WTK_XFR__{PERIOD}"
+ transit_transfer_wait_time: "WTK_XWT__{PERIOD}"
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ SCHOOLBUS:
+ output_prefix: skim_
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
+ transit_bus_ivtt: "WTW_BUS__{PERIOD}"
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ transit_first_wait_time: "WTW_FWT__{PERIOD}"
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ fallbacks:
+ - matrix: "WTW_ACC__{PERIOD}"
+ origin: OTAZ
+ destination: DTAZ
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ fallbacks:
+ - matrix: "WTW_EGR__{PERIOD}"
+ origin: OTAZ
+ destination: DTAZ
+ TAXI:
+ output_prefix: skim_auto_
+ time: "SOV_{VOT}_TIME__{PERIOD}"
+ cost: "SOV_{VOT}_COST__{PERIOD}"
+ distance: "SOV_{VOT}_DIST__{PERIOD}"
+ TNC_SHARED:
+ output_prefix: skim_auto_
+ time: "SR2_{VOT}_TIME__{PERIOD}"
+ cost: "SR2_{VOT}_COST__{PERIOD}"
+ distance: "SR2_{VOT}_DIST__{PERIOD}"
+ TNC_SINGLE:
+ output_prefix: skim_auto_
+ time: "SOV_{VOT}_TIME__{PERIOD}"
+ cost: "SOV_{VOT}_COST__{PERIOD}"
+ distance: "SOV_{VOT}_DIST__{PERIOD}"
WALK:
output_prefix: skim_walk_
- distance: maz_maz_walk__DISTWALK
- actual: maz_maz_walk__actual
+ distance: WLK_DIST
+ maz_walk_distance:
+ output: skim_walk_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: maz_maz_walk__DISTWALK
+ maz_walk_actual:
+ output: skim_walk_maz_actual
+ origin: o_maz
+ destination: d_maz
+ matrix: maz_maz_walk__actual
+ WALK_TRANSIT:
+ output_prefix: skim_
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
+ transit_brt_ivtt: "WTW_BRT__{PERIOD}"
+ transit_bus_ivtt: "WTW_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "WTW_CRT__{PERIOD}"
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ transit_first_wait_time: "WTW_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "WTW_LRT__{PERIOD}"
+ transit_tiv: "WTW_TIV__{PERIOD}"
+ transit_num_transfers: "WTW_XFR__{PERIOD}"
+ transit_transfer_wait_time: "WTW_XWT__{PERIOD}"
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
BIKE:
output_prefix: skim_bike_
maz_distance:
@@ -107,3 +325,12 @@ modes:
lookup: key
key_column: d_maz
matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
+ transit_brt_ivtt: "WTW_BRT__{PERIOD}"
+ transit_bus_ivtt: "WTW_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "WTW_CRT__{PERIOD}"
+ transit_first_wait_time: "WTW_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "WTW_LRT__{PERIOD}"
+ transit_tiv: "WTW_TIV__{PERIOD}"
+ transit_num_transfers: "WTW_XFR__{PERIOD}"
+ transit_transfer_wait_time: "WTW_XWT__{PERIOD}"
diff --git a/simor_configs/lcog_configs/lcog_skimjoin_config_alternate_id_col.yaml b/simor_configs/lcog_configs/lcog_skimjoin_config_alternate_id_col.yaml
index e0e3a11..e7b653d 100644
--- a/simor_configs/lcog_configs/lcog_skimjoin_config_alternate_id_col.yaml
+++ b/simor_configs/lcog_configs/lcog_skimjoin_config_alternate_id_col.yaml
@@ -1,5 +1,6 @@
project:
skim_files:
+ - 'C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\*.omx'
- C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\maz_stop_walk.csv
- C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\maz_maz_walk.csv
- C:\Users\wesley.darling\OneDrive - Resource Systems Group, Inc\odot_joint_estimation_data\lcog_data\bike_maz_logsums_commute.csv
@@ -14,14 +15,32 @@ activitysim:
outbound_column: outbound
defaults:
- origin: o_maz
- destination: d_maz
+ origin: OTAZ
+ destination: DTAZ
output_prefix: skim_
sentinel_values:
- 9999
- 999999
+zone_mapping:
+ lookup_name: taz
+ missing_zone_policy: error
+
dimensions:
+ PERIOD:
+ source_columns:
+ trip_source_column: trip_period
+ outbound_tour_source_column: start_period
+ inbound_tour_source_column: first_inbound_trip_period
+ VOT:
+ source_columns:
+ trip_source_column: vot_bin
+ outbound_tour_source_column: vot_bin
+ inbound_tour_source_column: vot_bin
+ values:
+ L: L
+ M: M
+ H: H
BIKE_PURPOSE:
source_columns:
trip_source_column: tour_purpose
@@ -45,24 +64,223 @@ dimensions:
ignore_modes:
- ESCOOTER
- - HOV2
- - HOV3
- - KNR_TRANSIT
- MISSING
- OTHER
- - PNR_TRANSIT
- - SCHOOLBUS
- - SOV
- - TAXI
- - TNC_SHARED
- - TNC_SINGLE
- - WALK_TRANSIT
modes:
+ SOV:
+ output_prefix: skim_auto_
+ time: "SOV_{VOT}_TIME__{PERIOD}"
+ cost: "SOV_{VOT}_COST__{PERIOD}"
+ distance: "SOV_{VOT}_DIST__{PERIOD}"
+ HOV2:
+ output_prefix: skim_auto_
+ time: "SR2_{VOT}_TIME__{PERIOD}"
+ cost: "SR2_{VOT}_COST__{PERIOD}"
+ distance: "SR2_{VOT}_DIST__{PERIOD}"
+ HOV3:
+ output_prefix: skim_auto_
+ time: "SR3_{VOT}_TIME__{PERIOD}"
+ cost: "SR3_{VOT}_COST__{PERIOD}"
+ distance: "SR3_{VOT}_DIST__{PERIOD}"
+ KNR_TRANSIT:
+ output_prefix: skim_
+ segment_on: outbound
+ segments:
+ true:
+ auto_time: "KTW_ACC__{PERIOD}"
+ auto_distance: "KTW_TDD__{PERIOD}"
+ transit_auxiliary_walk_time: "KTW_AUX__{PERIOD}"
+ transit_brt_ivtt: "KTW_BRT__{PERIOD}"
+ transit_bus_ivtt: "KTW_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "KTW_CRT__{PERIOD}"
+ walk_time: "KTW_EGR__{PERIOD}"
+ transit_first_wait_time: "KTW_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "KTW_LRT__{PERIOD}"
+ transit_tiv: "KTW_TIV__{PERIOD}"
+ transit_num_transfers: "KTW_XFR__{PERIOD}"
+ transit_transfer_wait_time: "KTW_XWT__{PERIOD}"
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ false:
+ walk_time: "WTK_ACC__{PERIOD}"
+ transit_auxiliary_walk_time: "WTK_AUX__{PERIOD}"
+ transit_brt_ivtt: "WTK_BRT__{PERIOD}"
+ transit_bus_ivtt: "WTK_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "WTK_CRT__{PERIOD}"
+ auto_time: "WTK_EGR__{PERIOD}"
+ auto_distance: "WTK_TDD__{PERIOD}"
+ transit_first_wait_time: "WTK_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "WTK_LRT__{PERIOD}"
+ transit_tiv: "WTK_TIV__{PERIOD}"
+ transit_num_transfers: "WTK_XFR__{PERIOD}"
+ transit_transfer_wait_time: "WTK_XWT__{PERIOD}"
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ PNR_TRANSIT:
+ output_prefix: skim_
+ segment_on: outbound
+ segments:
+ true:
+ auto_time: "KTW_ACC__{PERIOD}"
+ auto_distance: "KTW_TDD__{PERIOD}"
+ transit_auxiliary_walk_time: "KTW_AUX__{PERIOD}"
+ transit_brt_ivtt: "KTW_BRT__{PERIOD}"
+ transit_bus_ivtt: "KTW_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "KTW_CRT__{PERIOD}"
+ walk_time: "KTW_EGR__{PERIOD}"
+ transit_first_wait_time: "KTW_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "KTW_LRT__{PERIOD}"
+ transit_tiv: "KTW_TIV__{PERIOD}"
+ transit_num_transfers: "KTW_XFR__{PERIOD}"
+ transit_transfer_wait_time: "KTW_XWT__{PERIOD}"
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ false:
+ walk_time: "WTK_ACC__{PERIOD}"
+ transit_auxiliary_walk_time: "WTK_AUX__{PERIOD}"
+ transit_brt_ivtt: "WTK_BRT__{PERIOD}"
+ transit_bus_ivtt: "WTK_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "WTK_CRT__{PERIOD}"
+ auto_time: "WTK_EGR__{PERIOD}"
+ auto_distance: "WTK_TDD__{PERIOD}"
+ transit_first_wait_time: "WTK_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "WTK_LRT__{PERIOD}"
+ transit_tiv: "WTK_TIV__{PERIOD}"
+ transit_num_transfers: "WTK_XFR__{PERIOD}"
+ transit_transfer_wait_time: "WTK_XWT__{PERIOD}"
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ SCHOOLBUS:
+ output_prefix: skim_
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
+ transit_bus_ivtt: "WTW_BUS__{PERIOD}"
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ transit_first_wait_time: "WTW_FWT__{PERIOD}"
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ fallbacks:
+ - matrix: "WTW_ACC__{PERIOD}"
+ origin: OTAZ
+ destination: DTAZ
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ fallbacks:
+ - matrix: "WTW_EGR__{PERIOD}"
+ origin: OTAZ
+ destination: DTAZ
+ TAXI:
+ output_prefix: skim_auto_
+ time: "SOV_{VOT}_TIME__{PERIOD}"
+ cost: "SOV_{VOT}_COST__{PERIOD}"
+ distance: "SOV_{VOT}_DIST__{PERIOD}"
+ TNC_SHARED:
+ output_prefix: skim_auto_
+ time: "SR2_{VOT}_TIME__{PERIOD}"
+ cost: "SR2_{VOT}_COST__{PERIOD}"
+ distance: "SR2_{VOT}_DIST__{PERIOD}"
+ TNC_SINGLE:
+ output_prefix: skim_auto_
+ time: "SOV_{VOT}_TIME__{PERIOD}"
+ cost: "SOV_{VOT}_COST__{PERIOD}"
+ distance: "SOV_{VOT}_DIST__{PERIOD}"
WALK:
output_prefix: skim_walk_
- distance: maz_maz_walk__DISTWALK
- actual: maz_maz_walk__actual
+ distance: WLK_DIST
+ maz_walk_distance:
+ output: skim_walk_maz_distance
+ origin: o_maz
+ destination: d_maz
+ matrix: maz_maz_walk__DISTWALK
+ maz_walk_actual:
+ output: skim_walk_maz_actual
+ origin: o_maz
+ destination: d_maz
+ matrix: maz_maz_walk__actual
+ WALK_TRANSIT:
+ output_prefix: skim_
+ access_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_ACC__{PERIOD}"
+ transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
+ transit_brt_ivtt: "WTW_BRT__{PERIOD}"
+ transit_bus_ivtt: "WTW_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "WTW_CRT__{PERIOD}"
+ egress_walk_time:
+ output: skim_walk_time
+ combine: sum
+ matrix: "WTW_EGR__{PERIOD}"
+ transit_first_wait_time: "WTW_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "WTW_LRT__{PERIOD}"
+ transit_tiv: "WTW_TIV__{PERIOD}"
+ transit_num_transfers: "WTW_XFR__{PERIOD}"
+ transit_transfer_wait_time: "WTW_XWT__{PERIOD}"
+ transit_o_maz_stop_walk_bus:
+ output: skim_transit_o_maz_stop_walk_bus
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_d_maz_stop_walk_bus:
+ output: skim_transit_d_maz_stop_walk_bus
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_local_bus
+ transit_o_maz_stop_walk_premium:
+ output: skim_transit_o_maz_stop_walk_premium
+ lookup: key
+ key_column: o_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_d_maz_stop_walk_premium:
+ output: skim_transit_d_maz_stop_walk_premium
+ lookup: key
+ key_column: d_maz
+ matrix: maz_stop_walk__walk_dist_premium_transit
BIKE:
output_prefix: skim_bike_
maz_distance:
@@ -107,3 +325,12 @@ modes:
lookup: key
key_column: d_maz
matrix: maz_stop_walk__walk_dist_premium_transit
+ transit_auxiliary_walk_time: "WTW_AUX__{PERIOD}"
+ transit_brt_ivtt: "WTW_BRT__{PERIOD}"
+ transit_bus_ivtt: "WTW_BUS__{PERIOD}"
+ transit_commuter_rail_ivtt: "WTW_CRT__{PERIOD}"
+ transit_first_wait_time: "WTW_FWT__{PERIOD}"
+ transit_light_rail_ivtt: "WTW_LRT__{PERIOD}"
+ transit_tiv: "WTW_TIV__{PERIOD}"
+ transit_num_transfers: "WTW_XFR__{PERIOD}"
+ transit_transfer_wait_time: "WTW_XWT__{PERIOD}"
diff --git a/simor_configs/skats_configs/skats_config.yaml b/simor_configs/skats_configs/skats_config.yaml
index 26312f1..273a502 100644
--- a/simor_configs/skats_configs/skats_config.yaml
+++ b/simor_configs/skats_configs/skats_config.yaml
@@ -519,13 +519,13 @@ display:
su: Single-Unit Truck
mu: Multi-Unit Truck
- # run_colors:
- # - "#298c8c" # Teal
- # - "#a00000" # Red
- # - "#b8b8b8" # Light gray
- # - "#384860" # Dark blue-gray
- # - "#ff7f0e" # Orange
- # - "#1f77b4"
+ run_colors:
+ - "#3A86FF" # Blue
+ - "#8338EC" # Purple
+ - "#FF006E" # Pink
+ - "#FB5607" # Orange
+ - "#06D6A0" # Mint
+ - "#FFBE0B" # Yellow
# - "#ff7f0e"
# - "#2ca02c"
# - "#d62728"
diff --git a/simor_configs/skats_configs/skats_skimjoin_config.yaml b/simor_configs/skats_configs/skats_skimjoin_config.yaml
index c5e4ff8..f4a6195 100644
--- a/simor_configs/skats_configs/skats_skimjoin_config.yaml
+++ b/simor_configs/skats_configs/skats_skimjoin_config.yaml
@@ -35,7 +35,7 @@ defaults:
zone_mapping:
lookup_name: taz
file_lookup_names:
- fares.omx: zone_number
+ fares.omx: taz
missing_zone_policy: error
dimensions:
@@ -76,9 +76,11 @@ dimensions:
ignore_modes:
- ESCOOTER
+ # SKATS has WTW skims only and no prepared drive-transit parking-zone field.
- KNR_TRANSIT
- MISSING
- OTHER
+ - PNR_TRANSIT
modes:
SOV:
@@ -96,181 +98,6 @@ modes:
time: "SR3_{VOT}_TIME__{PERIOD}"
cost: "SR3_{VOT}_COST__{PERIOD}"
distance: "SR3_{VOT}_DIST__{PERIOD}"
- PNR_TRANSIT:
- output_prefix: skim_
- segment_on: outbound
- segments:
- true:
- auto_time:
- matrix: "SOV_{VOT}_TIME__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- auto_distance:
- matrix: "SOV_{VOT}_DIST__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- auto_cost:
- matrix: "SOV_{VOT}_COST__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- access_walk_time:
- output: skim_walk_time
- combine: sum
- matrix: "WTW_ACC__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_auxiliary_walk_time:
- matrix: "WTW_AUX__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_brt_ivtt:
- matrix: "WTW_BRT__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_bus_ivtt:
- matrix: "WTW_BUS__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_commuter_rail_ivtt:
- matrix: "WTW_CRT__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- egress_walk_time:
- output: skim_walk_time
- combine: sum
- matrix: "WTW_EGR__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_first_wait_time:
- matrix: "WTW_FWT__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_light_rail_ivtt:
- matrix: "WTW_LRT__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_tiv:
- matrix: "WTW_TIV__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_num_transfers:
- matrix: "WTW_XFR__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_transfer_wait_time:
- matrix: "WTW_XWT__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_fare:
- matrix: "fare__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_o_maz_stop_walk_bus:
- output: skim_transit_o_maz_stop_walk_bus
- lookup: key
- key_column: pnr_zone_id
- matrix: maz_stop_walk__walk_dist_local_bus
- transit_d_maz_stop_walk_bus:
- output: skim_transit_d_maz_stop_walk_bus
- lookup: key
- key_column: d_maz
- matrix: maz_stop_walk__walk_dist_local_bus
- transit_o_maz_stop_walk_premium:
- output: skim_transit_o_maz_stop_walk_premium
- lookup: key
- key_column: pnr_zone_id
- matrix: maz_stop_walk__walk_dist_premium_transit
- transit_d_maz_stop_walk_premium:
- output: skim_transit_d_maz_stop_walk_premium
- lookup: key
- key_column: d_maz
- matrix: maz_stop_walk__walk_dist_premium_transit
- false:
- auto_time:
- matrix: "SOV_{VOT}_TIME__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- auto_distance:
- matrix: "SOV_{VOT}_DIST__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- auto_cost:
- matrix: "SOV_{VOT}_COST__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- access_walk_time:
- output: skim_walk_time
- combine: sum
- matrix: "WTW_ACC__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_auxiliary_walk_time:
- matrix: "WTW_AUX__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_brt_ivtt:
- matrix: "WTW_BRT__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_bus_ivtt:
- matrix: "WTW_BUS__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_commuter_rail_ivtt:
- matrix: "WTW_CRT__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- egress_walk_time:
- output: skim_walk_time
- combine: sum
- matrix: "WTW_EGR__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_first_wait_time:
- matrix: "WTW_FWT__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_light_rail_ivtt:
- matrix: "WTW_LRT__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_tiv:
- matrix: "WTW_TIV__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_num_transfers:
- matrix: "WTW_XFR__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_transfer_wait_time:
- matrix: "WTW_XWT__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_fare:
- matrix: "fare__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_o_maz_stop_walk_bus:
- output: skim_transit_o_maz_stop_walk_bus
- lookup: key
- key_column: o_maz
- matrix: maz_stop_walk__walk_dist_local_bus
- transit_d_maz_stop_walk_bus:
- output: skim_transit_d_maz_stop_walk_bus
- lookup: key
- key_column: pnr_zone_id
- matrix: maz_stop_walk__walk_dist_local_bus
- transit_o_maz_stop_walk_premium:
- output: skim_transit_o_maz_stop_walk_premium
- lookup: key
- key_column: o_maz
- matrix: maz_stop_walk__walk_dist_premium_transit
- transit_d_maz_stop_walk_premium:
- output: skim_transit_d_maz_stop_walk_premium
- lookup: key
- key_column: pnr_zone_id
- matrix: maz_stop_walk__walk_dist_premium_transit
-
SCHOOLBUS:
output_prefix: skim_
access_walk_time:
diff --git a/simor_configs/skats_configs/skats_skimjoin_config_alternate_id_col.yaml b/simor_configs/skats_configs/skats_skimjoin_config_alternate_id_col.yaml
index 9aa99f0..decd99f 100644
--- a/simor_configs/skats_configs/skats_skimjoin_config_alternate_id_col.yaml
+++ b/simor_configs/skats_configs/skats_skimjoin_config_alternate_id_col.yaml
@@ -35,7 +35,7 @@ defaults:
zone_mapping:
lookup_name: taz
file_lookup_names:
- fares.omx: zone_number
+ fares.omx: taz
missing_zone_policy: error
dimensions:
@@ -76,9 +76,11 @@ dimensions:
ignore_modes:
- ESCOOTER
+ # SKATS has WTW skims only and no prepared drive-transit parking-zone field.
- KNR_TRANSIT
- MISSING
- OTHER
+ - PNR_TRANSIT
modes:
SOV:
@@ -96,181 +98,6 @@ modes:
time: "SR3_{VOT}_TIME__{PERIOD}"
cost: "SR3_{VOT}_COST__{PERIOD}"
distance: "SR3_{VOT}_DIST__{PERIOD}"
- PNR_TRANSIT:
- output_prefix: skim_
- segment_on: outbound
- segments:
- true:
- auto_time:
- matrix: "SOV_{VOT}_TIME__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- auto_distance:
- matrix: "SOV_{VOT}_DIST__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- auto_cost:
- matrix: "SOV_{VOT}_COST__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- access_walk_time:
- output: skim_walk_time
- combine: sum
- matrix: "WTW_ACC__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_auxiliary_walk_time:
- matrix: "WTW_AUX__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_brt_ivtt:
- matrix: "WTW_BRT__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_bus_ivtt:
- matrix: "WTW_BUS__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_commuter_rail_ivtt:
- matrix: "WTW_CRT__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- egress_walk_time:
- output: skim_walk_time
- combine: sum
- matrix: "WTW_EGR__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_first_wait_time:
- matrix: "WTW_FWT__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_light_rail_ivtt:
- matrix: "WTW_LRT__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_tiv:
- matrix: "WTW_TIV__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_num_transfers:
- matrix: "WTW_XFR__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_transfer_wait_time:
- matrix: "WTW_XWT__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_fare:
- matrix: "fare__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- transit_o_maz_stop_walk_bus:
- output: skim_transit_o_maz_stop_walk_bus
- lookup: key
- key_column: pnr_zone_id
- matrix: maz_stop_walk__walk_dist_local_bus
- transit_d_maz_stop_walk_bus:
- output: skim_transit_d_maz_stop_walk_bus
- lookup: key
- key_column: d_maz
- matrix: maz_stop_walk__walk_dist_local_bus
- transit_o_maz_stop_walk_premium:
- output: skim_transit_o_maz_stop_walk_premium
- lookup: key
- key_column: pnr_zone_id
- matrix: maz_stop_walk__walk_dist_premium_transit
- transit_d_maz_stop_walk_premium:
- output: skim_transit_d_maz_stop_walk_premium
- lookup: key
- key_column: d_maz
- matrix: maz_stop_walk__walk_dist_premium_transit
- false:
- auto_time:
- matrix: "SOV_{VOT}_TIME__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- auto_distance:
- matrix: "SOV_{VOT}_DIST__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- auto_cost:
- matrix: "SOV_{VOT}_COST__{PERIOD}"
- origin: pnr_taz
- destination: DTAZ
- access_walk_time:
- output: skim_walk_time
- combine: sum
- matrix: "WTW_ACC__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_auxiliary_walk_time:
- matrix: "WTW_AUX__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_brt_ivtt:
- matrix: "WTW_BRT__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_bus_ivtt:
- matrix: "WTW_BUS__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_commuter_rail_ivtt:
- matrix: "WTW_CRT__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- egress_walk_time:
- output: skim_walk_time
- combine: sum
- matrix: "WTW_EGR__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_first_wait_time:
- matrix: "WTW_FWT__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_light_rail_ivtt:
- matrix: "WTW_LRT__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_tiv:
- matrix: "WTW_TIV__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_num_transfers:
- matrix: "WTW_XFR__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_transfer_wait_time:
- matrix: "WTW_XWT__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_fare:
- matrix: "fare__{PERIOD}"
- origin: OTAZ
- destination: pnr_taz
- transit_o_maz_stop_walk_bus:
- output: skim_transit_o_maz_stop_walk_bus
- lookup: key
- key_column: o_maz
- matrix: maz_stop_walk__walk_dist_local_bus
- transit_d_maz_stop_walk_bus:
- output: skim_transit_d_maz_stop_walk_bus
- lookup: key
- key_column: pnr_zone_id
- matrix: maz_stop_walk__walk_dist_local_bus
- transit_o_maz_stop_walk_premium:
- output: skim_transit_o_maz_stop_walk_premium
- lookup: key
- key_column: o_maz
- matrix: maz_stop_walk__walk_dist_premium_transit
- transit_d_maz_stop_walk_premium:
- output: skim_transit_d_maz_stop_walk_premium
- lookup: key
- key_column: pnr_zone_id
- matrix: maz_stop_walk__walk_dist_premium_transit
-
SCHOOLBUS:
output_prefix: skim_
access_walk_time:
From 5be64d55e43efc0e106a32bea3dcfcc06ed412fb Mon Sep 17 00:00:00 2001
From: Wesley Darling <98353044+wesley-darling@users.noreply.github.com>
Date: Mon, 10 Aug 2026 12:29:30 -0400
Subject: [PATCH 24/27] Added optional logo for dashboards
---
dashboard/app.py | 1 +
dashboard/export/assets/export.css | 26 ++++++++++++++
dashboard/export/assets/export_runtime.js | 16 ++++++++-
dashboard/export/js_runtime/renderers/app.js | 16 ++++++++-
dashboard/export/payload.py | 15 ++++++++
dashboard/export/types.py | 1 +
runtime/config/loader.py | 28 +++++++++++++++
runtime/config/models.py | 1 +
runtime/config/schema.py | 1 +
runtime/config/signatures.py | 1 +
simor_configs/assets/lcog_logo.jpg | Bin 0 -> 58504 bytes
simor_configs/assets/metro_logo.png | Bin 0 -> 10544 bytes
simor_configs/assets/skats_logo_white_bg.png | Bin 0 -> 127824 bytes
simor_configs/lcog_configs/lcog_config.yaml | 1 +
simor_configs/metro_configs/metro_config.yaml | 1 +
simor_configs/skats_configs/skats_config.yaml | 15 +++++---
tests/test_config_refactor_phase1.py | 29 +++++++++++++++
tests/test_dashboard_live.py | 5 ++-
tests/test_export_html.py | 34 ++++++++++++++++++
tests/test_export_html_smoke.py | 2 ++
tests/test_export_payload.py | 5 +++
wiki/13-configuration-reference.md | 1 +
22 files changed, 191 insertions(+), 8 deletions(-)
create mode 100644 simor_configs/assets/lcog_logo.jpg
create mode 100644 simor_configs/assets/metro_logo.png
create mode 100644 simor_configs/assets/skats_logo_white_bg.png
diff --git a/dashboard/app.py b/dashboard/app.py
index 126ae7a..9ad47a8 100644
--- a/dashboard/app.py
+++ b/dashboard/app.py
@@ -148,6 +148,7 @@ def _on_value_change(event) -> None:
template = pn.template.FastListTemplate(
title=config.dashboard_title,
+ logo=config.dashboard_logo or "",
sidebar=sidebar_items,
main=[main_content],
theme="default",
diff --git a/dashboard/export/assets/export.css b/dashboard/export/assets/export.css
index c34d8ad..81b4b9d 100644
--- a/dashboard/export/assets/export.css
+++ b/dashboard/export/assets/export.css
@@ -38,6 +38,22 @@ body {
gap: 16px;
}
+.export-brand {
+ display: flex;
+ align-items: center;
+ gap: 18px;
+ min-width: 0;
+}
+
+.export-logo {
+ display: block;
+ flex: 0 0 auto;
+ width: auto;
+ max-width: min(280px, 35vw);
+ max-height: 72px;
+ object-fit: contain;
+}
+
.export-header h1 {
margin: 0;
font-size: 30px;
@@ -577,6 +593,16 @@ table.export-table thead th {
grid-template-columns: 1fr;
}
+ .export-header-top {
+ align-items: flex-start;
+ flex-wrap: wrap;
+ }
+
+ .export-logo {
+ max-width: min(220px, 45vw);
+ max-height: 56px;
+ }
+
.page-panel {
padding: 16px;
}
diff --git a/dashboard/export/assets/export_runtime.js b/dashboard/export/assets/export_runtime.js
index 29b8322..d8441e3 100644
--- a/dashboard/export/assets/export_runtime.js
+++ b/dashboard/export/assets/export_runtime.js
@@ -2025,9 +2025,23 @@
context.plotManager.scheduleResize();
});
+ const brandChildren = [];
+ if (context.payload.logo) {
+ brandChildren.push(
+ el("img", {
+ className: "export-logo",
+ attrs: {
+ src: context.payload.logo,
+ alt: context.payload.title + " logo",
+ },
+ })
+ );
+ }
+ brandChildren.push(el("h1", { text: context.payload.title }));
+
const headerChildren = [
el("div", { className: "export-header-top" }, [
- el("h1", { text: context.payload.title }),
+ el("div", { className: "export-brand" }, brandChildren),
railToggle,
]),
];
diff --git a/dashboard/export/js_runtime/renderers/app.js b/dashboard/export/js_runtime/renderers/app.js
index 2297aec..18b7c52 100644
--- a/dashboard/export/js_runtime/renderers/app.js
+++ b/dashboard/export/js_runtime/renderers/app.js
@@ -281,9 +281,23 @@
context.plotManager.scheduleResize();
});
+ const brandChildren = [];
+ if (context.payload.logo) {
+ brandChildren.push(
+ el("img", {
+ className: "export-logo",
+ attrs: {
+ src: context.payload.logo,
+ alt: context.payload.title + " logo",
+ },
+ })
+ );
+ }
+ brandChildren.push(el("h1", { text: context.payload.title }));
+
const headerChildren = [
el("div", { className: "export-header-top" }, [
- el("h1", { text: context.payload.title }),
+ el("div", { className: "export-brand" }, brandChildren),
railToggle,
]),
];
diff --git a/dashboard/export/payload.py b/dashboard/export/payload.py
index 3673810..f6449e9 100644
--- a/dashboard/export/payload.py
+++ b/dashboard/export/payload.py
@@ -2,7 +2,10 @@
from __future__ import annotations
+import base64
import json
+import mimetypes
+from pathlib import Path
from typing import Any
from runtime.logging import get_logger
@@ -49,6 +52,17 @@
PAGE_WARNING_BYTES = 10 * 1024 * 1024
STATIC_REGION_WARNING_BYTES = 5 * 1024 * 1024
SELECTOR_REGION_WARNING_BYTES = 1 * 1024 * 1024
+
+
+def _dashboard_logo_data_uri(path: str | None) -> str | None:
+ if path is None:
+ return None
+ logo_path = Path(path)
+ media_type, _ = mimetypes.guess_type(logo_path.name)
+ encoded = base64.b64encode(logo_path.read_bytes()).decode("ascii")
+ return f"data:{media_type};base64,{encoded}"
+
+
def _build_validation_page(
page_def: DashboardPageDefinition,
config: Config,
@@ -103,6 +117,7 @@ def build_export_artifacts(
payload: ExportPayload = {
"schema_version": EXPORT_SCHEMA_VERSION,
"title": config.dashboard_title,
+ "logo": _dashboard_logo_data_uri(config.dashboard_logo),
"runs_loaded": run_legend_entries(
RenderContext.from_dashboard(config, chrome_state)
),
diff --git a/dashboard/export/types.py b/dashboard/export/types.py
index 950abaa..86cf02a 100644
--- a/dashboard/export/types.py
+++ b/dashboard/export/types.py
@@ -185,6 +185,7 @@ class PageExportSupportPayload(TypedDict):
class ExportPayload(TypedDict):
title: str
+ logo: str | None
runs_loaded: list[dict[str, str]]
chrome: ExportChrome
dashboard_controls: DashboardControlsPayload
diff --git a/runtime/config/loader.py b/runtime/config/loader.py
index bce8873..44b2f7b 100644
--- a/runtime/config/loader.py
+++ b/runtime/config/loader.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import hashlib
+import mimetypes
from pathlib import Path
from typing import TypeVar
@@ -48,6 +49,28 @@
ConfigT = TypeVar("ConfigT", bound=Config)
+
+def _normalize_dashboard_logo(raw_value, *, config_dir: Path) -> str | None:
+ if raw_value is None:
+ return None
+ if not isinstance(raw_value, str) or not raw_value.strip():
+ raise ValueError("dashboard.logo must be a non-empty image path when provided.")
+
+ logo_path = Path(raw_value.strip())
+ if not logo_path.is_absolute():
+ logo_path = config_dir / logo_path
+ logo_path = logo_path.resolve()
+ if not logo_path.is_file():
+ raise ValueError(f"dashboard.logo file does not exist: {logo_path}")
+
+ media_type, _ = mimetypes.guess_type(logo_path.name)
+ if media_type is None or not media_type.startswith("image/"):
+ raise ValueError(
+ f"dashboard.logo must reference a recognized image file: {logo_path}"
+ )
+ return str(logo_path)
+
+
def load_config_from_yaml(path: str | Path, *, cls: type[ConfigT] = Config) -> ConfigT:
config_path = Path(path).resolve()
config_bytes = config_path.read_bytes()
@@ -182,6 +205,10 @@ def load_config_from_yaml(path: str | Path, *, cls: type[ConfigT] = Config) -> C
)
dashboard_title = dashboard_cfg.get("title", "ActivitySim Visualizer")
+ dashboard_logo = _normalize_dashboard_logo(
+ dashboard_cfg.get("logo"),
+ config_dir=config_path.parent,
+ )
log_level = str(raw.get("log_level", "INFO")).strip().upper()
if log_level not in {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"}:
raise ValueError(
@@ -271,6 +298,7 @@ def load_config_from_yaml(path: str | Path, *, cls: type[ConfigT] = Config) -> C
presentation_config_digest="",
name=raw.get("name", ""),
dashboard_title=str(dashboard_title),
+ dashboard_logo=dashboard_logo,
log_level=log_level,
pipeline=pipeline,
dashboard_pages=dashboard_pages,
diff --git a/runtime/config/models.py b/runtime/config/models.py
index 60d51a6..8f7f345 100644
--- a/runtime/config/models.py
+++ b/runtime/config/models.py
@@ -350,6 +350,7 @@ class Config:
presentation_config_digest: str
name: str
dashboard_title: str
+ dashboard_logo: str | None
log_level: str
pipeline: PipelineSettings
dashboard_pages: list[DashboardPageConfigEntry] | None
diff --git a/runtime/config/schema.py b/runtime/config/schema.py
index 2b82fe4..209eab0 100644
--- a/runtime/config/schema.py
+++ b/runtime/config/schema.py
@@ -125,6 +125,7 @@ def validate_canonical_config(raw: Mapping[str, object]) -> None:
field_name="dashboard",
allowed={
"title",
+ "logo",
"live",
"export",
"host",
diff --git a/runtime/config/signatures.py b/runtime/config/signatures.py
index 5919a31..4b5c5b7 100644
--- a/runtime/config/signatures.py
+++ b/runtime/config/signatures.py
@@ -300,6 +300,7 @@ def segmentation_unit_signature_payload(
def presentation_signature_payload(config: Config) -> dict[str, Any]:
return {
"dashboard_title": config.dashboard_title,
+ "dashboard_logo": config.dashboard_logo,
"log_level": config.log_level,
"dashboard_pages": (
[
diff --git a/simor_configs/assets/lcog_logo.jpg b/simor_configs/assets/lcog_logo.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..7656ee2f76ac10dbb590087b44076adce518f04e
GIT binary patch
literal 58504
zcmeFYbzED~w=Woq6)#1KhXO5DptzGl3lu1>#ZuheH8{l!6e(J)6iq4aPN2AyP>Kha
zlAysS{oUW(d*93-bMJd^?!5VY-r=mn$YppG7?eALqZtiXc@IYBXNdbU`g#~zr
z`2*ao-uo)=YiA7rsHy_E006*!01g%@;2!3V6ac_djjU7gt^hdC1^A!;o8GXz
z|ElVL>+Ronm^Xlz0Cj*OfEnNjZ~@q0h#A0xA&eF=Jr(Dft*57}1RtNX2d|l>i@6oA
zg^LrPubC?!KksusfE38r)y%@d%9GjL%Er!Fn*Fe~lbzYlQkq>~SoOK8>q{$JyVw5i
zR@(k*Iu`y87UGudAQ=)uDPIX+Cs!vcPcvp;Cr4)w314ZJe>`0RbN{!Pk3~kx-O^e@
z>y^SkYhXS}v;316A0HoHA30B1vlQg#u@n@v
z6cy$d6R_qtXZh#rEiL}_?XF(#j{kVKr3Igrqm`4Dv!@3}8~lHHwzQD&wDWYd`WNnH
z9X^`?mn1tefJwPm0;hy_xHpcIG7XnJ}xc}4lW@+KHhy|LSkYfLLwp((uWU7NXbcw
zh#pWsAg7?DqM{-udqhJ`N%N4Bit_J6u&^<|!NDcK#U-F5AtIstPak((0P_3yM6vm>
zu~-22$g!}=vF`c+K#aA-#aQ9LhWMWc);$c5c=-1T2#GKi>K_2^VPRw6!@>T`HKukD
z=6e7RIqt*9{IYlynr8ScZj=I{$p!aWKd)1R`*mNmW_`*
z=k9H@xv~m;j`SEN+)Jo3QE;8T)KSm^tr)Z4N}IB;LD3u|2`wN}U+EY48~Hk|Y&YJ_
zcYt3NQ$^|KD8iGKeZRD0DIOAbVUL3YM56ym(TIj%9``X(-*Q8KzdHFRY94z=Cj7yp
zx+@)%PNc|iutM837+tb<2cY~hUg4Xq+c`A&=#g5t*Z_bhG~fQr&UN#t|Ckb!uqkW7
zBU|~}(^_5BR@BLJm?r#Wn4~k{K-3MEG!lzA1#;f_
zIj;&ajrz96Ah*PM2QV$&x6VL;x-Wm1+ud?Z9$Grb;9jf2kR(|<)V2%vs*9Oq@vDM4
z*{XnC`^~I>QaV423D*?*ZuJE-cm@d?&(xjKcGtK0){Hkma^2LU?f_eh;I4yc@;iXE
z;F*!@fJaglZrAP$=nss@x3hNusy3-E#kJM$htjRqr;OaTGmf{C;qvIRs4&etHM!mhG@}
zyna;t$29IM_{Kj1INQ!3f=v+zX-&)>BUS*BMd|0%%Z$yOW;3!3{ef1m)HiCTJ{zM_
zTyyBUWx4$wd=F`);#8`{_x?~VkgJ#l8~dY)p?BPUMxpRspj4UrrmtZF=ZcmVmUrz3O}oB6vlE!reKQF
z+X*{80Uz8+@0aoXc~YBZ6{ZlTyJ{*T&Ul>1KdLCtB1
z11QdfG`%71bsN0C4=t3!zXRZsd{5Kjdt^&y+htXKa(?`1Ax#XF?yx#9f!c!^>!*Ub
zd(6FZg#Xk@OVH+V6IT(~&E0q+`Cvet>zDB&F>6Fh8{;BJq8VH#G+cAiud9zonA-H$
z>g(6feC|2;6^CiKv&H;)E{-qGkQdetU2QxFN8ACTH))=9(#__D+ck7tyg%391M|hM
z&c_)`wj9f36ZJi}pNq6OxfYYY0}z_@bhx6ySI%heffAn63>T7RhI@@ted@JjdtlkQ
zUA>m7HXjimaucK1qHGgQUS|99fmM#ujzZ1$1F)HizNtP~*2ik&!6Kob?*g`Vl=-tZ
zTLKtOi@yGUJ|I-tf@}f2xOnokTcuCScm30}OwH~PHOCwaHh^$5#BbQi1$pMzzq
zLg9hb6$1*bHbR}<4D1J-C$c}2W-RthGL0DnEt_pCZlkH$X%z{W?5mf{PRdXb<}#lIB$x)1-S#tPpFPM
zX&Iqo+#0z42*u8Sf=_S(GaG?G5c+=r6u=r_+(Xygj}qiyGKU
zbGWLL{)x`&U_q%t*M9Htc_qjpG@FB~Tz_}o=%yv+tcS**6vke@@7LCgiW|Hr8=CRS
ztHku`!oQu_OrBB$>mS4z5RRVQ8Qe-9CA^9BTHq)lR1;f2VXChw_T@80;Z#uP7J^%UIeUPVDMIn=peUZq<}J$4)Y}9IDgA0+$e9MZ6uQ#Cs*){#+F{jr2bdVV
z)m_@Y1AN2pQ&lu;Z=F1uc)d{j#*!J*r4hT5XBAP3(ou92xYkvV%9gQV4v&9#spTbh
zxC6GbTSlt!D8IUQ2N=+6SPxHjF}I{0XM7tFps#3g2f*2wwBX{CvI>&813c`vO%A^U
zB+;#<`-LX`zNY=fkU`H^gU%-)Rg=7=1F+XzSDet1q1lr{Cq;&Nco&y+zdQKeus
zdw%bPOps9@-$_#l6pQpI72b=wPZxKsoMSt4}(k*ohd`RfP
z#H@bh8+`|e|7iq8v%7?`y)%cOW_BGsj=2Mz$;zz}Q(US4d2~iCcmbg}K0n5vn0kn>
zom3Sx9)MwcZTboReur>NMCu)Y$B`V<+8zBKmr2(Y*#>d=>ZqWPn3B9RC^_hS$bLzD
z2@mXov=$husks8ipoUfutRO?xUa>*w7vcOliZzk!rg`HX!w-v7L{V~z*TjUY!kxbq
z*Yd)QRaPZiBe0)TF52Lbohw!U@JoDH+N;Un(PdtCtznaVHi`{(8E6zv!uE~pJrb}t
zd~y?`9Yyi``#Xemzd03`s{&bx|AUmgZw`mz48}6TE+6t1K9JGE)T?|Tq%vZ@!>c9}
z8M3oaqX6zsPcBxS`NJaa09>ojUys|Xc0CjzV64^Fi+A9C+p|hIRl;lgW8?-0t#svU
z%fx2u*k45zkifV2-GIDa>4gEe*}ekL|sF>dDiken8Wa|*XcYsnOLQqLfu
zFXzsyk5{2EGMYg4wzqX8hPGAbR+ZKkY-Ia7C)eETH969R8R@h~9gAe;-CM_BKDrU^
zHDB)1P=t7>fCaCqZtB|vQ8@jjz6sjB{xEFMq?RQ;7_t$0D}0jH-lxycmaP1Jpx3>H
zA&TkqIWz_uRS}iJV=#ARNRwrdd93ZjUYJGqkpa&0$BRgwQv1Jj0eb+?t9g}A+iC{-sn<3)Mc3R&`=Qyhb<^Fx
z;6t^>cVOv8h^NnH8_`+hO=Dzp^F-jiHN}gfeBBRBWN#Ee&Y)b-ii`uW$&k?lRl1k3~Jsi0reM7lHx&i`f(b*{u_j;0Uka!ATJYRU8+>%gATZaw!5{zR3=4M7x!%fmPAGm-4ulrTirexL(OJH6FUQwzuRn`Cx~RX<7tbL6T5oL47){0dQENqg#L0T
zY3DmAKHB!#En8DYpAUrd=>5@WE%8-Fc2s{5#p)rxZ3kt8krS&*{q#FV>Lun(FlBkRM5=nft(3_P{?qhfQRUtnn0(BgTIUd5oQi18ui{UYSOs!7vVyE9)8
z#YOxl3Y0UtDc|?JiwDSrS4p}E?aH}Ib5px)Gc#?*(m%D-3kY)D1*BthGuA
zDpa2a_OC(b9(5TGl9Z?h6eZ(S(MJ^RM(Ir~nw@iX+D9F3vsx!x1
zgtx{A1#zF12N+aq6eq_}p6b(5Z7zx3WXk}3FKBjU7#m_&Qi7-LJ9I+w-AqqOSrdP*
zo-^SFGiA+2+#Vl!YykMT$>>8?ZXD)ih;M(&@Szn@^E^Ydo&tUKk%)Bi_(aW?@We&#
z`QXPGw^!Pk#ij`v+*bYhA&>Cm@RPUr$`{$NWn3vItPc9=u43J!-(Jc2nuHDrssw0$4{DDZ{t3eQCAQ*Z~_ibNBwy`NIE}YNtDd
z3SC>gd!F^=MD#Lf;Vi@Ln`PGGw_gmWhhVEfyqo49LHRGyApy8a1~F>x>hv=ZelSnc
z;Fj{~^QEKxAO&p6NEPkA0BG~s_PcV+>;7G;me`j3RKJY8i#CLXe4Df
z)}32*e1tDXQHN*c@Z0B$N>-Lfwo433MDlh-YfA80?Y(`mYO*!h0NI0Ua}>eM>2S(m
z{w~jhWJVAV$&*noPu2~yG|Z|gD3>B{?6^>
za#Eh^WLCz>?kh%JT^Qd9z)_f32hbcZ*h`E`j9y-@_xI_>zz&D~5SqgYJ8vhvoNAnw
z_?7qY1smvU)pga>JKkpkBT5)MgoX?Vg=Rshm_D!b}{d{Kh;Cz5C)b!>mYi
zi`>>!0z$0(+~@3!Kcnf>W7(g%A;2m(y7Pff;_ZTuru5AjXdbxF-c(Zs!gbE;>Q}M#
z7LD`xi^1Zhj>3)r%zE99dnk_g(-$_Co2#gy%N2d1%0NAoT75Eal@JFcY~kYN3DvP!
z9#^-<6w|S|O?)Y1QsrTN1&OeSmmhA^Y5I%K5janIpanAVQ@iEQwPYB*ncj!ggAn<^
zPaa;%!j>dN2x-EiMKKl{|=7^h0|0OIcu26@pr+a?!_vPX$nY1{ptq)~aKXQHl~f0lrH
zR(#>z63E&(xUaZobg35vgLOU{seE{JpJ03PcNy#L<@ga)qOk44#xMT7a^$u~{~~ew
z+8I>_I-&!q*S@?1^l1STI)p&Sw4j7G(6zklj2>`4^YG$^@0uX>pd2D7nT%>22zj@=
z8ePzUaWUUl5kcm$`#;=V@K^&6y6|tqc*D06l+DeoV?i~jE`RiRF=bkqN|hF1iURj
zwRH>TqSFoY7D*IwSW{fI6A=5+#C%C-@&&FD|AHg)1_8n(DX{H7TKnYJ$-J&S2Z{S-p4kSC4y{k~vm#Qy%|>ydna$kP}2?4CisRG01w@`^0!FJW7zOOHhR
z%^aJXGpv0xEAxqBS{u3lB>gxH2iJk)g7NKp?&lr~A1`ObNEQV%r
zTCuCe0Qg`B|4l|P>opEMwJ(^sHAA?xGf6(_bi#R3rP59wSNhRrre{ShDw&}H8Oc&wR?rO$5OrgW%2p8Cm6`5
zy=V|~4fz-PjEF~a0Q%of*R_tCpK(jlzpegL@~(`S<83y3BEF0N?(L&%7SyBugMceO
z*;+f1j)zOQOKDUWKwpO#+Y3}H$;5&p^rJJaKYmjl
ze(NorgZI{m?$6Jn1tc^C{B5m|=d_cF@z3iN>8Gq+h@|J|r(MwRLRC!Ta^Itd)$3x%
zrXEK`QQa?>4yg!vXP)!(Vt|`opH`GBaj)^^i6bv*Iw_&hZ2LeO$tWN8Z%QV4SuY=w
z9Mxw>#rzz@lHUOOxtwg(i&YB;N<1n!p{jnm4{F{4{`l*s`8FcX5{eF^-{5OEGgw+_
zsyD+xk-o5)4lduge$qWR--#EHjd&ne608iG26N*jyT8DEv9VDzq)nxkmmw340`)JG
zqU~r+(I#r3jkE@at%XEfKhYSFZIITLr@!uZg8|4!alRmC*7gG$H(HFjU5`FhbjP28
zcUg#K)5barEk6rP;{C$w;V-DJj}obXDqD7THZ`%MRZtgtE?9~$f})R$53Q*qE%wSusHWA6Yr3emxcYSmVb>%l66U^3r<4MCts@2fG@s(1(YupSvFQ!ScYv47*7!
zojqfU{2~<`)sn86lz-g}@!+2Oxj!z{PXpDw14Ox_6oKv2&3Aw=qnL?bNBv65w}uR8
zX12=?x_(^+s__6`9P>2=_dt*Gtde&@65JD;XcENz+PK%2Fwa~_8C9+uGGIWjqWMHz
z`nuV|WrDxbD$SyxvP|WV~#sh0Dy?Y&W^@ue&P!3R31(Qx%Y%
z=ayQfN5cJFRA#7tXmYleQNc%z`{LdBOi{7jr8l-}zauG0)$rM;+c{|lZ1n>kUlPbu
zYRxbeU>3PecvveZm)4bU?H?IB7OJSk@Z|VJ#%4j~b=Wy1`{(TH8=}9U9S}?Hg*dV3
z>Qd+XI73HGKuy#3RrLf@CKT&47x9{=70R4v+Jp@D=s62bodI#!P}kJBMH*C9fN2`Le-
zOQnns*+}&SDGQ|DsweRMGKzmfd!r5|#QY&@{~I1
z|L=I}uhnn?IH9vyV}QMvd5aYoPuA{mUSW7;Gh#kCpJl-}{lJZw%iT;H16a{d|CUA>
zX%tDl`6=O`=K+Tj3o$Auy#$^W-;`C!P#O$>-qz;XigkX3?=tf2;~jwBnmmk(Oa*4makJnHs%_;ueF+Z`bPN6GAI_Bgpc+&`~?zp1k&bu`Q3=XhDU
z@Z}e9MlUX@PWx8#g*ON&Wx0@j?vG2UXTS2?<{?oC$MLVPI|CAR1!v9>Up$ODYsvN-15jbN(hhHx9IrXC
z$(G2YT3O3W8{AT-Y5Ri{w!w+ewpZxu#qKQMVx_JCqy5=?!DQV<+j>ei4i5m|fKsJbWB#coqS*P$T614sBkV33
zuHaj#AHOV=G@qlwB6KVYU!KP!+7+coM38kUgEa#ZwG3EjI2j@-E=x4r!Zv={sU7w}
zPIewLFWmwU-&+zv>Gx$fB=WnRd5fty$V)21XFH6ZkAFie?%P%S=Vn~wobvOR7JJ!L
zp_dU$p9-WJS_~m}CQ$l;SIR--ke)WazFW!JHZW(994e35o%#Ji3}gj#<6#mV4YC=9
z=;#E@#;LwPt~*-(aCiz;N1k^9$?R}V_m#@tNd9o``~Xd${ozkaMd$XWNMvs9(AACCd@vvexXBe*>B7D4uSUF15S=@W%kX
z6>#ANx4br9JF)0}O_iI%k{gnuki=8Q@q1I9)&XTSB1M&KDvLa83lKq*;7?c5Xls~R
zjtThN-je2CWy^1MOY#H6mjE_EXZGLPqi
zg$?)KG_=JRpP|%Bzk1>TyMA^^+mfVq51v&nJm!me$_@5(H_Y#W67BvH+@4(>N68T@
z9&3@f8$$OHpBJMXBr95{&j#|BqrF|Y
zHEPm-hVE*t+zNlnzin&}%-I>sY3AaV8CkQd!uU?xn7Ro}Js8*N1J)KCS!_j;+sk%d
zQkdz_s2(Bn#G!*EO$O#^)j)$c>ux9@-5Ps|-E{|`Jpohihw0yrce3NrCSo&qOLK2n
zOgAEw6DOZ@^uJzb5+IxHNa7uL^9#__+Kk3n6`dNvlI@_G?|3_-yesm7-7COEy8NiO
zJi;uOQF&s-TLSEerH&}%a_q@6<*z5Bkv%_(-P_^2+^2z!;=R_Ib-?zyvmO0NT~8+_
zH${J2NZ?eWn!~ba{a>hL;D7oA{5KMN`~q(^OF>M_){##b7O0KREe_;B25mN17(a8M
z`ArG#e-<6`Iq3EH=kkarTFN=1)*fYa8eZ(*#a-qp5Y_oeqaGJGwwR|i?q4J&-vyg5
zidsx~U2`9u_cL+KwAoZI?$NbRFYZ2iWBS(kNX?&2_F$?(hT~f4XE3+#zZuAa@s?VG_?pR6Aa=(@$7YiME+5Rzr>9_4O~lR$$}1wd1t
zQ5=SyClA1vjv^IIod|$W-2uM)>~}3w@*+@ATj~2(fBRu2ifEAxCqZ>@xarSKZa)3#
zc)k}?zB?)17+JYMOWSce>~#eH{UyLc_)@)ihuWVivOOWP&AY*S`ymIbF&76T0Gc%B
z0k*mwm*4w&h6(EjIjg6&K!48~dYWL;8;HKxrOyk;i|FZ1&d}msyH{+UT4D);~OR$ogqAbT;$K{`DgjCeq8jDE8gyU
zQqA-~?5)?l_n8LZir>HutB8gs1lDG-1@Qd(SiqGfo%93=+eekY6&SBFmMN9NvyfsI
zW4z4E-J3pXCxppInt!j0T?DKsG~GBT#P0NL%o)?&j64jwOe55Y?ul{|KU7u_FV$8d
z_JYcn%*;pzXy#}=(jb%OPC!BWJBfUsJi1cXNg4%t(|E9^o=~}#N+eF`vuMS8rPln4
z*4uWMmgoP&nm|;Gt2t!ud%d@@OdUK6KY!TfkC(w%X%@xHJD#~OK19?-h3H^G$E<;=
z$#1GEjTgTf`vv^?EIuxqn4@2#$OYAYYwM~5(~gSMJ^($$;4jgU3oyURE-Oa!kc));
zH@ps`_4)J5FB|o5N2mTAvgvgRIwib5UCFx8Na@>Xq!_0DV<(dnKzO>dL22H;7)t-g9Ln&G_A+npLSjIztuN=HVd^DRIe}{;`XwTgd|vLnh++49Z=dF
zurGAp0SHJd?f|mfy>|eo-`5U$WHBF2sgLO&s+mOpQZlsF?4~E
z(c86#Ih+erK*r7Kfy&z_0CX2PA=U$TXI<7;VM5Ws)%oOPuj@cpn4$NOK!y*
zsPhchu0N%F&e7R4C1P8)`s(G)*Fa!7gK>717*bG3K6n9
zU^`KyGH`t2fNbCrKjyj6sOi*2(*?^%YTC0sBeQTOT_|S52*q^8%)PB<(iIA#^q8&K^^V9$*Qy`dMRL
zv+XMUmDb$CQb@XPy%44nH+gH#WL~SAuYuwsDLDmq0MD~)hhxX#
z$npv2YN})2MfIl*gqpLTR^DFA)aj0DE45F$Sk`wmq%5tf5w_%y=*kpHCj>IA>@Gq67~4tiQ2!@C3DXj!=xZ;s>Rz{ubf1auf`cYz@e(-(7C#JRl8J`8vAdgtx-T
z`>S($E1?tqMx5Y;2AZPyvdc7wG?)8?A@$jr>*c6!#oYBBKw_l5@)oMJ-ALNy@7YhO
z34Rxthp`?1%r>CZxJOZElQht7bLNL8D+_c+x&MCZm!Ig6jJ87V
z>7YvxM2y(V@9W?s$3GOb*w!;#oI$0AH`-`bWUC@tag|;bUonEsEq;{$>UcQm)7%|^
zPZlhmW37TuW<{28i$AeJ>jQssVBOy5jJD|2lX);vC6vqi%c=4}Ly^HD@dbi?CTo`V
z!uJlar@ajwSzjn#zsxUD;NsUb!E5@F9KV?rv+-4<;KhC$VM=T^rBM6?)?ld?(?`P<
zoKV?!oCVP}=hf=4dv1Bku^CrVI+2#u6Dn`7#NAjWG3=bI
zW6=9fpp@ha^Mqft_6*^M)4_XYp!YUN;^J@(bU+669k6NqmVA;c_&JvK3I;q7MeBcT
zt&2B_dA6t|&c5rg{q2{EcUn*qOM}Gxi<3i!;E4_nls^q~rX96SrAVc$*`FJS5!$#~
ze)BQdv(cG+sL)6K$*Cejt_eMPylI3>8H^Vq81J)#!T3rJ9SHY6
z2X`ui)T^9)1G@c}V~cgCPFl0O6xIeKIA~$)rG}>v(%>JaY%_t)!ra3_!YY=931g2g
z^m*Qx5P~Ta-)oj>uPR0d=WUe$!m!4vPRC-dIDGBEky)V{(>`*C^jQ=sP!MIw9xxm86!9%;%}*nELLtjw5*Lx
zSkBDby}OoI!JqZA@N(aU@>KGy8gG&JeTHx~)s!P2a9!y$)Rvx^p>xTHuWDBeIw;XS
zfsV+{&%aJ0W;&6j*A908xq%W4%tI1{)B!51hEM2$ehWj=9-ZFzk->L~Fcp%2wnR7c
zf^p<&g$i=KUt=pcH2AZ-QUrXPwEIBTklI7LQUhWGYrYuejVzPvQz~72jE^Hk3@Xu1
z5si{dVP#EPwPJ?v;55{{;b&960~~S)e+pP>C)0WcH&o3T=_MY&w?2y+b86!`7DzX}
ze0_}^P;mPoEw5_lBb?jk@mrOry(gzKxkgWWmmloB+gW=UTCbbxoxs?Zgjsqsn@0q3
z$IfyU6tO#Gz945cj>3xUtpZ2zrtM{iy`H`m
z2eVWAy%uQ+gK{s=_D{dE#9)-!-P?9u@Eot4tvD%?65p}$!5Jsrxu7BasF%AWfg70Y
zf(Qv{_*1(gy^iV3+*Fp0j9)k0d-p(tU!8fT*4SfT=_d@(8hwBeUMx(r^=>MkZRC4h
z=G1>fmbVJ7CU%`kZ$rJ%*+|iQs*1halqrTPrpvle;w<)tCt5(mBr4cuNS9sYF*T%BD9`r5>b;hLGgt=>99wVV(%c}a;yCTqU=WE**=TUP
zLLGOv&X+%*{`4R7T7BE)C~V~Tm$>BL*yR5RB{$)P(2YFzm_UK0g5jOON2}n7#H253
zpw-7SGlfoj#6im_TI@a3UKBkn3(}i(-(KRZl%<%uEKR}bi{dR6P&*Wy3L(FSlm)7x
zf(zD2)bvvvN_oG+ScJ!@^yf&qx4%2Kuk8lBg}MdXjM>nfMwRY=zS;CheRKYB=oxfV
zdA+uIcSN1-YMP_4Q}w&jlkXtmKg)k}E<^rl4W~N72=jgnAhcUzXJ=`7**}G`xe?bs
zbOoG$6>P*W+XdWk?;p3>`kjWL1t2tQCcS-W}f9^;toF
z@1NBvIzEx^Ju^V#yC8jfh&=q50u{?RQVxF8Mp(Shm>?}(Y9GQA`%-?eqvb<&@x*?r
zPsi)&sm<;Q2aHTrJ8{sEqp(KeUMwe(dC0vk;?z)0W8;1oq9N{x1iVdeF@V>QOj^cT
zKDAxP!)dnmnQFcJA8eo_%d$%y(r75^w;np~aYMb$bX~+VYD)7eKF#LAQ^pp3???Fh
z4^QUStKKK@NaPivT+TA%)a^Vw2|qSx(*!A&b0Q)b$H>4)Xtr>W^OZ@fL+jNi-fH@=
zqXx|Am@PNZc-{sGHU%K^p34*+6?P1hMU+OxD%tx|?WtA#jCS$IWVm`SY{|+~et!4@
zTU>0#IzuHj$T4D^c3WDV6796noW>fpXcc0c1Z)Su#rKoA*pToOZ{Cjo$S?#T8MS
zVpeI=kW7soYC}aOn7fG-+;_CG4a}cv-ZIPu#VQef;`vpRWbpzq+tl2_>+45AAoM+S
zk;NIp|28GiVfuySGv1i=!vFH`j&mM*d*L1wESyROOmL5;BCK=s^VM3ge_^0d!^xzz
z!{`N0pDd}9W2leLP$+sRa1|0HhQD>onI`U;bv@P6>rKeTz`93$(
z#;%F$m;UI2TXJibA(aHC<9?+>2tkA|W^A&SZeV8a{yL9DG!_RY#|f1qg5dO_*SlRM
zU#fE_-b#-Pp}_k$E0`>K7oTbQwC0j=PqFY%|WsW0Y+MMwP!I*~>*=$v9esY!o;Jz^t^nz$6jxXb9%YiJY06FgdOAPXTF(DNio^jK;;
zl_57pvua8tg`MIzzD#|Z?7dM{yoGid)gROd>ShUnQ+Os?Zt8
zFT*gsW!o1#r>|v?Ho*wsqKn}L>Z{_3eEfdpUaUyovz%(KiXV(O9~H>$EBek-D6%UxMzXID8^2
zjO8}_hr#-5wEnwDi^&!+E(CXdkaPPqtqCVz83~%-LpVYRJAlI$7GiW~)m0?l%Pxx4
zusioc(TIYouakK!#da^fJ!YW{(xO^HWSN~#>WLPxB_0`?>8Hy@c3Kbiqgm{twNLP4
zFS7Ajb8a9XO5312HZ*@QQRXK#;A2O5;0qBQ}5i{Zpj#9)d6Fpl+hF
z#S+)1Jy(%1*ZWqQ{}5d6&ztPJq{PL4~#g6
z*k`Wv`-8br&Ls#{YZDcn%u&e`U@zU7E>qgximO21a2A+g%iN7CWg~GjsSo&GBEQeX
zUS&~E??{$~J_u&%uoJq}?9%%g2AYcH-rwmFDjLQuI6g|vZp8qul^=>&n_;wDv4Tl&
zt}fwb$mP|<3KXA?$Sk3GqndS_*N@U~GW8TWcJe=5}E$j
zJlN$K@n_DUWx>?JqgjyXR}#_hsk|6p0fw<56_M@j5V{lf=zL<&{OBwl?^%t?44RvR
z;jc>vdwDHD9^yl`;l^mxEjv^2wNbmJpj?Gd?FELAoS}{1q!BN!|I_S=d#a1HA*eTw*i_9Aj
z(?~mHYwuZT=cLr)V&E)~zIGQAP&TyWcYZlM&WI#)O(ZM!==n#k;j)#+mb0R-`~h)0
z8y3HV0p*xH7|j8}M2O)f-|Yiw{M%5y<@0_HByZS#&@VjsV0!V=Nmd@|)?i1n*ae{$%47)u%zf9R;--5Sr7
ziQbD<%k@1Nr~32o8aQDbm^@6&ct54aC76Jhpmv7VI0X{eT&?`+DKBHOCGscvhJo0)
z{fi({GC0+5doY1;_R|p-$$Fw8gg!;?~OA3Gs7A1lN$l&hM8Tv7RhZU6K~
zpBR-ecZJ}#nF-T{`J8U459?bS9vbT&gNUW@qYq>_C;06ioImJH`%Gb~U#=D4p76mo
zhX}-are&}f+zv=5Yh@xg?5xg5uoCRG;Ta?bRZg^uEhuI$sIwoFRxc8Q(C^R<-Xp3b
zyMe9DuB5${CB84cx8ZeR@EnIB{)3y*JbR*WF2P4x-y;RVtL6?fux#`i6=j;&v7)h?
z$^lTp881HE)MXWuWlx}92lKq8lD
z33qOCj!lgX{!~KSNs$*vb*m3oztO13((UuqSJ1?SYn2n}GN*Ux%l8jAKQl-M9TuEc
z=;$t(z4q6de2w?P4Q99^!x_jQ4WqO9MDSqOH6F%`KYlv|l`VFTYUVdE^lec~(DfY5
zp`4DBq3xLdBUI7sMEWg#(CdG{GgS^E*btKBt>iuuJc+>296Nc1Y+genCWEfanlUQbi;2WERxMhw7lsLvHQ$
z;!EyCSt>E%D`KmWkExS+M?XzS3uC$dndbPPUrKP&MB?OyrDtsss@sd)%d4%JxAoUu
zDhPVsSXZA=lCGV@^>s)pz+ZoT3+57g4x;H`*`-^r9JcB6)(~-{WPKcfU@tqj02|yc
zVxm94UrzE|lYJSFYehb3x00%Smrx$u4k1K8BL?t?CjLBM0EMl`_01TzCxwjHbWEv`
z;TJAoT=I_LlB8seKL44hQ<$Vd()yl;BsHGt^wejc9cgs?5K1wMaMJrdGA-2b+dB2B
z)bH@Iea{|Z3q}-J1kK(fdMk-i$Sv6UYPx%-8}s@Wpw5FP&hP0o>Yzu{!B%16b)95(
z+tf=x9%Q$+LMVV`&S3@>mSLmH%Inw%+oQ6NH=sC
z3@j=%l%+{Mwv)2%Fq)2HRIsw~D2%xQf74=Dv9!4g1VsPo?J^{8JWnh@zS7*v2=1jq
z!r}vszWV+)Y5M6!n`LKZ@lEtK{5!PXe~y^4Mqo3)#}8epTwKLC*rvHwr8r>NH=kFE
z*=MD;+!L2arC54#%KkK#`@n+g{#V$`t!#H6GYr5eAeS@{J^}YZrmhAZ_CX%VKkQS;
zCb$DAWyTZxV=w1;R_@LuaCi)5+;1LYdyCdax@e&@0!L^oJ!iVoS4x)|d#R)6{Qz7R@EK!2ft1>B3
zjGFqp3=;xm^*mHtxuI{4;Nk!qc)Lt2;0?nhyq<@$65CB#o`=%#Dr6x*GuDgJ01vUJ
zAYC@2X^H)!h-qe*Gh|9^XMDCWZ&vdC4;8w>=vsiiKbNugWOUh}1BkfJ=`8{E=9Kf4
z5o75!cL-dDY}emcz(8VIapp4?@Kj9rU?yP#p8EdBw>eBimDoz*;dWr7kCk-90ZAVp
zaX8x80%No^P?(&YOCx>cL1Hsj;?d%hJAiGX;spGYZ|chR+3zn?>l6y&50qwt$owye
zt(6QDHq@75wq|;NfCx6{21luc7mLqC$#DCeD9e5tm}AiEQ}~!<_ffmFnuC!m&^VaN
zLwVMvPAQ^6l+U?qi_jOOsk2=kzd_(3KS$vn@5H)>2t3BT(&^Q9Q${hi|^lUF4FhM1PsXvQ@wb+s41gM
zcqqFp;?LIFSP`YYEI#Ns)ws}54?;dR8~W%|sGl&^2!R_mj3p{zU{-q~_NbPjBi&4P
zG7DaG%wM6Zo5U^i2#36TlxiFJ=$5H5TzFdgJ0=C{`*tez&sEHRBq)dhv0&Ya+rMi6
zZcjNubuDmVZIvAlk_*v
zhIbZKc#1AHlkZ9xF&lO
zXRcm+t2Df}Dp3?AZ!7>NYv4^u`
z)M>}d*rkZ6`ngL&^!V8Yy7^3!o)5?FIFx6A*F|pq!KVwI8-KVn5h+R+q2vUgSQ%KO
zrj96=z63G{DxpRX`&{Vl^O!r5i?fd8y7FQ_T~PYJ(-(Xpf`JQmYruf(b5*C%*y{lL7~htoueYYrdzsG!dL|Quk58#%Iq5?j-X9YGB=In;pAf3Z
z-fPBjm5ahmYG1_LweDOnH+dlZNzB2ktgy}WUc(OE=K2*7
zw!Mrr3b!NMg5Vv@^Lu_P9rh4A{C8E*u?Y1N}4Yos|d@$KNw2_tt0exAeXF|kG8cshtsCXOgt2QhN~h!QK-ko$-9|Kkp^
zVyRd|js-kKtQ;xliALQ7G2a7UMRgPduJdPYW}3J9QNRRWcw8VBWHm|v_ewtB?m^IU
zRS>SGDxvYK$-&UcCazv)ctXN32ZFkMyg5=?^V887giF%8{Ur=UBvrt3`ysx!Vm=WQ
z(6Dand1aeZ6c#U0;i@>Jzh%(&ZctZUi&asV$#^IzE+UbT?t#JsLDI|
zAv!JKv%pKwQ6HO|>GKZ(`m~CyrH0!D@5`bxghkB}qWGRc{Dph7{x^ljhHuzj&iGLM
zAJn~fR8#%7E*wNfsVX2vASxhDKt!Y`mKP9s0YRFys3-_X6Of(&3P|W(YJ`AvLX{#A
zI!NzPKuYL6fdB!Lc-Onn+54V*_Br?4XMBI%G02E47b{t7%{9w2pZWZ9Yfbv07vh~dBTm78d0Y4?DT;{qyLuRh}WB-`PRUbsb~>QA_BfmVG%9d5)n
z)oGL0Vpn2y+dV4BNuvaCI;}+K;4MFstZ6%(#BO(vgmt1*XA_jj8t67(VzZ%JZ%}dX
z2hVmRfGSTuPd~xMQ9Q_SxyGv-mt*O@9r`{sOM(%-C4L&0mK?YqZrlL?%nkv#RJ4kB
z^26oLVf8s(g@ct6PV(~Lla^ZG;R*Shx0xhFBc=R=OQvS0%IPe;!!mh%z$4|aH#IO|
zt&t4P;nV^Kn~)s*cJx;j_BtRhf4<7Nv(p|lIt1n`GD8E5++v{*aEM6l+t
zrC{0PVZ!a)m{)4g@%y&piwZCY^NmYpg4v)TF&)WkYp_gk)>%97cB+A^>5K$X67zw?
z)Tq0AFZNyv4^420tq*CZD962hIyt&maq8cv3Ps6v!vU^0+WwCS{nKG~T0{3gKtd~tws`$uH&DPlG
z4PYuhBDeI_z?g}^fR!;g-@;_mXKG5*CCSJNuOh>*Pdub_QtOuGI_0g$g4BSguO&J(
z-tRu~!^I2m7W-bg-m-ND0?i9kTU;VoUV}WRZ;h0x=F|ZBm<-s|ev^L1u7aAQ>Cxwy?jltCevK|<8pA{
z+iO>z>Yn@Z*yl;O-L$7_cr)};vLW>OIRB+pzMJR(iV0dmtdq@3&95{U{cQc7$KtIy
zT)v+QIc^?3T5ad~5-&DY>d)d|YLYHy7U{X_In)lDPT6?M@x-W&xuE;bCJ9riyS_@I
zuhvj;5R!_@N(^Dl+?SNAe>~!IYxdzHCfthA`K(uxVHoTLGM92qXi-c=$mAkw1$jJ+
zuX)ejF>|>t(nS7oLELqvMGtVIUmUUO?d>x09+&3hC<(&$)!s~cj>RUP_2|qFg9Eqv
zbzz`$ebe>dA+(2d#u{PoJGZW2`bzT>1snYx2&m4D1u)AWyL?ttfgG!PE(*?FC4TX1
zmY7y3kNkT-J@)QJGYk0kGd&(vwp~m?w=6?Ir|9>SxZw;hv!;P}zF-TRq^uy1+OU@o
zcuZejTr-GDJn?&(lMqXiuLUlx>Hj1*IW0mj$2TgZFIzrYcw2K-Y=lp&`8x^2MJ;I(
zVhe?x_3%U*g+CtaI`AmqGf>EH@@wcic+>>KgE%|8~LW1AGjU-8o+
zSC`uNwExV(zN&tbeh%G}ML)X?jr6X*I`~Dk3!Jy14J#kc$#3PD_zsRSF4!v6Lazhg
zwxsXD-&3i`+**D;J$p9#J4F{WKG=t-!%qdqX|8qL7&H1Y^7fs^1h$N;?d(K@Wj?uQ
z_mX4P*skRJ8w}~wajF3QDqUl%Na
zm(((X73?dQ=0qmU^!@?IB+XB7v2aWNs0vAnjXLYlr>b8vwA}QBnhzI(n7
z;aFVmpA2|+_#SrX@pa6xot?3FG92LQ4ZH%q6WI*L5isvPn2z?fH^RS#N~-Mgyc9_<
zKRm$rinaM_*2n!6&{e7tUsQ9E*%$?}Cg`jAhniB88DN-vKf1+n_$y>1;Q>CO
zZHSMV=4!d?Z~3wBc&6f@Kb?sfQxoG(R)?j4jz_N)1Y^{E8QNfApr}e1!Avvdg1^Q5
z25|!l1O8w2QobZI4qAm~g<=rzG;$5tABjn99%lw94EW-DQG5-2D)RVv?^9b9W9MU-
zT`D{=`GdKxvz#gdO}7Y%OZE9D7Wew;t%Z~hC>r2Mzh&>hV;ZZz_DodKqT35GUsKZ~
zjtf3MHb@>TdGU?vy%6rsb92!HNXiY(j@qWseuqM!JpLQUu~}4c3s>r|x|+MDDKcbk
z@BfgX+zBPbdA&57hB&_|zv6b7T#nIVh>=)y{y-B4aEYdohc_hiuJZ({T=1naaPC*V
z;XBK4l_a=_nO5+;WjcL598S7pJN(>Uto4`XYe1Hxvg}`N*8(+%*~5_lUP=LGPQOwoS5T75ut8<*Ojz&2}OV3lQ3e?bYw4kmg{z9)GZgX3Xc8?YyBKM-~wJtl|2BJJGKwrcu%#xH=Fe|*~*3R!#h
zA6l2=@=T4QS-H;anOT2-KLX9V7_A$Kt8lb%hUMeFs
z01oD7nE!Pc=43tPQQZIWJKvF5%sJBOMO<_m9#ZllA!&EmZxANdMz!VSZcHjZPPoR+
zAFPsYFak_!>%xmFn3qFNz63b1b<%`FTj88z9-3*dYd$6(TDJl^TdvV630(SG7Uge&
z2w%|PR6mP!Bx!qxo3HhlsWDvbnVmc1<^ObS|9rKhI{sk3FWy4=(bl?VzcFTrK=Kb3
zZ9Xy^@wl-FSg!l0GaQd0gkz-KDm^}IDMj#p3QvHQ>~|G+OaqPGOI-Stmw21rW0eEg
zU7PKJ>C_{{fippY=ow|O>%Mohs+vpcgUf(h8Q;kqjq~86l?K1Wl3RqvokZ@|LMEIt?t|$l_3V`f;3APUWeC|AK~-f?~T%XAg9P!3NBgl
z$~mohh2xd#M@+%g5s=VW5+>JG^?QQaopwzVP(;-rZp#KUlQrXuv<
z>(3GTFoveCG~hmqUf}g_7%>}&|D)wbWJLw*xx+8|`-?_5M(e?rY<>V7$w9oER3wq#
z6duNPGP?zro7g|b^$YM%ED@FqI5$PkJeq+vpnhSpt0E2fUvx6nUJ<*#-IZ4jk4>oSTWndux5OzO%Y+a)V%1aRgvQu2dN}TAsNf?_V8yaz0
zB2bWXAKVFmzWj*R+k=;WgK~bt_x0XlJB|_JM{i_Qjxlm%*o9~C+(^J;%+M<`cYqF$
zBTyD|?ulIe_Y15Fk_|dH7sTqXy0IbsentWUkgF?uX4e3?zAnMGR0-VEq@#k|m;MPL
z6hIdjBCg~&V0>kN@-vB*PH^v3Q)Ngw>Pct5=1Ilq@<0PnJ|%7Y%ctw7s$C1n#Y{zK
zU;eA*_aB&T|LwmYO|zMqsjuAwE5oOnlmQ-#-dunGhf9tP6F1C__*R}BrPBi7az}-9
z3}e{U0M40rNhvs8Oca<2UF3KNNL!}2C
z>d9ZfF{#~{Ul*NLRr6=f02u#mHT$lwbAVs3gyy4le}e+dk87(8oCLBX+s5?s5Y3WQ
z3o|i!HhuN`3(!PfPc~TJFkv}dKFouvtXXpYy=hZ73rvvDq(ffci`hjS6#Lq@NUUyTs8dVF2TbXD26
zejxn_<=;m3WHcxdE)F#iP?Hq)3)^oq#W%T9ODM1<(=|AM@UD%un^Fj0aQ9=4I$$uU
z*0-I1Os<)C<7x
z=1D?~#unM&8jrn>-2qsAYdfuRFr#u4c*MCZ+pJpdb*ihY-@EF`VWLS`Y0c}*yghm+
z)2|@+tTDav892H*j~*Vsjs4wQJS+o1s9JfO~&50haC_v}zsJ
z{A(}3FyQWf*sI?*#n!Ce6M(!W50;_i&G7Tf5)*=H;TV^>O44
zNFD3Q8dNuYj1{_FRH__@%}y5@O=Eg<-QV-|!KRq_EU0
ze@fCP3CiWL>UfK6Zs7$B$Yzd88oz`i@|^UPuiS)Mj&HocW>rwMI-c8&Tk7v~d?~yy
zGX_+KHd+7Gef-bcbX>HE6B%kC@8xdu)EEM>jlAV4CgGn#DU90DeZ
z*!>2PfgfDnu+ctXG{627hRFZ_wd1i+6R>^dwvE@4}Qy;W(W@uba%QqNB!
zXi(an|NM~vR1c|y(csX=(DB!Z>nE|xU_#$4GjvsV6i}E1esY|DCeMd6BU-QI(Uh^B
zjSv3M{iB|=AedET2!~q?ZS|60<7@$b%rgt;gVKD1VN)$#r(I7Q;)iD?oG*drR`aD>
zYx0E#15vMO&R44E