Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 42 additions & 12 deletions crates/cost/src/node/state_bytes/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,15 @@ pub fn sketch_state_bytes(params: &SketchParams, input_rows: StateInputRows) ->
SketchParams::DDSketch { alpha } if !(*alpha > 0.0 && *alpha < 1.0) => {
std::mem::size_of::<DdSketch>() as u64
}
SketchParams::DDSketch { alpha } => ddsketch_state_bytes(*alpha, input_rows),
// Over the declared magnitude range, or the one assumed when none
// is declared (`assumed_magnitude_range`).
SketchParams::DDSketch { alpha } => ddsketch_state_bytes(
*alpha,
input_rows,
input_rows
.value_magnitude_range
.or_else(|| assumed_magnitude_range(params, input_rows)),
),
SketchParams::Hll { precision, .. } => {
HyperLogLog::<Classic>::estimated_memory_bytes(usize::from(*precision)) as u64
}
Expand Down Expand Up @@ -314,6 +322,27 @@ const ASSUMED_HEAP_KEY_BYTES: usize = 3 * (1 + 4 + 16);
/// A sample value's key owns no bytes, so this bounds it too.
const ASSUMED_VALUE_KEY_BYTES: usize = 1 + 16;

/// The magnitude range a DDSketch state of `params` over `input_rows` is
/// costed for when its values declare none, the one place that default is
/// applied: [`DDSKETCH_ASSUMED_MAGNITUDE_RANGE`] for a DDSketch with α in
/// (0, 1) whose values declare no range; `None` for a declared range, an α
/// costed as an empty sketch, or any other sketch, whose bytes assume
/// none. E.g. a DDSketch at α = 0.01 over a `value` without
/// `column_magnitudes` gives `Some([1, 1e9])`.
pub fn assumed_magnitude_range(
params: &SketchParams,
input_rows: StateInputRows,
) -> Option<ValueMagnitudeRange> {
match params {
SketchParams::DDSketch { alpha }
if *alpha > 0.0 && *alpha < 1.0 && input_rows.value_magnitude_range.is_none() =>
{
Some(DDSKETCH_ASSUMED_MAGNITUDE_RANGE)
}
_ => None,
}
}

/// The range of value magnitudes a DDSketch is assumed to see at most when
/// its column declares no `column_magnitudes`: nine decades, e.g.
/// latencies from 1 ns to 1 s, costed as magnitudes in `[1, 1e9]` (its
Expand Down Expand Up @@ -354,11 +383,11 @@ fn exponential_values_magnitude_ratio(rows: u64) -> f64 {
/// counter per bucket its values span beyond the first.
///
/// The span is that of the values the state folds, not of the column: the
/// smallest of the magnitude ratio its rows are assumed to span
/// ([`exponential_values_magnitude_ratio`]), the ratio of its column's
/// declared `column_magnitudes` (`input_rows.value_magnitude_range`) and
/// that of `DDSKETCH_ASSUMED_MAGNITUDE_RANGE`; without known rows, the
/// smaller of the last two. The library's worst case over every order of
/// smaller of the magnitude ratio its rows are assumed to span
/// ([`exponential_values_magnitude_ratio`]) and the ratio of
/// `column_range`, its column's declared `column_magnitudes`, or the range
/// [`assumed_magnitude_range`] assumes when none is declared; without known
/// rows, the ratio of `column_range`. The library's worst case over every order of
/// inserts and merges (`estimated_memory_bytes`) is about twice the span
/// over the whole range, which is not what a state holds on average.
///
Expand All @@ -368,7 +397,11 @@ fn exponential_values_magnitude_ratio(rows: u64) -> f64 {
/// against 5,769 held. The library's bound for either is 18,752. A gauge
/// declared within [90, 110] spans a ratio of 1.22, 11 buckets: 1,248
/// bytes for any number of rows.
fn ddsketch_state_bytes(alpha: f64, input_rows: StateInputRows) -> u64 {
fn ddsketch_state_bytes(
alpha: f64,
input_rows: StateInputRows,
column_range: Option<ValueMagnitudeRange>,
) -> u64 {
use asap_sketchlib::DdSketch;
let struct_bytes = std::mem::size_of::<DdSketch>() as u64;
// One value allocates one store's first block.
Expand All @@ -393,12 +426,9 @@ fn ddsketch_state_bytes(alpha: f64, input_rows: StateInputRows) -> u64 {
let rows_per_store = input_rows
.rows_per_state
.map(|rows| rows.div_ceil(stores).max(1));
// The most the column's values span: its declared range, at most the
// The most the column's values span: its declared range, else the
// assumed one.
let column_ratio = input_rows
.value_magnitude_range
.map_or(f64::INFINITY, |range| range.magnitude_ratio())
.min(DDSKETCH_ASSUMED_MAGNITUDE_RANGE.magnitude_ratio());
let column_ratio = column_range.map_or(f64::INFINITY, |range| range.magnitude_ratio());
let span = buckets_spanned(rows_per_store.map_or(column_ratio, |rows| {
exponential_values_magnitude_ratio(rows).min(column_ratio)
}));
Expand Down
43 changes: 43 additions & 0 deletions crates/cost/src/node/state_bytes/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,49 @@ fn ddsketch_state_bytes_match_the_library_on_latency_like_values() {
assert!(priced(Some(10), true) < priced(Some(3_000), true));
}

/// The range a DDSketch is costed for when its values declare none is the
/// one `assumed_magnitude_range` names, and only when the sketch is costed
/// over a range: at α = 0.01 with no declared range it is [1, 1e9]; a
/// declared range, another sketch, or α = 1.5 (costed as an empty sketch)
/// assumes none.
#[test]
fn the_assumed_ddsketch_range_is_named_only_when_the_bytes_use_it() {
let rows = |value_magnitude_range| StateInputRows {
rows_per_state: Some(600),
non_negative: true,
value_magnitude_range,
};
let declared = ValueMagnitudeRange {
min_abs: 0.001,
max_abs: 10_000.0,
};
let ddsketch = |alpha| SketchParams::DDSketch { alpha };
assert_eq!(
assumed_magnitude_range(&ddsketch(0.01), rows(None)),
Some(DDSKETCH_ASSUMED_MAGNITUDE_RANGE)
);
assert_eq!(
sketch_state_bytes(&ddsketch(0.01), rows(None)),
sketch_state_bytes(
&ddsketch(0.01),
rows(Some(DDSKETCH_ASSUMED_MAGNITUDE_RANGE))
)
);
assert_eq!(
assumed_magnitude_range(&ddsketch(0.01), rows(Some(declared))),
None
);
assert_eq!(assumed_magnitude_range(&ddsketch(1.5), rows(None)), None);
assert_eq!(
sketch_state_bytes(&ddsketch(1.5), rows(None)),
std::mem::size_of::<asap_sketchlib::DdSketch>() as u64
);
assert_eq!(
assumed_magnitude_range(&SketchParams::Kll { k: 200 }, rows(None)),
None
);
}

/// A DDSketch's span is the smallest of the exponential spread of its
/// rows, its column's declared `column_magnitudes` and the assumed nine
/// decades. At α = 0.01, non-negative values:
Expand Down
74 changes: 56 additions & 18 deletions crates/cost/src/node/usage/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ use crate::node::operator_statistics::{
EdgeStatistics, OperatorStatistics, PartitionStatistics, UnaryEdgeStatistics,
};
use crate::node::state_bytes::{
keeps_every_value, row_bytes, sketch_state_bytes, state_bytes_per_row,
assumed_magnitude_range, keeps_every_value, row_bytes, sketch_state_bytes, state_bytes_per_row,
state_row_bytes_beside_state, summary_state_bytes, StateInputRows,
};
use crate::{aggregate_lookup, lookup, StateFamily};
Expand All @@ -56,7 +56,7 @@ use asap_types::workload::bounds::{
declared_non_negative, declared_series, declared_value_magnitude_range, group_count_bound,
source_name, source_rows_per_ms, unchanged_source_columns,
};
use asap_types::workload::{DataWorkload, ONE_SECOND_MS};
use asap_types::workload::{AssumedDataFact, DataFactAssumption, DataWorkload, ONE_SECOND_MS};

/// One node's cost per second of wall time, with how it was measured. A
/// node is one operator node of the candidate's physical plan DAG
Expand Down Expand Up @@ -90,6 +90,9 @@ pub struct NodeCost {
/// Whether its CPU operations were costed from the measured table or
/// from the formulas.
pub cost_source: NodeCostSource,
/// The data facts the data workload does not declare that its cost
/// assumed, each once, e.g. a DDSketch's value magnitude range.
pub assumed_data_facts: Vec<AssumedDataFact>,
}

/// The state family of `node` whose first producer has `producer_family`:
Expand Down Expand Up @@ -554,6 +557,8 @@ pub fn node_resource_usage_and_cost<'a>(
.and_then(|p| p.memory_bytes_per_state)
.map(|bytes| bytes.round() as u64);
let mut cost_source = NodeCostSource::Analytical;
// The undeclared data facts its cost assumed.
let mut assumed_data_facts = Vec::new();
let input = inputs
.first()
.copied()
Expand All @@ -567,22 +572,23 @@ pub fn node_resource_usage_and_cost<'a>(
.ok_or(AnalyticalCostError::Overflow)?,
})
};
// What the sources declare about the values a summary built here folds:
// whether every one is non-negative, and their magnitude range, both
// from the one source column it folds
// (`ProducerOutput::unchanged_column`).
// The one source column a summary built here folds
// (`ProducerOutput::unchanged_column`; `None` for computed values), and
// what the sources declare about its values: whether every one is
// non-negative, and their magnitude range.
let folded_values = |update: &SummaryUpdate| {
producers
let column = producers
.first()
.and_then(|producer| producer.unchanged_column(&update.weight))
.map_or((false, None), |column| {
let non_negative = !sources.is_empty()
&& sources
.iter()
.all(|source| declared_non_negative(data, source, column));
let range = declared_value_magnitude_range(data, sources.iter().copied(), column);
(non_negative, range)
})
.and_then(|producer| producer.unchanged_column(&update.weight));
let (non_negative, range) = column.map_or((false, None), |column| {
let non_negative = !sources.is_empty()
&& sources
.iter()
.all(|source| declared_non_negative(data, source, column));
let range = declared_value_magnitude_range(data, sources.iter().copied(), column);
(non_negative, range)
});
(column, non_negative, range)
};
// The distinct values a source column passed on unchanged takes per
// evaluation, as every source declares them
Expand Down Expand Up @@ -633,6 +639,35 @@ pub fn node_resource_usage_and_cost<'a>(
state_bytes_per_row(producer.output, producer.output_schema)
})
};
// The magnitude range a summary built here over `input_rows` of
// `column` is costed for when its values declare none
// (`assumed_magnitude_range`), named with the sources that declare no
// range for that column (every source when it folds computed values,
// `column` `None`). None when the measured table gives its bytes, which
// assume no range.
let assumed_magnitude_range_fact = |column: Option<&str>, input_rows: StateInputRows| {
if measured_state_bytes.is_some() {
return None;
}
let assumed_range = assumed_magnitude_range(node.parameters.as_ref()?, input_rows)?;
let undeclared = sources
.iter()
.copied()
.filter(|&source| {
column.is_none_or(|column| {
declared_value_magnitude_range(data, [source], column).is_none()
})
})
.map(|source| source.to_string())
.collect();
Some(AssumedDataFact {
sources: undeclared,
assumption: DataFactAssumption::DDSketchValueMagnitudeRange {
column: column.map(str::to_string),
assumed_range,
},
})
};
let unary = |output| UnaryEdgeStatistics {
input,
output,
Expand Down Expand Up @@ -1041,7 +1076,7 @@ pub fn node_resource_usage_and_cost<'a>(
// At query time, each state sees its group's input rows; at
// ingestion time, the rows of a window one second of
// arriving rows does not bound.
let (non_negative, value_magnitude_range) = folded_values(update);
let (column, non_negative, value_magnitude_range) = folded_values(update);
let input_rows = StateInputRows {
rows_per_state: (!ingestion).then(|| input.rows.div_ceil(group_count.max(1))),
non_negative,
Expand All @@ -1053,6 +1088,7 @@ pub fn node_resource_usage_and_cost<'a>(
node.parameters.as_ref(),
input_rows,
)?);
assumed_data_facts.extend(assumed_magnitude_range_fact(column, input_rows));
let update_ops = update_operations_per_row(
family,
node.parameters.as_ref(),
Expand Down Expand Up @@ -1110,7 +1146,7 @@ pub fn node_resource_usage_and_cost<'a>(
.clamp(1, input.rows.max(1))
}),
};
let (non_negative, value_magnitude_range) = folded_values(update);
let (column, non_negative, value_magnitude_range) = folded_values(update);
let input_rows = StateInputRows {
rows_per_state: Some(rows_per_tumbling_window.div_ceil(group_count.max(1))),
non_negative,
Expand All @@ -1122,6 +1158,7 @@ pub fn node_resource_usage_and_cost<'a>(
node.parameters.as_ref(),
input_rows,
)?);
assumed_data_facts.extend(assumed_magnitude_range_fact(column, input_rows));
let update_ops = update_operations_per_row(
family,
node.parameters.as_ref(),
Expand Down Expand Up @@ -1599,6 +1636,7 @@ pub fn node_resource_usage_and_cost<'a>(
multiplier,
detail,
cost_source,
assumed_data_facts,
},
})
}
Expand Down
Loading
Loading