From a6664b3066a7b97d1d3148b5c04487f3c6d1e53b Mon Sep 17 00:00:00 2001 From: "jeremy.barisch.rooney@channable.com" Date: Thu, 6 Aug 2026 13:33:37 +0200 Subject: [PATCH 01/17] Add failing PreferDistinct fairness test --- libs/opsqueue_python/tests/test_roundtrip.py | 29 ++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/libs/opsqueue_python/tests/test_roundtrip.py b/libs/opsqueue_python/tests/test_roundtrip.py index 23d9a14..2e887ac 100644 --- a/libs/opsqueue_python/tests/test_roundtrip.py +++ b/libs/opsqueue_python/tests/test_roundtrip.py @@ -667,3 +667,32 @@ def test_lookup_too_many_submission_ids_by_strategic_metadata() -> None: ) assert exc.type is TooManyMatchingSubmissionsError assert exc.value.max_submissions == max_ + + +def test_prefer_distinct_strategy_fairness(opsqueue: OpsqueueProcess) -> None: + """Test the PreferDistinct strategy fairly interleaves chunks from different + submissions based on their strategic metadata. + + """ + url = "file:///tmp/opsqueue/test_prefer_distinct_fairness" + producer_client = ProducerClient(f"localhost:{opsqueue.port}", url) + consumer_client = ConsumerClient(f"localhost:{opsqueue.port}", url) + company_ids = [1, 2, 3] + chunks_per_company = 4 + company_id_per_submission = {} + for company_id in company_ids: + sub_id = producer_client.insert_submission( + [company_id] * chunks_per_company, + chunk_size=1, + strategic_metadata={"company_id": company_id}, + ) + company_id_per_submission[sub_id] = company_id + strategy = strategy_from_description(("PreferDistinct", "company_id", "Oldest")) + reserved_company_order = [] + # Fetch 1 chunk at a time. Because we don't complete chunks, opsqueue's + # metastate tracks them as reserved, increasing the busy count for that + # company. + for _ in range(len(company_ids) * chunks_per_company): + [chunk] = consumer_client.reserve_chunks(strategy=strategy) + reserved_company_order.append(company_id_per_submission[chunk.submission_id]) + assert reserved_company_order == [1, 2, 3] * chunks_per_company From ada870942119ef16e41fedd13b773a9578a9b409 Mon Sep 17 00:00:00 2001 From: "jeremy.barisch.rooney@channable.com" Date: Thu, 6 Aug 2026 14:16:19 +0200 Subject: [PATCH 02/17] PreferDistinct sorts submissions by metadata --- opsqueue/src/consumer/dispatcher/metastate.rs | 2 +- opsqueue/src/consumer/strategy.rs | 232 +++++++++--------- 2 files changed, 119 insertions(+), 115 deletions(-) diff --git a/opsqueue/src/consumer/dispatcher/metastate.rs b/opsqueue/src/consumer/dispatcher/metastate.rs index f96534c..851ed8b 100644 --- a/opsqueue/src/consumer/dispatcher/metastate.rs +++ b/opsqueue/src/consumer/dispatcher/metastate.rs @@ -69,7 +69,7 @@ pub type MetaStateVal = i64; #[derive(Debug, Default)] pub struct MetaStateField { - vals_to_counts: DashMap, + pub vals_to_counts: DashMap, counts_to_vals: SkipSet<(usize, MetaStateVal)>, } diff --git a/opsqueue/src/consumer/strategy.rs b/opsqueue/src/consumer/strategy.rs index a725d13..7b88555 100644 --- a/opsqueue/src/consumer/strategy.rs +++ b/opsqueue/src/consumer/strategy.rs @@ -28,65 +28,105 @@ impl Strategy { qb: &'a mut QueryBuilder, metastate: &MetaState, ) -> &'a mut QueryBuilder { - let qb = self.build_query_snippet(qb, metastate); + let qb = self.build_query_snippet(qb, metastate, false); tracing::trace!("sql: {:?}", qb.sql()); qb } + /// * `submissions` - Select submissions (true) or chunks (false). fn build_query_snippet<'a>( &'a self, qb: &'a mut QueryBuilder, metastate: &MetaState, + submissions: bool, ) -> &'a mut QueryBuilder { use Strategy::{Newest, Oldest, PreferDistinct, Random}; match self { - Oldest => qb.push("SELECT * FROM chunks ORDER BY submission_id ASC"), - Newest => qb.push("SELECT * FROM chunks ORDER BY submission_id DESC"), + Oldest => { + if submissions { + qb.push("SELECT id as submission_id FROM submissions ORDER BY id ASC") + } else { + qb.push("SELECT * FROM chunks ORDER BY submission_id ASC") + } + } + Newest => { + if submissions { + qb.push("SELECT id as submission_id FROM submissions ORDER BY id DESC") + } else { + qb.push("SELECT * FROM chunks ORDER BY submission_id DESC") + } + } Random => { - let random_offset: u16 = rand::random(); - qb.push("SELECT * FROM chunks WHERE random_order >= ") - .push_bind(random_offset) - .push(" UNION ALL SELECT * FROM chunks WHERE random_order < ") - .push_bind(random_offset) + if submissions { + panic!("Random underlying strategy not supported") + } else { + let random_offset: u16 = rand::random(); + qb.push("SELECT * FROM chunks WHERE random_order >= ") + .push_bind(random_offset) + .push(" UNION ALL SELECT * FROM chunks WHERE random_order < ") + .push_bind(random_offset) + } } - PreferDistinct { meta_key, underlying, } => { - let qb = qb.push(format_args!("WITH inner_{meta_key} AS NOT MATERIALIZED (")); - let qb = underlying.build_query_snippet(qb, metastate); - qb.push(format_args!( - r"), - taken_{meta_key} AS ( - SELECT * FROM submissions_metadata - WHERE - submissions_metadata.metadata_key = ", - )); - qb.push_bind(meta_key); - qb.push( - r" AND submissions_metadata.metadata_value IN (SELECT value FROM json_each(", - ); + // Unique submission IDs from the underlying strategy. + let qb = qb.push("WITH inner AS NOT MATERIALIZED ("); + let qb = underlying.build_query_snippet(qb, metastate, true); + qb.push("),"); + // Count of in-flight chunks per submission. + qb.push("counts AS (SELECT key, value FROM json_each("); match metastate.get(meta_key) { None => { - tracing::trace!("No metastatefield for key: {meta_key}"); + tracing::trace!("No metastate field for key: {meta_key}"); + qb.push_bind("{}"); } Some(field) => { - let taken_values: Vec<_> = field.too_high_counts(1).collect(); - let taken_values_string = - serde_json::to_string(&taken_values).expect("Always valid JSON"); + let counts_map: std::collections::HashMap<_, _> = field + .vals_to_counts + .iter() + .map(|kv| (*kv.key(), *kv.value())) + .collect(); + let counts_json = + serde_json::to_string(&counts_map).expect("Always valid JSON"); tracing::trace!( - "Taken values that are left out of PreferDistinct: {taken_values_string:?}" + "Granular active counts for PreferDistinct: {counts_json:?}" ); - qb.push_bind(taken_values_string); + qb.push_bind(counts_json); } } - qb.push(format_args!(")) - ) - SELECT * FROM inner_{meta_key} WHERE NOT EXISTS (SELECT 1 FROM taken_{meta_key} WHERE inner_{meta_key}.submission_id = taken_{meta_key}.submission_id) - UNION ALL - SELECT * FROM inner_{meta_key} WHERE EXISTS (SELECT 1 FROM taken_{meta_key} WHERE inner_{meta_key}.submission_id = taken_{meta_key}.submission_id) - ")) + qb.push(")),"); + // Submissions ranked by in-flight chunks. + qb.push( + // MATERIALIZED is necessary to preserve the order. + "ranked_submissions AS MATERIALIZED ( + SELECT inner.submission_id + FROM inner + LEFT JOIN submissions_metadata sm + ON inner.submission_id = sm.submission_id + AND sm.metadata_key = ", + ); + qb.push_bind(meta_key); + qb.push( + " + LEFT JOIN counts c + ON sm.metadata_value = c.key + ORDER BY c.value ASC NULLS FIRST + )", + ); + if submissions { + qb.push(" SELECT submission_id FROM ranked_submissions") + } else { + // In SQLite, CROSS JOIN ON does NOT produce N x M rows, it + // forces the query planner to use 'ranked_submissions' as + // the outer loop, preserving the sort order. + qb.push( + " SELECT chunks.* + FROM ranked_submissions + CROSS JOIN chunks ON chunks.submission_id = ranked_submissions.submission_id", + ) + } } } } @@ -249,55 +289,37 @@ pub mod test { ); insta::assert_snapshot!(formatted_query, @" WITH - inner_company_id AS NOT MATERIALIZED ( + inner AS NOT MATERIALIZED ( SELECT - * + id as submission_id FROM - chunks + submissions ORDER BY - submission_id ASC + id ASC ), - taken_company_id AS ( + counts AS ( SELECT - * + key, + value FROM - submissions_metadata - WHERE - submissions_metadata.metadata_key = ? - AND submissions_metadata.metadata_value IN ( - SELECT - value - FROM - json_each() - ) + json_each(?) + ), + ranked_submissions AS MATERIALIZED ( + SELECT + inner.submission_id + FROM + inner + LEFT JOIN submissions_metadata sm ON inner.submission_id = sm.submission_id + AND sm.metadata_key = ? + LEFT JOIN counts c ON sm.metadata_value = c.key + ORDER BY + c.value ASC NULLS FIRST ) SELECT - * + chunks.* FROM - inner_company_id - WHERE - NOT EXISTS ( - SELECT - 1 - FROM - taken_company_id - WHERE - inner_company_id.submission_id = taken_company_id.submission_id - ) - UNION ALL - SELECT - * - FROM - inner_company_id - WHERE - EXISTS ( - SELECT - 1 - FROM - taken_company_id - WHERE - inner_company_id.submission_id = taken_company_id.submission_id - ) + ranked_submissions + CROSS JOIN chunks ON chunks.submission_id = ranked_submissions.submission_id "); let explained = explain(qb, &mut conn).await; @@ -339,55 +361,37 @@ pub mod test { ); insta::assert_snapshot!(formatted_query, @" WITH - inner_company_id AS NOT MATERIALIZED ( + inner AS NOT MATERIALIZED ( SELECT - * + id as submission_id FROM - chunks + submissions ORDER BY - submission_id DESC + id DESC ), - taken_company_id AS ( + counts AS ( SELECT - * + key, + value FROM - submissions_metadata - WHERE - submissions_metadata.metadata_key = ? - AND submissions_metadata.metadata_value IN ( - SELECT - value - FROM - json_each() - ) + json_each(?) + ), + ranked_submissions AS MATERIALIZED ( + SELECT + inner.submission_id + FROM + inner + LEFT JOIN submissions_metadata sm ON inner.submission_id = sm.submission_id + AND sm.metadata_key = ? + LEFT JOIN counts c ON sm.metadata_value = c.key + ORDER BY + c.value ASC NULLS FIRST ) SELECT - * + chunks.* FROM - inner_company_id - WHERE - NOT EXISTS ( - SELECT - 1 - FROM - taken_company_id - WHERE - inner_company_id.submission_id = taken_company_id.submission_id - ) - UNION ALL - SELECT - * - FROM - inner_company_id - WHERE - EXISTS ( - SELECT - 1 - FROM - taken_company_id - WHERE - inner_company_id.submission_id = taken_company_id.submission_id - ) + ranked_submissions + CROSS JOIN chunks ON chunks.submission_id = ranked_submissions.submission_id "); let explained = explain(qb, &mut conn).await; From b40b1066228cde60cceb9d30ef555bf576b14e87 Mon Sep 17 00:00:00 2001 From: "jeremy.barisch.rooney@channable.com" Date: Thu, 6 Aug 2026 14:52:22 +0200 Subject: [PATCH 03/17] fixup! PreferDistinct sorts submissions by metadata Separate chunk/submission selection --- opsqueue/src/consumer/strategy.rs | 201 ++++++++++++++++-------------- 1 file changed, 107 insertions(+), 94 deletions(-) diff --git a/opsqueue/src/consumer/strategy.rs b/opsqueue/src/consumer/strategy.rs index 7b88555..471e664 100644 --- a/opsqueue/src/consumer/strategy.rs +++ b/opsqueue/src/consumer/strategy.rs @@ -28,52 +28,64 @@ impl Strategy { qb: &'a mut QueryBuilder, metastate: &MetaState, ) -> &'a mut QueryBuilder { - let qb = self.build_query_snippet(qb, metastate, false); + let qb = self.build_query_snippet_returning_chunks(qb, metastate); tracing::trace!("sql: {:?}", qb.sql()); qb } - /// * `submissions` - Select submissions (true) or chunks (false). - fn build_query_snippet<'a>( + fn build_query_snippet_returning_chunks<'a>( &'a self, qb: &'a mut QueryBuilder, metastate: &MetaState, - submissions: bool, ) -> &'a mut QueryBuilder { use Strategy::{Newest, Oldest, PreferDistinct, Random}; match self { - Oldest => { - if submissions { - qb.push("SELECT id as submission_id FROM submissions ORDER BY id ASC") - } else { - qb.push("SELECT * FROM chunks ORDER BY submission_id ASC") - } - } - Newest => { - if submissions { - qb.push("SELECT id as submission_id FROM submissions ORDER BY id DESC") - } else { - qb.push("SELECT * FROM chunks ORDER BY submission_id DESC") - } - } + Oldest => qb.push("SELECT * FROM chunks ORDER BY submission_id ASC"), + Newest => qb.push("SELECT * FROM chunks ORDER BY submission_id DESC"), Random => { - if submissions { - panic!("Random underlying strategy not supported") - } else { - let random_offset: u16 = rand::random(); - qb.push("SELECT * FROM chunks WHERE random_order >= ") - .push_bind(random_offset) - .push(" UNION ALL SELECT * FROM chunks WHERE random_order < ") - .push_bind(random_offset) - } + let random_offset: u16 = rand::random(); + qb.push("SELECT * FROM chunks WHERE random_order >= ") + .push_bind(random_offset) + .push(" UNION ALL SELECT * FROM chunks WHERE random_order < ") + .push_bind(random_offset) } + PreferDistinct { .. } => { + // Unique submission IDs from the underlying strategy. + let qb = qb.push("WITH underlying_submission_ids AS MATERIALIZED ("); + let qb = self.build_query_snippet_returning_submission_ids(qb, metastate); + qb.push(") "); + // In SQLite, CROSS JOIN ON/WHERE does NOT produce N x M + // rows, it acts as an INNER JOIN forcing the query planner to + // use '
' as the outer loop, preserving its sort order. + // c.f. + // https://sqlite.org/optoverview.html#manual_control_of_query_plans_using_cross_join + qb.push( + " SELECT chunks.* + FROM underlying_submission_ids + CROSS JOIN chunks + ON chunks.submission_id = underlying_submission_ids.submission_id", + ) + } + } + } + + fn build_query_snippet_returning_submission_ids<'a>( + &'a self, + qb: &'a mut QueryBuilder, + metastate: &MetaState, + ) -> &'a mut QueryBuilder { + use Strategy::{Newest, Oldest, PreferDistinct, Random}; + match self { + Oldest => qb.push("SELECT id as submission_id FROM submissions ORDER BY id ASC"), + Newest => qb.push("SELECT id as submission_id FROM submissions ORDER BY id DESC"), + Random => panic!("Random underlying strategy not supported"), PreferDistinct { meta_key, underlying, } => { // Unique submission IDs from the underlying strategy. let qb = qb.push("WITH inner AS NOT MATERIALIZED ("); - let qb = underlying.build_query_snippet(qb, metastate, true); + let qb = underlying.build_query_snippet_returning_submission_ids(qb, metastate); qb.push("),"); // Count of in-flight chunks per submission. qb.push("counts AS (SELECT key, value FROM json_each("); @@ -109,24 +121,11 @@ impl Strategy { ); qb.push_bind(meta_key); qb.push( - " - LEFT JOIN counts c - ON sm.metadata_value = c.key - ORDER BY c.value ASC NULLS FIRST + " LEFT JOIN counts c ON sm.metadata_value = c.key + ORDER BY c.value ASC NULLS FIRST )", ); - if submissions { - qb.push(" SELECT submission_id FROM ranked_submissions") - } else { - // In SQLite, CROSS JOIN ON does NOT produce N x M rows, it - // forces the query planner to use 'ranked_submissions' as - // the outer loop, preserving the sort order. - qb.push( - " SELECT chunks.* - FROM ranked_submissions - CROSS JOIN chunks ON chunks.submission_id = ranked_submissions.submission_id", - ) - } + qb.push(" SELECT submission_id FROM ranked_submissions") } } } @@ -289,37 +288,44 @@ pub mod test { ); insta::assert_snapshot!(formatted_query, @" WITH - inner AS NOT MATERIALIZED ( - SELECT - id as submission_id - FROM - submissions - ORDER BY - id ASC - ), - counts AS ( - SELECT - key, - value - FROM - json_each(?) - ), - ranked_submissions AS MATERIALIZED ( + underlying_submission_ids AS MATERIALIZED ( + WITH + inner AS NOT MATERIALIZED ( + SELECT + id as submission_id + FROM + submissions + ORDER BY + id ASC + ), + counts AS ( + SELECT + key, + value + FROM + json_each(?) + ), + ranked_submissions AS MATERIALIZED ( + SELECT + inner.submission_id + FROM + inner + LEFT JOIN submissions_metadata sm ON inner.submission_id = sm.submission_id + AND sm.metadata_key = ? + LEFT JOIN counts c ON sm.metadata_value = c.key + ORDER BY + c.value ASC NULLS FIRST + ) SELECT - inner.submission_id + submission_id FROM - inner - LEFT JOIN submissions_metadata sm ON inner.submission_id = sm.submission_id - AND sm.metadata_key = ? - LEFT JOIN counts c ON sm.metadata_value = c.key - ORDER BY - c.value ASC NULLS FIRST + ranked_submissions ) SELECT chunks.* FROM - ranked_submissions - CROSS JOIN chunks ON chunks.submission_id = ranked_submissions.submission_id + underlying_submission_ids + CROSS JOIN chunks ON chunks.submission_id = underlying_submission_ids.submission_id "); let explained = explain(qb, &mut conn).await; @@ -361,37 +367,44 @@ pub mod test { ); insta::assert_snapshot!(formatted_query, @" WITH - inner AS NOT MATERIALIZED ( - SELECT - id as submission_id - FROM - submissions - ORDER BY - id DESC - ), - counts AS ( - SELECT - key, - value - FROM - json_each(?) - ), - ranked_submissions AS MATERIALIZED ( + underlying_submission_ids AS MATERIALIZED ( + WITH + inner AS NOT MATERIALIZED ( + SELECT + id as submission_id + FROM + submissions + ORDER BY + id DESC + ), + counts AS ( + SELECT + key, + value + FROM + json_each(?) + ), + ranked_submissions AS MATERIALIZED ( + SELECT + inner.submission_id + FROM + inner + LEFT JOIN submissions_metadata sm ON inner.submission_id = sm.submission_id + AND sm.metadata_key = ? + LEFT JOIN counts c ON sm.metadata_value = c.key + ORDER BY + c.value ASC NULLS FIRST + ) SELECT - inner.submission_id + submission_id FROM - inner - LEFT JOIN submissions_metadata sm ON inner.submission_id = sm.submission_id - AND sm.metadata_key = ? - LEFT JOIN counts c ON sm.metadata_value = c.key - ORDER BY - c.value ASC NULLS FIRST + ranked_submissions ) SELECT chunks.* FROM - ranked_submissions - CROSS JOIN chunks ON chunks.submission_id = ranked_submissions.submission_id + underlying_submission_ids + CROSS JOIN chunks ON chunks.submission_id = underlying_submission_ids.submission_id "); let explained = explain(qb, &mut conn).await; From 31cf874416fcb125802ea105d1032929cc4c54fb Mon Sep 17 00:00:00 2001 From: "jeremy.barisch.rooney@channable.com" Date: Thu, 6 Aug 2026 15:33:41 +0200 Subject: [PATCH 04/17] fixup! PreferDistinct sorts submissions by metadata Support random submission selection --- ...random_order_index_to_submissions.down.sql | 2 + ...d_random_order_index_to_submissions.up.sql | 9 + opsqueue/opsqueue_example_database_schema.db | Bin 102400 -> 102400 bytes opsqueue/src/consumer/strategy.rs | 263 ++++++++---------- 4 files changed, 128 insertions(+), 146 deletions(-) create mode 100644 opsqueue/migrations/20260803133844_add_random_order_index_to_submissions.down.sql create mode 100644 opsqueue/migrations/20260803133844_add_random_order_index_to_submissions.up.sql diff --git a/opsqueue/migrations/20260803133844_add_random_order_index_to_submissions.down.sql b/opsqueue/migrations/20260803133844_add_random_order_index_to_submissions.down.sql new file mode 100644 index 0000000..6432542 --- /dev/null +++ b/opsqueue/migrations/20260803133844_add_random_order_index_to_submissions.down.sql @@ -0,0 +1,2 @@ +DROP INDEX random_submissions_order; +ALTER TABLE submissions DROP COLUMN random_order; diff --git a/opsqueue/migrations/20260803133844_add_random_order_index_to_submissions.up.sql b/opsqueue/migrations/20260803133844_add_random_order_index_to_submissions.up.sql new file mode 100644 index 0000000..3797c3f --- /dev/null +++ b/opsqueue/migrations/20260803133844_add_random_order_index_to_submissions.up.sql @@ -0,0 +1,9 @@ +-- Uses the same formula as '20250803174028_better_random_order_formula.down.sql'. +ALTER TABLE submissions ADD COLUMN random_order INTEGER NOT NULL GENERATED ALWAYS AS ( + (((id + (id >> 22)) % 65536) * 40503) % 65536 +) VIRTUAL; + +CREATE INDEX random_submissions_order ON submissions ( + random_order + , id +); diff --git a/opsqueue/opsqueue_example_database_schema.db b/opsqueue/opsqueue_example_database_schema.db index 83e09941a8e01e276a01805f785de1774aa7e3b4..1883e20f03a4f77b6ace20bdab657f48099e2f34 100644 GIT binary patch delta 543 zcmZozz}B#UZGyC*Gy?;JBoK=OF%uB$Ow=)El-`)IgkO%AtDb>>0lzEXUA{ctU%dUi zT0FaW0=Tbn=WzYvs^6?A5X)6x$;-|fZp@j(B9t5Y({V~-N{T{JVqQvqu0noMN@|fp zW?o8ag+fWbLUCzQZf0?DW`16=k%5t!u7L#*DHs}C85>&}8fUr|sUA%ZVS2t>pmf5U zFH#>bypnI_j%eR^`d&(G=-(Gj5$nrz=5(^G-z;}hEo;6O!=fPF0v>kO&Htp>Smk*6 zix~L7@xSJOz<-th6#styE&MC_=kZVG@8GZHFWRiA5WvsP#mveGaU<{MTK)b9Ol(`X z^9eArv$1f_U|29+QjBp9SFzSBX*KMD##JGyJzL71}xMnD4_Q-gxPA99c@XmBZj0n`Gpp+KSO7jzg|STa+%G`C;SU^F*p MWZce?!1zNS00u9pKL7v# delta 353 zcmZozz}B#UZGyC*Bm)Bj2#W(TGZ1S{)G=n1+?cS0UxtV47X$wSepkM`e0jXTc>8&^ zcy{pwa9`uj;rg}NP~jHW=3i1=tTH_OwG8~<_+Rrs;J?a$ihn=<7XFp|^Y|z8cktJ4 z7Bq qb.push("SELECT * FROM chunks ORDER BY submission_id ASC"), Newest => qb.push("SELECT * FROM chunks ORDER BY submission_id DESC"), - Random => { - let random_offset: u16 = rand::random(); - qb.push("SELECT * FROM chunks WHERE random_order >= ") - .push_bind(random_offset) - .push(" UNION ALL SELECT * FROM chunks WHERE random_order < ") - .push_bind(random_offset) - } + Random => Self::push_random_order_query(qb, "*", "chunks"), PreferDistinct { .. } => { // Unique submission IDs from the underlying strategy. let qb = qb.push("WITH underlying_submission_ids AS MATERIALIZED ("); let qb = self.build_query_snippet_returning_submission_ids(qb, metastate); qb.push(") "); - // In SQLite, CROSS JOIN
ON/WHERE does NOT produce N x M - // rows, it acts as an INNER JOIN forcing the query planner to - // use '
' as the outer loop, preserving its sort order. - // c.f. - // https://sqlite.org/optoverview.html#manual_control_of_query_plans_using_cross_join + // In SQLite, CROSS JOIN ON/WHERE does NOT produce N + // x M rows, it acts as an INNER JOIN but forces the query + // planner to use '' as the outer loop, preserving the + // underlying sort order. + // c.f. https://sqlite.org/optoverview.html#manual_control_of_query_plans_using_cross_join qb.push( " SELECT chunks.* FROM underlying_submission_ids @@ -78,7 +72,7 @@ impl Strategy { match self { Oldest => qb.push("SELECT id as submission_id FROM submissions ORDER BY id ASC"), Newest => qb.push("SELECT id as submission_id FROM submissions ORDER BY id DESC"), - Random => panic!("Random underlying strategy not supported"), + Random => Self::push_random_order_query(qb, "id as submission_id", "submissions"), PreferDistinct { meta_key, underlying, @@ -129,6 +123,24 @@ impl Strategy { } } } + + /// Append a query snippet to select from the `random_order` column on the + /// given table using the "cutting the deck" technique. + fn push_random_order_query<'a>( + qb: &'a mut QueryBuilder, + columns: &str, + table_name: &str, + ) -> &'a mut QueryBuilder { + let random_offset: u16 = rand::random(); + qb.push(format!( + "SELECT {columns} FROM {table_name} WHERE random_order >= " + )) + .push_bind(random_offset) + .push(format!( + " UNION ALL SELECT {columns} FROM {table_name} WHERE random_order < " + )) + .push_bind(random_offset) + } } #[cfg(feature = "server-logic")] @@ -446,62 +458,51 @@ pub mod test { ); insta::assert_snapshot!(formatted_query, @" WITH - inner_company_id AS NOT MATERIALIZED ( - SELECT - * - FROM - chunks - WHERE - random_order >= ? - UNION ALL - SELECT - * - FROM - chunks - WHERE - random_order < ? - ), - taken_company_id AS ( - SELECT - * - FROM - submissions_metadata - WHERE - submissions_metadata.metadata_key = ? - AND submissions_metadata.metadata_value IN ( - SELECT - value - FROM - json_each() - ) - ) - SELECT - * - FROM - inner_company_id - WHERE - NOT EXISTS ( + underlying_submission_ids AS MATERIALIZED ( + WITH + inner AS NOT MATERIALIZED ( SELECT - 1 + id as submission_id FROM - taken_company_id + submissions WHERE - inner_company_id.submission_id = taken_company_id.submission_id - ) - UNION ALL - SELECT - * - FROM - inner_company_id - WHERE - EXISTS ( + random_order >= ? + UNION ALL SELECT - 1 + id as submission_id FROM - taken_company_id + submissions WHERE - inner_company_id.submission_id = taken_company_id.submission_id + random_order < ? + ), + counts AS ( + SELECT + key, + value + FROM + json_each(?) + ), + ranked_submissions AS MATERIALIZED ( + SELECT + inner.submission_id + FROM + inner + LEFT JOIN submissions_metadata sm ON inner.submission_id = sm.submission_id + AND sm.metadata_key = ? + LEFT JOIN counts c ON sm.metadata_value = c.key + ORDER BY + c.value ASC NULLS FIRST ) + SELECT + submission_id + FROM + ranked_submissions + ) + SELECT + chunks.* + FROM + underlying_submission_ids + CROSS JOIN chunks ON chunks.submission_id = underlying_submission_ids.submission_id "); let explained = explain(qb, &mut conn).await; @@ -561,106 +562,76 @@ pub mod test { ); insta::assert_snapshot!(formatted_query, @" WITH - inner_company_id AS NOT MATERIALIZED ( + underlying_submission_ids AS MATERIALIZED ( WITH - inner_priority AS NOT MATERIALIZED ( - SELECT - * - FROM - chunks - WHERE - random_order >= ? - UNION ALL - SELECT - * - FROM - chunks - WHERE - random_order < ? - ), - taken_priority AS ( - SELECT - * - FROM - submissions_metadata - WHERE - submissions_metadata.metadata_key = ? - AND submissions_metadata.metadata_value IN ( - SELECT - value - FROM - json_each() - ) - ) - SELECT - * - FROM - inner_priority - WHERE - NOT EXISTS ( + inner AS NOT MATERIALIZED ( + WITH + inner AS NOT MATERIALIZED ( SELECT - 1 + id as submission_id FROM - taken_priority + submissions WHERE - inner_priority.submission_id = taken_priority.submission_id - ) - UNION ALL - SELECT - * - FROM - inner_priority - WHERE - EXISTS ( + random_order >= ? + UNION ALL SELECT - 1 + id as submission_id FROM - taken_priority + submissions WHERE - inner_priority.submission_id = taken_priority.submission_id - ) - ), - taken_company_id AS ( - SELECT - * - FROM - submissions_metadata - WHERE - submissions_metadata.metadata_key = ? - AND submissions_metadata.metadata_value IN ( + random_order < ? + ), + counts AS ( SELECT + key, value FROM - json_each() + json_each(?) + ), + ranked_submissions AS MATERIALIZED ( + SELECT + inner.submission_id + FROM + inner + LEFT JOIN submissions_metadata sm ON inner.submission_id = sm.submission_id + AND sm.metadata_key = ? + LEFT JOIN counts c ON sm.metadata_value = c.key + ORDER BY + c.value ASC NULLS FIRST ) - ) - SELECT - * - FROM - inner_company_id - WHERE - NOT EXISTS ( SELECT - 1 + submission_id FROM - taken_company_id - WHERE - inner_company_id.submission_id = taken_company_id.submission_id - ) - UNION ALL - SELECT - * - FROM - inner_company_id - WHERE - EXISTS ( + ranked_submissions + ), + counts AS ( + SELECT + key, + value + FROM + json_each(?) + ), + ranked_submissions AS MATERIALIZED ( SELECT - 1 + inner.submission_id FROM - taken_company_id - WHERE - inner_company_id.submission_id = taken_company_id.submission_id + inner + LEFT JOIN submissions_metadata sm ON inner.submission_id = sm.submission_id + AND sm.metadata_key = ? + LEFT JOIN counts c ON sm.metadata_value = c.key + ORDER BY + c.value ASC NULLS FIRST ) + SELECT + submission_id + FROM + ranked_submissions + ) + SELECT + chunks.* + FROM + underlying_submission_ids + CROSS JOIN chunks ON chunks.submission_id = underlying_submission_ids.submission_id "); let explained = explain(qb, &mut conn).await; From 7c1d688a444bbe7b5884cb0cdff4c7ab41c36efd Mon Sep 17 00:00:00 2001 From: "jeremy.barisch.rooney@channable.com" Date: Thu, 6 Aug 2026 16:12:38 +0200 Subject: [PATCH 05/17] fixup! PreferDistinct sorts submissions by metadata Add assert_streaming_chunks --- opsqueue/src/consumer/strategy.rs | 244 ++++++++++-------------------- 1 file changed, 78 insertions(+), 166 deletions(-) diff --git a/opsqueue/src/consumer/strategy.rs b/opsqueue/src/consumer/strategy.rs index cd34d88..401914f 100644 --- a/opsqueue/src/consumer/strategy.rs +++ b/opsqueue/src/consumer/strategy.rs @@ -194,6 +194,27 @@ pub mod test { ); } + /// A weaker version of `assert_streaming_query`, for `PreferDistinct`. + /// + /// `PreferDistinct` cannot stream: to rank submissions by how many of their + /// chunks are already in flight, it has to sort the `submissions` table. We + /// accept that cost, because there are fewer submissions than chunks. + /// + /// What we do not accept is doing the same to `chunks`, so we only require + /// that `chunks` is reached by an index seek. + fn assert_streaming_chunks(qb: &sqlx::QueryBuilder, explained: &str) { + let query_binding = qb.sql(); + let query = query_binding.as_str(); + assert!( + !explained.contains("SCAN chunks"), + "Query should never scan the whole `chunks` backlog, but it did.\n\nQuery: {query}\n\nPlan: \n\n{explained}" + ); + assert!( + explained.contains("SEARCH chunks"), + "Query should reach `chunks` via an index seek, but it did not.\n\nQuery: {query}\n\nPlan: \n\n{explained}" + ); + } + #[sqlx::test(migrator = "crate::MIGRATOR")] pub async fn test_query_plan_oldest(db: sqlx::SqlitePool) { let mut conn = db.acquire().await.unwrap(); @@ -341,21 +362,17 @@ pub mod test { "); let explained = explain(qb, &mut conn).await; - assert_streaming_query(qb, &explained); + assert_streaming_chunks(qb, &explained); insta::assert_snapshot!(explained, @" - 1, 0, COMPOUND QUERY - 2, 1, LEFT-MOST SUBQUERY - 5, 2, SCAN chunks - 8, 2, CORRELATED SCALAR SUBQUERY 4 - 12, 8, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 22, 8, LIST SUBQUERY 2 - 25, 22, SCAN json_each VIRTUAL TABLE INDEX 0: - 33, 22, CREATE BLOOM FILTER - 62, 1, UNION ALL - 65, 62, SCAN chunks - 68, 62, CORRELATED SCALAR SUBQUERY 6 - 72, 68, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 80, 68, REUSE LIST SUBQUERY 2 + 3, 0, MATERIALIZE underlying_submission_ids + 6, 3, MATERIALIZE ranked_submissions + 12, 6, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 + 14, 6, SEARCH sm USING PRIMARY KEY (submission_id=? AND metadata_key=?) LEFT-JOIN + 23, 6, SCAN json_each VIRTUAL TABLE INDEX 1: LEFT-JOIN + 46, 6, USE TEMP B-TREE FOR ORDER BY + 58, 3, SCAN ranked_submissions + 69, 0, SCAN underlying_submission_ids + 71, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) "); } @@ -420,21 +437,17 @@ pub mod test { "); let explained = explain(qb, &mut conn).await; - assert_streaming_query(qb, &explained); + assert_streaming_chunks(qb, &explained); insta::assert_snapshot!(explained, @" - 1, 0, COMPOUND QUERY - 2, 1, LEFT-MOST SUBQUERY - 5, 2, SCAN chunks - 8, 2, CORRELATED SCALAR SUBQUERY 4 - 12, 8, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 22, 8, LIST SUBQUERY 2 - 25, 22, SCAN json_each VIRTUAL TABLE INDEX 0: - 33, 22, CREATE BLOOM FILTER - 62, 1, UNION ALL - 65, 62, SCAN chunks - 68, 62, CORRELATED SCALAR SUBQUERY 6 - 72, 68, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 80, 68, REUSE LIST SUBQUERY 2 + 3, 0, MATERIALIZE underlying_submission_ids + 6, 3, MATERIALIZE ranked_submissions + 12, 6, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 + 14, 6, SEARCH sm USING PRIMARY KEY (submission_id=? AND metadata_key=?) LEFT-JOIN + 23, 6, SCAN json_each VIRTUAL TABLE INDEX 1: LEFT-JOIN + 46, 6, USE TEMP B-TREE FOR ORDER BY + 58, 3, SCAN ranked_submissions + 69, 0, SCAN underlying_submission_ids + 71, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) "); } @@ -506,35 +519,23 @@ pub mod test { "); let explained = explain(qb, &mut conn).await; - assert_streaming_query(qb, &explained); + assert_streaming_chunks(qb, &explained); insta::assert_snapshot!(explained, @" - 1, 0, COMPOUND QUERY - 2, 1, LEFT-MOST SUBQUERY - 3, 2, COMPOUND QUERY - 4, 3, LEFT-MOST SUBQUERY - 7, 4, SEARCH chunks USING INDEX random_chunks_order (random_order>?) - 16, 4, CORRELATED SCALAR SUBQUERY 5 - 20, 16, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 30, 16, LIST SUBQUERY 3 - 33, 30, SCAN json_each VIRTUAL TABLE INDEX 0: - 41, 30, CREATE BLOOM FILTER - 62, 3, UNION ALL - 65, 62, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 121, 109, CORRELATED SCALAR SUBQUERY 7 - 125, 121, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 133, 121, REUSE LIST SUBQUERY 3 - 153, 108, UNION ALL - 156, 153, SEARCH chunks USING INDEX random_chunks_order (random_order?) + 22, 9, UNION ALL + 25, 22, SEARCH submissions USING INDEX random_submissions_order (random_order?) - 18, 6, CORRELATED SCALAR SUBQUERY 5 - 22, 18, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 32, 18, LIST SUBQUERY 3 - 35, 32, SCAN json_each VIRTUAL TABLE INDEX 0: - 43, 32, CREATE BLOOM FILTER - 56, 6, CORRELATED SCALAR SUBQUERY 11 - 60, 56, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 70, 56, LIST SUBQUERY 9 - 73, 70, SCAN json_each VIRTUAL TABLE INDEX 0: - 81, 70, CREATE BLOOM FILTER - 102, 5, UNION ALL - 105, 102, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 213, 201, CORRELATED SCALAR SUBQUERY 7 - 217, 213, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 227, 213, LIST SUBQUERY 3 - 230, 227, SCAN json_each VIRTUAL TABLE INDEX 0: - 238, 227, CREATE BLOOM FILTER - 251, 201, CORRELATED SCALAR SUBQUERY 11 - 255, 251, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 265, 251, LIST SUBQUERY 9 - 268, 265, SCAN json_each VIRTUAL TABLE INDEX 0: - 276, 265, CREATE BLOOM FILTER - 297, 200, UNION ALL - 300, 297, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 410, 398, CORRELATED SCALAR SUBQUERY 5 - 414, 410, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 424, 410, LIST SUBQUERY 3 - 427, 424, SCAN json_each VIRTUAL TABLE INDEX 0: - 435, 424, CREATE BLOOM FILTER - 448, 398, CORRELATED SCALAR SUBQUERY 13 - 452, 448, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 462, 448, LIST SUBQUERY 9 - 465, 462, SCAN json_each VIRTUAL TABLE INDEX 0: - 473, 462, CREATE BLOOM FILTER - 494, 397, UNION ALL - 497, 494, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 605, 593, CORRELATED SCALAR SUBQUERY 7 - 609, 605, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 619, 605, LIST SUBQUERY 3 - 622, 619, SCAN json_each VIRTUAL TABLE INDEX 0: - 630, 619, CREATE BLOOM FILTER - 643, 593, CORRELATED SCALAR SUBQUERY 13 - 647, 643, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 657, 643, LIST SUBQUERY 9 - 660, 657, SCAN json_each VIRTUAL TABLE INDEX 0: - 668, 657, CREATE BLOOM FILTER - 689, 592, UNION ALL - 692, 689, SEARCH chunks USING INDEX random_chunks_order (random_order?) + 25, 12, UNION ALL + 28, 25, SEARCH submissions USING INDEX random_submissions_order (random_order Date: Mon, 13 Jul 2026 16:24:29 +0200 Subject: [PATCH 06/17] SQL: Prevent duplicated binds --- opsqueue/src/common/submission.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/opsqueue/src/common/submission.rs b/opsqueue/src/common/submission.rs index 991744d..0b6091a 100644 --- a/opsqueue/src/common/submission.rs +++ b/opsqueue/src/common/submission.rs @@ -580,12 +580,10 @@ pub mod db { r#" SELECT id AS "id: SubmissionId" FROM submissions WHERE prefix = $1 UNION ALL - SELECT id AS "id: SubmissionId" FROM submissions_completed WHERE prefix = $2 + SELECT id AS "id: SubmissionId" FROM submissions_completed WHERE prefix = $1 UNION ALL - SELECT id AS "id: SubmissionId" FROM submissions_failed WHERE prefix = $3 + SELECT id AS "id: SubmissionId" FROM submissions_failed WHERE prefix = $1 "#, - prefix, - prefix, prefix ) .fetch_optional(conn.get_inner()) From db1de2946243520de0b5d3fbe988a6cba68d990c Mon Sep 17 00:00:00 2001 From: Reinier Maas Date: Mon, 13 Jul 2026 17:02:48 +0200 Subject: [PATCH 07/17] SQL: Filter already reserved chunks from `Chunk` selection --- Cargo.lock | 4 +- Cargo.toml | 1 + opsqueue/Cargo.toml | 2 + opsqueue/src/common/chunk.rs | 12 ++ opsqueue/src/consumer/dispatcher/mod.rs | 158 ++++++++++++++++++- opsqueue/src/consumer/dispatcher/reserver.rs | 5 + opsqueue/src/consumer/strategy.rs | 99 +++++++++--- workspace-hack/Cargo.toml | 11 +- 8 files changed, 260 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d5960f..cf1f582 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2119,6 +2119,7 @@ dependencies = [ "humantime", "insta", "itertools 0.15.0", + "libsqlite3-sys", "moka", "moro-local", "object_store", @@ -3233,7 +3234,6 @@ dependencies = [ "log", "memchr", "percent-encoding", - "rustls", "serde", "serde_json", "sha2 0.10.9", @@ -3243,7 +3243,6 @@ dependencies = [ "tokio-stream", "tracing", "url", - "webpki-roots", ] [[package]] @@ -4360,7 +4359,6 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" name = "workspace-hack" version = "0.1.0" dependencies = [ - "aws-lc-rs", "base64", "bitflags", "cc", diff --git a/Cargo.toml b/Cargo.toml index 68ec307..44ce405 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ http = { version = "1.4.0" } humantime = { version = "2.1.0" } insta = { version = "1.47.2" } itertools = { version = "0.15.0" } +libsqlite3-sys = { version = "0.30.1" } moka = { version = "0.12.15", features = ["sync"] } moro-local = { version = "0.4.0" } object_store = { version = "0.14.0", features = ["gcp", "http"] } diff --git a/opsqueue/Cargo.toml b/opsqueue/Cargo.toml index 82d562a..d8859fc 100644 --- a/opsqueue/Cargo.toml +++ b/opsqueue/Cargo.toml @@ -29,6 +29,7 @@ ux.workspace = true anyhow.workspace = true # Database: sqlx = { workspace = true, optional = true } +libsqlite3-sys = { workspace = true, optional = true } # Serialization: serde.workspace = true serde_json.workspace = true @@ -90,6 +91,7 @@ required-features = ["server-logic"] # Dependencies only in use by the server-logic: server-logic = [ "dep:sqlx", + "dep:libsqlite3-sys", "dep:opentelemetry-otlp", "dep:opentelemetry-semantic-conventions", "dep:moka", diff --git a/opsqueue/src/common/chunk.rs b/opsqueue/src/common/chunk.rs index c27c17e..9010b92 100644 --- a/opsqueue/src/common/chunk.rs +++ b/opsqueue/src/common/chunk.rs @@ -130,6 +130,18 @@ impl From for i64 { } } +impl TryFrom for ChunkIndex { + type Error = crate::common::errors::TryFromIntError; + + fn try_from(value: i64) -> Result { + if value < 0 { + return Err(crate::common::errors::TryFromIntError(())); + } + + Ok(Self(u63::new(value.cast_unsigned()))) + } +} + impl TryFrom for ChunkIndex { type Error = crate::common::errors::TryFromIntError; fn try_from(value: u64) -> Result { diff --git a/opsqueue/src/consumer/dispatcher/mod.rs b/opsqueue/src/consumer/dispatcher/mod.rs index 8ba9b03..db0609f 100644 --- a/opsqueue/src/consumer/dispatcher/mod.rs +++ b/opsqueue/src/consumer/dispatcher/mod.rs @@ -3,12 +3,13 @@ pub mod reserver; use crate::{ common::{ - chunk::{Chunk, ChunkId}, - submission::Submission, + chunk::{Chunk, ChunkId, ChunkIndex}, + submission::{Submission, SubmissionId}, }, db::{Connection, Pool, ReaderPool, magic::Bool}, }; use futures::stream::{StreamExt as _, TryStreamExt as _}; +use libsqlite3_sys as ffi; use metastate::MetaState; use reserver::Reserver; use sqlx::QueryBuilder; @@ -21,6 +22,65 @@ use std::sync::Arc; use super::strategy; use crate::common::StrategicMetadataMap; +unsafe extern "C" fn sqlite_reserved_chunk_lookup( + context: *mut ffi::sqlite3_context, + n_args: i32, + args: *mut *mut ffi::sqlite3_value, +) { + if n_args != 2 { + tracing::error!( + n_args, + "opsqueue_is_reserved called with unexpected argument count" + ); + // Fail open: this callback is an optimization only. + unsafe { ffi::sqlite3_result_int(context, 0) }; + return; + } + + let user_data = unsafe { ffi::sqlite3_user_data(context) } + .cast_const() + .cast::>(); + if user_data.is_null() { + tracing::error!("opsqueue_is_reserved called without registered reserver user_data"); + // Fail open: this callback is an optimization only. + unsafe { ffi::sqlite3_result_int(context, 0) }; + return; + } + + let submission_id_raw = unsafe { ffi::sqlite3_value_int64(*args.add(0)) }; + let chunk_index_raw = unsafe { ffi::sqlite3_value_int64(*args.add(1)) }; + + let Ok(submission_id) = SubmissionId::try_from(submission_id_raw) else { + tracing::error!( + submission_id_raw, + "opsqueue_is_reserved got invalid submission_id" + ); + // Fail open: this callback is an optimization only. + unsafe { ffi::sqlite3_result_int(context, 0) }; + return; + }; + let Ok(chunk_index) = ChunkIndex::try_from(chunk_index_raw) else { + tracing::error!( + chunk_index_raw, + "opsqueue_is_reserved got invalid chunk_index" + ); + // Fail open: this callback is an optimization only. + unsafe { ffi::sqlite3_result_int(context, 0) }; + return; + }; + + let chunk_id = ChunkId::from((submission_id, chunk_index)); + let is_reserved = unsafe { &*user_data }.is_reserved(&chunk_id); + unsafe { ffi::sqlite3_result_int(context, i32::from(is_reserved)) }; +} + +unsafe extern "C" fn sqlite_reserved_chunk_lookup_destructor(ptr: *mut std::ffi::c_void) { + if ptr.is_null() { + return; + } + let _boxed: Box> = unsafe { Box::from_raw(ptr.cast()) }; +} + #[derive(Debug, Clone)] pub struct Dispatcher { reserver: Reserver, @@ -74,6 +134,8 @@ impl Dispatcher { stale_chunks_notifier: &UnboundedSender, ) -> Result, sqlx::Error> { let mut conn = pool.reader_conn().await?; + self.register_reserved_chunk_lookup(conn.get_inner()) + .await?; let mut query_builder = QueryBuilder::new(""); let stream = strategy .build_query(&mut query_builder, &self.metastate) @@ -89,6 +151,43 @@ impl Dispatcher { .await } + async fn register_reserved_chunk_lookup( + &self, + conn: &mut sqlx::SqliteConnection, + ) -> Result<(), sqlx::Error> { + let mut handle = conn.lock_handle().await?; + let sqlite = handle.as_raw_handle().as_ptr(); + let function_name = b"opsqueue_is_reserved\0"; + + // Register the current reserver state on this connection. + // Re-registering replaces any previous callback on this handle. + let user_data = Box::new(self.reserver.clone()); + let user_data = Box::into_raw(user_data).cast::(); + + let rc = unsafe { + ffi::sqlite3_create_function_v2( + sqlite, + function_name.as_ptr().cast(), + 2, + ffi::SQLITE_UTF8, + user_data, + Some(sqlite_reserved_chunk_lookup), + None, + None, + Some(sqlite_reserved_chunk_lookup_destructor), + ) + }; + + if rc != ffi::SQLITE_OK { + unsafe { sqlite_reserved_chunk_lookup_destructor(user_data) }; + return Err(sqlx::Error::Protocol(format!( + "sqlite3_create_function_v2 failed with rc={rc}" + ))); + } + + Ok(()) + } + fn reserve_chunk( &self, chunk: Chunk, @@ -152,3 +251,58 @@ impl Dispatcher { .run_pending_tasks_periodically(cancellation_token); } } + +#[cfg(test)] +#[cfg(feature = "server-logic")] +mod test { + use super::*; + use crate::common::chunk::ChunkId; + use crate::common::chunk::ChunkSize; + use crate::db::DBPools; + use tokio::sync::mpsc::unbounded_channel; + use ux::u63; + + #[sqlx::test(migrator = "crate::MIGRATOR")] + async fn fetch_and_reserve_chunks_excludes_already_reserved(db: sqlx::SqlitePool) { + let pools = DBPools::from_test_pool(&db); + let dispatcher = Dispatcher::new(Duration::from_mins(1)); + let (stale_chunks_notifier, mut _stale_chunks_receiver) = unbounded_channel::(); + + let mut writer_conn = pools.writer_conn().await.unwrap(); + let submission_id = crate::common::submission::db::insert_submission_from_chunks( + None, + vec![Some("a".into()), Some("b".into()), Some("c".into())], + None, + StrategicMetadataMap::default(), + ChunkSize::default(), + &mut writer_conn, + ) + .await + .unwrap(); + + let pre_reserved_chunk = ChunkId::from((submission_id, u63::new(0).into())); + dispatcher + .reserver() + .try_reserve( + pre_reserved_chunk, + pre_reserved_chunk, + &stale_chunks_notifier, + ) + .expect("precondition: pre-reserving chunk should succeed"); + + let reserved = dispatcher + .fetch_and_reserve_chunks( + pools.reader_pool(), + strategy::Strategy::Oldest, + 10, + &stale_chunks_notifier, + ) + .await + .unwrap(); + + assert_eq!(reserved.len(), 2); + assert!(reserved.iter().all(|(chunk, _submission)| { + ChunkId::from((chunk.submission_id, chunk.chunk_index)) != pre_reserved_chunk + })); + } +} diff --git a/opsqueue/src/consumer/dispatcher/reserver.rs b/opsqueue/src/consumer/dispatcher/reserver.rs index 0699857..91b93e6 100644 --- a/opsqueue/src/consumer/dispatcher/reserver.rs +++ b/opsqueue/src/consumer/dispatcher/reserver.rs @@ -83,6 +83,11 @@ where } } + /// Returns whether a key currently has an active reservation. + pub fn is_reserved(&self, key: &K) -> bool { + self.reservations.contains_key(key) + } + /// Removes a particular key-val from the reserver. /// Afterwards, it is possible to reserve it again. /// diff --git a/opsqueue/src/consumer/strategy.rs b/opsqueue/src/consumer/strategy.rs index 401914f..e554d1e 100644 --- a/opsqueue/src/consumer/strategy.rs +++ b/opsqueue/src/consumer/strategy.rs @@ -39,10 +39,17 @@ impl Strategy { metastate: &MetaState, ) -> &'a mut QueryBuilder { use Strategy::{Newest, Oldest, PreferDistinct, Random}; + let ffi_is_reserved = "opsqueue_is_reserved(chunks.submission_id, chunks.chunk_index) = 0"; match self { - Oldest => qb.push("SELECT * FROM chunks ORDER BY submission_id ASC"), - Newest => qb.push("SELECT * FROM chunks ORDER BY submission_id DESC"), - Random => Self::push_random_order_query(qb, "*", "chunks"), + Oldest => qb + .push("SELECT * FROM chunks") + .push(format!(" WHERE {ffi_is_reserved}")) + .push(" ORDER BY submission_id ASC"), + Newest => qb + .push("SELECT * FROM chunks") + .push(format!(" WHERE {ffi_is_reserved}")) + .push(" ORDER BY submission_id DESC"), + Random => Self::push_random_order_query(qb, "*", "chunks", Some(ffi_is_reserved)), PreferDistinct { .. } => { // Unique submission IDs from the underlying strategy. let qb = qb.push("WITH underlying_submission_ids AS MATERIALIZED ("); @@ -53,12 +60,13 @@ impl Strategy { // planner to use '' as the outer loop, preserving the // underlying sort order. // c.f. https://sqlite.org/optoverview.html#manual_control_of_query_plans_using_cross_join - qb.push( + qb.push(format!( " SELECT chunks.* FROM underlying_submission_ids CROSS JOIN chunks - ON chunks.submission_id = underlying_submission_ids.submission_id", - ) + ON chunks.submission_id = underlying_submission_ids.submission_id + AND {ffi_is_reserved}", + )) } } } @@ -72,7 +80,7 @@ impl Strategy { match self { Oldest => qb.push("SELECT id as submission_id FROM submissions ORDER BY id ASC"), Newest => qb.push("SELECT id as submission_id FROM submissions ORDER BY id DESC"), - Random => Self::push_random_order_query(qb, "id as submission_id", "submissions"), + Random => Self::push_random_order_query(qb, "id as submission_id", "submissions", None), PreferDistinct { meta_key, underlying, @@ -130,16 +138,22 @@ impl Strategy { qb: &'a mut QueryBuilder, columns: &str, table_name: &str, + condition: Option<&str>, ) -> &'a mut QueryBuilder { let random_offset: u16 = rand::random(); - qb.push(format!( - "SELECT {columns} FROM {table_name} WHERE random_order >= " - )) - .push_bind(random_offset) - .push(format!( - " UNION ALL SELECT {columns} FROM {table_name} WHERE random_order < " - )) - .push_bind(random_offset) + let push_select = |qb: &mut QueryBuilder, operator: &str| { + qb.push(format!( + "SELECT {columns} FROM {table_name} WHERE random_order {operator} " + )) + .push_bind(random_offset); + if let Some(condition_) = condition { + qb.push(format!(" AND {condition_}")); + } + }; + push_select(qb, ">="); + qb.push(" UNION ALL "); + push_select(qb, "<"); + qb } } @@ -149,22 +163,49 @@ pub type ChunkStream<'a> = BoxStream<'a, Result>; #[cfg(test)] #[cfg(feature = "server-logic")] pub mod test { + use super::*; use crate::common::StrategicMetadataMap; use crate::common::chunk::ChunkSize; - - use super::*; use itertools::Itertools; + use libsqlite3_sys as ffi; use sqlformat::{FormatOptions, QueryParams, format}; use sqlx::Row; use sqlx::{QueryBuilder, Sqlite, SqliteConnection}; + unsafe extern "C" fn sqlite_reserved_chunk_lookup_noop( + context: *mut ffi::sqlite3_context, + _n_args: i32, + _args: *mut *mut ffi::sqlite3_value, + ) { + unsafe { ffi::sqlite3_result_int(context, 0) }; + } + + async fn register_reserved_lookup_noop(conn: &mut SqliteConnection) { + let mut handle = conn.lock_handle().await.unwrap(); + let sqlite = handle.as_raw_handle().as_ptr(); + let function_name = b"opsqueue_is_reserved\0"; + let rc = unsafe { + ffi::sqlite3_create_function_v2( + sqlite, + function_name.as_ptr().cast(), + 2, + ffi::SQLITE_UTF8, + std::ptr::null_mut(), + Some(sqlite_reserved_chunk_lookup_noop), + None, + None, + None, + ) + }; + assert_eq!(rc, ffi::SQLITE_OK, "register opsqueue_is_reserved failed"); + } + async fn explain(qb: &mut sqlx::QueryBuilder, conn: &mut SqliteConnection) -> String { let formatted_query = format( qb.sql().as_str(), &QueryParams::None, &FormatOptions::default(), ); - sqlx::raw_sql(sqlx::AssertSqlSafe(format!( "EXPLAIN QUERY PLAN {formatted_query}" ))) @@ -218,6 +259,7 @@ pub mod test { #[sqlx::test(migrator = "crate::MIGRATOR")] pub async fn test_query_plan_oldest(db: sqlx::SqlitePool) { let mut conn = db.acquire().await.unwrap(); + register_reserved_lookup_noop(&mut conn).await; let mut qb = QueryBuilder::new(""); let metastate = MetaState::default(); @@ -229,6 +271,8 @@ pub mod test { * FROM chunks + WHERE + opsqueue_is_reserved(chunks.submission_id, chunks.chunk_index) = 0 ORDER BY submission_id ASC "); @@ -241,6 +285,7 @@ pub mod test { #[sqlx::test(migrator = "crate::MIGRATOR")] pub async fn test_query_plan_newest(db: sqlx::SqlitePool) { let mut conn = db.acquire().await.unwrap(); + register_reserved_lookup_noop(&mut conn).await; let mut qb = QueryBuilder::new(""); let metastate = MetaState::default(); @@ -252,6 +297,8 @@ pub mod test { * FROM chunks + WHERE + opsqueue_is_reserved(chunks.submission_id, chunks.chunk_index) = 0 ORDER BY submission_id DESC "); @@ -264,6 +311,7 @@ pub mod test { #[sqlx::test(migrator = "crate::MIGRATOR")] pub async fn test_query_plan_random(db: sqlx::SqlitePool) { let mut conn = db.acquire().await.unwrap(); + register_reserved_lookup_noop(&mut conn).await; let metastate = MetaState::default(); let mut qb = QueryBuilder::new(""); @@ -281,6 +329,7 @@ pub mod test { chunks WHERE random_order >= ? + AND opsqueue_is_reserved(chunks.submission_id, chunks.chunk_index) = 0 UNION ALL SELECT * @@ -288,6 +337,7 @@ pub mod test { chunks WHERE random_order < ? + AND opsqueue_is_reserved(chunks.submission_id, chunks.chunk_index) = 0 "); let explained = explain(qb, &mut conn).await; @@ -296,8 +346,8 @@ pub mod test { 1, 0, COMPOUND QUERY 2, 1, LEFT-MOST SUBQUERY 5, 2, SEARCH chunks USING INDEX random_chunks_order (random_order>?) - 22, 1, UNION ALL - 25, 22, SEARCH chunks USING INDEX random_chunks_order (random_order = Strategy::Random .build_query(&mut query_builder, &MetaState::default()) diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index c4bae4e..af0c120 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -15,7 +15,6 @@ publish = false ### BEGIN HAKARI SECTION [dependencies] -aws-lc-rs = { version = "1", default-features = false, features = ["aws-lc-sys", "prebuilt-nasm"] } base64 = { version = "0.22" } chrono = { version = "0.4", features = ["serde"] } crossbeam-epoch = { version = "0.9" } @@ -28,7 +27,7 @@ futures-io = { version = "0.3" } futures-sink = { version = "0.3" } futures-util = { version = "0.3", features = ["channel", "io", "sink"] } hyper = { version = "1", features = ["client", "http1", "http2", "server"] } -libsqlite3-sys = { version = "0.30", default-features = false, features = ["bundled", "pkg-config", "unlock_notify", "vcpkg"] } +libsqlite3-sys = { version = "0.30", features = ["bundled", "unlock_notify"] } log = { version = "0.4", default-features = false, features = ["std"] } num-traits = { version = "0.2", default-features = false, features = ["std"] } opentelemetry = { version = "0.32" } @@ -45,7 +44,7 @@ serde_json = { version = "1", features = ["raw_value"] } sha2 = { version = "0.10" } slab = { version = "0.4" } smallvec = { version = "1", default-features = false, features = ["const_new"] } -sqlx-core = { version = "0.9", features = ["_rt-tokio", "_tls-rustls-aws-lc-rs", "any", "chrono", "json", "migrate", "offline"] } +sqlx-core = { version = "0.9", features = ["_rt-tokio", "any", "chrono", "json", "migrate", "offline"] } sqlx-sqlite = { version = "0.9", default-features = false, features = ["any", "bundled", "chrono", "deserialize", "json", "load-extension", "migrate", "offline", "unlock-notify"] } thiserror = { version = "2" } tokio = { version = "1", features = ["fs", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] } @@ -58,7 +57,6 @@ uuid = { version = "1", features = ["fast-rng", "serde", "v4", "v7"] } zerocopy = { version = "0.8", default-features = false, features = ["derive", "simd"] } [build-dependencies] -aws-lc-rs = { version = "1", default-features = false, features = ["aws-lc-sys", "prebuilt-nasm"] } base64 = { version = "0.22" } chrono = { version = "0.4", features = ["serde"] } crossbeam-utils = { version = "0.8" } @@ -68,17 +66,16 @@ futures-channel = { version = "0.3", features = ["sink"] } futures-io = { version = "0.3" } futures-sink = { version = "0.3" } futures-util = { version = "0.3", features = ["channel", "io", "sink"] } -libsqlite3-sys = { version = "0.30", default-features = false, features = ["bundled", "pkg-config", "unlock_notify", "vcpkg"] } +libsqlite3-sys = { version = "0.30", features = ["bundled", "unlock_notify"] } log = { version = "0.4", default-features = false, features = ["std"] } num-traits = { version = "0.2", default-features = false, features = ["std"] } -rustls-pki-types = { version = "1", features = ["std"] } serde = { version = "1", features = ["alloc", "derive", "rc"] } serde_core = { version = "1", features = ["alloc", "rc"] } serde_json = { version = "1", features = ["raw_value"] } sha2 = { version = "0.10" } slab = { version = "0.4" } smallvec = { version = "1", default-features = false, features = ["const_new"] } -sqlx-core = { version = "0.9", features = ["_rt-tokio", "_tls-rustls-aws-lc-rs", "any", "chrono", "json", "migrate", "offline"] } +sqlx-core = { version = "0.9", features = ["_rt-tokio", "any", "chrono", "json", "migrate", "offline"] } sqlx-sqlite = { version = "0.9", default-features = false, features = ["any", "bundled", "chrono", "deserialize", "json", "load-extension", "migrate", "offline", "unlock-notify"] } syn = { version = "3", features = ["full", "visit-mut"] } thiserror = { version = "2" } From 7dfef50fa5c773ffe65ed39cc3c24a7d01753c60 Mon Sep 17 00:00:00 2001 From: "jeremy.barisch.rooney@channable.com" Date: Thu, 6 Aug 2026 17:31:54 +0200 Subject: [PATCH 08/17] fixup! SQL: Filter already reserved chunks from `Chunk` selection No explicit destructor call --- opsqueue/src/consumer/dispatcher/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opsqueue/src/consumer/dispatcher/mod.rs b/opsqueue/src/consumer/dispatcher/mod.rs index db0609f..0d1fa63 100644 --- a/opsqueue/src/consumer/dispatcher/mod.rs +++ b/opsqueue/src/consumer/dispatcher/mod.rs @@ -179,7 +179,8 @@ impl Dispatcher { }; if rc != ffi::SQLITE_OK { - unsafe { sqlite_reserved_chunk_lookup_destructor(user_data) }; + // We don't need to explicitly call the destructor. + // c.f. https://sqlite.org/c3ref/create_function.html return Err(sqlx::Error::Protocol(format!( "sqlite3_create_function_v2 failed with rc={rc}" ))); From b48dd48baadf838f3acc5f0a353fddfe1c6e0d0a Mon Sep 17 00:00:00 2001 From: Reinier Maas Date: Wed, 15 Jul 2026 10:52:12 +0200 Subject: [PATCH 09/17] SQL: Add metadata count FFI function --- opsqueue/src/consumer/dispatcher/metastate.rs | 75 +++++---------- opsqueue/src/consumer/dispatcher/mod.rs | 95 +++++++++++++++++-- 2 files changed, 113 insertions(+), 57 deletions(-) diff --git a/opsqueue/src/consumer/dispatcher/metastate.rs b/opsqueue/src/consumer/dispatcher/metastate.rs index 851ed8b..7043631 100644 --- a/opsqueue/src/consumer/dispatcher/metastate.rs +++ b/opsqueue/src/consumer/dispatcher/metastate.rs @@ -1,4 +1,3 @@ -use crossbeam_skiplist::SkipSet; use dashmap::{DashMap, Entry}; use rustc_hash::FxBuildHasher; use tracing; @@ -70,7 +69,6 @@ pub type MetaStateVal = i64; #[derive(Debug, Default)] pub struct MetaStateField { pub vals_to_counts: DashMap, - counts_to_vals: SkipSet<(usize, MetaStateVal)>, } impl MetaStateField { @@ -80,55 +78,31 @@ impl MetaStateField { } fn increment(&self, val: MetaStateVal) { - match self.vals_to_counts.entry(val) { - Entry::Vacant(entry) => { - self.counts_to_vals.insert((1, *entry.key())); - entry.insert(1); - } - Entry::Occupied(mut entry) => { - // The entry is now locked, so we can also safely update the relevant element of the SkipSet - let count = entry.get(); - let mut set_entry = (*count, *entry.key()); - self.counts_to_vals.remove(&set_entry); - set_entry.0 += 1; - self.counts_to_vals.insert(set_entry); - *entry.get_mut() += 1; - } - } + self.vals_to_counts + .entry(val) + .and_modify(|count| *count += 1) + .or_insert(1); } fn decrement(&self, val: MetaStateVal) { - match self.vals_to_counts.entry(val) { - Entry::Vacant(_entry) => { - unreachable!() - } - Entry::Occupied(mut entry) => { - // The entry is now locked, so we can also safely update the relevant element of the SkipSet - let count = entry.get(); - let mut set_entry = (*count, *entry.key()); - if *count == 1 { - *entry.get_mut() -= 1; - self.counts_to_vals.remove(&set_entry); - entry.remove(); - } else { - *entry.get_mut() -= 1; - self.counts_to_vals.remove(&set_entry); - set_entry.0 -= 1; - self.counts_to_vals.insert(set_entry); - } - } + if let Entry::Occupied(entry) = self + .vals_to_counts + .entry(val) + .and_modify(|count| *count -= 1) + && *entry.get() == 0 + { + entry.remove(); } } + #[must_use] pub fn is_empty(&self) -> bool { self.vals_to_counts.is_empty() } - pub fn too_high_counts(&self, max: usize) -> impl Iterator + '_ { - tracing::debug!("metastate: {self:?}"); - self.counts_to_vals - .range((max, 0)..) - .map(|entry| entry.value().1) + #[must_use] + pub fn get(&self, val: &MetaStateVal) -> Option { + self.vals_to_counts.get(val).map(|count| *count) } } @@ -158,14 +132,17 @@ mod tests { sut.increment(key, val); } - dbg!(&sut); - - let too_highs: Vec<_> = sut - .get(key) - .expect("Should exist at this stage") - .too_high_counts(group_size) - .collect(); - assert_eq!(too_highs.len(), n_groups); + { + // We have to release the selected state_field before we can decrement it, otherwise we + // would deadlock on the DashMap lock. + let state_field = sut.get(key).expect("Should exist at this stage"); + for group in 0..n_groups { + assert_eq!( + state_field.get(&i64::try_from(group).unwrap()), + Some(group_size) + ); + } + } // Decrement in a different order vals.shuffle(&mut rand::rng()); diff --git a/opsqueue/src/consumer/dispatcher/mod.rs b/opsqueue/src/consumer/dispatcher/mod.rs index 0d1fa63..6b798b8 100644 --- a/opsqueue/src/consumer/dispatcher/mod.rs +++ b/opsqueue/src/consumer/dispatcher/mod.rs @@ -13,6 +13,7 @@ use libsqlite3_sys as ffi; use metastate::MetaState; use reserver::Reserver; use sqlx::QueryBuilder; +use std::ffi::CStr; use std::time::{Duration, Instant}; use tokio::sync::mpsc::UnboundedSender; use tokio_util::sync::CancellationToken; @@ -81,6 +82,61 @@ unsafe extern "C" fn sqlite_reserved_chunk_lookup_destructor(ptr: *mut std::ffi: let _boxed: Box> = unsafe { Box::from_raw(ptr.cast()) }; } +unsafe extern "C" fn sqlite_metadata_count_lookup( + context: *mut ffi::sqlite3_context, + n_args: i32, + args: *mut *mut ffi::sqlite3_value, +) { + if n_args != 2 { + tracing::error!( + n_args, + "opsqueue_metadata_count called with unexpected argument count" + ); + unsafe { ffi::sqlite3_result_null(context) }; + return; + } + + let user_data = unsafe { ffi::sqlite3_user_data(context) } + .cast_const() + .cast::>(); + if user_data.is_null() { + tracing::error!("opsqueue_metadata_count called without registered metastate user_data"); + unsafe { ffi::sqlite3_result_null(context) }; + return; + } + + let metadata_key_ptr = unsafe { ffi::sqlite3_value_text(*args.add(0)) }; + if metadata_key_ptr.is_null() { + unsafe { ffi::sqlite3_result_null(context) }; + return; + } + let Ok(metadata_key) = unsafe { CStr::from_ptr(metadata_key_ptr.cast()) }.to_str() else { + tracing::error!("opsqueue_metadata_count got non-utf8 metadata_key"); + unsafe { ffi::sqlite3_result_null(context) }; + return; + }; + + let metadata_value = unsafe { ffi::sqlite3_value_int64(*args.add(1)) }; + + if let Some(meta_count) = unsafe { &*user_data } + .get(metadata_key) + .and_then(|meta_keys| meta_keys.get(&metadata_value)) + { + unsafe { + ffi::sqlite3_result_int64(context, i64::try_from(meta_count).unwrap_or(i64::MAX)); + }; + } else { + unsafe { ffi::sqlite3_result_null(context) }; + } +} + +unsafe extern "C" fn sqlite_metadata_count_lookup_destructor(ptr: *mut std::ffi::c_void) { + if ptr.is_null() { + return; + } + let _boxed: Box> = unsafe { Box::from_raw(ptr.cast()) }; +} + #[derive(Debug, Clone)] pub struct Dispatcher { reserver: Reserver, @@ -134,8 +190,7 @@ impl Dispatcher { stale_chunks_notifier: &UnboundedSender, ) -> Result, sqlx::Error> { let mut conn = pool.reader_conn().await?; - self.register_reserved_chunk_lookup(conn.get_inner()) - .await?; + self.register_lookups(conn.get_inner()).await?; let mut query_builder = QueryBuilder::new(""); let stream = strategy .build_query(&mut query_builder, &self.metastate) @@ -151,13 +206,11 @@ impl Dispatcher { .await } - async fn register_reserved_chunk_lookup( - &self, - conn: &mut sqlx::SqliteConnection, - ) -> Result<(), sqlx::Error> { + async fn register_lookups(&self, conn: &mut sqlx::SqliteConnection) -> Result<(), sqlx::Error> { let mut handle = conn.lock_handle().await?; let sqlite = handle.as_raw_handle().as_ptr(); - let function_name = b"opsqueue_is_reserved\0"; + let reserved_function_name = b"opsqueue_is_reserved\0"; + let metadata_count_function_name = b"opsqueue_metadata_count\0"; // Register the current reserver state on this connection. // Re-registering replaces any previous callback on this handle. @@ -167,7 +220,7 @@ impl Dispatcher { let rc = unsafe { ffi::sqlite3_create_function_v2( sqlite, - function_name.as_ptr().cast(), + reserved_function_name.as_ptr().cast(), 2, ffi::SQLITE_UTF8, user_data, @@ -186,6 +239,32 @@ impl Dispatcher { ))); } + // Register metadata count lookup backed by current metastate. + // Re-registering replaces any previous callback on this handle. + let user_data = Box::new(self.metastate.clone()); + let user_data = Box::into_raw(user_data).cast::(); + + let rc = unsafe { + ffi::sqlite3_create_function_v2( + sqlite, + metadata_count_function_name.as_ptr().cast(), + 2, + ffi::SQLITE_UTF8, + user_data, + Some(sqlite_metadata_count_lookup), + None, + None, + Some(sqlite_metadata_count_lookup_destructor), + ) + }; + + if rc != ffi::SQLITE_OK { + unsafe { sqlite_metadata_count_lookup_destructor(user_data) }; + return Err(sqlx::Error::Protocol(format!( + "sqlite3_create_function_v2 failed with rc={rc}" + ))); + } + Ok(()) } From 573b2d62cab8b52ba02ef9d95c0bd8563af48cc6 Mon Sep 17 00:00:00 2001 From: "jeremy.barisch.rooney@channable.com" Date: Thu, 6 Aug 2026 18:51:49 +0200 Subject: [PATCH 10/17] fixup! SQL: Add metadata count FFI function Call the metadata count FFI function --- Cargo.lock | 12 - opsqueue/Cargo.toml | 1 - opsqueue/src/consumer/dispatcher/metastate.rs | 2 +- opsqueue/src/consumer/dispatcher/mod.rs | 2 +- opsqueue/src/consumer/strategy.rs | 255 +++++++++--------- workspace-hack/Cargo.toml | 1 - 6 files changed, 125 insertions(+), 148 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cf1f582..fe470f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -595,16 +595,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "crossbeam-skiplist" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - [[package]] name = "crossbeam-utils" version = "0.8.22" @@ -2111,7 +2101,6 @@ dependencies = [ "chrono", "ciborium", "clap", - "crossbeam-skiplist", "dashmap", "either", "futures", @@ -4363,7 +4352,6 @@ dependencies = [ "bitflags", "cc", "chrono", - "crossbeam-epoch", "crossbeam-utils", "either", "event-listener", diff --git a/opsqueue/Cargo.toml b/opsqueue/Cargo.toml index d8859fc..75daf32 100644 --- a/opsqueue/Cargo.toml +++ b/opsqueue/Cargo.toml @@ -70,7 +70,6 @@ clap.workspace = true humantime.workspace = true dashmap.workspace = true -crossbeam-skiplist.workspace = true sqlformat.workspace = true workspace-hack.workspace = true diff --git a/opsqueue/src/consumer/dispatcher/metastate.rs b/opsqueue/src/consumer/dispatcher/metastate.rs index 7043631..20fbb40 100644 --- a/opsqueue/src/consumer/dispatcher/metastate.rs +++ b/opsqueue/src/consumer/dispatcher/metastate.rs @@ -68,7 +68,7 @@ pub type MetaStateVal = i64; #[derive(Debug, Default)] pub struct MetaStateField { - pub vals_to_counts: DashMap, + vals_to_counts: DashMap, } impl MetaStateField { diff --git a/opsqueue/src/consumer/dispatcher/mod.rs b/opsqueue/src/consumer/dispatcher/mod.rs index 6b798b8..28bb1d8 100644 --- a/opsqueue/src/consumer/dispatcher/mod.rs +++ b/opsqueue/src/consumer/dispatcher/mod.rs @@ -193,7 +193,7 @@ impl Dispatcher { self.register_lookups(conn.get_inner()).await?; let mut query_builder = QueryBuilder::new(""); let stream = strategy - .build_query(&mut query_builder, &self.metastate) + .build_query(&mut query_builder) .build_query_as() .fetch(conn.get_inner()); stream diff --git a/opsqueue/src/consumer/strategy.rs b/opsqueue/src/consumer/strategy.rs index e554d1e..016efb6 100644 --- a/opsqueue/src/consumer/strategy.rs +++ b/opsqueue/src/consumer/strategy.rs @@ -7,9 +7,6 @@ use sqlx::{QueryBuilder, Sqlite}; #[cfg(feature = "server-logic")] use crate::common::chunk::Chunk; -#[cfg(feature = "server-logic")] -use super::dispatcher::metastate::MetaState; - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum Strategy { Oldest, @@ -26,9 +23,8 @@ impl Strategy { pub fn build_query<'a>( &'a self, qb: &'a mut QueryBuilder, - metastate: &MetaState, ) -> &'a mut QueryBuilder { - let qb = self.build_query_snippet_returning_chunks(qb, metastate); + let qb = self.build_query_snippet_returning_chunks(qb); tracing::trace!("sql: {:?}", qb.sql()); qb } @@ -36,7 +32,6 @@ impl Strategy { fn build_query_snippet_returning_chunks<'a>( &'a self, qb: &'a mut QueryBuilder, - metastate: &MetaState, ) -> &'a mut QueryBuilder { use Strategy::{Newest, Oldest, PreferDistinct, Random}; let ffi_is_reserved = "opsqueue_is_reserved(chunks.submission_id, chunks.chunk_index) = 0"; @@ -53,7 +48,7 @@ impl Strategy { PreferDistinct { .. } => { // Unique submission IDs from the underlying strategy. let qb = qb.push("WITH underlying_submission_ids AS MATERIALIZED ("); - let qb = self.build_query_snippet_returning_submission_ids(qb, metastate); + let qb = self.build_query_snippet_returning_submission_ids(qb); qb.push(") "); // In SQLite, CROSS JOIN ON/WHERE does NOT produce N // x M rows, it acts as an INNER JOIN but forces the query @@ -74,7 +69,6 @@ impl Strategy { fn build_query_snippet_returning_submission_ids<'a>( &'a self, qb: &'a mut QueryBuilder, - metastate: &MetaState, ) -> &'a mut QueryBuilder { use Strategy::{Newest, Oldest, PreferDistinct, Random}; match self { @@ -87,44 +81,25 @@ impl Strategy { } => { // Unique submission IDs from the underlying strategy. let qb = qb.push("WITH inner AS NOT MATERIALIZED ("); - let qb = underlying.build_query_snippet_returning_submission_ids(qb, metastate); + let qb = underlying.build_query_snippet_returning_submission_ids(qb); qb.push("),"); - // Count of in-flight chunks per submission. - qb.push("counts AS (SELECT key, value FROM json_each("); - match metastate.get(meta_key) { - None => { - tracing::trace!("No metastate field for key: {meta_key}"); - qb.push_bind("{}"); - } - Some(field) => { - let counts_map: std::collections::HashMap<_, _> = field - .vals_to_counts - .iter() - .map(|kv| (*kv.key(), *kv.value())) - .collect(); - let counts_json = - serde_json::to_string(&counts_map).expect("Always valid JSON"); - tracing::trace!( - "Granular active counts for PreferDistinct: {counts_json:?}" - ); - qb.push_bind(counts_json); - } - } - qb.push(")),"); - // Submissions ranked by in-flight chunks. + // In-flight chunk count per submission, read via FFI. + qb.push("counts AS (SELECT submission_id, opsqueue_metadata_count("); + qb.push_bind(meta_key); + qb.push( + ", metadata_value) AS count FROM submissions_metadata WHERE metadata_key = ", + ); + qb.push_bind(meta_key); + qb.push("),"); + // Submissions ranked by in-flight chunks. Submissions without a + // value for this key get a NULL count and so are ranked first. qb.push( // MATERIALIZED is necessary to preserve the order. "ranked_submissions AS MATERIALIZED ( SELECT inner.submission_id FROM inner - LEFT JOIN submissions_metadata sm - ON inner.submission_id = sm.submission_id - AND sm.metadata_key = ", - ); - qb.push_bind(meta_key); - qb.push( - " LEFT JOIN counts c ON sm.metadata_value = c.key - ORDER BY c.value ASC NULLS FIRST + LEFT JOIN counts c ON inner.submission_id = c.submission_id + ORDER BY c.count ASC NULLS FIRST )", ); qb.push(" SELECT submission_id FROM ranked_submissions") @@ -180,7 +155,15 @@ pub mod test { unsafe { ffi::sqlite3_result_int(context, 0) }; } - async fn register_reserved_lookup_noop(conn: &mut SqliteConnection) { + unsafe extern "C" fn sqlite_metadata_count_lookup_noop( + context: *mut ffi::sqlite3_context, + _n_args: i32, + _args: *mut *mut ffi::sqlite3_value, + ) { + unsafe { ffi::sqlite3_result_null(context) }; + } + + async fn register_lookup_noops(conn: &mut SqliteConnection) { let mut handle = conn.lock_handle().await.unwrap(); let sqlite = handle.as_raw_handle().as_ptr(); let function_name = b"opsqueue_is_reserved\0"; @@ -198,6 +181,26 @@ pub mod test { ) }; assert_eq!(rc, ffi::SQLITE_OK, "register opsqueue_is_reserved failed"); + + let function_name = b"opsqueue_metadata_count\0"; + let rc = unsafe { + ffi::sqlite3_create_function_v2( + sqlite, + function_name.as_ptr().cast(), + 2, + ffi::SQLITE_UTF8, + std::ptr::null_mut(), + Some(sqlite_metadata_count_lookup_noop), + None, + None, + None, + ) + }; + assert_eq!( + rc, + ffi::SQLITE_OK, + "register opsqueue_metadata_count failed" + ); } async fn explain(qb: &mut sqlx::QueryBuilder, conn: &mut SqliteConnection) -> String { @@ -259,11 +262,10 @@ pub mod test { #[sqlx::test(migrator = "crate::MIGRATOR")] pub async fn test_query_plan_oldest(db: sqlx::SqlitePool) { let mut conn = db.acquire().await.unwrap(); - register_reserved_lookup_noop(&mut conn).await; + register_lookup_noops(&mut conn).await; let mut qb = QueryBuilder::new(""); - let metastate = MetaState::default(); - let qb = Strategy::Oldest.build_query(&mut qb, &metastate); + let qb = Strategy::Oldest.build_query(&mut qb); let options = FormatOptions::default(); let formatted_query = format(qb.sql().as_str(), &QueryParams::None, &options); insta::assert_snapshot!(formatted_query, @" @@ -285,11 +287,10 @@ pub mod test { #[sqlx::test(migrator = "crate::MIGRATOR")] pub async fn test_query_plan_newest(db: sqlx::SqlitePool) { let mut conn = db.acquire().await.unwrap(); - register_reserved_lookup_noop(&mut conn).await; + register_lookup_noops(&mut conn).await; let mut qb = QueryBuilder::new(""); - let metastate = MetaState::default(); - let qb = Strategy::Newest.build_query(&mut qb, &metastate); + let qb = Strategy::Newest.build_query(&mut qb); let options = FormatOptions::default(); let formatted_query = format(qb.sql().as_str(), &QueryParams::None, &options); insta::assert_snapshot!(formatted_query, @" @@ -311,11 +312,10 @@ pub mod test { #[sqlx::test(migrator = "crate::MIGRATOR")] pub async fn test_query_plan_random(db: sqlx::SqlitePool) { let mut conn = db.acquire().await.unwrap(); - register_reserved_lookup_noop(&mut conn).await; - let metastate = MetaState::default(); + register_lookup_noops(&mut conn).await; let mut qb = QueryBuilder::new(""); - let qb = Strategy::Random.build_query(&mut qb, &metastate); + let qb = Strategy::Random.build_query(&mut qb); let formatted_query = format( qb.sql().as_str(), @@ -355,15 +355,14 @@ pub mod test { pub async fn test_query_plan_prefer_distinct_oldest(db: sqlx::SqlitePool) { use Strategy::*; let mut conn = db.acquire().await.unwrap(); - register_reserved_lookup_noop(&mut conn).await; - let metastate = MetaState::default(); + register_lookup_noops(&mut conn).await; let strategy = PreferDistinct { meta_key: "company_id".to_string(), underlying: Box::new(Oldest), }; let mut qb = QueryBuilder::new(""); - let qb = strategy.build_query(&mut qb, &metastate); + let qb = strategy.build_query(&mut qb); let formatted_query = format( qb.sql().as_str(), @@ -384,21 +383,21 @@ pub mod test { ), counts AS ( SELECT - key, - value + submission_id, + opsqueue_metadata_count(?, metadata_value) AS count FROM - json_each(?) + submissions_metadata + WHERE + metadata_key = ? ), ranked_submissions AS MATERIALIZED ( SELECT inner.submission_id FROM inner - LEFT JOIN submissions_metadata sm ON inner.submission_id = sm.submission_id - AND sm.metadata_key = ? - LEFT JOIN counts c ON sm.metadata_value = c.key + LEFT JOIN counts c ON inner.submission_id = c.submission_id ORDER BY - c.value ASC NULLS FIRST + c.count ASC NULLS FIRST ) SELECT submission_id @@ -418,13 +417,12 @@ pub mod test { insta::assert_snapshot!(explained, @" 3, 0, MATERIALIZE underlying_submission_ids 6, 3, MATERIALIZE ranked_submissions - 12, 6, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 - 14, 6, SEARCH sm USING PRIMARY KEY (submission_id=? AND metadata_key=?) LEFT-JOIN - 23, 6, SCAN json_each VIRTUAL TABLE INDEX 1: LEFT-JOIN - 46, 6, USE TEMP B-TREE FOR ORDER BY - 58, 3, SCAN ranked_submissions - 69, 0, SCAN underlying_submission_ids - 71, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) + 11, 6, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 + 13, 6, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) LEFT-JOIN + 35, 6, USE TEMP B-TREE FOR ORDER BY + 47, 3, SCAN ranked_submissions + 58, 0, SCAN underlying_submission_ids + 60, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) "); } @@ -432,15 +430,14 @@ pub mod test { pub async fn test_query_plan_prefer_distinct_newest(db: sqlx::SqlitePool) { use Strategy::*; let mut conn = db.acquire().await.unwrap(); - register_reserved_lookup_noop(&mut conn).await; - let metastate = MetaState::default(); + register_lookup_noops(&mut conn).await; let strategy = PreferDistinct { meta_key: "company_id".to_string(), underlying: Box::new(Newest), }; let mut qb = QueryBuilder::new(""); - let qb = strategy.build_query(&mut qb, &metastate); + let qb = strategy.build_query(&mut qb); let formatted_query = format( qb.sql().as_str(), @@ -461,21 +458,21 @@ pub mod test { ), counts AS ( SELECT - key, - value + submission_id, + opsqueue_metadata_count(?, metadata_value) AS count FROM - json_each(?) + submissions_metadata + WHERE + metadata_key = ? ), ranked_submissions AS MATERIALIZED ( SELECT inner.submission_id FROM inner - LEFT JOIN submissions_metadata sm ON inner.submission_id = sm.submission_id - AND sm.metadata_key = ? - LEFT JOIN counts c ON sm.metadata_value = c.key + LEFT JOIN counts c ON inner.submission_id = c.submission_id ORDER BY - c.value ASC NULLS FIRST + c.count ASC NULLS FIRST ) SELECT submission_id @@ -495,13 +492,12 @@ pub mod test { insta::assert_snapshot!(explained, @" 3, 0, MATERIALIZE underlying_submission_ids 6, 3, MATERIALIZE ranked_submissions - 12, 6, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 - 14, 6, SEARCH sm USING PRIMARY KEY (submission_id=? AND metadata_key=?) LEFT-JOIN - 23, 6, SCAN json_each VIRTUAL TABLE INDEX 1: LEFT-JOIN - 46, 6, USE TEMP B-TREE FOR ORDER BY - 58, 3, SCAN ranked_submissions - 69, 0, SCAN underlying_submission_ids - 71, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) + 11, 6, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 + 13, 6, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) LEFT-JOIN + 35, 6, USE TEMP B-TREE FOR ORDER BY + 47, 3, SCAN ranked_submissions + 58, 0, SCAN underlying_submission_ids + 60, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) "); } @@ -509,15 +505,14 @@ pub mod test { pub async fn test_query_plan_prefer_distinct_random(db: sqlx::SqlitePool) { use Strategy::*; let mut conn = db.acquire().await.unwrap(); - register_reserved_lookup_noop(&mut conn).await; - let metastate = MetaState::default(); + register_lookup_noops(&mut conn).await; let strategy = PreferDistinct { meta_key: "company_id".to_string(), underlying: Box::new(Random), }; let mut qb = QueryBuilder::new(""); - let qb = strategy.build_query(&mut qb, &metastate); + let qb = strategy.build_query(&mut qb); let formatted_query = format( qb.sql().as_str(), @@ -545,21 +540,21 @@ pub mod test { ), counts AS ( SELECT - key, - value + submission_id, + opsqueue_metadata_count(?, metadata_value) AS count FROM - json_each(?) + submissions_metadata + WHERE + metadata_key = ? ), ranked_submissions AS MATERIALIZED ( SELECT inner.submission_id FROM inner - LEFT JOIN submissions_metadata sm ON inner.submission_id = sm.submission_id - AND sm.metadata_key = ? - LEFT JOIN counts c ON sm.metadata_value = c.key + LEFT JOIN counts c ON inner.submission_id = c.submission_id ORDER BY - c.value ASC NULLS FIRST + c.count ASC NULLS FIRST ) SELECT submission_id @@ -585,13 +580,12 @@ pub mod test { 13, 10, SEARCH submissions USING INDEX random_submissions_order (random_order>?) 22, 9, UNION ALL 25, 22, SEARCH submissions USING INDEX random_submissions_order (random_order?) 25, 12, UNION ALL 28, 25, SEARCH submissions USING INDEX random_submissions_order (random_order = Strategy::Random - .build_query(&mut query_builder, &MetaState::default()) + .build_query(&mut query_builder) .build_query_as() .fetch(conn.get_inner()) .try_collect() @@ -757,7 +748,7 @@ pub mod test { let mut query_builder = QueryBuilder::default(); let vals2: Vec = Strategy::Random - .build_query(&mut query_builder, &MetaState::default()) + .build_query(&mut query_builder) .build_query_as() .fetch(conn.get_inner()) .try_collect() diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index af0c120..3dc7427 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -17,7 +17,6 @@ publish = false [dependencies] base64 = { version = "0.22" } chrono = { version = "0.4", features = ["serde"] } -crossbeam-epoch = { version = "0.9" } crossbeam-utils = { version = "0.8" } either = { version = "1", features = ["serde", "use_std"] } event-listener = { version = "5" } From 1d2f36abf0c183e04ab99c9a5ecaee5dac754fe1 Mon Sep 17 00:00:00 2001 From: "jeremy.barisch.rooney@channable.com" Date: Thu, 6 Aug 2026 19:16:29 +0200 Subject: [PATCH 11/17] Flatten PreferDistinct into a single query level --- opsqueue/src/consumer/strategy.rs | 207 ++++++++++++++++++------------ 1 file changed, 123 insertions(+), 84 deletions(-) diff --git a/opsqueue/src/consumer/strategy.rs b/opsqueue/src/consumer/strategy.rs index 016efb6..0cb04c9 100644 --- a/opsqueue/src/consumer/strategy.rs +++ b/opsqueue/src/consumer/strategy.rs @@ -18,6 +18,48 @@ pub enum Strategy { }, } +/// Iterator over the `meta_key`s of a chain of nested +/// [`Strategy::PreferDistinct`]. outermost first. Stops at the first +/// non-`PreferDistinct` strategy, which can afterwards be retrieved with +/// [`MetaKeysIter::take`]. +pub struct MetaKeysIter<'a> { + strategy: &'a Strategy, +} + +impl<'a> MetaKeysIter<'a> { + /// The first non-[`Strategy::PreferDistinct`] strategy in the chain. + #[must_use] + pub fn take(self) -> &'a Strategy { + self.strategy + } +} + +impl<'a> Iterator for MetaKeysIter<'a> { + type Item = &'a str; + + fn next(&mut self) -> Option { + match self.strategy { + Strategy::Oldest | Strategy::Newest | Strategy::Random => None, + Strategy::PreferDistinct { + meta_key, + underlying, + } => { + self.strategy = underlying.as_ref(); + Some(meta_key.as_str()) + } + } + } +} + +impl Strategy { + /// Iterate over the `meta_key`s of this chain of nested + /// [`Strategy::PreferDistinct`], outermost first. + #[must_use] + pub fn meta_keys(&self) -> MetaKeysIter<'_> { + MetaKeysIter { strategy: self } + } +} + #[cfg(feature = "server-logic")] impl Strategy { pub fn build_query<'a>( @@ -75,34 +117,48 @@ impl Strategy { Oldest => qb.push("SELECT id as submission_id FROM submissions ORDER BY id ASC"), Newest => qb.push("SELECT id as submission_id FROM submissions ORDER BY id DESC"), Random => Self::push_random_order_query(qb, "id as submission_id", "submissions", None), - PreferDistinct { - meta_key, - underlying, - } => { + PreferDistinct { .. } => { + // Nested `PreferDistinct`s are flattened into a single query + // level: rather than one CTE per level, we emit one `counts_N` + // CTE per meta key and a single multi-key `ORDER BY`. + let mut meta_keys_iter = self.meta_keys(); + let meta_keys: Vec<&str> = meta_keys_iter.by_ref().collect(); + let underlying = meta_keys_iter.take(); + // Unique submission IDs from the underlying strategy. let qb = qb.push("WITH inner AS NOT MATERIALIZED ("); let qb = underlying.build_query_snippet_returning_submission_ids(qb); - qb.push("),"); - // In-flight chunk count per submission, read via FFI. - qb.push("counts AS (SELECT submission_id, opsqueue_metadata_count("); - qb.push_bind(meta_key); - qb.push( - ", metadata_value) AS count FROM submissions_metadata WHERE metadata_key = ", - ); - qb.push_bind(meta_key); - qb.push("),"); + qb.push(")"); + // In-flight chunk count per submission, per meta key, read via FFI. + for (i, meta_key) in meta_keys.iter().enumerate() { + qb.push(format!( + ", counts_{i} AS (SELECT submission_id, opsqueue_metadata_count(" + )); + qb.push_bind(*meta_key); + qb.push( + ", metadata_value) AS count FROM submissions_metadata WHERE metadata_key = ", + ); + qb.push_bind(*meta_key); + qb.push(")"); + } // Submissions ranked by in-flight chunks. Submissions without a - // value for this key get a NULL count and so are ranked first. + // value for a key get a NULL count and so are ranked first. qb.push( // MATERIALIZED is necessary to preserve the order. - "ranked_submissions AS MATERIALIZED ( + ", ranked_submissions AS MATERIALIZED ( SELECT inner.submission_id - FROM inner - LEFT JOIN counts c ON inner.submission_id = c.submission_id - ORDER BY c.count ASC NULLS FIRST - )", + FROM inner", ); - qb.push(" SELECT submission_id FROM ranked_submissions") + for i in 0..meta_keys.len() { + qb.push(format!( + " LEFT JOIN counts_{i} ON inner.submission_id = counts_{i}.submission_id" + )); + } + for i in 0..meta_keys.len() { + qb.push(if i == 0 { " ORDER BY " } else { ", " }); + qb.push(format!("counts_{i}.count ASC NULLS FIRST")); + } + qb.push(") SELECT submission_id FROM ranked_submissions") } } } @@ -381,7 +437,7 @@ pub mod test { ORDER BY id ASC ), - counts AS ( + counts_0 AS ( SELECT submission_id, opsqueue_metadata_count(?, metadata_value) AS count @@ -395,9 +451,9 @@ pub mod test { inner.submission_id FROM inner - LEFT JOIN counts c ON inner.submission_id = c.submission_id + LEFT JOIN counts_0 ON inner.submission_id = counts_0.submission_id ORDER BY - c.count ASC NULLS FIRST + counts_0.count ASC NULLS FIRST ) SELECT submission_id @@ -456,7 +512,7 @@ pub mod test { ORDER BY id DESC ), - counts AS ( + counts_0 AS ( SELECT submission_id, opsqueue_metadata_count(?, metadata_value) AS count @@ -470,9 +526,9 @@ pub mod test { inner.submission_id FROM inner - LEFT JOIN counts c ON inner.submission_id = c.submission_id + LEFT JOIN counts_0 ON inner.submission_id = counts_0.submission_id ORDER BY - c.count ASC NULLS FIRST + counts_0.count ASC NULLS FIRST ) SELECT submission_id @@ -538,7 +594,7 @@ pub mod test { WHERE random_order < ? ), - counts AS ( + counts_0 AS ( SELECT submission_id, opsqueue_metadata_count(?, metadata_value) AS count @@ -552,9 +608,9 @@ pub mod test { inner.submission_id FROM inner - LEFT JOIN counts c ON inner.submission_id = c.submission_id + LEFT JOIN counts_0 ON inner.submission_id = counts_0.submission_id ORDER BY - c.count ASC NULLS FIRST + counts_0.count ASC NULLS FIRST ) SELECT submission_id @@ -616,46 +672,30 @@ pub mod test { underlying_submission_ids AS MATERIALIZED ( WITH inner AS NOT MATERIALIZED ( - WITH - inner AS NOT MATERIALIZED ( - SELECT - id as submission_id - FROM - submissions - WHERE - random_order >= ? - UNION ALL - SELECT - id as submission_id - FROM - submissions - WHERE - random_order < ? - ), - counts AS ( - SELECT - submission_id, - opsqueue_metadata_count(?, metadata_value) AS count - FROM - submissions_metadata - WHERE - metadata_key = ? - ), - ranked_submissions AS MATERIALIZED ( - SELECT - inner.submission_id - FROM - inner - LEFT JOIN counts c ON inner.submission_id = c.submission_id - ORDER BY - c.count ASC NULLS FIRST - ) SELECT - submission_id + id as submission_id FROM - ranked_submissions + submissions + WHERE + random_order >= ? + UNION ALL + SELECT + id as submission_id + FROM + submissions + WHERE + random_order < ? ), - counts AS ( + counts_0 AS ( + SELECT + submission_id, + opsqueue_metadata_count(?, metadata_value) AS count + FROM + submissions_metadata + WHERE + metadata_key = ? + ), + counts_1 AS ( SELECT submission_id, opsqueue_metadata_count(?, metadata_value) AS count @@ -669,9 +709,11 @@ pub mod test { inner.submission_id FROM inner - LEFT JOIN counts c ON inner.submission_id = c.submission_id + LEFT JOIN counts_0 ON inner.submission_id = counts_0.submission_id + LEFT JOIN counts_1 ON inner.submission_id = counts_1.submission_id ORDER BY - c.count ASC NULLS FIRST + counts_0.count ASC NULLS FIRST, + counts_1.count ASC NULLS FIRST ) SELECT submission_id @@ -691,22 +733,19 @@ pub mod test { insta::assert_snapshot!(explained, @" 3, 0, MATERIALIZE underlying_submission_ids 6, 3, MATERIALIZE ranked_submissions - 9, 6, MATERIALIZE ranked_submissions - 11, 9, CO-ROUTINE inner - 12, 11, COMPOUND QUERY - 13, 12, LEFT-MOST SUBQUERY - 16, 13, SEARCH submissions USING INDEX random_submissions_order (random_order>?) - 25, 12, UNION ALL - 28, 25, SEARCH submissions USING INDEX random_submissions_order (random_order?) + 22, 9, UNION ALL + 25, 22, SEARCH submissions USING INDEX random_submissions_order (random_order Date: Thu, 6 Aug 2026 19:22:01 +0200 Subject: [PATCH 12/17] fixup! Flatten PreferDistinct into a single query level Avoid duplicate MATERIALIZE --- opsqueue/src/consumer/strategy.rs | 151 +++++++++++------------------- 1 file changed, 57 insertions(+), 94 deletions(-) diff --git a/opsqueue/src/consumer/strategy.rs b/opsqueue/src/consumer/strategy.rs index 0cb04c9..a89bfd4 100644 --- a/opsqueue/src/consumer/strategy.rs +++ b/opsqueue/src/consumer/strategy.rs @@ -143,12 +143,7 @@ impl Strategy { } // Submissions ranked by in-flight chunks. Submissions without a // value for a key get a NULL count and so are ranked first. - qb.push( - // MATERIALIZED is necessary to preserve the order. - ", ranked_submissions AS MATERIALIZED ( - SELECT inner.submission_id - FROM inner", - ); + qb.push(" SELECT inner.submission_id FROM inner"); for i in 0..meta_keys.len() { qb.push(format!( " LEFT JOIN counts_{i} ON inner.submission_id = counts_{i}.submission_id" @@ -158,7 +153,7 @@ impl Strategy { qb.push(if i == 0 { " ORDER BY " } else { ", " }); qb.push(format!("counts_{i}.count ASC NULLS FIRST")); } - qb.push(") SELECT submission_id FROM ranked_submissions") + qb } } } @@ -445,20 +440,14 @@ pub mod test { submissions_metadata WHERE metadata_key = ? - ), - ranked_submissions AS MATERIALIZED ( - SELECT - inner.submission_id - FROM - inner - LEFT JOIN counts_0 ON inner.submission_id = counts_0.submission_id - ORDER BY - counts_0.count ASC NULLS FIRST ) SELECT - submission_id + inner.submission_id FROM - ranked_submissions + inner + LEFT JOIN counts_0 ON inner.submission_id = counts_0.submission_id + ORDER BY + counts_0.count ASC NULLS FIRST ) SELECT chunks.* @@ -472,13 +461,11 @@ pub mod test { assert_streaming_chunks(qb, &explained); insta::assert_snapshot!(explained, @" 3, 0, MATERIALIZE underlying_submission_ids - 6, 3, MATERIALIZE ranked_submissions - 11, 6, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 - 13, 6, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) LEFT-JOIN - 35, 6, USE TEMP B-TREE FOR ORDER BY - 47, 3, SCAN ranked_submissions - 58, 0, SCAN underlying_submission_ids - 60, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) + 8, 3, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 + 10, 3, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) LEFT-JOIN + 32, 3, USE TEMP B-TREE FOR ORDER BY + 44, 0, SCAN underlying_submission_ids + 46, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) "); } @@ -520,20 +507,14 @@ pub mod test { submissions_metadata WHERE metadata_key = ? - ), - ranked_submissions AS MATERIALIZED ( - SELECT - inner.submission_id - FROM - inner - LEFT JOIN counts_0 ON inner.submission_id = counts_0.submission_id - ORDER BY - counts_0.count ASC NULLS FIRST ) SELECT - submission_id + inner.submission_id FROM - ranked_submissions + inner + LEFT JOIN counts_0 ON inner.submission_id = counts_0.submission_id + ORDER BY + counts_0.count ASC NULLS FIRST ) SELECT chunks.* @@ -547,13 +528,11 @@ pub mod test { assert_streaming_chunks(qb, &explained); insta::assert_snapshot!(explained, @" 3, 0, MATERIALIZE underlying_submission_ids - 6, 3, MATERIALIZE ranked_submissions - 11, 6, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 - 13, 6, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) LEFT-JOIN - 35, 6, USE TEMP B-TREE FOR ORDER BY - 47, 3, SCAN ranked_submissions - 58, 0, SCAN underlying_submission_ids - 60, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) + 8, 3, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 + 10, 3, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) LEFT-JOIN + 32, 3, USE TEMP B-TREE FOR ORDER BY + 44, 0, SCAN underlying_submission_ids + 46, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) "); } @@ -602,20 +581,14 @@ pub mod test { submissions_metadata WHERE metadata_key = ? - ), - ranked_submissions AS MATERIALIZED ( - SELECT - inner.submission_id - FROM - inner - LEFT JOIN counts_0 ON inner.submission_id = counts_0.submission_id - ORDER BY - counts_0.count ASC NULLS FIRST ) SELECT - submission_id + inner.submission_id FROM - ranked_submissions + inner + LEFT JOIN counts_0 ON inner.submission_id = counts_0.submission_id + ORDER BY + counts_0.count ASC NULLS FIRST ) SELECT chunks.* @@ -629,19 +602,17 @@ pub mod test { assert_streaming_chunks(qb, &explained); insta::assert_snapshot!(explained, @" 3, 0, MATERIALIZE underlying_submission_ids - 6, 3, MATERIALIZE ranked_submissions - 8, 6, CO-ROUTINE inner - 9, 8, COMPOUND QUERY - 10, 9, LEFT-MOST SUBQUERY - 13, 10, SEARCH submissions USING INDEX random_submissions_order (random_order>?) - 22, 9, UNION ALL - 25, 22, SEARCH submissions USING INDEX random_submissions_order (random_order?) + 19, 6, UNION ALL + 22, 19, SEARCH submissions USING INDEX random_submissions_order (random_order?) - 22, 9, UNION ALL - 25, 22, SEARCH submissions USING INDEX random_submissions_order (random_order?) + 19, 6, UNION ALL + 22, 19, SEARCH submissions USING INDEX random_submissions_order (random_order Date: Thu, 6 Aug 2026 21:07:25 +0200 Subject: [PATCH 13/17] SQL: Deduplicate FFI calls per distinct metadata value --- opsqueue/benches/chunks_select_bench.svg | 6020 +++++++++++----------- opsqueue/src/consumer/strategy.rs | 188 +- 2 files changed, 3071 insertions(+), 3137 deletions(-) diff --git a/opsqueue/benches/chunks_select_bench.svg b/opsqueue/benches/chunks_select_bench.svg index 1df4bd6..9bc581f 100644 --- a/opsqueue/benches/chunks_select_bench.svg +++ b/opsqueue/benches/chunks_select_bench.svg @@ -21,182 +21,182 @@ - - - - - + + - - - + + - + - - + - - - @@ -208,49 +208,49 @@ z - + - + - @@ -262,36 +262,36 @@ z - + - + - @@ -303,42 +303,42 @@ z - + - + - @@ -350,47 +350,47 @@ z - + - + - @@ -402,462 +402,462 @@ z - + - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -865,356 +865,356 @@ L 529.92655 48.482812 - - - - - - - - - - - - - - - - - @@ -1251,23 +1251,23 @@ z - + - - + - + @@ -1276,18 +1276,18 @@ L -3.5 0 - + - + - + @@ -1296,18 +1296,18 @@ L 532.078437 270.441087 - + - + - + @@ -1316,18 +1316,18 @@ L 532.078437 214.951518 - + - + - + @@ -1336,18 +1336,18 @@ L 532.078437 159.46195 - + - + - + @@ -1356,27 +1356,27 @@ L 532.078437 103.972381 - + - + - @@ -1388,558 +1388,486 @@ z - + - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -1947,219 +1875,219 @@ L 532.078437 51.021876 - - - - - - - - - - - - @@ -2218,261 +2146,261 @@ z - - + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - @@ -2504,116 +2432,116 @@ z - - - + - + - - - - - - @@ -2657,45 +2585,45 @@ z - - + - + - @@ -2711,105 +2639,105 @@ z - - - - + + - - - + + - - + + - + - + @@ -2822,14 +2750,14 @@ L 598.962647 48.482812 - - + + - + - + @@ -2842,14 +2770,14 @@ L 700.518017 48.482812 - - + + - + - + @@ -2862,14 +2790,14 @@ L 802.073386 48.482812 - - + + - + - + @@ -2882,14 +2810,14 @@ L 903.628756 48.482812 - - + + - + - + @@ -2902,458 +2830,458 @@ L 1005.184125 48.482812 - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + @@ -3391,115 +3319,115 @@ L 1066.32655 48.482812 - - - + + + - + - + - + - - - + + + - + - + - + - - - + + + - + - + - + - - - + + + - + - + - + - - - + + + - + - + - + - - - + + + - + - + @@ -3511,543 +3439,507 @@ L 1068.478437 48.482812 - - - + + + - + - + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -4109,94 +4001,94 @@ L 1068.478437 51.064955 - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - @@ -4230,25 +4122,25 @@ L 1068.478437 48.482812 - - - + - + @@ -4294,13 +4186,13 @@ L 877.634844 60.271406 - - + - + @@ -4318,137 +4210,137 @@ L 877.634844 73.772109 - - - - + + - - - + + - - + + - + - + @@ -4461,14 +4353,14 @@ L 1141.75686 48.482812 - - + + - + - + @@ -4481,14 +4373,14 @@ L 1241.929174 48.482812 - - + + - + - + @@ -4501,14 +4393,14 @@ L 1342.101487 48.482812 - - + + - + - + @@ -4521,14 +4413,14 @@ L 1442.273801 48.482812 - - + + - + - + @@ -4541,470 +4433,470 @@ L 1542.446114 48.482812 - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + - - + + - + - + @@ -5042,115 +4934,115 @@ L 1602.755856 48.482812 - - - + + + - + - + - + - - - + + + - + - + - + - - - + + + - + - + - + - - - + + + - + - + - + - - - + + + - + - + - + - - - + + + - + - + @@ -5162,567 +5054,507 @@ L 1604.878437 48.482812 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -5784,126 +5616,126 @@ L 1604.878437 50.987189 - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - @@ -5922,25 +5754,25 @@ L 1604.878437 48.482812 - - - + - + @@ -5986,13 +5818,13 @@ L 1414.034844 60.271406 - - + - + @@ -6012,30 +5844,30 @@ L 1414.034844 73.772109 - @@ -6084,13 +5916,13 @@ z - + - + - + diff --git a/opsqueue/src/consumer/strategy.rs b/opsqueue/src/consumer/strategy.rs index a89bfd4..17e3c11 100644 --- a/opsqueue/src/consumer/strategy.rs +++ b/opsqueue/src/consumer/strategy.rs @@ -132,11 +132,23 @@ impl Strategy { // In-flight chunk count per submission, per meta key, read via FFI. for (i, meta_key) in meta_keys.iter().enumerate() { qb.push(format!( - ", counts_{i} AS (SELECT submission_id, opsqueue_metadata_count(" + ", counts_{i} AS ( + SELECT sm.submission_id, ffi_counts.count + FROM submissions_metadata sm + JOIN ( + SELECT metadata_value, opsqueue_metadata_count(" )); qb.push_bind(*meta_key); qb.push( - ", metadata_value) AS count FROM submissions_metadata WHERE metadata_key = ", + ", metadata_value) AS count + FROM submissions_metadata + WHERE metadata_key = ", + ); + qb.push_bind(*meta_key); + qb.push( + " GROUP BY metadata_value + ) ffi_counts ON sm.metadata_value = ffi_counts.metadata_value + WHERE sm.metadata_key = ", ); qb.push_bind(*meta_key); qb.push(")"); @@ -434,12 +446,23 @@ pub mod test { ), counts_0 AS ( SELECT - submission_id, - opsqueue_metadata_count(?, metadata_value) AS count + sm.submission_id, + ffi_counts.count FROM - submissions_metadata + submissions_metadata sm + JOIN ( + SELECT + metadata_value, + opsqueue_metadata_count(?, metadata_value) AS count + FROM + submissions_metadata + WHERE + metadata_key = ? + GROUP BY + metadata_value + ) ffi_counts ON sm.metadata_value = ffi_counts.metadata_value WHERE - metadata_key = ? + sm.metadata_key = ? ) SELECT inner.submission_id @@ -461,11 +484,18 @@ pub mod test { assert_streaming_chunks(qb, &explained); insta::assert_snapshot!(explained, @" 3, 0, MATERIALIZE underlying_submission_ids - 8, 3, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 - 10, 3, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) LEFT-JOIN - 32, 3, USE TEMP B-TREE FOR ORDER BY - 44, 0, SCAN underlying_submission_ids - 46, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) + 6, 3, MATERIALIZE counts_0 + 8, 6, CO-ROUTINE ffi_counts + 14, 8, SEARCH submissions_metadata USING COVERING INDEX lookup_submission_by_metadata (metadata_key=?) + 48, 6, SEARCH sm USING COVERING INDEX lookup_submission_by_metadata (metadata_key=?) + 56, 6, BLOOM FILTER ON ffi_counts (metadata_value=?) + 68, 6, SEARCH ffi_counts USING AUTOMATIC COVERING INDEX (metadata_value=?) + 85, 3, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 + 91, 3, BLOOM FILTER ON counts_0 (submission_id=?) + 101, 3, SEARCH counts_0 USING AUTOMATIC COVERING INDEX (submission_id=?) LEFT-JOIN + 119, 3, USE TEMP B-TREE FOR ORDER BY + 131, 0, SCAN underlying_submission_ids + 133, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) "); } @@ -501,12 +531,23 @@ pub mod test { ), counts_0 AS ( SELECT - submission_id, - opsqueue_metadata_count(?, metadata_value) AS count + sm.submission_id, + ffi_counts.count FROM - submissions_metadata + submissions_metadata sm + JOIN ( + SELECT + metadata_value, + opsqueue_metadata_count(?, metadata_value) AS count + FROM + submissions_metadata + WHERE + metadata_key = ? + GROUP BY + metadata_value + ) ffi_counts ON sm.metadata_value = ffi_counts.metadata_value WHERE - metadata_key = ? + sm.metadata_key = ? ) SELECT inner.submission_id @@ -528,11 +569,18 @@ pub mod test { assert_streaming_chunks(qb, &explained); insta::assert_snapshot!(explained, @" 3, 0, MATERIALIZE underlying_submission_ids - 8, 3, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 - 10, 3, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) LEFT-JOIN - 32, 3, USE TEMP B-TREE FOR ORDER BY - 44, 0, SCAN underlying_submission_ids - 46, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) + 6, 3, MATERIALIZE counts_0 + 8, 6, CO-ROUTINE ffi_counts + 14, 8, SEARCH submissions_metadata USING COVERING INDEX lookup_submission_by_metadata (metadata_key=?) + 48, 6, SEARCH sm USING COVERING INDEX lookup_submission_by_metadata (metadata_key=?) + 56, 6, BLOOM FILTER ON ffi_counts (metadata_value=?) + 68, 6, SEARCH ffi_counts USING AUTOMATIC COVERING INDEX (metadata_value=?) + 85, 3, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 + 91, 3, BLOOM FILTER ON counts_0 (submission_id=?) + 101, 3, SEARCH counts_0 USING AUTOMATIC COVERING INDEX (submission_id=?) LEFT-JOIN + 119, 3, USE TEMP B-TREE FOR ORDER BY + 131, 0, SCAN underlying_submission_ids + 133, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) "); } @@ -575,12 +623,23 @@ pub mod test { ), counts_0 AS ( SELECT - submission_id, - opsqueue_metadata_count(?, metadata_value) AS count + sm.submission_id, + ffi_counts.count FROM - submissions_metadata + submissions_metadata sm + JOIN ( + SELECT + metadata_value, + opsqueue_metadata_count(?, metadata_value) AS count + FROM + submissions_metadata + WHERE + metadata_key = ? + GROUP BY + metadata_value + ) ffi_counts ON sm.metadata_value = ffi_counts.metadata_value WHERE - metadata_key = ? + sm.metadata_key = ? ) SELECT inner.submission_id @@ -608,11 +667,18 @@ pub mod test { 10, 7, SEARCH submissions USING INDEX random_submissions_order (random_order>?) 19, 6, UNION ALL 22, 19, SEARCH submissions USING INDEX random_submissions_order (random_order?) 19, 6, UNION ALL 22, 19, SEARCH submissions USING INDEX random_submissions_order (random_order Date: Thu, 6 Aug 2026 21:18:27 +0200 Subject: [PATCH 14/17] SQL: Single FFI call --- opsqueue/benches/chunks_select_bench.svg | 5344 ++++++++--------- opsqueue/src/consumer/dispatcher/metastate.rs | 16 + opsqueue/src/consumer/dispatcher/mod.rs | 72 + opsqueue/src/consumer/strategy.rs | 222 +- 4 files changed, 2840 insertions(+), 2814 deletions(-) diff --git a/opsqueue/benches/chunks_select_bench.svg b/opsqueue/benches/chunks_select_bench.svg index 9bc581f..840118c 100644 --- a/opsqueue/benches/chunks_select_bench.svg +++ b/opsqueue/benches/chunks_select_bench.svg @@ -21,182 +21,182 @@ - - - - - + + - - - + + - + - - + - - - @@ -208,49 +208,49 @@ z - + - + - @@ -262,36 +262,36 @@ z - + - + - @@ -303,42 +303,42 @@ z - + - + - @@ -350,47 +350,47 @@ z - + - + - @@ -402,462 +402,462 @@ z - + - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -865,356 +865,356 @@ L 529.92655 48.482812 - - - - - - - - - - - - - - - - - @@ -1251,23 +1251,23 @@ z - + - - + - + @@ -1276,18 +1276,18 @@ L -3.5 0 - + - + - + @@ -1296,18 +1296,18 @@ L 532.078437 290.128701 - + - + - + @@ -1316,18 +1316,18 @@ L 532.078437 229.717229 - + - + - + @@ -1336,18 +1336,18 @@ L 532.078437 169.305757 - + - + - + @@ -1356,27 +1356,27 @@ L 532.078437 108.894285 - + - + - @@ -1388,486 +1388,486 @@ z - + - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -1875,219 +1875,219 @@ L 532.078437 51.24709 - - - - - - - - - - - - @@ -2147,260 +2147,260 @@ z - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - @@ -2432,116 +2432,116 @@ z - - - + - - - - - - @@ -2586,44 +2586,44 @@ z - - + - @@ -2639,105 +2639,105 @@ z - - - - + + - - - + + - + - + @@ -2751,13 +2751,13 @@ L 598.962647 48.482812 - + - + @@ -2771,13 +2771,13 @@ L 700.518017 48.482812 - + - + @@ -2791,13 +2791,13 @@ L 802.073386 48.482812 - + - + @@ -2811,13 +2811,13 @@ L 903.628756 48.482812 - + - + @@ -2831,457 +2831,457 @@ L 1005.184125 48.482812 - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -3321,18 +3321,18 @@ L 1066.32655 48.482812 - + - + - + @@ -3341,18 +3341,18 @@ L 1068.478437 343.203221 - + - + - + @@ -3361,18 +3361,18 @@ L 1068.478437 284.259139 - + - + - + @@ -3381,18 +3381,18 @@ L 1068.478437 225.315058 - + - + - + @@ -3401,18 +3401,18 @@ L 1068.478437 166.370976 - + - + - + @@ -3421,13 +3421,13 @@ L 1068.478437 107.426894 - + - + @@ -3441,505 +3441,493 @@ L 1068.478437 48.482812 - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - + @@ -4001,94 +3989,94 @@ L 1068.478437 51.179946 - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - @@ -4122,25 +4110,25 @@ L 1068.478437 48.482812 - - - + - + @@ -4186,13 +4174,13 @@ L 877.634844 60.271406 - - + - + @@ -4210,137 +4198,137 @@ L 877.634844 73.772109 - - - - + + - - - + + - - + + - + - + @@ -4353,14 +4341,14 @@ L 1141.75686 48.482812 - - + + - + - + @@ -4373,14 +4361,14 @@ L 1241.929174 48.482812 - - + + - + - + @@ -4393,14 +4381,14 @@ L 1342.101487 48.482812 - - + + - + - + @@ -4413,14 +4401,14 @@ L 1442.273801 48.482812 - - + + - + - + @@ -4433,470 +4421,470 @@ L 1542.446114 48.482812 + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - + @@ -4934,115 +4922,115 @@ L 1602.755856 48.482812 - - - + + + - + - + - + - - - + + + - + - + - + - - - + + + - + - + - + - - - + + + - + - + - + - - - + + + - + - + - + - - - + + + - + - + @@ -5054,507 +5042,495 @@ L 1604.878437 48.482812 + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - + @@ -5616,126 +5592,126 @@ L 1604.878437 51.194772 - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - @@ -5754,25 +5730,25 @@ L 1604.878437 48.482812 - - - + - + @@ -5818,13 +5794,13 @@ L 1414.034844 60.271406 - - + - + @@ -5844,30 +5820,30 @@ L 1414.034844 73.772109 - @@ -5916,13 +5892,13 @@ z - + - + - + diff --git a/opsqueue/src/consumer/dispatcher/metastate.rs b/opsqueue/src/consumer/dispatcher/metastate.rs index 20fbb40..9eb9993 100644 --- a/opsqueue/src/consumer/dispatcher/metastate.rs +++ b/opsqueue/src/consumer/dispatcher/metastate.rs @@ -104,6 +104,22 @@ impl MetaStateField { pub fn get(&self, val: &MetaStateVal) -> Option { self.vals_to_counts.get(val).map(|count| *count) } + + /// The whole value -> count map as a JSON object, for handing to `SQLite` + /// in a single FFI call. + #[must_use] + pub fn to_json(&self) -> String { + use std::fmt::Write as _; + let mut out = String::from("{"); + for entry in &self.vals_to_counts { + if out.len() > 1 { + out.push(','); + } + let _ = write!(out, "\"{}\":{}", entry.key(), entry.value()); + } + out.push('}'); + out + } } #[cfg(test)] diff --git a/opsqueue/src/consumer/dispatcher/mod.rs b/opsqueue/src/consumer/dispatcher/mod.rs index 28bb1d8..a8f36d6 100644 --- a/opsqueue/src/consumer/dispatcher/mod.rs +++ b/opsqueue/src/consumer/dispatcher/mod.rs @@ -130,6 +130,53 @@ unsafe extern "C" fn sqlite_metadata_count_lookup( } } +/// Returns the whole value -> count map for one metadata key as a JSON object, +/// so the caller can obtain every count in a single FFI call. +unsafe extern "C" fn sqlite_metadata_counts_lookup( + context: *mut ffi::sqlite3_context, + n_args: i32, + args: *mut *mut ffi::sqlite3_value, +) { + if n_args != 1 { + tracing::error!( + n_args, + "opsqueue_metadata_counts called with unexpected argument count" + ); + unsafe { ffi::sqlite3_result_null(context) }; + return; + } + + let user_data = unsafe { ffi::sqlite3_user_data(context) } + .cast_const() + .cast::>(); + if user_data.is_null() { + tracing::error!("opsqueue_metadata_counts called without registered metastate user_data"); + unsafe { ffi::sqlite3_result_null(context) }; + return; + } + + let metadata_key_ptr = unsafe { ffi::sqlite3_value_text(*args.add(0)) }; + if metadata_key_ptr.is_null() { + unsafe { ffi::sqlite3_result_null(context) }; + return; + } + let Ok(metadata_key) = unsafe { CStr::from_ptr(metadata_key_ptr.cast()) }.to_str() else { + tracing::error!("opsqueue_metadata_counts got non-utf8 metadata_key"); + unsafe { ffi::sqlite3_result_null(context) }; + return; + }; + + let json = match unsafe { &*user_data }.get(metadata_key) { + Some(field) => field.to_json(), + None => "{}".to_string(), + }; + let len = i32::try_from(json.len()).unwrap_or(i32::MAX); + unsafe { + // SQLITE_TRANSIENT tells SQLite to copy the bytes before we drop them. + ffi::sqlite3_result_text(context, json.as_ptr().cast(), len, ffi::SQLITE_TRANSIENT()); + }; +} + unsafe extern "C" fn sqlite_metadata_count_lookup_destructor(ptr: *mut std::ffi::c_void) { if ptr.is_null() { return; @@ -265,6 +312,31 @@ impl Dispatcher { ))); } + // Register the bulk metadata counts lookup backed by current metastate. + let counts_function_name = b"opsqueue_metadata_counts\0"; + let user_data = Box::new(self.metastate.clone()); + let user_data = Box::into_raw(user_data).cast::(); + + let rc = unsafe { + ffi::sqlite3_create_function_v2( + sqlite, + counts_function_name.as_ptr().cast(), + 1, + ffi::SQLITE_UTF8, + user_data, + Some(sqlite_metadata_counts_lookup), + None, + None, + Some(sqlite_metadata_count_lookup_destructor), + ) + }; + + if rc != ffi::SQLITE_OK { + return Err(sqlx::Error::Protocol(format!( + "sqlite3_create_function_v2 failed with rc={rc}" + ))); + } + Ok(()) } diff --git a/opsqueue/src/consumer/strategy.rs b/opsqueue/src/consumer/strategy.rs index 17e3c11..250c99b 100644 --- a/opsqueue/src/consumer/strategy.rs +++ b/opsqueue/src/consumer/strategy.rs @@ -129,25 +129,22 @@ impl Strategy { let qb = qb.push("WITH inner AS NOT MATERIALIZED ("); let qb = underlying.build_query_snippet_returning_submission_ids(qb); qb.push(")"); - // In-flight chunk count per submission, per meta key, read via FFI. + // In-flight chunk count per submission, per meta key. + // + // The FFI call returns all counts as JSON in a single call. + // The CROSS JOIN ON ensures the json_each is the outer loop, + // and only performed once. for (i, meta_key) in meta_keys.iter().enumerate() { qb.push(format!( ", counts_{i} AS ( - SELECT sm.submission_id, ffi_counts.count - FROM submissions_metadata sm - JOIN ( - SELECT metadata_value, opsqueue_metadata_count(" + SELECT sm.submission_id, je.value AS count + FROM json_each(opsqueue_metadata_counts(" )); qb.push_bind(*meta_key); qb.push( - ", metadata_value) AS count - FROM submissions_metadata - WHERE metadata_key = ", - ); - qb.push_bind(*meta_key); - qb.push( - " GROUP BY metadata_value - ) ffi_counts ON sm.metadata_value = ffi_counts.metadata_value + ")) je + CROSS JOIN submissions_metadata sm + ON sm.metadata_value = CAST(je.key AS INTEGER) WHERE sm.metadata_key = ", ); qb.push_bind(*meta_key); @@ -218,6 +215,16 @@ pub mod test { unsafe { ffi::sqlite3_result_int(context, 0) }; } + unsafe extern "C" fn sqlite_metadata_counts_lookup_noop( + context: *mut ffi::sqlite3_context, + _n_args: i32, + _args: *mut *mut ffi::sqlite3_value, + ) { + unsafe { + ffi::sqlite3_result_text(context, c"{}".as_ptr(), 2, ffi::SQLITE_TRANSIENT()); + }; + } + unsafe extern "C" fn sqlite_metadata_count_lookup_noop( context: *mut ffi::sqlite3_context, _n_args: i32, @@ -264,6 +271,26 @@ pub mod test { ffi::SQLITE_OK, "register opsqueue_metadata_count failed" ); + + let function_name = b"opsqueue_metadata_counts\0"; + let rc = unsafe { + ffi::sqlite3_create_function_v2( + sqlite, + function_name.as_ptr().cast(), + 1, + ffi::SQLITE_UTF8, + std::ptr::null_mut(), + Some(sqlite_metadata_counts_lookup_noop), + None, + None, + None, + ) + }; + assert_eq!( + rc, + ffi::SQLITE_OK, + "register opsqueue_metadata_counts failed" + ); } async fn explain(qb: &mut sqlx::QueryBuilder, conn: &mut SqliteConnection) -> String { @@ -447,20 +474,10 @@ pub mod test { counts_0 AS ( SELECT sm.submission_id, - ffi_counts.count + je.value AS count FROM - submissions_metadata sm - JOIN ( - SELECT - metadata_value, - opsqueue_metadata_count(?, metadata_value) AS count - FROM - submissions_metadata - WHERE - metadata_key = ? - GROUP BY - metadata_value - ) ffi_counts ON sm.metadata_value = ffi_counts.metadata_value + json_each(opsqueue_metadata_counts(?)) je + CROSS JOIN submissions_metadata sm ON sm.metadata_value = CAST(je.key AS INTEGER) WHERE sm.metadata_key = ? ) @@ -485,17 +502,14 @@ pub mod test { insta::assert_snapshot!(explained, @" 3, 0, MATERIALIZE underlying_submission_ids 6, 3, MATERIALIZE counts_0 - 8, 6, CO-ROUTINE ffi_counts - 14, 8, SEARCH submissions_metadata USING COVERING INDEX lookup_submission_by_metadata (metadata_key=?) - 48, 6, SEARCH sm USING COVERING INDEX lookup_submission_by_metadata (metadata_key=?) - 56, 6, BLOOM FILTER ON ffi_counts (metadata_value=?) - 68, 6, SEARCH ffi_counts USING AUTOMATIC COVERING INDEX (metadata_value=?) - 85, 3, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 - 91, 3, BLOOM FILTER ON counts_0 (submission_id=?) - 101, 3, SEARCH counts_0 USING AUTOMATIC COVERING INDEX (submission_id=?) LEFT-JOIN - 119, 3, USE TEMP B-TREE FOR ORDER BY - 131, 0, SCAN underlying_submission_ids - 133, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) + 10, 6, SCAN je VIRTUAL TABLE INDEX 1: + 15, 6, SEARCH sm USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=?) + 35, 3, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 + 41, 3, BLOOM FILTER ON counts_0 (submission_id=?) + 51, 3, SEARCH counts_0 USING AUTOMATIC COVERING INDEX (submission_id=?) LEFT-JOIN + 69, 3, USE TEMP B-TREE FOR ORDER BY + 81, 0, SCAN underlying_submission_ids + 83, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) "); } @@ -532,20 +546,10 @@ pub mod test { counts_0 AS ( SELECT sm.submission_id, - ffi_counts.count + je.value AS count FROM - submissions_metadata sm - JOIN ( - SELECT - metadata_value, - opsqueue_metadata_count(?, metadata_value) AS count - FROM - submissions_metadata - WHERE - metadata_key = ? - GROUP BY - metadata_value - ) ffi_counts ON sm.metadata_value = ffi_counts.metadata_value + json_each(opsqueue_metadata_counts(?)) je + CROSS JOIN submissions_metadata sm ON sm.metadata_value = CAST(je.key AS INTEGER) WHERE sm.metadata_key = ? ) @@ -570,17 +574,14 @@ pub mod test { insta::assert_snapshot!(explained, @" 3, 0, MATERIALIZE underlying_submission_ids 6, 3, MATERIALIZE counts_0 - 8, 6, CO-ROUTINE ffi_counts - 14, 8, SEARCH submissions_metadata USING COVERING INDEX lookup_submission_by_metadata (metadata_key=?) - 48, 6, SEARCH sm USING COVERING INDEX lookup_submission_by_metadata (metadata_key=?) - 56, 6, BLOOM FILTER ON ffi_counts (metadata_value=?) - 68, 6, SEARCH ffi_counts USING AUTOMATIC COVERING INDEX (metadata_value=?) - 85, 3, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 - 91, 3, BLOOM FILTER ON counts_0 (submission_id=?) - 101, 3, SEARCH counts_0 USING AUTOMATIC COVERING INDEX (submission_id=?) LEFT-JOIN - 119, 3, USE TEMP B-TREE FOR ORDER BY - 131, 0, SCAN underlying_submission_ids - 133, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) + 10, 6, SCAN je VIRTUAL TABLE INDEX 1: + 15, 6, SEARCH sm USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=?) + 35, 3, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 + 41, 3, BLOOM FILTER ON counts_0 (submission_id=?) + 51, 3, SEARCH counts_0 USING AUTOMATIC COVERING INDEX (submission_id=?) LEFT-JOIN + 69, 3, USE TEMP B-TREE FOR ORDER BY + 81, 0, SCAN underlying_submission_ids + 83, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) "); } @@ -624,20 +625,10 @@ pub mod test { counts_0 AS ( SELECT sm.submission_id, - ffi_counts.count + je.value AS count FROM - submissions_metadata sm - JOIN ( - SELECT - metadata_value, - opsqueue_metadata_count(?, metadata_value) AS count - FROM - submissions_metadata - WHERE - metadata_key = ? - GROUP BY - metadata_value - ) ffi_counts ON sm.metadata_value = ffi_counts.metadata_value + json_each(opsqueue_metadata_counts(?)) je + CROSS JOIN submissions_metadata sm ON sm.metadata_value = CAST(je.key AS INTEGER) WHERE sm.metadata_key = ? ) @@ -668,17 +659,14 @@ pub mod test { 19, 6, UNION ALL 22, 19, SEARCH submissions USING INDEX random_submissions_order (random_order Date: Mon, 10 Aug 2026 16:37:40 +0200 Subject: [PATCH 15/17] Break ties of PreferDistinct submissions with equal counts --- opsqueue/benches/chunks_select_bench.svg | 2876 +++++++++++----------- opsqueue/src/consumer/strategy.rs | 268 +- 2 files changed, 1610 insertions(+), 1534 deletions(-) diff --git a/opsqueue/benches/chunks_select_bench.svg b/opsqueue/benches/chunks_select_bench.svg index 840118c..fdcebee 100644 --- a/opsqueue/benches/chunks_select_bench.svg +++ b/opsqueue/benches/chunks_select_bench.svg @@ -39,84 +39,84 @@ z - - - + + - - - + + @@ -124,16 +124,16 @@ z +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - - + @@ -210,11 +210,11 @@ z +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -264,11 +264,11 @@ z +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -305,11 +305,11 @@ z +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -352,11 +352,11 @@ z +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -404,16 +404,16 @@ z +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - - + @@ -421,11 +421,11 @@ L 0 2 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -433,11 +433,11 @@ L 52.720915 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -445,11 +445,11 @@ L 57.915728 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -457,11 +457,11 @@ L 93.13386 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -469,11 +469,11 @@ L 111.016873 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -481,11 +481,11 @@ L 123.705072 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -493,11 +493,11 @@ L 133.546804 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -505,11 +505,11 @@ L 141.588085 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -517,11 +517,11 @@ L 148.386891 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -529,11 +529,11 @@ L 154.276285 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -541,11 +541,11 @@ L 159.471098 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -553,11 +553,11 @@ L 194.689229 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -565,11 +565,11 @@ L 212.572242 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -577,11 +577,11 @@ L 225.260442 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -589,11 +589,11 @@ L 235.102174 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -601,11 +601,11 @@ L 243.143455 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -613,11 +613,11 @@ L 249.942261 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -625,11 +625,11 @@ L 255.831654 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -637,11 +637,11 @@ L 261.026468 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -649,11 +649,11 @@ L 296.244599 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -661,11 +661,11 @@ L 314.127612 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -673,11 +673,11 @@ L 326.815811 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -685,11 +685,11 @@ L 336.657543 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -697,11 +697,11 @@ L 344.698824 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -709,11 +709,11 @@ L 351.49763 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -721,11 +721,11 @@ L 357.387024 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -733,11 +733,11 @@ L 362.581837 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -745,11 +745,11 @@ L 397.799968 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -757,11 +757,11 @@ L 415.682981 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -769,11 +769,11 @@ L 428.371181 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -781,11 +781,11 @@ L 438.212913 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -793,11 +793,11 @@ L 446.254194 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -805,11 +805,11 @@ L 453.053 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -817,11 +817,11 @@ L 458.942393 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -829,11 +829,11 @@ L 464.137207 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -841,11 +841,11 @@ L 499.355338 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -853,11 +853,11 @@ L 517.238351 48.482812 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -1251,23 +1251,23 @@ z - + - - + - + @@ -1276,18 +1276,18 @@ L -3.5 0 - + - + - + @@ -1296,18 +1296,18 @@ L 532.078437 290.575574 - + - + - + @@ -1316,18 +1316,18 @@ L 532.078437 230.052384 - + - + - + @@ -1336,18 +1336,18 @@ L 532.078437 169.529193 - + - + - + @@ -1358,11 +1358,11 @@ L 532.078437 109.006003 +" clip-path="url(#p9766e11a9e)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -1388,486 +1388,486 @@ z - + - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -2147,24 +2147,24 @@ z - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + @@ -2450,7 +2450,7 @@ L 332.234844 60.271406 L 341.234844 60.271406 " style="fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: square"/> - + @@ -2591,7 +2591,7 @@ L 332.234844 73.772109 L 341.234844 73.772109 " style="fill: none; stroke: #ff7f0e; stroke-width: 1.5; stroke-linecap: square"/> - + @@ -2648,84 +2648,84 @@ z - - - + + - - - + + @@ -2733,11 +2733,11 @@ z +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -2753,11 +2753,11 @@ L 598.962647 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -2773,11 +2773,11 @@ L 700.518017 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -2793,11 +2793,11 @@ L 802.073386 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -2813,11 +2813,11 @@ L 903.628756 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -2833,11 +2833,11 @@ L 1005.184125 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -2845,11 +2845,11 @@ L 583.231522 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -2857,11 +2857,11 @@ L 589.120915 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -2869,11 +2869,11 @@ L 594.315728 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -2881,11 +2881,11 @@ L 629.53386 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -2893,11 +2893,11 @@ L 647.416873 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -2905,11 +2905,11 @@ L 660.105072 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -2917,11 +2917,11 @@ L 669.946804 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -2929,11 +2929,11 @@ L 677.988085 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -2941,11 +2941,11 @@ L 684.786891 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -2953,11 +2953,11 @@ L 690.676285 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -2965,11 +2965,11 @@ L 695.871098 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -2977,11 +2977,11 @@ L 731.089229 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -2989,11 +2989,11 @@ L 748.972242 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3001,11 +3001,11 @@ L 761.660442 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3013,11 +3013,11 @@ L 771.502174 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3025,11 +3025,11 @@ L 779.543455 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3037,11 +3037,11 @@ L 786.342261 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3049,11 +3049,11 @@ L 792.231654 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3061,11 +3061,11 @@ L 797.426468 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3073,11 +3073,11 @@ L 832.644599 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3085,11 +3085,11 @@ L 850.527612 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3097,11 +3097,11 @@ L 863.215811 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3109,11 +3109,11 @@ L 873.057543 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3121,11 +3121,11 @@ L 881.098824 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3133,11 +3133,11 @@ L 887.89763 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3145,11 +3145,11 @@ L 893.787024 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3157,11 +3157,11 @@ L 898.981837 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3169,11 +3169,11 @@ L 934.199968 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3181,11 +3181,11 @@ L 952.082981 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3193,11 +3193,11 @@ L 964.771181 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3205,11 +3205,11 @@ L 974.612913 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3217,11 +3217,11 @@ L 982.654194 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3229,11 +3229,11 @@ L 989.453 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3241,11 +3241,11 @@ L 995.342393 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3253,11 +3253,11 @@ L 1000.537207 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3265,11 +3265,11 @@ L 1035.755338 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3277,11 +3277,11 @@ L 1053.638351 48.482812 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3321,18 +3321,18 @@ L 1066.32655 48.482812 - + - + - + @@ -3341,18 +3341,18 @@ L 1068.478437 347.114709 - + - + - + @@ -3361,18 +3361,18 @@ L 1068.478437 287.38833 - + - + - + @@ -3381,18 +3381,18 @@ L 1068.478437 227.66195 - + - + - + @@ -3401,18 +3401,18 @@ L 1068.478437 167.935571 - + - + - + @@ -3423,11 +3423,11 @@ L 1068.478437 108.209192 +" clip-path="url(#pba62c91b52)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + @@ -3441,493 +3441,505 @@ L 1068.478437 48.482812 - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + @@ -3989,74 +4001,74 @@ L 1068.478437 51.215742 - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + @@ -4122,13 +4134,13 @@ Q 856.034844 82.684219 857.834844 82.684219 z " style="fill: #ffffff; opacity: 0.8; stroke: #cccccc; stroke-linejoin: miter"/> - + - + @@ -4174,13 +4186,13 @@ L 877.634844 60.271406 - + - + @@ -4207,128 +4219,128 @@ z - - - + + - - - + + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + @@ -4341,14 +4353,14 @@ L 1141.75686 48.482812 - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + @@ -4361,14 +4373,14 @@ L 1241.929174 48.482812 - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + @@ -4381,14 +4393,14 @@ L 1342.101487 48.482812 - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + @@ -4401,14 +4413,14 @@ L 1442.273801 48.482812 - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + @@ -4421,470 +4433,470 @@ L 1542.446114 48.482812 - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + - + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + @@ -4922,115 +4934,115 @@ L 1602.755856 48.482812 - - - + + + - + - + - + - - - + + + - + - + - + - - - + + + - + - + - + - - - + + + - + - + - + - - - + + + - + - + - + - - + + +" clip-path="url(#p05342ab53a)" style="fill: none; stroke-dasharray: 0.8,1.32; stroke-dashoffset: 0; stroke: #b0b0b0; stroke-opacity: 0.5; stroke-width: 0.8"/> - + - + @@ -5042,495 +5054,507 @@ L 1604.878437 48.482812 - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + @@ -5592,106 +5616,106 @@ L 1604.878437 51.205686 - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -5742,13 +5766,13 @@ Q 1392.434844 82.684219 1394.234844 82.684219 z " style="fill: #ffffff; opacity: 0.8; stroke: #cccccc; stroke-linejoin: miter"/> - + - + @@ -5794,13 +5818,13 @@ L 1414.034844 60.271406 - + - + @@ -5892,13 +5916,13 @@ z - + - + - + diff --git a/opsqueue/src/consumer/strategy.rs b/opsqueue/src/consumer/strategy.rs index 250c99b..c37ccd0 100644 --- a/opsqueue/src/consumer/strategy.rs +++ b/opsqueue/src/consumer/strategy.rs @@ -118,17 +118,18 @@ impl Strategy { Newest => qb.push("SELECT id as submission_id FROM submissions ORDER BY id DESC"), Random => Self::push_random_order_query(qb, "id as submission_id", "submissions", None), PreferDistinct { .. } => { - // Nested `PreferDistinct`s are flattened into a single query - // level: rather than one CTE per level, we emit one `counts_N` - // CTE per meta key and a single multi-key `ORDER BY`. let mut meta_keys_iter = self.meta_keys(); let meta_keys: Vec<&str> = meta_keys_iter.by_ref().collect(); let underlying = meta_keys_iter.take(); - // Unique submission IDs from the underlying strategy. + // Unique submission IDs from the underlying strategy. Note how + // we also keep the row number from the underlying query, this + // is used as a tie-breaker if metadata counts are equal. let qb = qb.push("WITH inner AS NOT MATERIALIZED ("); + qb.push("SELECT submission_id, ROW_NUMBER() OVER () as underlying_row FROM ( "); let qb = underlying.build_query_snippet_returning_submission_ids(qb); - qb.push(")"); + qb.push(" ))"); + // In-flight chunk count per submission, per meta key. // // The FFI call returns all counts as JSON in a single call. @@ -137,19 +138,21 @@ impl Strategy { for (i, meta_key) in meta_keys.iter().enumerate() { qb.push(format!( ", counts_{i} AS ( - SELECT sm.submission_id, je.value AS count + SELECT sm.submission_id, ffi_counts.value AS count FROM json_each(opsqueue_metadata_counts(" )); qb.push_bind(*meta_key); qb.push( - ")) je + ")) + ffi_counts CROSS JOIN submissions_metadata sm - ON sm.metadata_value = CAST(je.key AS INTEGER) + ON sm.metadata_value = CAST(ffi_counts.key AS INTEGER) WHERE sm.metadata_key = ", ); qb.push_bind(*meta_key); qb.push(")"); } + // Submissions ranked by in-flight chunks. Submissions without a // value for a key get a NULL count and so are ranked first. qb.push(" SELECT inner.submission_id FROM inner"); @@ -162,6 +165,15 @@ impl Strategy { qb.push(if i == 0 { " ORDER BY " } else { ", " }); qb.push(format!("counts_{i}.count ASC NULLS FIRST")); } + + if meta_keys.is_empty() { + panic!("`PreferDistinct` always yields at least one meta key.") + } else { + // Ensure that submissions with equal metadata counts use the + // ordering of the underlying strategy as a tie-breaker. + qb.push(", inner.underlying_row ASC"); + } + qb } } @@ -465,19 +477,25 @@ pub mod test { WITH inner AS NOT MATERIALIZED ( SELECT - id as submission_id + submission_id, + ROW_NUMBER() OVER () as underlying_row FROM - submissions - ORDER BY - id ASC + ( + SELECT + id as submission_id + FROM + submissions + ORDER BY + id ASC + ) ), counts_0 AS ( SELECT sm.submission_id, - je.value AS count + ffi_counts.value AS count FROM - json_each(opsqueue_metadata_counts(?)) je - CROSS JOIN submissions_metadata sm ON sm.metadata_value = CAST(je.key AS INTEGER) + json_each(opsqueue_metadata_counts(?)) ffi_counts + CROSS JOIN submissions_metadata sm ON sm.metadata_value = CAST(ffi_counts.key AS INTEGER) WHERE sm.metadata_key = ? ) @@ -487,7 +505,8 @@ pub mod test { inner LEFT JOIN counts_0 ON inner.submission_id = counts_0.submission_id ORDER BY - counts_0.count ASC NULLS FIRST + counts_0.count ASC NULLS FIRST, + inner.underlying_row ASC ) SELECT chunks.* @@ -501,15 +520,19 @@ pub mod test { assert_streaming_chunks(qb, &explained); insta::assert_snapshot!(explained, @" 3, 0, MATERIALIZE underlying_submission_ids - 6, 3, MATERIALIZE counts_0 - 10, 6, SCAN je VIRTUAL TABLE INDEX 1: - 15, 6, SEARCH sm USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=?) - 35, 3, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 - 41, 3, BLOOM FILTER ON counts_0 (submission_id=?) - 51, 3, SEARCH counts_0 USING AUTOMATIC COVERING INDEX (submission_id=?) LEFT-JOIN - 69, 3, USE TEMP B-TREE FOR ORDER BY - 81, 0, SCAN underlying_submission_ids - 83, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) + 5, 3, CO-ROUTINE inner + 8, 5, CO-ROUTINE (subquery-6) + 11, 8, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 + 22, 5, SCAN (subquery-6) + 59, 3, MATERIALIZE counts_0 + 63, 59, SCAN ffi_counts VIRTUAL TABLE INDEX 1: + 68, 59, SEARCH sm USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=?) + 87, 3, SCAN inner + 94, 3, BLOOM FILTER ON counts_0 (submission_id=?) + 104, 3, SEARCH counts_0 USING AUTOMATIC COVERING INDEX (submission_id=?) LEFT-JOIN + 124, 3, USE TEMP B-TREE FOR ORDER BY + 136, 0, SCAN underlying_submission_ids + 138, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) "); } @@ -537,19 +560,25 @@ pub mod test { WITH inner AS NOT MATERIALIZED ( SELECT - id as submission_id + submission_id, + ROW_NUMBER() OVER () as underlying_row FROM - submissions - ORDER BY - id DESC + ( + SELECT + id as submission_id + FROM + submissions + ORDER BY + id DESC + ) ), counts_0 AS ( SELECT sm.submission_id, - je.value AS count + ffi_counts.value AS count FROM - json_each(opsqueue_metadata_counts(?)) je - CROSS JOIN submissions_metadata sm ON sm.metadata_value = CAST(je.key AS INTEGER) + json_each(opsqueue_metadata_counts(?)) ffi_counts + CROSS JOIN submissions_metadata sm ON sm.metadata_value = CAST(ffi_counts.key AS INTEGER) WHERE sm.metadata_key = ? ) @@ -559,7 +588,8 @@ pub mod test { inner LEFT JOIN counts_0 ON inner.submission_id = counts_0.submission_id ORDER BY - counts_0.count ASC NULLS FIRST + counts_0.count ASC NULLS FIRST, + inner.underlying_row ASC ) SELECT chunks.* @@ -573,15 +603,19 @@ pub mod test { assert_streaming_chunks(qb, &explained); insta::assert_snapshot!(explained, @" 3, 0, MATERIALIZE underlying_submission_ids - 6, 3, MATERIALIZE counts_0 - 10, 6, SCAN je VIRTUAL TABLE INDEX 1: - 15, 6, SEARCH sm USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=?) - 35, 3, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 - 41, 3, BLOOM FILTER ON counts_0 (submission_id=?) - 51, 3, SEARCH counts_0 USING AUTOMATIC COVERING INDEX (submission_id=?) LEFT-JOIN - 69, 3, USE TEMP B-TREE FOR ORDER BY - 81, 0, SCAN underlying_submission_ids - 83, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) + 5, 3, CO-ROUTINE inner + 8, 5, CO-ROUTINE (subquery-6) + 11, 8, SCAN submissions USING COVERING INDEX sqlite_autoindex_submissions_1 + 22, 5, SCAN (subquery-6) + 59, 3, MATERIALIZE counts_0 + 63, 59, SCAN ffi_counts VIRTUAL TABLE INDEX 1: + 68, 59, SEARCH sm USING COVERING INDEX lookup_submission_by_metadata (metadata_key=? AND metadata_value=?) + 87, 3, SCAN inner + 94, 3, BLOOM FILTER ON counts_0 (submission_id=?) + 104, 3, SEARCH counts_0 USING AUTOMATIC COVERING INDEX (submission_id=?) LEFT-JOIN + 124, 3, USE TEMP B-TREE FOR ORDER BY + 136, 0, SCAN underlying_submission_ids + 138, 0, SEARCH chunks USING PRIMARY KEY (submission_id=?) "); } @@ -609,26 +643,32 @@ pub mod test { WITH inner AS NOT MATERIALIZED ( SELECT - id as submission_id + submission_id, + ROW_NUMBER() OVER () as underlying_row FROM - submissions - WHERE - random_order >= ? - UNION ALL - SELECT - id as submission_id - FROM - submissions - WHERE - random_order < ? + ( + SELECT + id as submission_id + FROM + submissions + WHERE + random_order >= ? + UNION ALL + SELECT + id as submission_id + FROM + submissions + WHERE + random_order < ? + ) ), counts_0 AS ( SELECT sm.submission_id, - je.value AS count + ffi_counts.value AS count FROM - json_each(opsqueue_metadata_counts(?)) je - CROSS JOIN submissions_metadata sm ON sm.metadata_value = CAST(je.key AS INTEGER) + json_each(opsqueue_metadata_counts(?)) ffi_counts + CROSS JOIN submissions_metadata sm ON sm.metadata_value = CAST(ffi_counts.key AS INTEGER) WHERE sm.metadata_key = ? ) @@ -638,7 +678,8 @@ pub mod test { inner LEFT JOIN counts_0 ON inner.submission_id = counts_0.submission_id ORDER BY - counts_0.count ASC NULLS FIRST + counts_0.count ASC NULLS FIRST, + inner.underlying_row ASC ) SELECT chunks.* @@ -653,20 +694,22 @@ pub mod test { insta::assert_snapshot!(explained, @" 3, 0, MATERIALIZE underlying_submission_ids 5, 3, CO-ROUTINE inner - 6, 5, COMPOUND QUERY - 7, 6, LEFT-MOST SUBQUERY - 10, 7, SEARCH submissions USING INDEX random_submissions_order (random_order>?) - 19, 6, UNION ALL - 22, 19, SEARCH submissions USING INDEX random_submissions_order (random_order?) + 22, 9, UNION ALL + 25, 22, SEARCH submissions USING INDEX random_submissions_order (random_order= ? - UNION ALL - SELECT - id as submission_id - FROM - submissions - WHERE - random_order < ? + ( + SELECT + id as submission_id + FROM + submissions + WHERE + random_order >= ? + UNION ALL + SELECT + id as submission_id + FROM + submissions + WHERE + random_order < ? + ) ), counts_0 AS ( SELECT sm.submission_id, - je.value AS count + ffi_counts.value AS count FROM - json_each(opsqueue_metadata_counts(?)) je - CROSS JOIN submissions_metadata sm ON sm.metadata_value = CAST(je.key AS INTEGER) + json_each(opsqueue_metadata_counts(?)) ffi_counts + CROSS JOIN submissions_metadata sm ON sm.metadata_value = CAST(ffi_counts.key AS INTEGER) WHERE sm.metadata_key = ? ), counts_1 AS ( SELECT sm.submission_id, - je.value AS count + ffi_counts.value AS count FROM - json_each(opsqueue_metadata_counts(?)) je - CROSS JOIN submissions_metadata sm ON sm.metadata_value = CAST(je.key AS INTEGER) + json_each(opsqueue_metadata_counts(?)) ffi_counts + CROSS JOIN submissions_metadata sm ON sm.metadata_value = CAST(ffi_counts.key AS INTEGER) WHERE sm.metadata_key = ? ) @@ -739,7 +788,8 @@ pub mod test { LEFT JOIN counts_1 ON inner.submission_id = counts_1.submission_id ORDER BY counts_0.count ASC NULLS FIRST, - counts_1.count ASC NULLS FIRST + counts_1.count ASC NULLS FIRST, + inner.underlying_row ASC ) SELECT chunks.* @@ -754,25 +804,27 @@ pub mod test { insta::assert_snapshot!(explained, @" 3, 0, MATERIALIZE underlying_submission_ids 5, 3, CO-ROUTINE inner - 6, 5, COMPOUND QUERY - 7, 6, LEFT-MOST SUBQUERY - 10, 7, SEARCH submissions USING INDEX random_submissions_order (random_order>?) - 19, 6, UNION ALL - 22, 19, SEARCH submissions USING INDEX random_submissions_order (random_order?) + 22, 9, UNION ALL + 25, 22, SEARCH submissions USING INDEX random_submissions_order (random_order Date: Mon, 10 Aug 2026 16:39:42 +0200 Subject: [PATCH 16/17] fixup! SQL: Add metadata count FFI function Don't call destructor twice --- opsqueue/src/consumer/dispatcher/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opsqueue/src/consumer/dispatcher/mod.rs b/opsqueue/src/consumer/dispatcher/mod.rs index a8f36d6..728566b 100644 --- a/opsqueue/src/consumer/dispatcher/mod.rs +++ b/opsqueue/src/consumer/dispatcher/mod.rs @@ -306,7 +306,8 @@ impl Dispatcher { }; if rc != ffi::SQLITE_OK { - unsafe { sqlite_metadata_count_lookup_destructor(user_data) }; + // We don't need to explicitly call the destructor. + // c.f. https://sqlite.org/c3ref/create_function.html return Err(sqlx::Error::Protocol(format!( "sqlite3_create_function_v2 failed with rc={rc}" ))); From 7903f8b2a52e620f4c018c412775c5fde76ee718 Mon Sep 17 00:00:00 2001 From: "jeremy.barisch.rooney@channable.com" Date: Mon, 10 Aug 2026 17:13:19 +0200 Subject: [PATCH 17/17] fixup! fixup! PreferDistinct sorts submissions by metadata Fix comment in migration file --- ...d_random_order_index_to_submissions.up.sql | 2 +- opsqueue/opsqueue_example_database_schema.db | Bin 102400 -> 102400 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/opsqueue/migrations/20260803133844_add_random_order_index_to_submissions.up.sql b/opsqueue/migrations/20260803133844_add_random_order_index_to_submissions.up.sql index 3797c3f..ae5ee78 100644 --- a/opsqueue/migrations/20260803133844_add_random_order_index_to_submissions.up.sql +++ b/opsqueue/migrations/20260803133844_add_random_order_index_to_submissions.up.sql @@ -1,4 +1,4 @@ --- Uses the same formula as '20250803174028_better_random_order_formula.down.sql'. +-- Uses the same formula as '20250803174028_better_random_order_formula.up.sql'. ALTER TABLE submissions ADD COLUMN random_order INTEGER NOT NULL GENERATED ALWAYS AS ( (((id + (id >> 22)) % 65536) * 40503) % 65536 ) VIRTUAL; diff --git a/opsqueue/opsqueue_example_database_schema.db b/opsqueue/opsqueue_example_database_schema.db index 1883e20f03a4f77b6ace20bdab657f48099e2f34..e13b54bbbbf1d9dd72f8879ba985bd6e9ff8c8dc 100644 GIT binary patch delta 502 zcmZozz}B#UO(rqcc&>VCPCm~JbN24$bF4ku>~{+Oif>M8QY}AxTy@U@ zDVrRHy{jkxk}~44&_x(B*+SZoN5Q*1otvH26l_VUv;wM%h0=!15}AgRuSr|+B5O6B zEGuKcEas;)IY34WO{N_r!y`C(my7|L%y*!S@z0>iTC%EWGBL6S%<{J*Cilx~qsbfx z%7}T~ocv4H1Wm?T&VX6^>bl8!a{6d8tAH{R+JTer%4wj<2+130OEacNGO@Etf_yBL vvHVv=Vsb`lUUspOfsvW60je)^<$-o|^3XBB+=Ov1V delta 504 zcmZozz}B#UO(r=UUt@SW6m6w$z4*$_68OThQ?OL z##V;LnXW~uN0URCp6?bYo$%(1)W-|2_BTEHCQ!5igD zO=cBP#z|j$@?AL%G#Mdz18whRdm@?GStWs9W){j={wpFeIioZ$yV%IU$V}G&)i1g7 c2FxzGOD8XoS3ooAEKnvYwW>*BnF3=00QhZ=g8%>k