Chunkified queries - scratchpad

ARCHITECTURE.md

@@ -161,6 +161,7 @@ Update instructions:
 |----------------------|--------------------------------------------------------------------------|
 | re_entity_db | In-memory storage of Rerun entities |
 | re_query | Querying data in the re_chunk_store |
+| re_query2 | Querying data in the re_chunk_store |
 | re_types | The built-in Rerun data types, component types, and archetypes. |
 | re_types_blueprint | The core traits and types that power Rerun's Blueprint sub-system. |
 | re_log_encoding | Helpers for encoding and transporting Rerun log messages |

Cargo.lock

@@ -4311,6 +4311,7 @@ dependencies = [
  "rand",
  "re_arrow2",
  "re_build_info",
+ "re_error",
  "re_format",
  "re_format_arrow",
  "re_log",
@@ -4526,6 +4527,7 @@ dependencies = [
  "re_log_encoding",
  "re_log_types",
  "re_query",
+ "re_query2",
  "re_smart_channel",
  "re_tracing",
  "re_types",
@@ -4714,6 +4716,39 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "re_query2"
+version = "0.18.0-alpha.1+dev"
+dependencies = [
+ "ahash",
+ "anyhow",
+ "backtrace",
+ "criterion",
+ "indent",
+ "indexmap 2.1.0",
+ "itertools 0.13.0",
+ "mimalloc",
+ "nohash-hasher",
+ "parking_lot",
+ "paste",
+ "rand",
+ "re_arrow2",
+ "re_chunk",
+ "re_chunk_store",
+ "re_error",
+ "re_format",
+ "re_log",
+ "re_log_types",
+ "re_tracing",
+ "re_tuid",
+ "re_types",
+ "re_types_core",
+ "seq-macro",
+ "similar-asserts",
+ "static_assertions",
+ "thiserror",
+]
+
 [[package]]
 name = "re_renderer"
 version = "0.18.0-alpha.1+dev"
@@ -4891,6 +4926,7 @@ dependencies = [
  "re_log",
  "re_log_types",
  "re_query",
+ "re_query2",
  "re_tracing",
  "re_types_core",
  "re_ui",
@@ -5049,7 +5085,7 @@ dependencies = [
  "re_format",
  "re_log",
  "re_log_types",
- "re_query",
+ "re_query2",
  "re_renderer",
  "re_space_view",
  "re_tracing",
@@ -5278,6 +5314,7 @@ dependencies = [
  "re_log_types",
  "re_memory",
  "re_query",
+ "re_query2",
  "re_renderer",
  "re_sdk_comms",
  "re_selection_panel",
@@ -5349,6 +5386,7 @@ dependencies = [
  "re_log_types",
  "re_math",
  "re_query",
+ "re_query2",
  "re_renderer",
  "re_smart_channel",
  "re_string_interner",

Cargo.toml

@@ -49,6 +49,7 @@ re_format_arrow = { path = "crates/store/re_format_arrow", version = "=0.18.0-al
 re_log_encoding = { path = "crates/store/re_log_encoding", version = "=0.18.0-alpha.1", default-features = false }
 re_log_types = { path = "crates/store/re_log_types", version = "=0.18.0-alpha.1", default-features = false }
 re_query = { path = "crates/store/re_query", version = "=0.18.0-alpha.1", default-features = false }
+re_query2 = { path = "crates/store/re_query2", version = "=0.18.0-alpha.1", default-features = false }
 re_sdk_comms = { path = "crates/store/re_sdk_comms", version = "=0.18.0-alpha.1", default-features = false }
 re_types = { path = "crates/store/re_types", version = "=0.18.0-alpha.1", default-features = false }
 re_types_blueprint = { path = "crates/store/re_types_blueprint", version = "=0.18.0-alpha.1", default-features = false }

crates/store/re_chunk/Cargo.toml

@@ -37,6 +37,7 @@ serde = [
 
 # Rerun
 re_build_info.workspace = true
+re_error.workspace = true
 re_format.workspace = true
 re_format_arrow.workspace = true
 re_log.workspace = true

crates/store/re_chunk/src/chunk.rs

@@ -3,14 +3,19 @@ use std::{
  sync::atomic::{AtomicU64, Ordering},
 };
 
-use arrow2::array::{
- Array as ArrowArray, ListArray as ArrowListArray, PrimitiveArray as ArrowPrimitiveArray,
- StructArray as ArrowStructArray,
+use arrow2::{
+ array::{
+ Array as ArrowArray, ListArray as ArrowListArray, PrimitiveArray as ArrowPrimitiveArray,
+ StructArray as ArrowStructArray,
+ },
+ Either,
 };
 
 use itertools::{izip, Itertools};
 use re_log_types::{EntityPath, ResolvedTimeRange, Time, TimeInt, TimePoint, Timeline};
-use re_types_core::{ComponentName, Loggable, LoggableBatch, SerializationError, SizeBytes};
+use re_types_core::{
+ ComponentName, DeserializationError, Loggable, LoggableBatch, SerializationError, SizeBytes,
+};
 
 use crate::{ChunkId, RowId};
 
@@ -26,8 +31,18 @@ pub enum ChunkError {
  #[error(transparent)]
  Arrow(#[from] arrow2::error::Error),
 
+ #[error("{kind} index out of bounds: {index} (len={len})")]
+ IndexOutOfBounds {
+ kind: String,
+ len: usize,
+ index: usize,
+ },
+
  #[error(transparent)]
  Serialization(#[from] SerializationError),
+
+ #[error(transparent)]
+ Deserialization(#[from] DeserializationError),
 }
 
 pub type ChunkResult<T> = Result<T, ChunkError>;
@@ -822,6 +837,32 @@ impl Chunk {
  .map(|(&time, &counter)| RowId::from_u128((time as u128) << 64 | (counter as u128)))
  }
 
+ /// Returns an iterator over the [`RowId`]s of a [`Chunk`], for a given component.
+ ///
+ /// This is different than [`Self::row_ids`]: it will only yield `RowId`s for rows at which
+ /// there is data for the specified `component_name`.
+ #[inline]
+ pub fn component_row_ids(
+ &self,
+ component_name: &ComponentName,
+ ) -> impl Iterator<Item = RowId> + '_ {
+ let Some(list_array) = self.components.get(component_name) else {
+ return Either::Left(std::iter::empty());
+ };
+
+ let row_ids = self.row_ids();
+
+ if let Some(validity) = list_array.validity() {
+ Either::Right(Either::Left(
+ row_ids
+ .enumerate()
+ .filter_map(|(i, o)| validity.get_bit(i).then_some(o)),
+ ))
+ } else {
+ Either::Right(Either::Right(row_ids))
+ }
+ }
+
  /// Returns the [`RowId`]-range covered by this [`Chunk`].
  ///
  /// `None` if the chunk `is_empty`.