csv+serde based writing, new SumNotEmpty to mimic bedtools map's sum.

 - New `SerializableDatumType` and `DatumType::into_serializable()` that
   wraps in a runtime serialization config, e.g. for precision, missing
   values, etc.

 - Removed trait `TsvSerialize` and `to_tsv()` methods.
 - New special `Operation::SumNotEmpty` that returns a `DatumType::NoValue`
   to mimic bedtools map sum's behavior. This was to simplify testing and
   some users may want this behavior.
 - Renamed `deserialize_bed_missing` to `bed_missing` for ergonomics.
 - New parse capacity, reusing buffer (didn't seem to speed up much?).
 - Much better testing against bedtools map.
 - New testing utility functions: `head_file()` file looking at temp
   files, assert macros for floats and option floats.
 - New test case against bedtools map with multiple operations (before,
   only had unchecked benchmark! yikes)
 - Added a header to the `example.bed` BED file to mess stuff up :)

src/commands.rs

@@ -1,19 +1,19 @@
 //! Command functions that implement each of the `granges` subcommands.
 
-use std::path::PathBuf;
+use std::{path::PathBuf, io, fs::File};
+
+use csv::WriterBuilder;
 
 use crate::{
- data::{operations::Operation, DatumType},
+ data::{operations::Operation, SerializableDatumType},
  io::{
  parsers::{Bed5Iterator, GenomicRangesParser},
  tsv::BED_TSV,
- OutputStream,
  },
  prelude::*,
- ranges::{operations::adjust_range, GenomicRangeEmptyRecord, GenomicRangeRecord},
+ ranges::{operations::adjust_range, GenomicRangeRecordEmpty, GenomicRangeRecord},
  reporting::{CommandOutput, Report},
  test_utilities::{random_granges, random_granges_mock_bed5},
- traits::TsvSerialize,
  Position, PositionOffset,
 };
 
@@ -57,15 +57,18 @@ pub fn granges_adjust(
 ) -> Result<CommandOutput<()>, GRangesError> {
  let genome = read_seqlens(seqlens)?;
 
- // Setup Output stream -- header is None for now (TODO).
- let output_stream = output.map_or(OutputStream::new_stdout(None), |file| {
- OutputStream::new(file, None)
- });
- let mut writer = output_stream.writer()?;
+ let writer_boxed: Box<dyn io::Write> = match output {
+ Some(path) => Box::new(File::create(path)?),
+ None => Box::new(io::stdout()),
+ };
+
+ let mut writer = WriterBuilder::new()
+ .delimiter(b'\t')
+ .has_headers(false)
+ .from_writer(writer_boxed);
 
  // For reporting stuff to the user.
  let mut report = Report::new();
-
  let mut skipped_ranges = 0;
 
  if !sort {
@@ -84,16 +87,16 @@ pub fn granges_adjust(
  let possibly_adjusted_range = adjust_range(range, -both, both, length);
 
  if let Some(range_adjusted) = possibly_adjusted_range {
- writeln!(writer, "{}", &range_adjusted.to_tsv(&BED_TSV))?;
+ writer.serialize(range_adjusted)?;
  } else {
  skipped_ranges += 1;
  }
 
  if skipped_ranges > 0 {
  report.add_issue(format!(
- "{} ranges were removed because their widths after adjustment were ≤ 0",
- skipped_ranges
- ))
+  "{} ranges were removed because their widths after adjustment were ≤ 0",
+  skipped_ranges
+  ))
  }
  }
  } else {
@@ -158,7 +161,7 @@ pub fn granges_filter(
  right_path: &PathBuf,
  output: Option<&PathBuf>,
  skip_missing: bool,
-) -> Result<CommandOutput<()>, GRangesError> {
+ ) -> Result<CommandOutput<()>, GRangesError> {
  let genome = read_seqlens(seqlens)?;
  let seqnames: Vec<String> = genome.keys().cloned().collect();
 
@@ -197,7 +200,7 @@ pub fn granges_filter(
  right_gr = GRanges::from_iter(
  right.try_unwrap_data().retain_seqnames(&seqnames),
  &genome,
- )?;
+  )?;
  } else {
  left_gr = GRangesEmpty::from_iter(left, &genome)?;
  right_gr = GRanges::from_iter(right.try_unwrap_data(), &genome)?;
@@ -240,7 +243,7 @@ pub fn granges_filter(
  right_gr = GRanges::from_iter(
  right.try_unwrap_data().retain_seqnames(&seqnames),
  &genome,
- )?;
+  )?;
  } else {
  left_gr = GRanges::from_iter(left.try_unwrap_data(), &genome)?;
  right_gr = GRanges::from_iter(right.try_unwrap_data(), &genome)?;
@@ -289,7 +292,7 @@ pub fn granges_flank(
  output: Option<&PathBuf>,
  skip_missing: bool,
  mode: ProcessingMode,
-) -> Result<CommandOutput<()>, GRangesError> {
+ ) -> Result<CommandOutput<()>, GRangesError> {
  let genome = read_seqlens(seqlens)?;
  let seqnames: Vec<String> = genome.keys().cloned().collect();
  let ranges_iter = GenomicRangesFile::parsing_iterator(bedfile)?;
@@ -326,11 +329,15 @@ pub fn granges_flank(
  }
  },
  ProcessingMode::Streaming => {
- // Setup Output stream -- header is None for now (TODO).
- let output_stream = output.map_or(OutputStream::new_stdout(None), |file| {
- OutputStream::new(file, None)
- });
- let mut writer = output_stream.writer()?;
+ let writer_boxed: Box<dyn io::Write> = match output {
+ Some(path) => Box::new(File::create(path)?),
+ None => Box::new(io::stdout()),
+ };
+
+ let mut writer = WriterBuilder::new()
+ .delimiter(b'\t')
+ .has_headers(false)
+ .from_writer(writer_boxed);
 
  match ranges_iter {
  // FIXME: code redundancy. But too early now to design traits, etc.
@@ -346,7 +353,7 @@ pub fn granges_flank(
  let flanking_ranges = range
  .flanking_ranges::<GenomicRangeRecord<String>>(left, right, length);
  for flanking_range in flanking_ranges {
- writeln!(writer, "{}", &flanking_range.to_tsv(&BED_TSV))?;
+ writer.serialize(flanking_range)?;
  }
  }
  } else {
@@ -358,9 +365,9 @@ pub fn granges_flank(
  .ok_or(GRangesError::MissingSequence(seqname.to_string()))?;
 
  let flanking_ranges = range
- .flanking_ranges::<GenomicRangeEmptyRecord>(left, right, length);
+ .flanking_ranges::<GenomicRangeRecordEmpty>(left, right, length);
  for flanking_range in flanking_ranges {
- writeln!(writer, "{}", &flanking_range.to_tsv(&BED_TSV))?;
+ writer.serialize(flanking_range)?;
  }
  }
  }
@@ -380,7 +387,7 @@ pub fn granges_flank(
  let flanking_ranges = range
  .flanking_ranges::<GenomicRangeRecord<String>>(left, right, length);
  for flanking_range in flanking_ranges {
- writeln!(writer, "{}", &flanking_range.to_tsv(&BED_TSV))?;
+ writer.serialize(flanking_range)?;
  }
  }
  } else {
@@ -392,9 +399,9 @@ pub fn granges_flank(
  .ok_or(GRangesError::MissingSequence(seqname.to_string()))?;
 
  let flanking_ranges = range
- .flanking_ranges::<GenomicRangeEmptyRecord>(left, right, length);
+ .flanking_ranges::<GenomicRangeRecordEmpty>(left, right, length);
  for flanking_range in flanking_ranges {
- writeln!(writer, "{}", &flanking_range.to_tsv(&BED_TSV))?;
+ writer.serialize(flanking_range)?;
  }
  }
  }
@@ -417,7 +424,7 @@ pub fn granges_map(
  operations: Vec<Operation>,
  output: Option<&PathBuf>,
  skip_missing: bool,
-) -> Result<CommandOutput<()>, GRangesError> {
+ ) -> Result<CommandOutput<()>, GRangesError> {
  let genome = read_seqlens(seqlens)?;
  let seqnames: Vec<String> = genome.keys().cloned().collect();
  let report = Report::new();
@@ -470,8 +477,10 @@ pub fn granges_map(
  // Run all operations on the scores.
  operations
  .iter()
- .map(|operation| operation.run(&mut overlap_scores))
- .collect::<Vec<DatumType>>()
+ .map(|operation| {
+ operation.run(&mut overlap_scores).into_serializable(&BED_TSV)
+ })
+ .collect::<Vec<SerializableDatumType>>()
  })?;
 
  result_gr.write_to_tsv(output, &BED_TSV)?;
@@ -486,7 +495,7 @@ pub fn granges_windows(
  step: Option<Position>,
  chop: bool,
  output: Option<impl Into<PathBuf>>,
-) -> Result<CommandOutput<()>, GRangesError> {
+ ) -> Result<CommandOutput<()>, GRangesError> {
  let genome = read_seqlens(seqlens)?;
  GRangesEmpty::from_windows(&genome, width, step, chop)?.write_to_tsv(output, &BED_TSV)?;
  let report = Report::new();
@@ -500,7 +509,7 @@ pub fn granges_random_bed(
  output: Option<impl Into<PathBuf>>,
  sort: bool,
  bed5: bool,
-) -> Result<CommandOutput<()>, GRangesError> {
+ ) -> Result<CommandOutput<()>, GRangesError> {
  // get the genome info
  let genome = read_seqlens(seqlens)?;
 

src/data/mod.rs

@@ -1,7 +1,12 @@
 //! Data container implementations.
 //!
 
-use crate::traits::{DataContainer, IntoDatumType};
+use crate::{
+ io::TsvConfig,
+ traits::{DataContainer, IntoDatumType},
+};
+use serde::ser::Serializer;
+use serde::Serialize;
 
 #[cfg(feature = "ndarray")]
 pub mod ndarray;
@@ -12,9 +17,9 @@ impl<U> DataContainer for Vec<U> {}
 impl DataContainer for () {}
 
 /// These are core supported data types stored in an `enum`, to
-/// unify the types that come out of standard operations like
-/// `select()`.
-#[derive(Debug, Clone)]
+/// unify the types that come out of standard operations of
+/// heterogeneous output types.
+#[derive(Debug, Clone, Serialize)]
 pub enum DatumType {
  Float32(f32),
  Float64(f64),
@@ -26,6 +31,39 @@ pub enum DatumType {
  NoValue,
 }
 
+impl DatumType {
+ pub fn into_serializable(self, config: &TsvConfig) -> SerializableDatumType {
+ SerializableDatumType {
+ datum: self,
+ config,
+ }
+ }
+}
+
+#[derive(Debug, Clone)]
+pub struct SerializableDatumType<'a> {
+ pub datum: DatumType,
+ pub config: &'a TsvConfig,
+}
+
+impl<'a> Serialize for SerializableDatumType<'a> {
+ fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+ where
+ S: Serializer,
+ {
+ match &self.datum {
+ DatumType::NoValue => serializer.serialize_str(&self.config.no_value_string),
+ DatumType::Float32(value) => serializer.serialize_str(&value.to_string()),
+ DatumType::Float64(value) => serializer.serialize_str(&value.to_string()),
+ DatumType::String(value) => serializer.serialize_str(value),
+ DatumType::Integer32(value) => serializer.serialize_str(&value.to_string()),
+ DatumType::Integer64(value) => serializer.serialize_str(&value.to_string()),
+ DatumType::Unsigned32(value) => serializer.serialize_str(&value.to_string()),
+ DatumType::Unsigned64(value) => serializer.serialize_str(&value.to_string()),
+ }
+ }
+}
+
 impl IntoDatumType for f64 {
  fn into_data_type(self) -> DatumType {
  DatumType::Float64(self)

src/data/ndarray.rs

@@ -2,9 +2,8 @@
 
 use crate::error::GRangesError;
 use crate::granges::GRanges;
-use crate::io::TsvConfig;
-use crate::traits::{DataContainer, IndexedDataContainer, RangeContainer, TsvSerialize};
-use ndarray::{Array1, Array2, ArrayBase, ArrayView1, Dim, OwnedRepr};
+use crate::traits::{DataContainer, IndexedDataContainer, RangeContainer};
+use ndarray::{Array1, Array2, ArrayView1};
 
 impl<T> DataContainer for Array1<T> {}
 impl<T> DataContainer for Array2<T> {}
@@ -210,15 +209,6 @@ where
  }
 }
 
-impl<T: TsvSerialize> TsvSerialize for ArrayBase<OwnedRepr<T>, Dim<[usize; 1]>> {
- fn to_tsv(&self, config: &TsvConfig) -> String {
- self.iter()
- .map(|x| x.to_tsv(config))
- .collect::<Vec<_>>()
- .join("\t")
- }
-}
-
 #[cfg(test)]
 mod tests {
  use crate::test_utilities::granges_test_case_01;

src/data/operations.rs

@@ -34,11 +34,19 @@ pub fn median<F: Float + Sum>(numbers: &mut [F]) -> Option<F> {
 /// The (subset of) standard `bedtools map` operations.
 #[derive(Clone, Debug, ValueEnum)]
 pub enum Operation {
+ /// Calculate the sum of all values (a set of zero elements has sum 0.0).
  Sum,
+ /// Calculate the sum of all values, but set of zero elements is a missing value, not 0.0.
+ SumNotEmpty,
+ /// Calculate the minimum of values.
  Min,
+ /// Calculate the maximum of values.
  Max,
+ /// Calculate the mean of values.
  Mean,
+ /// Calculate the median of values.
  Median,
+ /// Concatenate all values into a string separated by commas.
  Collapse,
 }
 
@@ -53,6 +61,13 @@ impl Operation {
  let sum: T = data.iter().copied().sum();
  sum.into_data_type()
  }
+ Operation::SumNotEmpty => {
+ if data.is_empty() {
+ return DatumType::NoValue;
+ }
+ let sum: T = data.iter().copied().sum();
+ sum.into_data_type()
+ }
  Operation::Min => {
  let min = data
  .iter()