Merge pull request #44 from ginkgobioworks/separate-operations

Separate operation table POC

Cargo.toml

@@ -15,3 +15,4 @@ sha2 = "0.10.8"
 noodles = { version = "0.78.0", features = ["core", "vcf", "fasta", "async"] }
 petgraph = "0.6.5"
 chrono = "0.4.38"
+tempdir = "0.3.7"

migrations/operations/01-initial/up.sql

@@ -0,0 +1,28 @@
+CREATE TABLE operation_state (
+ db_uuid TEXT PRIMARY KEY NOT NULL,
+ operation_id INTEGER,
+ FOREIGN KEY(operation_id) REFERENCES operation(id)
+) STRICT;
+
+CREATE TABLE operation (
+ id INTEGER PRIMARY KEY NOT NULL,
+ db_uuid TEXT NOT NULL,
+ parent_id INTEGER,
+ collection_name TEXT NOT NULL,
+ change_type TEXT NOT NULL,
+ change_id INTEGER NOT NULL,
+ FOREIGN KEY(parent_id) REFERENCES operation(id)
+) STRICT;
+
+CREATE TABLE file_addition (
+ id INTEGER PRIMARY KEY NOT NULL,
+ file_path TEXT NOT NULL,
+ file_type TEXT NOT NULL
+) STRICT;
+
+CREATE TABLE operation_summary (
+ id INTEGER PRIMARY KEY NOT NULL,
+ operation_id INTEGER NOT NULL,
+ summary TEXT NOT NULL,
+ FOREIGN KEY(operation_id) REFERENCES operation(id)
+) STRICT;

src/config.rs

@@ -1,28 +1,45 @@
+use crate::migrations::run_operation_migrations;
+use crate::models::operations::Operation;
 use rusqlite::Connection;
 use std::io::{IsTerminal, Read, Write};
+use std::string::ToString;
+use std::sync::RwLock;
 use std::{
  env, fs,
  path::{Path, PathBuf},
+ sync::LazyLock,
 };
 
-use crate::models::metadata;
+thread_local! {
+pub static BASE_DIR: LazyLock<RwLock<PathBuf>> =
+ LazyLock::new(|| RwLock::new(env::current_dir().unwrap()));
+}
+
+pub fn get_operation_connection() -> Connection {
+ let db_path = get_gen_db_path();
+ let mut conn =
+ Connection::open(&db_path).unwrap_or_else(|_| panic!("Error connecting to {:?}", &db_path));
+ run_operation_migrations(&mut conn);
+ conn
+}
 
 fn ensure_dir(path: &PathBuf) {
  if !path.is_dir() {
  fs::create_dir_all(path).unwrap();
  }
 }
 
-pub fn get_or_create_gen_dir() {
- let start_dir = env::current_dir().unwrap();
+pub fn get_or_create_gen_dir() -> PathBuf {
+ let start_dir = BASE_DIR.with(|v| v.read().unwrap().clone());
  let mut cur_dir = start_dir.as_path();
  let gen_path = cur_dir.join(".gen");
  ensure_dir(&gen_path);
+ gen_path
 }
 
 // TODO: maybe just store all these things in a sqlite file too in .gen
 pub fn get_gen_dir() -> String {
- let start_dir = env::current_dir().unwrap();
+ let start_dir = BASE_DIR.with(|v| v.read().unwrap().clone());
  let mut cur_dir = start_dir.as_path();
  let mut gen_path = cur_dir.join(".gen");
  while !gen_path.is_dir() {
@@ -40,16 +57,14 @@ pub fn get_gen_dir() -> String {
  return gen_path.to_str().unwrap().to_string();
 }
 
-pub fn get_operation_path(conn: &Connection) -> PathBuf {
- let db_id = metadata::get_db_uuid(conn);
- let path = Path::new(&get_gen_dir()).join(db_id);
- ensure_dir(&path);
- path.join("operation")
+pub fn get_gen_db_path() -> PathBuf {
+ Path::new(&get_gen_dir()).join("gen.db")
 }
 
-pub fn get_changeset_path(conn: &Connection) -> PathBuf {
- let db_id = metadata::get_db_uuid(conn);
- let path = Path::new(&get_gen_dir()).join(db_id).join("changeset");
+pub fn get_changeset_path(operation: &Operation) -> PathBuf {
+ let path = Path::new(&get_gen_dir())
+ .join(operation.db_uuid.clone())
+ .join("changeset");
  ensure_dir(&path);
  path
 }

src/imports/fasta.rs

@@ -5,21 +5,29 @@ use std::str;
 use crate::models::file_types::FileTypes;
 use crate::models::operations::{FileAddition, Operation, OperationSummary};
 use crate::models::{
- self, block_group::BlockGroup, block_group_edge::BlockGroupEdge, edge::Edge, path::Path,
- sequence::Sequence, strand::Strand,
+ self, block_group::BlockGroup, block_group_edge::BlockGroupEdge, edge::Edge, metadata,
+ path::Path, sequence::Sequence, strand::Strand,
 };
 use crate::operation_management;
 use noodles::fasta;
 use rusqlite::{session, Connection};
 
-pub fn import_fasta(fasta: &String, name: &str, shallow: bool, conn: &Connection) {
+pub fn import_fasta(
+ fasta: &String,
+ name: &str,
+ shallow: bool,
+ conn: &Connection,
+ operation_conn: &Connection,
+) {
  let mut session = session::Session::new(conn).unwrap();
  operation_management::attach_session(&mut session);
- let change = FileAddition::create(conn, fasta, FileTypes::Fasta);
+ let change = FileAddition::create(operation_conn, fasta, FileTypes::Fasta);
 
  let mut reader = fasta::io::reader::Builder.build_from_path(fasta).unwrap();
 
- let operation = Operation::create(conn, name, "fasta_addition", change.id);
+ let db_uuid = metadata::get_db_uuid(conn);
+
+ let operation = Operation::create(operation_conn, &db_uuid, name, "fasta_addition", change.id);
 
  if !models::Collection::exists(conn, name) {
  let collection = models::Collection::create(conn, name);
@@ -75,15 +83,11 @@ pub fn import_fasta(fasta: &String, name: &str, shallow: bool, conn: &Connection
  for (path_name, change_count) in summary.iter() {
  summary_str.push_str(&format!(" {path_name}: {change_count} changes.\n"));
  }
- OperationSummary::create(conn, operation.id, &summary_str);
+ OperationSummary::create(operation_conn, operation.id, &summary_str);
  println!("Created it");
  let mut output = Vec::new();
  session.changeset_strm(&mut output).unwrap();
- operation_management::write_changeset(
- conn,
- operation_management::get_operation(conn).unwrap(),
- &output,
- );
+ operation_management::write_changeset(&operation, &output);
  } else {
  println!("Collection {:1} already exists", name);
  }
@@ -93,21 +97,23 @@ pub fn import_fasta(fasta: &String, name: &str, shallow: bool, conn: &Connection
 mod tests {
  // Note this useful idiom: importing names from outer (for mod tests) scope.
  use super::*;
- use crate::operation_management;
- use crate::test_helpers::get_connection;
+ use crate::test_helpers::{get_connection, get_operation_connection, setup_gen_dir};
  use std::collections::HashSet;
 
  #[test]
  fn test_add_fasta() {
+ setup_gen_dir();
  let mut fasta_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
  fasta_path.push("fixtures/simple.fa");
  let conn = get_connection(None);
+ let op_conn = &get_operation_connection(None);
 
  import_fasta(
  &fasta_path.to_str().unwrap().to_string(),
  "test",
  false,
  &conn,
+ op_conn,
  );
  assert_eq!(
  BlockGroup::get_all_sequences(&conn, 1),

src/imports/gfa.rs

@@ -209,13 +209,14 @@ fn edge_data_from_fields(
 #[cfg(test)]
 mod tests {
  use super::*;
- use crate::test_helpers::get_connection;
+ use crate::test_helpers::{get_connection, setup_gen_dir};
  use rusqlite::{types::Value as SQLValue, Connection};
  use std::fs;
  use std::path::PathBuf;
 
  #[test]
  fn test_import_simple_gfa() {
+ setup_gen_dir();
  let mut gfa_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
  gfa_path.push("fixtures/simple.gfa");
  let collection_name = "test".to_string();