From 14e1ccfa406c671fee474dc602155813ebc670b9 Mon Sep 17 00:00:00 2001
From: Vince Buffalo <vsbuffalo@gmail.com>
Date: Sun, 3 Mar 2024 17:36:57 -0800
Subject: [PATCH] This changes the behavior of pull --overwrite mentioned in
 Issue #16.

Previously the code would remove any file that was soon to be
downloaded with --pull, if --overwrite was set. At the time, this
was intentional to save disk space and not double the usage temporarily.
 However, as this issue points out, an upstream API bug will prevent
a download from replacing these. The previous behavior was not wrong,
but is a bit less safe. Users wishing to avoid disk space usage can
manually remove files beforehand, or eventaully a --remove option can
be added.
---
 Cargo.lock          |  2 +-
 Cargo.toml          |  2 +-
 README.md           |  1 +
 src/lib/download.rs | 19 ++++++++++++++++---
 src/main.rs         |  5 +++++
 5 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index ab6acd2..5dfdfbb 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2184,7 +2184,7 @@ dependencies = [
 
 [[package]]
 name = "scidataflow"
-version = "0.8.9"
+version = "0.8.11"
 dependencies = [
  "anyhow",
  "chrono",
diff --git a/Cargo.toml b/Cargo.toml
index 6be681f..486dac7 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "scidataflow"
-version = "0.8.10"
+version = "0.8.11"
 edition = "2021"
 exclude = ["logo.png", "tests/test_data/**"]
 license = "MIT"
diff --git a/README.md b/README.md
index 273f0df..3643759 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,6 @@
 ![Crates.io](https://img.shields.io/crates/v/scidataflow) ![Crates.io](https://img.shields.io/crates/d/scidataflow) ![CI tests](https://github.com/vsbuffalo/sciflow/workflows/CI/badge.svg)
 
+
 ![SciDataFlow logo](https://raw.githubusercontent.com/vsbuffalo/scidataflow/main/logo.png)
 
 
diff --git a/src/lib/download.rs b/src/lib/download.rs
index 42fd17f..4bf5ac7 100644
--- a/src/lib/download.rs
+++ b/src/lib/download.rs
@@ -105,15 +105,19 @@ impl Downloads {
         let total_files = downloads.len();
         if !downloads.is_empty() {
             // Let's handle the file operations:
-            // 1) Delete the files if they exist, since if it's in the queue, it's
-            //    overwrite-safe.
+            // 1) Move all the files to temporary destinations
             // 2) Create the directory structure if it does not exist.
+            let mut temp_files = Vec::new();
             for file in downloads {
                 let path = PathBuf::from(&file.filename);
                 if path.exists() {
-                    fs::remove_file(&path)?;
+                    // rather than delete, we move the file
+                    let temp_file_path = path.with_extension(".tmp");
+                    fs::rename(&path, &temp_file_path)?;
+                    temp_files.push(temp_file_path);
                 }
 
+                // recreate the directory structure if not there
                 if let Some(parent_dir) = path.parent() {
                     if !parent_dir.exists() {
                         fs::create_dir_all(parent_dir)?;
@@ -124,7 +128,16 @@ impl Downloads {
             let downloader = DownloaderBuilder::new()
                 .style_options(self.default_style()?)
                 .build();
+
+            // download everything
             downloader.download(downloads).await;
+
+            // now remove the temp files
+            for temp_file_path in temp_files {
+                if temp_file_path.exists() {
+                    fs::remove_file(temp_file_path)?;
+                }
+            }
             if show_total {
                 let punc = if total_files > 0 { "." } else { ":" };
                 println!(
diff --git a/src/main.rs b/src/main.rs
index c3f4200..56c4ef2 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -199,6 +199,11 @@ enum Commands {
     /// Pull in all tracked files from the remote. If --urls is set,
     /// this will (re)-download all files (tracked or not) in that manifest
     /// from their URLs.
+    ///
+    /// Note that if --overwrite is set, this will append the suffix '.tmp'
+    /// to each file that will be replaced, and those files will be removed
+    /// after the download is successful. While safer, this does temporarily
+    /// increase disk usage.
     Pull {
         /// Overwrite local files if they exit.
         #[arg(long)]