This changes the behavior of pull --overwrite mentioned in Issue #16.

Previously the code would remove any file that was soon to be
downloaded with --pull, if --overwrite was set. At the time, this
was intentional to save disk space and not double the usage temporarily.
 However, as this issue points out, an upstream API bug will prevent
a download from replacing these. The previous behavior was not wrong,
but is a bit less safe. Users wishing to avoid disk space usage can
manually remove files beforehand, or eventaully a --remove option can
be added.

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "scidataflow"
-version = "0.8.10"
+version = "0.8.11"
 edition = "2021"
 exclude = ["logo.png", "tests/test_data/**"]
 license = "MIT"

README.md

@@ -1,5 +1,6 @@
 ![Crates.io](https://img.shields.io/crates/v/scidataflow) ![Crates.io](https://img.shields.io/crates/d/scidataflow) ![CI tests](https://github.com/vsbuffalo/sciflow/workflows/CI/badge.svg)
 
+
 ![SciDataFlow logo](https://raw.githubusercontent.com/vsbuffalo/scidataflow/main/logo.png)
 
 

src/lib/download.rs

@@ -105,15 +105,19 @@ impl Downloads {
  let total_files = downloads.len();
  if !downloads.is_empty() {
  // Let's handle the file operations:
- // 1) Delete the files if they exist, since if it's in the queue, it's
- // overwrite-safe.
+ // 1) Move all the files to temporary destinations
  // 2) Create the directory structure if it does not exist.
+ let mut temp_files = Vec::new();
  for file in downloads {
  let path = PathBuf::from(&file.filename);
  if path.exists() {
- fs::remove_file(&path)?;
+ // rather than delete, we move the file
+ let temp_file_path = path.with_extension(".tmp");
+ fs::rename(&path, &temp_file_path)?;
+ temp_files.push(temp_file_path);
  }
 
+ // recreate the directory structure if not there
  if let Some(parent_dir) = path.parent() {
  if !parent_dir.exists() {
  fs::create_dir_all(parent_dir)?;
@@ -124,7 +128,16 @@ impl Downloads {
  let downloader = DownloaderBuilder::new()
  .style_options(self.default_style()?)
  .build();
+
+ // download everything
  downloader.download(downloads).await;
+
+ // now remove the temp files
+ for temp_file_path in temp_files {
+ if temp_file_path.exists() {
+ fs::remove_file(temp_file_path)?;
+ }
+ }
  if show_total {
  let punc = if total_files > 0 { "." } else { ":" };
  println!(

src/main.rs

@@ -199,6 +199,11 @@ enum Commands {
  /// Pull in all tracked files from the remote. If --urls is set,
  /// this will (re)-download all files (tracked or not) in that manifest
  /// from their URLs.
+ ///
+ /// Note that if --overwrite is set, this will append the suffix '.tmp'
+ /// to each file that will be replaced, and those files will be removed
+ /// after the download is successful. While safer, this does temporarily
+ /// increase disk usage.
  Pull {
  /// Overwrite local files if they exit.
  #[arg(long)]