refactored download approach, zenodo downloads working

src/lib/api/figshare.rs

@@ -6,7 +6,7 @@
 
 use url::Url;
 use std::fs;
-use std::path::{Path,PathBuf};
+use std::path::Path;
 use std::io::{Read,Seek,SeekFrom};
 use anyhow::{anyhow,Result};
 #[allow(unused_imports)]
@@ -23,8 +23,8 @@ use tokio::io::AsyncWriteExt;
 
 #[allow(unused_imports)]
 use crate::{print_info,print_warn};
-use crate::lib::data::{DataFile, MergedFile};
-use crate::lib::remote::{AuthKeys, RemoteFile, DownloadInfo,RequestData};
+use crate::lib::data::DataFile;
+use crate::lib::remote::{AuthKeys, RemoteFile, RequestData};
 use crate::lib::project::LocalMetadata;
 
 pub const FIGSHARE_BASE_URL: &str = "https://api.figshare.com/v2/";
@@ -324,6 +324,8 @@ impl FigShareAPI {
 
 
  // Download a single file through the FigShare API
+ // NOTE: Mostly deprecated due to trauma-based downloads.
+ #[allow(dead_code)]
  async fn download_file(&self, url: &str, save_path: &Path) -> Result<()> {
  let response = reqwest::get(url).await?;
  let mut file = File::create(save_path).await?;
@@ -368,48 +370,8 @@ impl FigShareAPI {
  Ok(true)
  }
 
- // Get the RemoteFile.url and combine with the token to get
- // a private download link.
- //
- // Note: this is overwrite-safe: it will error out 
- // if file exists unless overwrite is true.
- //
- // Note: this cannot be moved to higher-level (e.g. Remote)
- // since each API implements authentication its own way. 
- pub fn get_download_info(&self, merged_file: &MergedFile, path_context: &Path, overwrite: bool) 
- -> Result<DownloadInfo> {
- // if local DataFile is none, not in manifest; 
- // do not download
- let data_file = match &merged_file.local {
- None => return Err(anyhow!("Cannot download() without local DataFile.")),
- Some(file) => file
- };
- // check to make sure we won't overwrite
- if data_file.is_alive(path_context) && !overwrite {
- return Err(anyhow!("Data file '{}' exists locally, and would be \
- overwritten by download. Use --overwrite to download.",
- data_file.path));
- }
- // if no remote, there is nothing to download,
- // silently return Ok. Get URL.
- let remote = merged_file.remote.as_ref().ok_or(anyhow!("Remote is None"))?;
- let url = remote.url.as_ref().ok_or(anyhow!("Cannot download; download URL not set."))?;
-
- // add the token in
- let url = format!("{}?token={}", url, self.token);
- let save_path = &data_file.full_path(path_context)?;
- Ok( DownloadInfo { url, path:save_path.to_string_lossy().to_string() })
- }
-
- // Download a single file.
- //
- // For the most part, this is deprecated, since we use the download manager 
- // "trauma" now.
- pub async fn download(&self, merged_file: &MergedFile, 
- path_context: &Path, overwrite: bool) -> Result<()>{
- let info = self.get_download_info(merged_file, path_context, overwrite)?;
- self.download_file(&info.url, &PathBuf::from(info.path)).await?;
- Ok(())
+ pub fn authenticate_url(&self, url: &str) -> Result<String> {
+ Ok(format!("{}?token={}", url, self.token))
  }
 
  pub async fn find_article(&self) -> Result<Option<FigShareArticle>> {

src/lib/api/zenodo.rs

@@ -14,7 +14,7 @@ use tokio_util::io::ReaderStream;
 use crate::{print_info,print_warn};
 
 
-use crate::lib::{data::{DataFile, MergedFile}, project::LocalMetadata, remote::DownloadInfo};
+use crate::lib::{data::DataFile, project::LocalMetadata};
 use crate::lib::remote::{AuthKeys,RemoteFile,RequestData};
 use crate::lib::utils::{ISSUE_URL, shorten};
 
@@ -462,39 +462,9 @@ impl ZenodoAPI {
  Ok(files_hash)
  }
 
-
- // Get the RemoteFile.url and combine with the token to get
- // a private download link.
- //
- // Note: this is overwrite-safe: it will error out 
- // if file exists unless overwrite is true.
- //
- // Note: this *cannot* be moved to higher-level (e.g. Remote)
- // since each API implements authentication its own way. 
- pub fn get_download_info(&self, merged_file: &MergedFile, path_context: &Path, overwrite: bool) 
- -> Result<DownloadInfo> {
- // if local DataFile is none, not in manifest; 
- // do not download
- let data_file = match &merged_file.local {
- None => return Err(anyhow!("Cannot download() without local DataFile.")),
- Some(file) => file
- };
- // check to make sure we won't overwrite
- if data_file.is_alive(path_context) && !overwrite {
- return Err(anyhow!("Data file '{}' exists locally, and would be \
- overwritten by download. Use --overwrite to download.",
- data_file.path));
- }
- // if no remote, there is nothing to download,
- // silently return Ok. Get URL.
- let remote = merged_file.remote.as_ref().ok_or(anyhow!("Remote is None"))?;
- let url = remote.url.as_ref().ok_or(anyhow!("Cannot download; download URL not set."))?;
-
- // add the token in
- let url = format!("{}?access_token={}", url, self.token);
- let save_path = &data_file.full_path(path_context)?;
- Ok( DownloadInfo { url, path:save_path.to_string_lossy().to_string() })
- }
+ pub fn authenticate_url(&self, url: &str) -> Result<String> {
+ Ok(format!("{}?access_token={}", url, self.token))
+ }
 }
 
 #[cfg(test)]

src/lib/data.rs

@@ -1023,6 +1023,11 @@ impl DataCollection {
  .build();
  downloader.download(&downloads).await;
  println!("Downloaded {}.", pluralize(total_files as u64, "file"));
+ for download in downloads {
+ let filename = PathBuf::from(&download.filename);
+ let name_str = filename.file_name().ok_or(anyhow!("Internal Error: could not extract filename from download"))?;
+ println!(" - {}", name_str.to_string_lossy());
+ }
  } else {
  println!("No files downloaded.");
  }

src/lib/remote.rs

@@ -8,7 +8,7 @@ use anyhow::{anyhow,Result};
 #[allow(unused_imports)]
 use log::{info, trace, debug};
 use std::collections::HashMap;
-use trauma::{download::Download};
+use trauma::download::Download;
 use serde_derive::{Serialize,Deserialize};
 use reqwest::Url;
 
@@ -205,11 +205,30 @@ impl Remote {
  // TODO: could be struct, if some APIs require more authentication
  // Note: requires each API actually *check* overwrite.
  pub fn get_download_info(&self, merged_file: &MergedFile, path_context: &Path, overwrite: bool) -> Result<DownloadInfo> {
- match self {
- Remote::FigShareAPI(fgsh_api) => fgsh_api.get_download_info(merged_file, path_context, overwrite),
- Remote::ZenodoAPI(_) => Err(anyhow!("ZenodoAPI does not support get_project method")),
- Remote::DataDryadAPI(_) => service_not_implemented!("DataDryad"),
+ // if local DataFile is none, not in manifest; 
+ // do not download
+ let data_file = match &merged_file.local {
+ None => return Err(anyhow!("Cannot download() without local DataFile.")),
+ Some(file) => file
+ };
+ // check to make sure we won't overwrite
+ if data_file.is_alive(path_context) && !overwrite {
+ return Err(anyhow!("Data file '{}' exists locally, and would be \
+ overwritten by download. Use --overwrite to download.",
+ data_file.path));
  }
+ // if no remote, there is nothing to download,
+ // silently return Ok. Get URL.
+ let remote = merged_file.remote.as_ref().ok_or(anyhow!("Remote is None"))?;
+ let url = remote.url.as_ref().ok_or(anyhow!("Cannot download; download URL not set."))?;
+
+ let authenticated_url = match self {
+ Remote::FigShareAPI(fgsh_api) => fgsh_api.authenticate_url(url),
+ Remote::ZenodoAPI(znd_api) => znd_api.authenticate_url(url),
+ Remote::DataDryadAPI(_) => service_not_implemented!("DataDryad"),
+ }?;
+ let save_path = &data_file.full_path(path_context)?;
+ Ok( DownloadInfo { url: authenticated_url, path:save_path.to_string_lossy().to_string() })
  }
 }