From 7a7bb41f020c9f0743df43560dde73cad8d8c1db Mon Sep 17 00:00:00 2001 From: Vince Buffalo Date: Wed, 30 Aug 2023 17:40:05 -0700 Subject: [PATCH] refactored download approach, zenodo downloads working --- src/lib/api/figshare.rs | 52 ++++++----------------------------------- src/lib/api/zenodo.rs | 38 ++++-------------------------- src/lib/data.rs | 5 ++++ src/lib/remote.rs | 29 +++++++++++++++++++---- 4 files changed, 40 insertions(+), 84 deletions(-) diff --git a/src/lib/api/figshare.rs b/src/lib/api/figshare.rs index c136b1c..96b548b 100644 --- a/src/lib/api/figshare.rs +++ b/src/lib/api/figshare.rs @@ -6,7 +6,7 @@ use url::Url; use std::fs; -use std::path::{Path,PathBuf}; +use std::path::Path; use std::io::{Read,Seek,SeekFrom}; use anyhow::{anyhow,Result}; #[allow(unused_imports)] @@ -23,8 +23,8 @@ use tokio::io::AsyncWriteExt; #[allow(unused_imports)] use crate::{print_info,print_warn}; -use crate::lib::data::{DataFile, MergedFile}; -use crate::lib::remote::{AuthKeys, RemoteFile, DownloadInfo,RequestData}; +use crate::lib::data::DataFile; +use crate::lib::remote::{AuthKeys, RemoteFile, RequestData}; use crate::lib::project::LocalMetadata; pub const FIGSHARE_BASE_URL: &str = "https://api.figshare.com/v2/"; @@ -324,6 +324,8 @@ impl FigShareAPI { // Download a single file through the FigShare API + // NOTE: Mostly deprecated due to trauma-based downloads. + #[allow(dead_code)] async fn download_file(&self, url: &str, save_path: &Path) -> Result<()> { let response = reqwest::get(url).await?; let mut file = File::create(save_path).await?; @@ -368,48 +370,8 @@ impl FigShareAPI { Ok(true) } - // Get the RemoteFile.url and combine with the token to get - // a private download link. - // - // Note: this is overwrite-safe: it will error out - // if file exists unless overwrite is true. - // - // Note: this cannot be moved to higher-level (e.g. Remote) - // since each API implements authentication its own way. - pub fn get_download_info(&self, merged_file: &MergedFile, path_context: &Path, overwrite: bool) - -> Result { - // if local DataFile is none, not in manifest; - // do not download - let data_file = match &merged_file.local { - None => return Err(anyhow!("Cannot download() without local DataFile.")), - Some(file) => file - }; - // check to make sure we won't overwrite - if data_file.is_alive(path_context) && !overwrite { - return Err(anyhow!("Data file '{}' exists locally, and would be \ - overwritten by download. Use --overwrite to download.", - data_file.path)); - } - // if no remote, there is nothing to download, - // silently return Ok. Get URL. - let remote = merged_file.remote.as_ref().ok_or(anyhow!("Remote is None"))?; - let url = remote.url.as_ref().ok_or(anyhow!("Cannot download; download URL not set."))?; - - // add the token in - let url = format!("{}?token={}", url, self.token); - let save_path = &data_file.full_path(path_context)?; - Ok( DownloadInfo { url, path:save_path.to_string_lossy().to_string() }) - } - - // Download a single file. - // - // For the most part, this is deprecated, since we use the download manager - // "trauma" now. - pub async fn download(&self, merged_file: &MergedFile, - path_context: &Path, overwrite: bool) -> Result<()>{ - let info = self.get_download_info(merged_file, path_context, overwrite)?; - self.download_file(&info.url, &PathBuf::from(info.path)).await?; - Ok(()) + pub fn authenticate_url(&self, url: &str) -> Result { + Ok(format!("{}?token={}", url, self.token)) } pub async fn find_article(&self) -> Result> { diff --git a/src/lib/api/zenodo.rs b/src/lib/api/zenodo.rs index 1e303a4..a303a55 100644 --- a/src/lib/api/zenodo.rs +++ b/src/lib/api/zenodo.rs @@ -14,7 +14,7 @@ use tokio_util::io::ReaderStream; use crate::{print_info,print_warn}; -use crate::lib::{data::{DataFile, MergedFile}, project::LocalMetadata, remote::DownloadInfo}; +use crate::lib::{data::DataFile, project::LocalMetadata}; use crate::lib::remote::{AuthKeys,RemoteFile,RequestData}; use crate::lib::utils::{ISSUE_URL, shorten}; @@ -462,39 +462,9 @@ impl ZenodoAPI { Ok(files_hash) } - - // Get the RemoteFile.url and combine with the token to get - // a private download link. - // - // Note: this is overwrite-safe: it will error out - // if file exists unless overwrite is true. - // - // Note: this *cannot* be moved to higher-level (e.g. Remote) - // since each API implements authentication its own way. - pub fn get_download_info(&self, merged_file: &MergedFile, path_context: &Path, overwrite: bool) - -> Result { - // if local DataFile is none, not in manifest; - // do not download - let data_file = match &merged_file.local { - None => return Err(anyhow!("Cannot download() without local DataFile.")), - Some(file) => file - }; - // check to make sure we won't overwrite - if data_file.is_alive(path_context) && !overwrite { - return Err(anyhow!("Data file '{}' exists locally, and would be \ - overwritten by download. Use --overwrite to download.", - data_file.path)); - } - // if no remote, there is nothing to download, - // silently return Ok. Get URL. - let remote = merged_file.remote.as_ref().ok_or(anyhow!("Remote is None"))?; - let url = remote.url.as_ref().ok_or(anyhow!("Cannot download; download URL not set."))?; - - // add the token in - let url = format!("{}?access_token={}", url, self.token); - let save_path = &data_file.full_path(path_context)?; - Ok( DownloadInfo { url, path:save_path.to_string_lossy().to_string() }) - } + pub fn authenticate_url(&self, url: &str) -> Result { + Ok(format!("{}?access_token={}", url, self.token)) + } } #[cfg(test)] diff --git a/src/lib/data.rs b/src/lib/data.rs index b44c5dc..c23651c 100644 --- a/src/lib/data.rs +++ b/src/lib/data.rs @@ -1023,6 +1023,11 @@ impl DataCollection { .build(); downloader.download(&downloads).await; println!("Downloaded {}.", pluralize(total_files as u64, "file")); + for download in downloads { + let filename = PathBuf::from(&download.filename); + let name_str = filename.file_name().ok_or(anyhow!("Internal Error: could not extract filename from download"))?; + println!(" - {}", name_str.to_string_lossy()); + } } else { println!("No files downloaded."); } diff --git a/src/lib/remote.rs b/src/lib/remote.rs index 41af647..d5767a3 100644 --- a/src/lib/remote.rs +++ b/src/lib/remote.rs @@ -8,7 +8,7 @@ use anyhow::{anyhow,Result}; #[allow(unused_imports)] use log::{info, trace, debug}; use std::collections::HashMap; -use trauma::{download::Download}; +use trauma::download::Download; use serde_derive::{Serialize,Deserialize}; use reqwest::Url; @@ -205,11 +205,30 @@ impl Remote { // TODO: could be struct, if some APIs require more authentication // Note: requires each API actually *check* overwrite. pub fn get_download_info(&self, merged_file: &MergedFile, path_context: &Path, overwrite: bool) -> Result { - match self { - Remote::FigShareAPI(fgsh_api) => fgsh_api.get_download_info(merged_file, path_context, overwrite), - Remote::ZenodoAPI(_) => Err(anyhow!("ZenodoAPI does not support get_project method")), - Remote::DataDryadAPI(_) => service_not_implemented!("DataDryad"), + // if local DataFile is none, not in manifest; + // do not download + let data_file = match &merged_file.local { + None => return Err(anyhow!("Cannot download() without local DataFile.")), + Some(file) => file + }; + // check to make sure we won't overwrite + if data_file.is_alive(path_context) && !overwrite { + return Err(anyhow!("Data file '{}' exists locally, and would be \ + overwritten by download. Use --overwrite to download.", + data_file.path)); } + // if no remote, there is nothing to download, + // silently return Ok. Get URL. + let remote = merged_file.remote.as_ref().ok_or(anyhow!("Remote is None"))?; + let url = remote.url.as_ref().ok_or(anyhow!("Cannot download; download URL not set."))?; + + let authenticated_url = match self { + Remote::FigShareAPI(fgsh_api) => fgsh_api.authenticate_url(url), + Remote::ZenodoAPI(znd_api) => znd_api.authenticate_url(url), + Remote::DataDryadAPI(_) => service_not_implemented!("DataDryad"), + }?; + let save_path = &data_file.full_path(path_context)?; + Ok( DownloadInfo { url: authenticated_url, path:save_path.to_string_lossy().to_string() }) } }