diff --git a/Cargo.lock b/Cargo.lock index 7c5621f..5208f7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aho-corasick" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] @@ -43,9 +43,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.4" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" dependencies = [ "anstyle", "anstyle-parse", @@ -63,37 +63,37 @@ checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" [[package]] name = "ascii-canvas" @@ -121,36 +121,49 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" dependencies = [ "concurrent-queue", - "event-listener", + "event-listener 2.5.3", "futures-core", ] +[[package]] +name = "async-channel" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca33f4bc4ed1babef42cad36cc1f51fa88be00420404e5b1e80ab1b18f7678c" +dependencies = [ + "concurrent-queue", + "event-listener 4.0.3", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-executor" -version = "1.5.4" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c1da3ae8dabd9c00f453a329dfe1fb28da3c0a72e2478cdcd93171740c20499" +checksum = "17ae5ebefcc48e7452b4987947920dac9450be1110cadf34d1b8c116bdbaf97c" dependencies = [ - "async-lock", + "async-lock 3.3.0", "async-task", "concurrent-queue", "fastrand 2.0.1", - "futures-lite", + "futures-lite 2.2.0", "slab", ] [[package]] name = "async-global-executor" -version = "2.3.1" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" dependencies = [ - "async-channel", + "async-channel 2.1.1", "async-executor", - "async-io", - "async-lock", + "async-io 2.3.0", + "async-lock 3.3.0", "blocking", - "futures-lite", + "futures-lite 2.2.0", "once_cell", ] @@ -160,27 +173,57 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" dependencies = [ - "async-lock", + "async-lock 2.8.0", "autocfg", "cfg-if", "concurrent-queue", - "futures-lite", + "futures-lite 1.13.0", "log", "parking", - "polling", - "rustix 0.37.24", + "polling 2.8.0", + "rustix 0.37.27", "slab", - "socket2 0.4.9", + "socket2 0.4.10", "waker-fn", ] +[[package]] +name = "async-io" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb41eb19024a91746eba0773aa5e16036045bbf45733766661099e182ea6a744" +dependencies = [ + "async-lock 3.3.0", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite 2.2.0", + "parking", + "polling 3.3.2", + "rustix 0.38.30", + "slab", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "async-lock" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" dependencies = [ - "event-listener", + "event-listener 2.5.3", +] + +[[package]] +name = "async-lock" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" +dependencies = [ + "event-listener 4.0.3", + "event-listener-strategy", + "pin-project-lite", ] [[package]] @@ -194,19 +237,36 @@ dependencies = [ [[package]] name = "async-process" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a9d28b1d97e08915212e2e45310d47854eafa69600756fc735fb788f75199c9" +checksum = "ea6438ba0a08d81529c69b36700fa2f95837bfe3e776ab39cde9c14d9149da88" dependencies = [ - "async-io", - "async-lock", - "autocfg", + "async-io 1.13.0", + "async-lock 2.8.0", + "async-signal", "blocking", "cfg-if", - "event-listener", - "futures-lite", - "rustix 0.37.24", - "signal-hook", + "event-listener 3.1.0", + "futures-lite 1.13.0", + "rustix 0.38.30", + "windows-sys 0.48.0", +] + +[[package]] +name = "async-signal" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e47d90f65a225c4527103a8d747001fc56e375203592b25ad103e1ca13124c5" +dependencies = [ + "async-io 2.3.0", + "async-lock 2.8.0", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix 0.38.30", + "signal-hook-registry", + "slab", "windows-sys 0.48.0", ] @@ -216,16 +276,16 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" dependencies = [ - "async-channel", + "async-channel 1.9.0", "async-global-executor", - "async-io", - "async-lock", + "async-io 1.13.0", + "async-lock 2.8.0", "async-process", "crossbeam-utils", "futures-channel", "futures-core", "futures-io", - "futures-lite", + "futures-lite 1.13.0", "gloo-timers", "kv-log-macro", "log", @@ -239,15 +299,15 @@ dependencies = [ [[package]] name = "async-task" -version = "4.4.1" +version = "4.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9441c6b2fe128a7c2bf680a44c34d0df31ce09e5b7e401fcca3faa483dbc921" +checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", @@ -283,15 +343,15 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.4" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "basic-cookies" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb53b6b315f924c7f113b162e53b3901c05fc9966baf84d201dfcc7432a4bb38" +checksum = "67bd8fd42c16bdb08688243dc5f0cc117a3ca9efeeaba3a345a18a6159ad96f7" dependencies = [ "lalrpop", "lalrpop-util", @@ -321,22 +381,22 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "blocking" -version = "1.4.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94c4ef1f913d78636d78d538eec1f18de81e481f44b1be0a81060090530846e1" +checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" dependencies = [ - "async-channel", - "async-lock", + "async-channel 2.1.1", + "async-lock 3.3.0", "async-task", "fastrand 2.0.1", "futures-io", - "futures-lite", + "futures-lite 2.2.0", "piper", "tracing", ] @@ -376,21 +436,21 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "41daef31d7a747c5c847246f36de49ced6f7403b4cdabc807a97b5cc184cda7a" dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] name = "clap" -version = "4.4.6" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" +checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" dependencies = [ "clap_builder", "clap_derive", @@ -398,9 +458,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.6" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" +checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" dependencies = [ "anstream", "anstyle", @@ -410,9 +470,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.2" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", "proc-macro2", @@ -422,9 +482,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "colorchoice" @@ -434,42 +494,41 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "colored" -version = "2.0.4" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2674ec482fbc38012cf31e6c42ba0177b431a0cb6f15fe40efa5aab1bda516f6" +checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" dependencies = [ - "is-terminal", "lazy_static", "windows-sys 0.48.0", ] [[package]] name = "concurrent-queue" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" dependencies = [ "crossbeam-utils", ] [[package]] name = "console" -version = "0.15.7" +version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ "encode_unicode", "lazy_static", "libc", "unicode-width", - "windows-sys 0.45.0", + "windows-sys 0.52.0", ] [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -477,9 +536,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "crc32fast" @@ -492,22 +551,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crunchy" @@ -517,9 +572,9 @@ checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] name = "csv" -version = "1.2.2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" dependencies = [ "csv-core", "itoa", @@ -529,9 +584,9 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" dependencies = [ "memchr", ] @@ -547,15 +602,15 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "socket2 0.4.9", + "socket2 0.4.10", "winapi", ] [[package]] name = "curl-sys" -version = "0.4.66+curl-8.3.0" +version = "0.4.70+curl-8.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70c44a72e830f0e40ad90dda8a6ab6ed6314d39776599a58a2e5e37fbc6db5b9" +checksum = "3c0333d8849afe78a4c8102a429a446bfdd055832af071945520e835ae2d841e" dependencies = [ "cc", "libc", @@ -569,9 +624,12 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] [[package]] name = "diff" @@ -653,9 +711,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" dependencies = [ "humantime", "is-terminal", @@ -672,30 +730,51 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.4" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ - "errno-dragonfly", "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] -name = "errno-dragonfly" -version = "0.1.2" +name = "event-listener" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "event-listener" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93877bcde0eb80ca09131a08d23f0a5c18a620b01db137dba666d18cd9b30c2" dependencies = [ - "cc", - "libc", + "concurrent-queue", + "parking", + "pin-project-lite", ] [[package]] name = "event-listener" -version = "2.5.3" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" +dependencies = [ + "event-listener 4.0.3", + "pin-project-lite", +] [[package]] name = "fastrand" @@ -720,9 +799,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "miniz_oxide", @@ -751,18 +830,18 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] [[package]] name = "futures" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -775,9 +854,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -785,15 +864,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -802,9 +881,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" @@ -821,11 +900,24 @@ dependencies = [ "waker-fn", ] +[[package]] +name = "futures-lite" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445ba825b27408685aaecefd65178908c36c6e96aaf6d8599419d46e624192ba" +dependencies = [ + "fastrand 2.0.1", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -834,21 +926,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -864,9 +956,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "js-sys", @@ -877,9 +969,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "gloo-timers" @@ -895,9 +987,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", @@ -905,7 +997,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.3", + "indexmap 2.1.0", "slab", "tokio", "tokio-util", @@ -920,9 +1012,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" [[package]] name = "heck" @@ -932,15 +1024,15 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" [[package]] name = "http" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ "bytes", "fnv", @@ -949,9 +1041,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http", @@ -1006,9 +1098,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes", "futures-channel", @@ -1021,7 +1113,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.5.5", "tokio", "tower-service", "tracing", @@ -1043,16 +1135,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.57" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows", + "windows-core", ] [[package]] @@ -1066,9 +1158,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -1086,12 +1178,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown 0.14.1", + "hashbrown 0.14.3", ] [[package]] @@ -1133,19 +1225,19 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" dependencies = [ "hermit-abi", - "rustix 0.38.15", - "windows-sys 0.48.0", + "rustix 0.38.30", + "windows-sys 0.52.0", ] [[package]] @@ -1154,19 +1246,19 @@ version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "334e04b4d781f436dc315cb1e7515bd96826426345d498149e4bde36b67f8ee9" dependencies = [ - "async-channel", + "async-channel 1.9.0", "castaway", "crossbeam-utils", "curl", "curl-sys", "encoding_rs", - "event-listener", - "futures-lite", + "event-listener 2.5.3", + "futures-lite 1.13.0", "http", "log", "mime", "once_cell", - "polling", + "polling 2.8.0", "slab", "sluice", "tracing", @@ -1186,15 +1278,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" dependencies = [ "wasm-bindgen", ] @@ -1210,9 +1302,9 @@ dependencies = [ [[package]] name = "lalrpop" -version = "0.19.12" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a1cbf952127589f2851ab2046af368fd20645491bb4b376f04b7f94d7a9837b" +checksum = "da4081d44f4611b66c6dd725e6de3169f9f63905421e8626fcb86b6a898998b8" dependencies = [ "ascii-canvas", "bit-set", @@ -1222,8 +1314,9 @@ dependencies = [ "itertools", "lalrpop-util", "petgraph", + "pico-args", "regex", - "regex-syntax 0.6.29", + "regex-syntax 0.7.5", "string_cache", "term", "tiny-keccak", @@ -1232,9 +1325,9 @@ dependencies = [ [[package]] name = "lalrpop-util" -version = "0.19.12" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3c48237b9604c5a4702de6b824e02006c3214327564636aef27c1028a8fa0ed" +checksum = "3f35c735096c0293d313e8f2a641627472b83d01b937177fe76e5e2708d31e0d" dependencies = [ "regex", ] @@ -1253,25 +1346,36 @@ checksum = "db13adb97ab515a3691f56e4dbab09283d0b86cb45abd991d8634a9d6f501760" [[package]] name = "libc" -version = "0.2.148" +version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "libnghttp2-sys" -version = "0.1.8+1.55.1" +version = "0.1.9+1.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fae956c192dadcdb5dace96db71fa0b827333cce7c7b38dc71446f024d8a340" +checksum = "b57e858af2798e167e709b9d969325b6d8e9d50232fcbc494d7d54f976854a64" dependencies = [ "cc", "libc", ] +[[package]] +name = "libredox" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +dependencies = [ + "bitflags 2.4.2", + "libc", + "redox_syscall 0.4.1", +] + [[package]] name = "libz-sys" -version = "1.1.12" +version = "1.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +checksum = "295c17e837573c8c821dbaeb3cceb3d745ad082f7572191409e69cbc1b3fd050" dependencies = [ "cc", "libc", @@ -1287,15 +1391,15 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.8" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -1324,9 +1428,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "mime" @@ -1355,9 +1459,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" dependencies = [ "libc", "wasi", @@ -1400,9 +1504,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", ] @@ -1434,26 +1538,26 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "openssl" -version = "0.10.57" +version = "0.10.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.2", "cfg-if", "foreign-types", "libc", @@ -1481,9 +1585,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.93" +version = "0.9.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" dependencies = [ "cc", "libc", @@ -1564,9 +1668,9 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "ordered-float" -version = "3.9.1" +version = "3.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a54938017eacd63036332b4ae5c8a49fc8c0c1d6d629893057e4f13609edd06" +checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc" dependencies = [ "num-traits", ] @@ -1579,9 +1683,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "parking" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e52c774a4c39359c1d1c52e43f73dd91a75a614652c825408eec30c95a9b2067" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" @@ -1601,7 +1705,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.8", + "parking_lot_core 0.9.9", ] [[package]] @@ -1620,22 +1724,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall 0.4.1", "smallvec", "windows-targets 0.48.5", ] [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" @@ -1644,7 +1748,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.0.2", + "indexmap 2.1.0", ] [[package]] @@ -1656,6 +1760,12 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pico-args" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315" + [[package]] name = "pin-project" version = "1.1.3" @@ -1701,9 +1811,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" [[package]] name = "polling" @@ -1721,11 +1831,31 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "polling" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545c980a3880efd47b2e262f6a4bb6daad6555cf3367aa9c4e52895f69537a41" +dependencies = [ + "cfg-if", + "concurrent-queue", + "pin-project-lite", + "rustix 0.38.30", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "portable-atomic" -version = "1.4.3" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" @@ -1741,18 +1871,18 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -1798,64 +1928,64 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "redox_users" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" dependencies = [ "getrandom", - "redox_syscall 0.2.16", + "libredox", "thiserror", ] [[package]] name = "regex" -version = "1.9.6" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", "regex-automata", - "regex-syntax 0.7.5", + "regex-syntax 0.8.2", ] [[package]] name = "regex-automata" -version = "0.3.9" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" +checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.5", + "regex-syntax 0.8.2", ] [[package]] name = "regex-syntax" -version = "0.6.29" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.20" +version = "0.11.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" +checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ "base64", "bytes", @@ -1879,6 +2009,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "system-configuration", "tokio", "tokio-native-tls", "tokio-socks", @@ -1894,9 +2025,9 @@ dependencies = [ [[package]] name = "reqwest-middleware" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff44108c7925d082f2861e683a88618b68235ad9cdc60d64d9d1188efc951cdb" +checksum = "88a3e86aa6053e59030e7ce2d2a3b258dd08fc2d337d52f73f6cb480f5858690" dependencies = [ "anyhow", "async-trait", @@ -1932,9 +2063,9 @@ dependencies = [ [[package]] name = "reqwest-tracing" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14b1e66540e0cac90acadaf7109bf99c90d95abcc94b4c096bfa16a2d7aa7a71" +checksum = "5a0152176687dd5cfe7f507ac1cb1a491c679cfe483afd133a7db7aaea818bb3" dependencies = [ "anyhow", "async-trait", @@ -1950,9 +2081,9 @@ dependencies = [ [[package]] name = "retry-policies" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a715dc4d0e8aea3085a9a94d76e79c79c7df7c9f6be609da841a6d2489ca3687" +checksum = "17dd00bff1d737c40dbcd47d4375281bf4c17933f9eef0a185fc7bacca23ecbd" dependencies = [ "anyhow", "chrono", @@ -1967,9 +2098,9 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.37.24" +version = "0.37.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4279d76516df406a8bd37e7dff53fd37d1a093f997a3c34a5c21658c126db06d" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" dependencies = [ "bitflags 1.3.2", "errno", @@ -1981,15 +2112,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.15" +version = "0.38.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f9da0cbd88f9f09e7814e388301c8414c51c62aa6ce1e4b5c551d49d96e531" +checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.2", "errno", "libc", - "linux-raw-sys 0.4.8", - "windows-sys 0.48.0", + "linux-raw-sys 0.4.13", + "windows-sys 0.52.0", ] [[package]] @@ -2000,22 +2131,22 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "scidataflow" -version = "0.8.6" +version = "0.8.7" dependencies = [ "anyhow", "chrono", @@ -2078,18 +2209,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.188" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", @@ -2098,9 +2229,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -2131,11 +2262,11 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.25" +version = "0.9.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" +checksum = "b1bf28c79a99f70ee1f1d83d10c875d2e70618417fda01ad1785e027579d9d38" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.1.0", "itoa", "ryu", "serde", @@ -2144,23 +2275,13 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1b21f559e07218024e7e9f90f96f601825397de0e25420135f7f952453fed0b" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] -[[package]] -name = "signal-hook" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" -dependencies = [ - "libc", - "signal-hook-registry", -] - [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -2172,9 +2293,9 @@ dependencies = [ [[package]] name = "similar" -version = "2.2.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" +checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21" [[package]] name = "siphasher" @@ -2197,22 +2318,22 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d7400c0eff44aa2fcb5e31a5f24ba9716ed90138769e4977a2ba6014ae63eb5" dependencies = [ - "async-channel", + "async-channel 1.9.0", "futures-core", "futures-io", ] [[package]] name = "smallvec" -version = "1.11.1" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "socket2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -2220,9 +2341,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", "windows-sys 0.48.0", @@ -2249,15 +2370,36 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "2.0.37" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "task-local-extensions" version = "0.1.4" @@ -2269,15 +2411,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.0" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if", "fastrand 2.0.1", - "redox_syscall 0.3.5", - "rustix 0.38.15", - "windows-sys 0.48.0", + "redox_syscall 0.4.1", + "rustix 0.38.30", + "windows-sys 0.52.0", ] [[package]] @@ -2293,27 +2435,27 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] [[package]] name = "thiserror" -version = "1.0.49" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.49" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", @@ -2332,14 +2474,15 @@ dependencies = [ [[package]] name = "time" -version = "0.3.29" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" +checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" dependencies = [ "deranged", "itoa", "libc", "num_threads", + "powerfmt", "serde", "time-core", "time-macros", @@ -2353,9 +2496,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" dependencies = [ "time-core", ] @@ -2392,9 +2535,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.32.0" +version = "1.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" dependencies = [ "backtrace", "bytes", @@ -2404,16 +2547,16 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.4", + "socket2 0.5.5", "tokio-macros", "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", @@ -2444,9 +2587,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -2464,11 +2607,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -2477,9 +2619,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", @@ -2488,9 +2630,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", "valuable", @@ -2508,12 +2650,23 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" dependencies = [ - "lazy_static", "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", "tracing-core", ] @@ -2527,7 +2680,7 @@ dependencies = [ "opentelemetry 0.17.0", "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.1.4", "tracing-subscriber", ] @@ -2543,22 +2696,22 @@ dependencies = [ "smallvec", "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.1.4", "tracing-subscriber", ] [[package]] name = "tracing-subscriber" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ "nu-ansi-term", "sharded-slab", "smallvec", "thread_local", "tracing-core", - "tracing-log", + "tracing-log 0.2.0", ] [[package]] @@ -2584,9 +2737,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "unicase" @@ -2599,9 +2752,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -2632,15 +2785,15 @@ checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "unsafe-libyaml" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" +checksum = "ab4c90930b95a82d00dc9e9ac071b4991924390d46cbd0dfe566148667605e4b" [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -2667,9 +2820,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value-bag" -version = "1.4.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d92ccd67fb88503048c01b59152a04effd0782d035a83a6d256ce6085f08f4a3" +checksum = "7cdbaf5e132e593e9fc1de6a15bbec912395b11fb9719e061cf64f804524c503" [[package]] name = "vcpkg" @@ -2706,9 +2859,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2716,9 +2869,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" dependencies = [ "bumpalo", "log", @@ -2731,9 +2884,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.37" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" dependencies = [ "cfg-if", "js-sys", @@ -2743,9 +2896,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2753,9 +2906,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", @@ -2766,9 +2919,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "wasm-streams" @@ -2800,9 +2953,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" dependencies = [ "js-sys", "wasm-bindgen", @@ -2840,21 +2993,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.48.0" +name = "windows-core" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", + "windows-targets 0.52.0", ] [[package]] @@ -2867,18 +3011,12 @@ dependencies = [ ] [[package]] -name = "windows-targets" -version = "0.42.2" +name = "windows-sys" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "windows-targets 0.52.0", ] [[package]] @@ -2897,10 +3035,19 @@ dependencies = [ ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" +name = "windows-targets" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] [[package]] name = "windows_aarch64_gnullvm" @@ -2909,10 +3056,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" +name = "windows_aarch64_gnullvm" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" [[package]] name = "windows_aarch64_msvc" @@ -2921,10 +3068,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] -name = "windows_i686_gnu" -version = "0.42.2" +name = "windows_aarch64_msvc" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" [[package]] name = "windows_i686_gnu" @@ -2933,10 +3080,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] -name = "windows_i686_msvc" -version = "0.42.2" +name = "windows_i686_gnu" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" [[package]] name = "windows_i686_msvc" @@ -2945,10 +3092,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" +name = "windows_i686_msvc" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" [[package]] name = "windows_x86_64_gnu" @@ -2957,10 +3104,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" +name = "windows_x86_64_gnu" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" [[package]] name = "windows_x86_64_gnullvm" @@ -2969,10 +3116,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" +name = "windows_x86_64_gnullvm" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" [[package]] name = "windows_x86_64_msvc" @@ -2980,6 +3127,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winreg" version = "0.50.0" diff --git a/src/lib.rs b/src/lib.rs index 17fefee..70535bc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,14 +5,14 @@ pub mod lib { pub mod figshare; pub mod zenodo; } - pub mod project; + pub mod assets; pub mod download; - pub mod progress; pub mod macros; + pub mod progress; + pub mod project; pub mod remote; - pub mod utils; - pub mod assets; pub mod test_utilities; + pub mod utils; } pub mod logging_setup; diff --git a/src/lib/api/dryad.rs b/src/lib/api/dryad.rs index ad0effa..e0b1ba4 100644 --- a/src/lib/api/dryad.rs +++ b/src/lib/api/dryad.rs @@ -1,11 +1,9 @@ -use serde_derive::{Serialize,Deserialize}; +use serde_derive::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] pub struct DataDryadAPI { base_url: String, #[serde(skip_serializing)] - token: String + token: String, } - - diff --git a/src/lib/api/figshare.rs b/src/lib/api/figshare.rs index 4947e9c..165ae84 100644 --- a/src/lib/api/figshare.rs +++ b/src/lib/api/figshare.rs @@ -1,31 +1,34 @@ // FigShare API // // Notes: -// FigShare's API design is, in my view, a bit awkward. -// There are articles, files, and projects. +// FigShare's API design is, in my view, a bit awkward. +// There are articles, files, and projects. -use url::Url; -use std::fs; -use std::path::Path; -use std::io::{Read,Seek,SeekFrom}; -use anyhow::{anyhow,Result}; -#[allow(unused_imports)] -use log::{info, trace, debug}; -use std::collections::HashMap; -use serde_derive::{Serialize,Deserialize}; -use serde_json::Value; -use reqwest::{Method, header::{HeaderMap, HeaderValue}}; -use reqwest::{Client, Response, Body}; +use anyhow::{anyhow, Result}; use colored::Colorize; use futures_util::StreamExt; +#[allow(unused_imports)] +use log::{debug, info, trace}; +use reqwest::{ + header::{HeaderMap, HeaderValue}, + Method, +}; +use reqwest::{Body, Client, Response}; +use serde_derive::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::HashMap; +use std::fs; +use std::io::{Read, Seek, SeekFrom}; +use std::path::Path; use tokio::fs::File; use tokio::io::AsyncWriteExt; +use url::Url; -#[allow(unused_imports)] -use crate::{print_info,print_warn}; use crate::lib::data::DataFile; -use crate::lib::remote::{AuthKeys, RemoteFile, RequestData}; use crate::lib::project::LocalMetadata; +use crate::lib::remote::{AuthKeys, RemoteFile, RequestData}; +#[allow(unused_imports)] +use crate::{print_info, print_warn}; pub const FIGSHARE_BASE_URL: &str = "https://api.figshare.com/v2/"; @@ -39,13 +42,13 @@ fn figshare_api_url() -> String { #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] pub struct FigShareAPI { - #[serde(skip_serializing, skip_deserializing,default="figshare_api_url")] + #[serde(skip_serializing, skip_deserializing, default = "figshare_api_url")] base_url: String, // one remote corresponds to a FigShare article article_id: Option, name: String, #[serde(skip_serializing, skip_deserializing)] - token: String + token: String, } pub struct FigShareUpload<'a> { @@ -74,9 +77,8 @@ pub struct FigShareFile { pub struct FigShareNewUpload { md5: String, name: String, - size: u64 + size: u64, } - /// This struct is for response to the initial GET using the /// upload_url. It contains more details about the actual upload. @@ -108,7 +110,6 @@ pub struct FigShareCompleteUpload { name: String, size: u64, } - /// Manage a FigShare Upload impl<'a> FigShareUpload<'a> { @@ -116,7 +117,10 @@ impl<'a> FigShareUpload<'a> { FigShareUpload { api_instance: api } } - async fn init_upload(&self, data_file: &DataFile) -> Result<(FigShareFile, FigSharePendingUploadInfo)> { + async fn init_upload( + &self, + data_file: &DataFile, + ) -> Result<(FigShareFile, FigSharePendingUploadInfo)> { debug!("initializing upload of '{:?}'", data_file); // Requires: article ID, in FigShareArticle struct // (0) create URL and data @@ -125,36 +129,39 @@ impl<'a> FigShareUpload<'a> { let data = FigShareNewUpload { name: data_file.basename()?, md5: data_file.md5.clone(), - size: data_file.size + size: data_file.size, }; // (1) issue POST to get location - let response = self.api_instance.issue_request(Method::POST, &url, Some(RequestData::Json(data))).await?; + let response = self + .api_instance + .issue_request(Method::POST, &url, Some(RequestData::Json(data))) + .await?; debug!("upload post response: {:?}", response); // (2) get location let data = response.json::().await?; let location_url = match data.get("location").and_then(|loc| loc.as_str()) { Some(loc) => Ok(loc), - None => Err(anyhow!("Response does not have 'location' set!")) + None => Err(anyhow!("Response does not have 'location' set!")), }?; // we need to extract out the non-domain part let parsed_url = Url::parse(location_url)?; - let location = parsed_url.path() - .to_string() - .replacen("/v2/", "/", 1); + let location = parsed_url.path().to_string().replacen("/v2/", "/", 1); debug!("upload location: {:?}", location); // (3) issue GET to retrieve upload info - let response = self.api_instance + let response = self + .api_instance .issue_request::>(Method::GET, &location, None) .await?; let upload_info: FigShareFile = response.json().await?; debug!("upload info: {:?}", upload_info); // (4) Now, we need to issue another GET to initiate upload. - // This returns the file parts info, which tells us how to split + // This returns the file parts info, which tells us how to split // the file. - let response = self.api_instance + let response = self + .api_instance .issue_request::>(Method::GET, &upload_info.upload_url, None) .await?; let pending_upload_info: FigSharePendingUploadInfo = response.json().await?; @@ -162,10 +169,13 @@ impl<'a> FigShareUpload<'a> { Ok((upload_info, pending_upload_info)) } - async fn upload_parts(&self, data_file: &DataFile, - upload_info: &FigShareFile, - pending_upload_info: &FigSharePendingUploadInfo, - path_context: &Path) -> Result<()> { + async fn upload_parts( + &self, + data_file: &DataFile, + upload_info: &FigShareFile, + pending_upload_info: &FigSharePendingUploadInfo, + path_context: &Path, + ) -> Result<()> { let full_path = path_context.join(&data_file.path); let url = &upload_info.upload_url; let mut file = fs::File::open(full_path)?; @@ -180,9 +190,18 @@ impl<'a> FigShareUpload<'a> { file.read_exact(&mut data)?; let part_url = format!("{}/{}", &url, part.part_no); - let _response = self.api_instance.issue_request::>(Method::PUT, &part_url, Some(RequestData::Binary(data))) + let _response = self + .api_instance + .issue_request::>( + Method::PUT, + &part_url, + Some(RequestData::Binary(data)), + ) .await?; - debug!("uploaded part {} (offsets {}:{})", part.part_no, start_offset, end_offset) + debug!( + "uploaded part {} (offsets {}:{})", + part.part_no, start_offset, end_offset + ) } Ok(()) @@ -194,15 +213,25 @@ impl<'a> FigShareUpload<'a> { let data = FigShareCompleteUpload { id: article_id, name: upload_info.name.clone(), - size: upload_info.size + size: upload_info.size, }; - self.api_instance.issue_request(Method::POST, &url, Some(RequestData::Json(data))).await?; + self.api_instance + .issue_request(Method::POST, &url, Some(RequestData::Json(data))) + .await?; Ok(()) } - pub async fn upload(&self, data_file: &DataFile, path_context: &Path, overwrite: bool) -> Result<()> { + pub async fn upload( + &self, + data_file: &DataFile, + path_context: &Path, + overwrite: bool, + ) -> Result<()> { if !data_file.is_alive(path_context) { - return Err(anyhow!("Cannot upload: file '{}' does not exist lcoally.", data_file.path)); + return Err(anyhow!( + "Cannot upload: file '{}' does not exist lcoally.", + data_file.path + )); } // check if any files are associated with this article let article_id = self.api_instance.get_article_id()?; @@ -210,18 +239,25 @@ impl<'a> FigShareUpload<'a> { let existing_file = self.api_instance.file_exists(&name).await?; if let Some(file) = existing_file { if !overwrite { - print_info!("FigShare::upload() found file '{}' in FigShare \ + print_info!( + "FigShare::upload() found file '{}' in FigShare \ Article ID={}. Since overwrite=false, this file will not be deleted and re-upload.", - name, article_id); + name, + article_id + ); } else { - info!("FigShare::upload() is deleting file '{}' since \ - overwrite=true.", name); + info!( + "FigShare::upload() is deleting file '{}' since \ + overwrite=true.", + name + ); self.api_instance.delete_article_file(&file).await?; - } - } + } + } let (upload_info, pending_upload_info) = self.init_upload(data_file).await?; - self.upload_parts(data_file, &upload_info, &pending_upload_info, path_context).await?; + self.upload_parts(data_file, &upload_info, &pending_upload_info, path_context) + .await?; self.complete_upload(&upload_info).await?; Ok(()) } @@ -234,7 +270,7 @@ impl From for RemoteFile { md5: Some(fgsh.computed_md5), size: Some(fgsh.size), remote_service: "FigShare".to_string(), - url: Some(fgsh.download_url) + url: Some(fgsh.download_url), } } } @@ -242,16 +278,16 @@ impl From for RemoteFile { #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] pub struct FigShareArticle { title: String, - id: u64 + id: u64, } impl FigShareAPI { pub fn new(name: &str, base_url: Option) -> Result { - // Note: this constructor is not called often, except through - // Project::link(), since serde is usually deserializing the + // Note: this constructor is not called often, except through + // Project::link(), since serde is usually deserializing the // new FigShareAPI Remote variant from the manifest. let auth_keys = if base_url.is_none() { - // using the default base_url means we're + // using the default base_url means we're // not using mock HTTP servers AuthKeys::new() } else { @@ -263,11 +299,11 @@ impl FigShareAPI { }; let token = auth_keys.get("figshare".to_string())?; let base_url = base_url.unwrap_or(FIGSHARE_BASE_URL.to_string()); - Ok(FigShareAPI { + Ok(FigShareAPI { base_url, article_id: None, - name: name.to_string(), - token + name: name.to_string(), + token, }) } @@ -279,11 +315,15 @@ impl FigShareAPI { self.base_url.clone() } - async fn issue_request(&self, method: Method, endpoint: &str, - data: Option>) -> Result { + async fn issue_request( + &self, + method: Method, + endpoint: &str, + data: Option>, + ) -> Result { let mut headers = HeaderMap::new(); - // FigShare will give download links outside the API, so we handle + // FigShare will give download links outside the API, so we handle // that possibility here. let url = if endpoint.starts_with("https://") || endpoint.starts_with("http://") { endpoint.to_string() @@ -296,7 +336,10 @@ impl FigShareAPI { let client = Client::new(); let mut request = client.request(method, &url); - headers.insert("Authorization", HeaderValue::from_str(&format!("token {}", self.token)).unwrap()); + headers.insert( + "Authorization", + HeaderValue::from_str(&format!("token {}", self.token)).unwrap(), + ); trace!("headers: {:?}", headers); request = request.headers(headers); @@ -308,8 +351,10 @@ impl FigShareAPI { let stream = tokio_util::io::ReaderStream::new(file); let body = Body::wrap_stream(stream); request.body(body) - }, - Some(RequestData::Empty) => request.json(&serde_json::Value::Object(serde_json::Map::new())), + } + Some(RequestData::Empty) => { + request.json(&serde_json::Value::Object(serde_json::Map::new())) + } None => request, }; @@ -318,11 +363,15 @@ impl FigShareAPI { if response_status.is_success() { Ok(response) } else { - Err(anyhow!("HTTP Error: {}\nurl: {:?}\n{:?}", response_status, url, response.text().await?)) + Err(anyhow!( + "HTTP Error: {}\nurl: {:?}\n{:?}", + response_status, + url, + response.text().await? + )) } } - // Download a single file through the FigShare API // NOTE: Mostly deprecated due to trauma-based downloads. #[allow(dead_code)] @@ -348,13 +397,17 @@ impl FigShareAPI { debug!("creating data for article: {:?}", data); // (2) issue request and parse out the article ID from location - let response = self.issue_request(Method::POST, endpoint, Some(RequestData::Json(data))).await?; + let response = self + .issue_request(Method::POST, endpoint, Some(RequestData::Json(data))) + .await?; let data = response.json::().await?; let article_id_result = match data.get("location").and_then(|loc| loc.as_str()) { Some(loc) => Ok(loc.split('/').last().unwrap_or_default().to_string()), - None => Err(anyhow!("Response does not have 'location' set!")) + None => Err(anyhow!("Response does not have 'location' set!")), }; - let article_id: u64 = article_id_result?.parse::().map_err(|_| anyhow!("Failed to parse article ID"))?; + let article_id: u64 = article_id_result? + .parse::() + .map_err(|_| anyhow!("Failed to parse article ID"))?; debug!("got article ID: {:?}", article_id); // (3) create and return the FigShareArticle @@ -364,9 +417,16 @@ impl FigShareAPI { }) } - pub async fn upload(&self, data_file: &DataFile, path_context: &Path, overwrite: bool) -> Result { + pub async fn upload( + &self, + data_file: &DataFile, + path_context: &Path, + overwrite: bool, + ) -> Result { let this_upload = FigShareUpload::new(self); - this_upload.upload(data_file, path_context, overwrite).await?; + this_upload + .upload(data_file, path_context, overwrite) + .await?; Ok(true) } @@ -376,10 +436,16 @@ impl FigShareAPI { pub async fn find_article(&self) -> Result> { let articles = self.get_articles().await?; - let matches_found: Vec<_> = articles.into_iter().filter(|a| a.title == self.name).collect(); + let matches_found: Vec<_> = articles + .into_iter() + .filter(|a| a.title == self.name) + .collect(); if !matches_found.is_empty() { if matches_found.len() > 1 { - Err(anyhow!("Found multiple FigShare Articles with the title '{}'", self.name)) + Err(anyhow!( + "Found multiple FigShare Articles with the title '{}'", + self.name + )) } else { Ok(Some(matches_found[0].clone())) } @@ -389,17 +455,24 @@ impl FigShareAPI { } // FigShare Remote initialization - // + // // This creates a FigShare article for the tracked directory. #[allow(unused)] - pub async fn remote_init(&mut self, local_metadata: LocalMetadata, link_only: bool) -> Result<()> { + pub async fn remote_init( + &mut self, + local_metadata: LocalMetadata, + link_only: bool, + ) -> Result<()> { // (1) Let's make sure there is no Article that exists // with this same name let found_match = self.find_article().await?; let article = if let Some(existing_info) = found_match { if !link_only { - return Err(anyhow!("An existing FigShare Article with the title \ - '{}' was found. Use --link-only to link.", self.name)); + return Err(anyhow!( + "An existing FigShare Article with the title \ + '{}' was found. Use --link-only to link.", + self.name + )); } existing_info } else { @@ -417,7 +490,9 @@ impl FigShareAPI { // TODO? does this get published data sets? async fn get_articles(&self) -> Result> { let url = "/account/articles"; - let response = self.issue_request::>(Method::GET, url, None).await?; + let response = self + .issue_request::>(Method::GET, url, None) + .await?; let articles: Vec = response.json().await?; Ok(articles) } @@ -430,9 +505,9 @@ impl FigShareAPI { // Get all files from a FigShare Article, in a HashMap // with file name as keys. - pub async fn get_files_hashmap(&self) -> Result> { + pub async fn get_files_hashmap(&self) -> Result> { let mut files: Vec = self.get_files().await?; - let mut files_hash: HashMap = HashMap::new(); + let mut files_hash: HashMap = HashMap::new(); for file in files.iter_mut() { files_hash.insert(file.name.clone(), file.clone()); } @@ -447,7 +522,9 @@ impl FigShareAPI { } pub fn get_article_id(&self) -> Result { - let article_id = self.article_id.ok_or(anyhow!("Internal Error: FigShare.article_id is None."))?; + let article_id = self + .article_id + .ok_or(anyhow!("Internal Error: FigShare.article_id is None."))?; Ok(article_id) } @@ -455,40 +532,44 @@ impl FigShareAPI { pub async fn get_files(&self) -> Result> { let article_id = self.get_article_id()?; let url = format!("/account/articles/{}/files", article_id); - let response = self.issue_request::>(Method::GET, &url, None).await?; + let response = self + .issue_request::>(Method::GET, &url, None) + .await?; let files: Vec = response.json().await?; Ok(files) } // Delete Article /* async fn delete_article(&self, article: &FigShareArticle) -> Result<()> { - let url = format!("account/articles/{}", article.id); - self.issue_request::>(Method::DELETE, &url, None).await?; - Ok(()) - } - */ + let url = format!("account/articles/{}", article.id); + self.issue_request::>(Method::DELETE, &url, None).await?; + Ok(()) + } + */ // Delete the specified file from the FigShare Article - // + // // Note: we require a &FigShareFile as a way to enforce it exists, // e.g. is the result of a previous query. async fn delete_article_file(&self, file: &FigShareFile) -> Result<()> { let article_id = self.get_article_id()?; let url = format!("account/articles/{}/files/{}", article_id, file.id); - self.issue_request::>(Method::DELETE, &url, None).await?; - info!("deleted FigShare file '{}' (Article ID={})", file.name, article_id); + self.issue_request::>(Method::DELETE, &url, None) + .await?; + info!( + "deleted FigShare file '{}' (Article ID={})", + file.name, article_id + ); Ok(()) } } - #[cfg(test)] mod tests { use super::*; + use crate::logging_setup::setup; use httpmock::prelude::*; use serde_json::json; - use crate::logging_setup::setup; - #[tokio::test] async fn test_create_article() { @@ -503,15 +584,17 @@ mod tests { let create_article_mock = server.mock(|when, then| { when.method(POST) .path("/account/articles") - .header("Authorization", &format!("token {}", TEST_TOKEN.to_string())) + .header( + "Authorization", + &format!("token {}", TEST_TOKEN.to_string()), + ) .json_body(json!({ "title": title.to_string(), "defined_type": "dataset" })); - then.status(201) - .json_body(json!({ - "location": format!("{}account/articles/{}", server.url(""), expected_id) - })); + then.status(201).json_body(json!({ + "location": format!("{}account/articles/{}", server.url(""), expected_id) + })); }); // Define a sample title for the article @@ -529,6 +612,5 @@ mod tests { // Verify that the mock was called exactly once create_article_mock.assert(); - } - + } } diff --git a/src/lib/api/zenodo.rs b/src/lib/api/zenodo.rs index 9fca56f..b725daf 100644 --- a/src/lib/api/zenodo.rs +++ b/src/lib/api/zenodo.rs @@ -1,23 +1,24 @@ -use anyhow::{anyhow,Result,Context}; -use std::path::Path; -use reqwest::{Method, header::{HeaderMap, HeaderValue, CONTENT_TYPE, CONTENT_LENGTH}}; -use reqwest::{Client, Response, Body}; -use std::collections::HashMap; -use serde_derive::{Serialize,Deserialize}; -#[allow(unused_imports)] -use log::{info, trace, debug}; +use anyhow::{anyhow, Context, Result}; use colored::Colorize; +#[allow(unused_imports)] +use log::{debug, info, trace}; +use reqwest::{ + header::{HeaderMap, HeaderValue, CONTENT_LENGTH, CONTENT_TYPE}, + Method, +}; +use reqwest::{Body, Client, Response}; +use serde_derive::{Deserialize, Serialize}; +use std::collections::HashMap; use std::convert::TryInto; +use std::path::Path; use tokio_util::io::ReaderStream; #[allow(unused_imports)] -use crate::{print_info,print_warn}; - +use crate::{print_info, print_warn}; +use crate::lib::remote::{AuthKeys, RemoteFile, RequestData}; +use crate::lib::utils::{shorten, ISSUE_URL}; use crate::lib::{data::DataFile, project::LocalMetadata}; -use crate::lib::remote::{AuthKeys,RemoteFile,RequestData}; -use crate::lib::utils::{ISSUE_URL, shorten}; - const BASE_URL: &str = "https://zenodo.org/api"; @@ -41,8 +42,7 @@ pub struct ZenodoDeposition { title: String, } - -#[allow(dead_code)] // used for deserialization of requests +#[allow(dead_code)] // used for deserialization of requests #[derive(Debug, Deserialize)] pub struct ZenodoFileUpload { key: String, @@ -57,7 +57,6 @@ pub struct ZenodoFileUpload { delete_marker: bool, } - #[derive(Debug, Serialize, Deserialize, Clone)] pub struct ZenodoFile { checksum: String, @@ -74,7 +73,7 @@ impl From for RemoteFile { md5: Some(znd.checksum), size: Some(znd.filesize as u64), remote_service: "Zenodo".to_string(), - url: znd.links.download + url: znd.links.download, } } } @@ -97,7 +96,7 @@ pub struct ZenodoLinks { #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] struct Creator { name: String, - affiliation: Option + affiliation: Option, } // We need this wrapper to provide the metadata @@ -121,17 +120,23 @@ impl TryInto for LocalMetadata { type Error = anyhow::Error; fn try_into(self) -> Result { - let name = self.author_name.ok_or_else(|| anyhow!("Author name is required"))?; + let name = self + .author_name + .ok_or_else(|| anyhow!("Author name is required"))?; // TODO? Warn user of default description? - let description = self.description.unwrap_or("Upload by SciDataFlow.".to_string()); + let description = self + .description + .unwrap_or("Upload by SciDataFlow.".to_string()); Ok(ZenodoDepositionData { metadata: ZenodoMetadata { prereserve_doi: None, - title: self.title.ok_or(anyhow!("Zenodo requires a title be set.\n\ + title: self.title.ok_or(anyhow!( + "Zenodo requires a title be set.\n\ Either: \n\ - set this manually in data_manifest.yml\n\ - - specify with 'sdf link --name '\n"))?, + - specify with 'sdf link --name '\n" + ))?, upload_type: Some("dataset".to_string()), description: Some(description), creators: Some(vec![Creator { @@ -149,13 +154,18 @@ struct PrereserveDoi { recid: usize, } -// Remove the BASE_URL from full URLs, e.g. for -// bucket_urls provided by Zenodo so they can go through the common +// Remove the BASE_URL from full URLs, e.g. for +// bucket_urls provided by Zenodo so they can go through the common // issue_request() method fn remove_base_url(full_url: &str) -> Result { - full_url.strip_prefix(BASE_URL).map(|s| s.to_string()) - .ok_or(anyhow!("Internal error: Zenodo BASE_URL not found in full URL: full_url={:?}, BASE_URL={:?}", - full_url, BASE_URL)) + full_url + .strip_prefix(BASE_URL) + .map(|s| s.to_string()) + .ok_or(anyhow!( + "Internal error: Zenodo BASE_URL not found in full URL: full_url={:?}, BASE_URL={:?}", + full_url, + BASE_URL + )) } // for serde deserialize default @@ -163,10 +173,9 @@ fn zenodo_api_url() -> String { BASE_URL.to_string() } - #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] pub struct ZenodoAPI { - #[serde(skip_serializing, skip_deserializing,default="zenodo_api_url")] + #[serde(skip_serializing, skip_deserializing, default = "zenodo_api_url")] base_url: String, name: String, #[serde(skip_serializing, skip_deserializing)] @@ -180,11 +189,11 @@ pub struct ZenodoAPI { impl ZenodoAPI { pub fn new(name: &str, base_url: Option) -> Result { - // Note: this constructor is not called often, except through - // Project::link(), since serde is usually deserializing the + // Note: this constructor is not called often, except through + // Project::link(), since serde is usually deserializing the // new ZenodoAPI Remote variant from the manifest. let auth_keys = if base_url.is_none() { - // using the default base_url means we're + // using the default base_url means we're // not using mock HTTP servers AuthKeys::new() } else { @@ -196,12 +205,12 @@ impl ZenodoAPI { }; let token = auth_keys.get("zenodo".to_string())?; let base_url = base_url.unwrap_or(BASE_URL.to_string()); - Ok(ZenodoAPI { + Ok(ZenodoAPI { base_url, - name: name.to_string(), + name: name.to_string(), token, deposition_id: None, - bucket_url: None + bucket_url: None, }) } @@ -213,11 +222,19 @@ impl ZenodoAPI { // TODO: this is the same as FigShareAPI's issue_request(). // Since APIs can have different authentication routines, we // should handle that part separately. - async fn issue_request(&self, method: Method, endpoint: &str, - headers: Option, - data: Option>) -> Result { - - let url = format!("{}/{}?access_token={}", self.base_url.trim_end_matches('/'), endpoint.trim_start_matches('/'), self.token); + async fn issue_request( + &self, + method: Method, + endpoint: &str, + headers: Option, + data: Option>, + ) -> Result { + let url = format!( + "{}/{}?access_token={}", + self.base_url.trim_end_matches('/'), + endpoint.trim_start_matches('/'), + self.token + ); trace!("request URL: {:?}", &url); let client = Client::new(); @@ -234,8 +251,10 @@ impl ZenodoAPI { let stream = ReaderStream::new(file); let body = Body::wrap_stream(stream); request.body(body) - }, - Some(RequestData::Empty) => request.json(&serde_json::Value::Object(serde_json::Map::new())), + } + Some(RequestData::Empty) => { + request.json(&serde_json::Value::Object(serde_json::Map::new())) + } None => request, }; @@ -247,34 +266,57 @@ impl ZenodoAPI { Ok(response) } else { let text = &response.text().await?; - Err(anyhow!("HTTP Error: {}\nurl: {:?}\n{:?}", response_status, &url, text)) + Err(anyhow!( + "HTTP Error: {}\nurl: {:?}\n{:?}", + response_status, + &url, + text + )) } } pub async fn get_depositions(&self) -> Result> { - let response = self.issue_request::>(Method::GET, "/deposit/depositions", None, None).await?; + let response = self + .issue_request::>( + Method::GET, + "/deposit/depositions", + None, + None, + ) + .await?; let info: Vec = response.json().await?; Ok(info) } pub async fn get_deposition_exists(&self) -> Result { let depositions = self.get_depositions().await?; - let matches_found: Vec<_> = depositions.iter().filter(|&a| a.title == self.name).collect(); - Ok(!matches_found.is_empty()) + let matches_found: Vec<_> = depositions + .iter() + .filter(|&a| a.title == self.name) + .collect(); + Ok(!matches_found.is_empty()) } pub async fn find_deposition(&self) -> Result> { let depositions = self.get_depositions().await?; - let mut matches_found: Vec<_> = depositions.into_iter().filter(|a| a.title == self.name).collect(); + let mut matches_found: Vec<_> = depositions + .into_iter() + .filter(|a| a.title == self.name) + .collect(); if !matches_found.is_empty() { if matches_found.len() > 1 { - Err(anyhow!("Found multiple Zenodo Depositions with the title '{}'", self.name)) + Err(anyhow!( + "Found multiple Zenodo Depositions with the title '{}'", + self.name + )) } else { // We need to do one more API call, to get the full listing // with the bucket URL. let partial_deposition = matches_found.remove(0); let url = format!("deposit/depositions/{}", partial_deposition.id); - let response = self.issue_request::>(Method::GET, &url, None, None).await?; + let response = self + .issue_request::>(Method::GET, &url, None, None) + .await?; let deposition: ZenodoDeposition = response.json().await?; Ok(Some(deposition)) } @@ -287,7 +329,10 @@ impl ZenodoAPI { // // Note that this uses LocalMetadata to propagate some of the Zenodo metadata fields // However, the title field is overwritten by ZenodoAPI.name. - pub async fn create_deposition(&self, local_metadata: LocalMetadata) -> Result { + pub async fn create_deposition( + &self, + local_metadata: LocalMetadata, + ) -> Result { let mut headers = HeaderMap::new(); headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); @@ -298,7 +343,9 @@ impl ZenodoAPI { let deposition_data: ZenodoDepositionData = metadata_copy.try_into()?; let data = Some(RequestData::Json(deposition_data)); - let response = self.issue_request(Method::POST, "/deposit/depositions", Some(headers), data).await?; + let response = self + .issue_request(Method::POST, "/deposit/depositions", Some(headers), data) + .await?; let deposition: ZenodoDeposition = response.json().await?; Ok(deposition) } @@ -306,14 +353,21 @@ impl ZenodoAPI { // Initialize the data collection on the Remote // // For Zenodo, this creates a new "deposition" - pub async fn remote_init(&mut self, local_metadata: LocalMetadata, link_only: bool) -> Result<()> { + pub async fn remote_init( + &mut self, + local_metadata: LocalMetadata, + link_only: bool, + ) -> Result<()> { // Step 1: Check if a deposition already exists let found_match = self.find_deposition().await?; let info = if let Some(existing_info) = found_match { if !link_only { - return Err(anyhow!("An existing Zenodo Deposition with the title \ - '{}' was found. Use --link-only to link.", self.name)); + return Err(anyhow!( + "An existing Zenodo Deposition with the title \ + '{}' was found. Use --link-only to link.", + self.name + )); } existing_info } else { @@ -331,7 +385,6 @@ impl ZenodoAPI { Ok(()) } - // Check if file exists, returning None if not, // and the ZenodoFile if so // TODO: could be part of higher Remote API, e.g. through generics? @@ -341,26 +394,38 @@ impl ZenodoAPI { } pub fn get_deposition_id(&self) -> Result { - self.deposition_id.ok_or(anyhow!("Internal Error: Zenodo deposition_id not set.")) + self.deposition_id + .ok_or(anyhow!("Internal Error: Zenodo deposition_id not set.")) } pub async fn delete_article_file(&self, file: &ZenodoFile) -> Result<()> { let id = self.get_deposition_id()?; let file_id = &file.id; let url = format!("{}/{}/files/{}", "/deposit/depositions", id, file_id); - self.issue_request::>(Method::DELETE, &url, None, None).await?; - info!("deleted Zenodo file '{}' (File ID={})", file.filename, file_id); + self.issue_request::>(Method::DELETE, &url, None, None) + .await?; + info!( + "deleted Zenodo file '{}' (File ID={})", + file.filename, file_id + ); Ok(()) } - // Upload the file, deleting any existing files if overwrite is true. // // Returns true/false if upload was completed or not. Will Error in other cases. - pub async fn upload(&self, data_file: &DataFile, path_context: &Path, overwrite: bool) -> Result { + pub async fn upload( + &self, + data_file: &DataFile, + path_context: &Path, + overwrite: bool, + ) -> Result { // (1) First, let's make sure that data_file isn't empty if data_file.size == 0 { - return Err(anyhow!("ZenodoAPI::upload() was called to upload an empty file: '{:?}'", data_file.full_path(path_context)?)) + return Err(anyhow!( + "ZenodoAPI::upload() was called to upload an empty file: '{:?}'", + data_file.full_path(path_context)? + )); } // (2) Get local file info @@ -369,7 +434,9 @@ impl ZenodoAPI { let file_size = data_file.size; // (3) Find the bucket url. - let bucket_url = self.bucket_url.as_ref().ok_or(anyhow!("Internal Error: Zenodo bucket_url not set. Please report."))?; + let bucket_url = self.bucket_url.as_ref().ok_or(anyhow!( + "Internal Error: Zenodo bucket_url not set. Please report." + ))?; // (4) Let's check if the file exists on the remote let existing_file = self.file_exists(&name).await?; @@ -378,47 +445,74 @@ impl ZenodoAPI { // (5) handle deleting files first if a file exists and overwrite is true if let Some(file) = existing_file { if !overwrite { - print_info!("Zenodo::upload() found file '{}' in Zenodo \ + print_info!( + "Zenodo::upload() found file '{}' in Zenodo \ Deposition ID={}. Since overwrite=false, this file will not be deleted and re-uploaded.", - name, id); + name, + id + ); return Ok(false); } else { - info!("FigShare::upload() is deleting file '{}' since \ - overwrite=true.", name); + info!( + "FigShare::upload() is deleting file '{}' since \ + overwrite=true.", + name + ); self.delete_article_file(&file).await?; - } - } + } + } - // (6) Build the headers -- note the content-length header is very important; + // (6) Build the headers -- note the content-length header is very important; // if not present, Zenodo will return "File is smaller than expected". reqwest // oddly attaches a wrong content-length header silently let mut headers = HeaderMap::new(); - headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/octet-stream")); - headers.insert(CONTENT_LENGTH, HeaderValue::from_str(&file_size.to_string()).unwrap()); + headers.insert( + CONTENT_TYPE, + HeaderValue::from_static("application/octet-stream"), + ); + headers.insert( + CONTENT_LENGTH, + HeaderValue::from_str(&file_size.to_string()).unwrap(), + ); // (7) we need to take the Zenodo bucket_url, remove the base since // issue_request adds it let bucket_endpoint = remove_base_url(bucket_url)?; let bucket_endpoint = format!("{}/{}", bucket_endpoint, name); - // (8) Prepare the file upload let file = tokio::fs::File::open(full_path).await?; - let response = self.issue_request::>(Method::PUT, &bucket_endpoint, - Some(headers), Some(RequestData::Stream(file))).await?; + let response = self + .issue_request::>( + Method::PUT, + &bucket_endpoint, + Some(headers), + Some(RequestData::Stream(file)), + ) + .await?; let info: ZenodoFileUpload = response.json().await?; // (9) After upload, compare the remote and local MD5s - let err_msg = format!("ZenodoAPI error: Zenodo did not provide a checksum that starts with 'md5:'\n\ - Please file an issue at: {}", ISSUE_URL); - let remote_md5 = info.checksum.strip_prefix("md5:").expect(&err_msg).to_owned(); + let err_msg = format!( + "ZenodoAPI error: Zenodo did not provide a checksum that starts with 'md5:'\n\ + Please file an issue at: {}", + ISSUE_URL + ); + let remote_md5 = info + .checksum + .strip_prefix("md5:") + .expect(&err_msg) + .to_owned(); let local_md5 = data_file.md5.clone(); - let msg = format!("After upload, the local ({}) and remote ({}) MD5s differed.\n\ - SciDataFlow automatically deletes the remote file in this case. \n", - shorten(&local_md5, Some(8)), shorten(&remote_md5, Some(8))); - + let msg = format!( + "After upload, the local ({}) and remote ({}) MD5s differed.\n\ + SciDataFlow automatically deletes the remote file in this case. \n", + shorten(&local_md5, Some(8)), + shorten(&remote_md5, Some(8)) + ); + // (10) Handle MD5 mismatch, deleting the remote file if they don't agree. // NOTE: this is not tested -- see note at test_upload() if remote_md5 != local_md5 { @@ -426,17 +520,19 @@ impl ZenodoAPI { match zenodo_file { None => { // The MD5s disagree, but when we try to get the file, we also cannot - // find it. This is an extreme corner case, likely due to issues on + // find it. This is an extreme corner case, likely due to issues on // Zenodo's end Err(anyhow!("{}However, in trying this, the remote file could not be found. This \n\ very likely reflects an internal error on Zenodo's end. Please log \n\ into Zenodo.org and manaually delete the file (if it exists) and \n\ try re-uploading.", msg)) - }, + } Some(file) => { - self.delete_article_file(&file).await - .context(format!("{}. However, SciDataFlow encountered an error while \ - trying to delete the file.", msg))?; + self.delete_article_file(&file).await.context(format!( + "{}. However, SciDataFlow encountered an error while \ + trying to delete the file.", + msg + ))?; Ok(false) } } @@ -449,22 +545,24 @@ impl ZenodoAPI { pub async fn get_files(&self) -> Result> { let id = self.get_deposition_id()?; let url = format!("{}/{}/files", "/deposit/depositions", id); - let response = self.issue_request::>(Method::GET, &url, None, None).await?; + let response = self + .issue_request::>(Method::GET, &url, None, None) + .await?; let files: Vec = response.json().await?; Ok(files) } pub async fn get_remote_files(&self) -> Result> { let articles = self.get_files().await?; - let remote_files:Vec = articles.into_iter().map(RemoteFile::from).collect(); + let remote_files: Vec = articles.into_iter().map(RemoteFile::from).collect(); Ok(remote_files) } // Get all files from a Zenodo Deposition, in a HashMap // with file name as keys. - pub async fn get_files_hashmap(&self) -> Result> { + pub async fn get_files_hashmap(&self) -> Result> { let mut files: Vec = self.get_files().await?; - let mut files_hash: HashMap = HashMap::new(); + let mut files_hash: HashMap = HashMap::new(); for file in files.iter_mut() { files_hash.insert(file.filename.clone(), file.clone()); } @@ -479,9 +577,9 @@ impl ZenodoAPI { #[cfg(test)] mod tests { use super::*; + use crate::logging_setup::setup; use httpmock::prelude::*; use serde_json::json; - use crate::logging_setup::setup; use std::io::Write; #[tokio::test] @@ -499,74 +597,72 @@ mod tests { title: Some("A *truly* reproducible project.".to_string()), email: None, affiliation: Some("UC Berkeley".to_string()), - description: Some("Let's build infrastructure so science can build off itself.".to_string()), + description: Some( + "Let's build infrastructure so science can build off itself.".to_string(), + ), }; // Create a mock deposition endpoint with a simulated success response let deposition_get_mock = server.mock(|when, then| { - when.method(GET) - .path("/deposit/depositions"); - then.status(200) - .json_body(json!([])); + when.method(GET).path("/deposit/depositions"); + then.status(200).json_body(json!([])); }); // Create a mock deposition endpoint with a simulated success response let deposition_mock = server.mock(|when, then| { - when.method(POST) - .path("/deposit/depositions"); + when.method(POST).path("/deposit/depositions"); // TODO probably could minimize this example - then.status(200) - .json_body(json!({ - "conceptrecid": "8266447", - "created": "2023-08-20T01:31:12.406094+00:00", + then.status(200).json_body(json!({ + "conceptrecid": "8266447", + "created": "2023-08-20T01:31:12.406094+00:00", + "doi": "", + "doi_url": "https://doi.org/", + "files": [], + "id": expected_id, + "links": { + "bucket": expected_bucket_url, + "discard": "https://zenodo.org/api/deposit/depositions/8266448/actions/discard", + "edit": "https://zenodo.org/api/deposit/depositions/8266448/actions/edit", + "files": "https://zenodo.org/api/deposit/depositions/8266448/files", + "html": "https://zenodo.org/deposit/8266448", + "latest_draft": "https://zenodo.org/api/deposit/depositions/8266448", + "latest_draft_html": "https://zenodo.org/deposit/8266448", + "publish": "https://zenodo.org/api/deposit/depositions/8266448/actions/publish", + "self": "https://zenodo.org/api/deposit/depositions/8266448" + }, + "metadata": { + "access_right": "open", + "creators": [ + { + "affiliation": local_metadata.affiliation, + "name": local_metadata.author_name, + } + ], + "description": "This is a description of my deposition", "doi": "", - "doi_url": "https://doi.org/", - "files": [], - "id": expected_id, - "links": { - "bucket": expected_bucket_url, - "discard": "https://zenodo.org/api/deposit/depositions/8266448/actions/discard", - "edit": "https://zenodo.org/api/deposit/depositions/8266448/actions/edit", - "files": "https://zenodo.org/api/deposit/depositions/8266448/files", - "html": "https://zenodo.org/deposit/8266448", - "latest_draft": "https://zenodo.org/api/deposit/depositions/8266448", - "latest_draft_html": "https://zenodo.org/deposit/8266448", - "publish": "https://zenodo.org/api/deposit/depositions/8266448/actions/publish", - "self": "https://zenodo.org/api/deposit/depositions/8266448" - }, - "metadata": { - "access_right": "open", - "creators": [ - { - "affiliation": local_metadata.affiliation, - "name": local_metadata.author_name, - } - ], - "description": "This is a description of my deposition", - "doi": "", - "license": "CC-BY-4.0", - "prereserve_doi": { - "doi": "10.5281/zenodo.8266448", - "recid": 8266448 - }, - "publication_date": "2023-08-20", - "title": "My Deposition Title", - "upload_type": "poster" + "license": "CC-BY-4.0", + "prereserve_doi": { + "doi": "10.5281/zenodo.8266448", + "recid": 8266448 }, - "modified": "2023-08-20T01:31:12.406103+00:00", - "owner": 110965, - "record_id": 8266448, - "state": "unsubmitted", - "submitted": false, - "title": "My Deposition Title" - })); + "publication_date": "2023-08-20", + "title": "My Deposition Title", + "upload_type": "poster" + }, + "modified": "2023-08-20T01:31:12.406103+00:00", + "owner": 110965, + "record_id": 8266448, + "state": "unsubmitted", + "submitted": false, + "title": "My Deposition Title" + })); }); // Create an instance of ZenodoAPI let mut api = ZenodoAPI::new("test", Some(server.url("/"))).unwrap(); // Main call to test - let _result = api.remote_init(local_metadata,false).await; + let _result = api.remote_init(local_metadata, false).await; //info!("result: {:?}", result); // ensure the specified mocks were called exactly one time (or fail). @@ -589,7 +685,7 @@ mod tests { filename: "fake_data.tsv".to_string(), id: "56789".to_string(), links: ZenodoLinks::default(), - filesize: 11 + filesize: 11, }; let expected_deposition_id = 1234564; @@ -597,8 +693,10 @@ mod tests { // Mock for delete_article_file let delete_file_mock = server.mock(|when, then| { when.method(DELETE) - .path(format!("/deposit/depositions/{}/files/{}", - expected_deposition_id, file.id)) + .path(format!( + "/deposit/depositions/{}/files/{}", + expected_deposition_id, file.id + )) .query_param("access_token", TEST_TOKEN); then.status(200); // Assuming a successful deletion returns a 200 status code }); @@ -612,18 +710,28 @@ mod tests { let result = api.delete_article_file(&file).await; // Assert that the result is OK - assert!(result.is_ok(), "Err encountered in Zenodo::delete_article_file(): {:?}", result); + assert!( + result.is_ok(), + "Err encountered in Zenodo::delete_article_file(): {:?}", + result + ); // Ensure the specified mock was called exactly once delete_file_mock.assert(); } - fn setup_get_files_mock<'a>(server: &'a MockServer, expected_deposition_id: u64, - remote_files: &'a Vec) -> httpmock::Mock<'a> { + fn setup_get_files_mock<'a>( + server: &'a MockServer, + expected_deposition_id: u64, + remote_files: &'a Vec, + ) -> httpmock::Mock<'a> { debug!("Setting up get_files mock"); server.mock(|when, then| { when.method(GET) - .path(format!("/deposit/depositions/{}/files", expected_deposition_id)) + .path(format!( + "/deposit/depositions/{}/files", + expected_deposition_id + )) .query_param("access_token", TEST_TOKEN); then.status(200) // return the files found, which depends on params of test @@ -631,9 +739,12 @@ mod tests { }) } - - fn setup_upload_file_mock<'a>(server: &'a MockServer, bucket_endpoint: &'a str, - md5: &'a str, size: usize) -> httpmock::Mock<'a> { + fn setup_upload_file_mock<'a>( + server: &'a MockServer, + bucket_endpoint: &'a str, + md5: &'a str, + size: usize, + ) -> httpmock::Mock<'a> { debug!("Setting up upload_file mock"); //let md5 = if !wrong_md5 { md5.to_owned() } else { md5.clone().chars().rev().collect::() }; let remote_md5 = format!("md5:{}", md5); @@ -661,24 +772,29 @@ mod tests { }) } - fn setup_delete_file_mock<'a>(server: &'a MockServer, zenodo_file: &'a ZenodoFile, - expected_deposition_id: u64) -> httpmock::Mock<'a> { + fn setup_delete_file_mock<'a>( + server: &'a MockServer, + zenodo_file: &'a ZenodoFile, + expected_deposition_id: u64, + ) -> httpmock::Mock<'a> { debug!("Setting up delete_file mock"); server.mock(|when, then| { let expected_file_id = &zenodo_file.id; when.method(DELETE) - .path(format!("/deposit/depositions/{}/files/{}", expected_deposition_id, expected_file_id)) + .path(format!( + "/deposit/depositions/{}/files/{}", + expected_deposition_id, expected_file_id + )) .query_param("access_token", TEST_TOKEN); then.status(204); // Typically, HTTP status 204 indicates that the server successfully processed the request and is not returning any content. }) } - // Main Test Function // // Note: this does *not* test wrong MD5s. It should, but this will require refactoring - // things quite a bit. The issue is that the vector remote_files will need to change - // mid-call to ZenodoAPI::upload(), since the file was uploaded but has wrong MD5, + // things quite a bit. The issue is that the vector remote_files will need to change + // mid-call to ZenodoAPI::upload(), since the file was uploaded but has wrong MD5, // and the upload() method then retrieves it async fn test_upload(file_exists: bool, overwrite: bool) -> Result { setup(); @@ -701,7 +817,7 @@ mod tests { tracked: true, md5: md5.to_string(), size, - url: None + url: None, }; let path_context = Path::new("path/to/datafile"); @@ -712,11 +828,11 @@ mod tests { // Mock for the get_files method let mut remote_files = Vec::new(); let zenodo_file = ZenodoFile { - checksum: md5.clone().to_string(), + checksum: md5.to_string(), filename: data_file.basename()?, filesize: size as usize, id: "4242".to_string(), - links: ZenodoLinks::default() + links: ZenodoLinks::default(), }; // Create a mock with the remote file there if we're testing this case. @@ -729,11 +845,16 @@ mod tests { // Mock for the upload method // NOTE: this mock does not test for binary files - let upload_file_mock = setup_upload_file_mock(&server, &bucket_endpoint, &md5, size as usize); + let upload_file_mock = + setup_upload_file_mock(&server, &bucket_endpoint, &md5, size as usize); // Mock for the delete_article_file method let delete_file_mock = if file_exists && overwrite { - Some(setup_delete_file_mock(&server, &zenodo_file, expected_deposition_id)) + Some(setup_delete_file_mock( + &server, + &zenodo_file, + expected_deposition_id, + )) } else { None }; @@ -746,7 +867,7 @@ mod tests { // Main call to test let result = api.upload(&data_file, &path_context, overwrite).await; - //println!("get_files_mock={:}?, upload_file_mock={:?}, delete_file_mock={:?}", + //println!("get_files_mock={:}?, upload_file_mock={:?}, delete_file_mock={:?}", // get_files_mock.hits(), upload_file_mock.hits(), delete_file_mock.unwrap().hits()); // Ensure the specified mocks were called exactly one time (or fail). @@ -759,14 +880,17 @@ mod tests { if file_exists && overwrite { delete_file_mock.unwrap().assert(); } - return result + return result; } #[tokio::test] async fn test_upload_no_overwrite_no_remote_files() -> Result<()> { let result = test_upload(false, false).await?; - assert!(result, "Zenodo::upload() failed (file_exists={:?}, overwrite={:?}0. Result: {:?}", - false, false, result); + assert!( + result, + "Zenodo::upload() failed (file_exists={:?}, overwrite={:?}0. Result: {:?}", + false, false, result + ); Ok(()) } @@ -774,17 +898,22 @@ mod tests { async fn test_upload_no_overwrite_with_remote_files() -> Result<()> { let result = test_upload(true, false).await?; // result should return false since no upload was done. - assert!(!result, "Zenodo::upload() failed (file_exists={:?}, overwrite={:?}). Result: {:?}", - true, false, result); + assert!( + !result, + "Zenodo::upload() failed (file_exists={:?}, overwrite={:?}). Result: {:?}", + true, false, result + ); Ok(()) } #[tokio::test] async fn test_upload_overwrite_with_remote_files() -> Result<()> { let result = test_upload(true, true).await?; - assert!(result, "Zenodo::upload() failed (file_exists={:?}, overwrite={:?}). Result: {:?}", - true, true, result); + assert!( + result, + "Zenodo::upload() failed (file_exists={:?}, overwrite={:?}). Result: {:?}", + true, true, result + ); Ok(()) } } - diff --git a/src/lib/assets.rs b/src/lib/assets.rs index 910bed1..bff64da 100644 --- a/src/lib/assets.rs +++ b/src/lib/assets.rs @@ -10,7 +10,10 @@ impl GitHubRepo { /// Create a new GitHubRepo from a URL string pub fn new(url_str: &str) -> Result { let parsed_url = Url::parse(url_str).map_err(|e| e.to_string())?; - let path_segments: Vec<&str> = parsed_url.path_segments().ok_or("Invalid path".to_string())?.collect(); + let path_segments: Vec<&str> = parsed_url + .path_segments() + .ok_or("Invalid path".to_string())? + .collect(); if path_segments.len() < 2 { return Err("URL should contain both username and repository".to_string()); diff --git a/src/lib/data.rs b/src/lib/data.rs index e147070..41a0767 100644 --- a/src/lib/data.rs +++ b/src/lib/data.rs @@ -1,35 +1,35 @@ -use std::path::{PathBuf,Path}; -use anyhow::{anyhow,Result}; -use std::fs::metadata; -use serde_derive::{Serialize,Deserialize}; -use serde; -use crate::lib::data::serde::{Serializer,Deserializer}; +use crate::lib::data::serde::{Deserializer, Serializer}; use crate::lib::download::Downloads; -#[allow(unused_imports)] -use log::{info, trace, debug}; +use anyhow::{anyhow, Result}; use chrono::prelude::*; -use std::collections::{HashMap,BTreeMap}; +use colored::*; +use futures::future::join_all; use futures::stream::FuturesUnordered; use futures::StreamExt; -use futures::future::join_all; +#[allow(unused_imports)] +use log::{debug, info, trace}; +use serde; +use serde_derive::{Deserialize, Serialize}; +use std::collections::{BTreeMap, HashMap}; use std::fs; -use colored::*; +use std::fs::metadata; +use std::path::{Path, PathBuf}; -use crate::{print_warn,print_info}; -use crate::lib::utils::{format_mod_time,compute_md5, md5_status,pluralize}; -use crate::lib::remote::{authenticate_remote,Remote,RemoteFile,RemoteStatusCode}; use crate::lib::progress::Progress; +use crate::lib::remote::{authenticate_remote, Remote, RemoteFile, RemoteStatusCode}; +use crate::lib::utils::{compute_md5, format_mod_time, md5_status, pluralize}; +use crate::{print_info, print_warn}; // The status of a local data file, *conditioned* on it being in the manifest. -#[derive(Debug,PartialEq,Clone)] +#[derive(Debug, PartialEq, Clone)] pub enum LocalStatusCode { - Current, // The MD5s between the file and manifest agree - Modified, // The MD5s disagree - Deleted, // The file is in the manifest but not file system - Invalid // Invalid state + Current, // The MD5s between the file and manifest agree + Modified, // The MD5s disagree + Deleted, // The file is in the manifest but not file system + Invalid, // Invalid state } -#[derive(Debug,Clone)] +#[derive(Debug, Clone)] pub struct StatusEntry { pub name: String, pub local_status: Option, @@ -39,15 +39,23 @@ pub struct StatusEntry { pub local_md5: Option, pub remote_md5: Option, pub manifest_md5: Option, - pub local_mod_time: Option> + pub local_mod_time: Option>, } impl StatusEntry { fn local_md5_column(&self, abbrev: Option) -> Result { - Ok(md5_status(self.local_md5.as_ref(), self.manifest_md5.as_ref(), abbrev)) + Ok(md5_status( + self.local_md5.as_ref(), + self.manifest_md5.as_ref(), + abbrev, + )) } fn remote_md5_column(&self, abbrev: Option) -> Result { - Ok(md5_status(self.remote_md5.as_ref(), self.manifest_md5.as_ref(), abbrev)) + Ok(md5_status( + self.remote_md5.as_ref(), + self.manifest_md5.as_ref(), + abbrev, + )) } // StatusEntry.remote_status can be set to None; if so the remote status // columns will no be displayed. @@ -60,23 +68,37 @@ impl StatusEntry { let local_status = &self.local_status; let remote_status = &self.remote_status; match (tracked, local_status, remote_status) { - (Some(true), Some(LocalStatusCode::Current), Some(RemoteStatusCode::Current)) => line.green().to_string(), + (Some(true), Some(LocalStatusCode::Current), Some(RemoteStatusCode::Current)) => { + line.green().to_string() + } (Some(true), Some(LocalStatusCode::Current), None) => line.green().to_string(), - (Some(false), Some(LocalStatusCode::Current), Some(RemoteStatusCode::NotExists)) => line.green().to_string(), - (Some(true), Some(LocalStatusCode::Current), Some(RemoteStatusCode::NotExists)) => line.yellow().to_string(), + (Some(false), Some(LocalStatusCode::Current), Some(RemoteStatusCode::NotExists)) => { + line.green().to_string() + } + (Some(true), Some(LocalStatusCode::Current), Some(RemoteStatusCode::NotExists)) => { + line.yellow().to_string() + } // not tracked, but on remote - (Some(false), Some(LocalStatusCode::Current), Some(RemoteStatusCode::Current)) => line.cyan().to_string(), + (Some(false), Some(LocalStatusCode::Current), Some(RemoteStatusCode::Current)) => { + line.cyan().to_string() + } // not tracked, not on remote (Some(false), Some(LocalStatusCode::Current), None) => line.green().to_string(), - // not tracked, no remote but everything is current + // not tracked, no remote but everything is current (None, Some(LocalStatusCode::Current), None) => line.green().to_string(), - (Some(true), Some(LocalStatusCode::Modified), _) => line.red().to_string(), - (Some(false), Some(LocalStatusCode::Modified), _) => line.red().to_string(), - (Some(true), Some(LocalStatusCode::Current), Some(RemoteStatusCode::Different)) => line.yellow().to_string(), + (Some(true), Some(LocalStatusCode::Modified), _) => line.red().to_string(), + (Some(false), Some(LocalStatusCode::Modified), _) => line.red().to_string(), + (Some(true), Some(LocalStatusCode::Current), Some(RemoteStatusCode::Different)) => { + line.yellow().to_string() + } // untracked, but exists on remote -- invalid - (Some(false), Some(LocalStatusCode::Current), Some(RemoteStatusCode::Different)) => line.cyan().to_string(), - (Some(false), Some(LocalStatusCode::Current), Some(RemoteStatusCode::Exists)) => line.cyan().to_string(), + (Some(false), Some(LocalStatusCode::Current), Some(RemoteStatusCode::Different)) => { + line.cyan().to_string() + } + (Some(false), Some(LocalStatusCode::Current), Some(RemoteStatusCode::Exists)) => { + line.cyan().to_string() + } _ => { //println!("{:?}: {:?}, {:?}, {:?}", self.name, tracked, local_status, remote_status); line.cyan().to_string() @@ -86,7 +108,9 @@ impl StatusEntry { pub fn columns(&self, abbrev: Option) -> Vec { let local_status = &self.local_status; - let md5_string = self.local_md5_column(abbrev).expect("Internal Error: StatusEntry::local_md5_column()."); + let md5_string = self + .local_md5_column(abbrev) + .expect("Internal Error: StatusEntry::local_md5_column()."); let mod_time_pretty = self.local_mod_time.map(format_mod_time).unwrap_or_default(); @@ -96,14 +120,14 @@ impl StatusEntry { Some(LocalStatusCode::Modified) => "changed", Some(LocalStatusCode::Deleted) => "deleted", Some(LocalStatusCode::Invalid) => "invalid", - _ => "no file" + _ => "no file", }; let tracked = match (self.include_remotes(), self.tracked) { (false, _) => "".to_string(), (true, Some(true)) => ", tracked".to_string(), (true, Some(false)) => ", untracked".to_string(), - (true, None) => ", not in manifest".to_string() + (true, None) => ", not in manifest".to_string(), }; let mut columns = vec![ self.name.clone(), @@ -117,14 +141,16 @@ impl StatusEntry { Some(RemoteStatusCode::Current) => "identical remote".to_string(), Some(RemoteStatusCode::MessyLocal) => "messy local".to_string(), Some(RemoteStatusCode::Different) => { - let remote_md5 = self.remote_md5_column(abbrev).expect("Internal Error: StatusEntry::remote_md5_column()."); + let remote_md5 = self + .remote_md5_column(abbrev) + .expect("Internal Error: StatusEntry::remote_md5_column()."); format!("different remote version ({:})", remote_md5) - }, + } Some(RemoteStatusCode::NotExists) => "not on remote".to_string(), Some(RemoteStatusCode::NoLocal) => "unknown (messy remote)".to_string(), Some(RemoteStatusCode::Exists) => "exists, no remote MD5".to_string(), Some(RemoteStatusCode::DeletedLocal) => "exists on remote".to_string(), - _ => "invalid".to_string() + _ => "invalid".to_string(), }; columns.push(remote_status_msg.to_string()); } @@ -138,14 +164,13 @@ pub struct DataFile { pub tracked: bool, pub md5: String, pub size: u64, - pub url: Option - //modified: Option>, + pub url: Option, //modified: Option>, } // A merged DataFile and RemoteFile -// -// remote_service: Some(String) remote name if this file's directory -// is linked to a remote. None if there is no remote. None distinguishes +// +// remote_service: Some(String) remote name if this file's directory +// is linked to a remote. None if there is no remote. None distinguishes // the important cases when remote = NotExists (there is no remote // file) due to there not being a remote tracking, and remote = NotExists // due to the remote being configured, but the file not existing (e.g. @@ -154,16 +179,19 @@ pub struct DataFile { pub struct MergedFile { pub local: Option, pub remote: Option, - pub remote_service: Option + pub remote_service: Option, } - impl MergedFile { - pub fn new(data_file: &DataFile, remote_file: &RemoteFile, remote_service: Option) -> Result { + pub fn new( + data_file: &DataFile, + remote_file: &RemoteFile, + remote_service: Option, + ) -> Result { Ok(MergedFile { local: Some(data_file.clone()), remote: Some(remote_file.clone()), - remote_service + remote_service, }) } @@ -176,7 +204,7 @@ impl MergedFile { } else { Err(anyhow!("Local and remote names do not match.")) } - }, + } (Some(local), None) => Ok(local.basename()?), (None, Some(remote)) => Ok(remote.name.clone()), (None, None) => Err(anyhow!("Invalid state: both local and remote are None.")), @@ -205,8 +233,7 @@ impl MergedFile { } pub fn remote_md5(&self) -> Option { - self.remote.as_ref() - .and_then(|remote| remote.get_md5()) + self.remote.as_ref().and_then(|remote| remote.get_md5()) } //pub fn local_md5_mismatch(&self, path_context: &PathBuf) -> Option { @@ -226,9 +253,9 @@ impl MergedFile { } pub fn local_mod_time(&self, path_context: &Path) -> Option> { - self.local.as_ref() - .and_then(|data_file| data_file - .get_mod_time(path_context).ok()) + self.local + .as_ref() + .and_then(|data_file| data_file.get_mod_time(path_context).ok()) } pub async fn status(&self, path_context: &Path) -> Result { @@ -247,7 +274,7 @@ impl MergedFile { let md5_mismatch = self.local_remote_md5_mismatch(path_context).await; if !self.has_remote().unwrap_or(false) { - return Ok(RemoteStatusCode::NotExists) + return Ok(RemoteStatusCode::NotExists); } // MergedFile has a remote, so get the remote status. @@ -255,42 +282,43 @@ impl MergedFile { (None, None) => { // no local file (so can't get MD5) RemoteStatusCode::NoLocal - }, - (Some(LocalStatusCode::Current), Some(false)) => { - RemoteStatusCode::Current - }, + } + (Some(LocalStatusCode::Current), Some(false)) => RemoteStatusCode::Current, (Some(LocalStatusCode::Current), Some(true)) => { - // Will pull with --overwrite. + // Will pull with --overwrite. // Will push with --overwrite. RemoteStatusCode::Different - }, + } (Some(LocalStatusCode::Current), None) => { - // We can't compare the MD5s, i.e. because remote + // We can't compare the MD5s, i.e. because remote // does not support them RemoteStatusCode::Exists - }, + } (Some(LocalStatusCode::Modified), _) => { // Messy local -- this will prevent syncing! // TODO: could compare the MD5s here further // and separate out modified local (manifest and remote agree) // and messy (manifest out of date)? RemoteStatusCode::MessyLocal - }, + } (Some(LocalStatusCode::Deleted), _) => { // Local file on file system does not exist, - // but exists in the manifest. If the file is in + // but exists in the manifest. If the file is in // the manifest and tracked a pull would pull it in. RemoteStatusCode::DeletedLocal - }, - (_, _) => RemoteStatusCode::Invalid + } + (_, _) => RemoteStatusCode::Invalid, }; Ok(status) } - // Create a StatusEntry, for printing the status to the user. - pub async fn status_entry(&self, path_context: &Path, include_remotes: bool) -> Result { + pub async fn status_entry( + &self, + path_context: &Path, + include_remotes: bool, + ) -> Result { let tracked = self.local.as_ref().map(|df| df.tracked); let local_status = if let Some(local) = self.local.as_ref() { local.status(path_context).await.ok() @@ -298,13 +326,23 @@ impl MergedFile { None }; - let remote_status = if include_remotes { Some(self.status(path_context).await?) } else { None }; + let remote_status = if include_remotes { + Some(self.status(path_context).await?) + } else { + None + }; //let remote_status = if self.remote_service.is_some() { Some(self.status(path_context)?) } else { None }; - let remote_service = if include_remotes { self.remote_service.clone() } else { None }; + let remote_service = if include_remotes { + self.remote_service.clone() + } else { + None + }; if self.local.is_none() && self.remote.is_none() { - return Err(anyhow!("Internal error: MergedFile with no RemoteFile and DataFile set. Please report.")); + return Err(anyhow!( + "Internal error: MergedFile with no RemoteFile and DataFile set. Please report." + )); } Ok(StatusEntry { @@ -316,17 +354,16 @@ impl MergedFile { local_md5: self.local_md5(path_context).await, remote_md5: self.remote_md5(), manifest_md5: self.manifest_md5(), - local_mod_time: self.local_mod_time(path_context) + local_mod_time: self.local_mod_time(path_context), }) } } - impl DataFile { pub async fn new(path: String, url: Option<&str>, path_context: &Path) -> Result { let full_path = path_context.join(&path); if !full_path.exists() { - return Err(anyhow!("File '{}' does not exist.", path)) + return Err(anyhow!("File '{}' does not exist.", path)); } let md5 = match compute_md5(&full_path).await? { Some(md5) => md5, @@ -338,7 +375,7 @@ impl DataFile { let maybe_url: Option = url.map(|s| s.to_string()); Ok(DataFile { path, - tracked: false, + tracked: false, md5, size, url: maybe_url, @@ -353,17 +390,18 @@ impl DataFile { let path = Path::new(&self.path); match path.file_name() { Some(basename) => Ok(basename.to_string_lossy().to_string()), - None => Err(anyhow!("could not get basename of '{}'", self.path)) + None => Err(anyhow!("could not get basename of '{}'", self.path)), } } pub fn directory(&self) -> Result { let path = std::path::Path::new(&self.path); - Ok(path.parent() - .unwrap_or(path) - .to_str() - .unwrap_or("") - .to_string()) + Ok(path + .parent() + .unwrap_or(path) + .to_str() + .unwrap_or("") + .to_string()) } pub async fn get_md5(&self, path_context: &Path) -> Result> { @@ -402,7 +440,7 @@ impl DataFile { let local_status = match (is_changed, is_alive) { (false, true) => LocalStatusCode::Current, (true, true) => LocalStatusCode::Modified, - (false, false) => LocalStatusCode::Deleted, // Invalid? (TODO) + (false, false) => LocalStatusCode::Deleted, // Invalid? (TODO) (true, false) => LocalStatusCode::Deleted, // incase a line gets dropped above #[allow(unreachable_patterns)] @@ -434,7 +472,10 @@ impl DataFile { /// Mark the file to track on the remote pub fn set_tracked(&mut self) -> Result<()> { if self.tracked { - return Err(anyhow!("file '{}' is already tracked on remote.", self.path)) + return Err(anyhow!( + "file '{}' is already tracked on remote.", + self.path + )); } self.tracked = true; Ok(()) @@ -442,21 +483,23 @@ impl DataFile { /// Mark the file to not track on the remote pub fn set_untracked(&mut self) -> Result<()> { if !self.tracked { - return Err(anyhow!("file '{}' is already not tracked on remote.", self.path)) + return Err(anyhow!( + "file '{}' is already not tracked on remote.", + self.path + )); } self.tracked = false; Ok(()) } } - #[derive(Debug, Serialize, Deserialize, Default, PartialEq, Clone)] pub struct DataCollectionMetadata { pub title: Option, pub description: Option, } -/// DataCollection structure for managing the data manifest +/// DataCollection structure for managing the data manifest /// and how it talks to the outside world. #[derive(Debug, PartialEq, Default)] pub struct DataCollection { @@ -470,60 +513,58 @@ pub struct MinimalDataCollection { pub files: Vec, pub remotes: HashMap, pub metadata: DataCollectionMetadata, - } impl serde::Serialize for DataCollection { fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - // Serialize `files` as a sorted vector - let sorted_files: Vec = self.files.values().cloned().collect(); - - // Construct a new struct to hold the serializable parts - let to_serialize = MinimalDataCollection { - files: sorted_files, - remotes: self.remotes.clone(), - metadata: self.metadata.clone(), - }; + where + S: Serializer, + { + // Serialize `files` as a sorted vector + let sorted_files: Vec = self.files.values().cloned().collect(); + + // Construct a new struct to hold the serializable parts + let to_serialize = MinimalDataCollection { + files: sorted_files, + remotes: self.remotes.clone(), + metadata: self.metadata.clone(), + }; - to_serialize.serialize(serializer) - } + to_serialize.serialize(serializer) + } } impl<'de> serde::Deserialize<'de> for DataCollection { fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - // Deserialize into a temporary struct - let temp = MinimalDataCollection::deserialize(deserializer)?; - - // Build the HashMap for files based on the path - let files = temp - .files - .into_iter() - .map(|df| (df.path.clone(), df)) - .collect(); - - Ok(DataCollection { - files, - remotes: temp.remotes, - metadata: temp.metadata, - }) - } -} + where + D: Deserializer<'de>, + { + // Deserialize into a temporary struct + let temp = MinimalDataCollection::deserialize(deserializer)?; + + // Build the HashMap for files based on the path + let files = temp + .files + .into_iter() + .map(|df| (df.path.clone(), df)) + .collect(); + Ok(DataCollection { + files, + remotes: temp.remotes, + metadata: temp.metadata, + }) + } +} -/// DataCollection methods: these should *only* be for +/// DataCollection methods: these should *only* be for /// interacting with the data manifest (including remotes). impl DataCollection { pub fn new() -> Self { Self { files: HashMap::new(), remotes: HashMap::new(), - metadata: DataCollectionMetadata::default() + metadata: DataCollectionMetadata::default(), } } @@ -544,9 +585,11 @@ impl DataCollection { e.insert(data_file); Ok(()) } else { - Err(anyhow!("File '{}' is already registered in the data manifest.\n\ + Err(anyhow!( + "File '{}' is already registered in the data manifest.\n\ If you wish to update the MD5 or metadata, use: sdf update FILE", - &data_file.path)) + &data_file.path + )) } } @@ -559,7 +602,10 @@ impl DataCollection { self.files.remove(filename); true } else { - println!("File '{}' is not registered in the manifest, so it was not removed.", filename); + println!( + "File '{}' is not registered in the manifest, so it was not removed.", + filename + ); false } } @@ -575,23 +621,20 @@ impl DataCollection { } } None => { - // + // let all_files: Vec<_> = self.files.keys().cloned().collect(); for file in all_files { if let Some(data_file) = self.files.get_mut(&file) { data_file.update(path_context).await?; debug!("rehashed file {:?}", data_file.path); } - } - } } Ok(()) } - - // Validate the directory as being tracked by a remote, + // Validate the directory as being tracked by a remote, // i.e. no nesting. pub fn validate_remote_directory(&self, dir: &String) -> Result<()> { let dir_path = Path::new(dir); @@ -604,7 +647,11 @@ impl DataCollection { for existing_dir in self.remotes.keys() { let existing_path = Path::new(existing_dir); if dir_path.starts_with(existing_path) { - return Err(anyhow!("Cannot add '{}' because its subdirectory '{}' is already tracked.", dir, existing_dir)); + return Err(anyhow!( + "Cannot add '{}' because its subdirectory '{}' is already tracked.", + dir, + existing_dir + )); } } @@ -620,7 +667,9 @@ impl DataCollection { pub fn get_this_files_remote(&self, data_file: &DataFile) -> Result> { let path = data_file.directory()?; - let res: Vec = self.remotes.iter() + let res: Vec = self + .remotes + .iter() .filter(|(r, _v)| PathBuf::from(&path).starts_with(r)) .map(|(_r, v)| v.name().to_string()) .collect(); @@ -652,23 +701,37 @@ impl DataCollection { let data_file = self.files.get_mut(filepath); // extract the directory from the filepath - let dir_path = Path::new(filepath).parent() + let dir_path = Path::new(filepath) + .parent() .ok_or_else(|| anyhow!("Failed to get directory for file '{}'", filepath))?; // check if the directory exists in self.remotes - if !self.remotes.contains_key(dir_path.to_str().unwrap_or_default()) { - return Err(anyhow!("Directory '{}' is not registered in remotes.", dir_path.display())); + if !self + .remotes + .contains_key(dir_path.to_str().unwrap_or_default()) + { + return Err(anyhow!( + "Directory '{}' is not registered in remotes.", + dir_path.display() + )); } match data_file { - None => Err(anyhow!("Data file '{}' is not in the data manifest. Add it first using:\n \ - $ sdf track {}\n", filepath, filepath)), + None => Err(anyhow!( + "Data file '{}' is not in the data manifest. Add it first using:\n \ + $ sdf track {}\n", + filepath, + filepath + )), Some(data_file) => { - // check that the file isn't empty + // check that the file isn't empty // (this is why a path_context is needed) let file_size = data_file.get_size(path_context)?; if file_size == 0 { - return Err(anyhow!("Cannot track an empty file, and '{}' has a file size of 0.", filepath)); + return Err(anyhow!( + "Cannot track an empty file, and '{}' has a file size of 0.", + filepath + )); } data_file.set_tracked() } @@ -677,14 +740,17 @@ impl DataCollection { pub fn untrack_file(&mut self, filepath: &String) -> Result<()> { let data_file = self.files.get_mut(filepath); match data_file { - None => Err(anyhow!("Cannot untrack data file '{}' since it was never added to\ - the data manifest.", filepath)), - Some(file) => file.set_untracked() + None => Err(anyhow!( + "Cannot untrack data file '{}' since it was never added to\ + the data manifest.", + filepath + )), + Some(file) => file.set_untracked(), } } // Get local DataFiles by directory - pub fn get_files_by_directory(&self) -> Result>> { + pub fn get_files_by_directory(&self) -> Result>> { let mut dir_map: HashMap> = HashMap::new(); for (path, data_file) in self.files.iter() { let path = Path::new(&path); @@ -699,7 +765,9 @@ impl DataCollection { // Fetch all remote files. // // (remote service, path) -> { filename -> RemoteFile, ... } - pub async fn fetch(&mut self) -> Result>> { + pub async fn fetch( + &mut self, + ) -> Result>> { self.authenticate_remotes()?; let mut all_remote_files = HashMap::new(); @@ -707,24 +775,29 @@ impl DataCollection { pb.bar.set_message("Fetching remote files..."); // Convert remotes into Futures, so that they can be awaited in parallel - let fetch_futures: Vec<_> = self.remotes.iter().map(|(path, remote)| { - let remote_name = remote.name().to_string(); - let path_clone = path.clone(); - async move { - let remote_files = remote.get_files_hashmap().await?; - Ok(((remote_name, path_clone), remote_files)) - } - }).collect(); + let fetch_futures: Vec<_> = self + .remotes + .iter() + .map(|(path, remote)| { + let remote_name = remote.name().to_string(); + let path_clone = path.clone(); + async move { + let remote_files = remote.get_files_hashmap().await?; + Ok(((remote_name, path_clone), remote_files)) + } + }) + .collect(); let results = join_all(fetch_futures).await; for result in results { match result { Ok((key, value)) => { - pb.bar.set_message(format!("Fetching remote files... {} done.", key.0)); + pb.bar + .set_message(format!("Fetching remote files... {} done.", key.0)); all_remote_files.insert(key, value); pb.bar.inc(1); - }, + } Err(e) => return Err(e), // Handle errors as needed } } @@ -735,16 +808,18 @@ impl DataCollection { // Merge all local and remote files. // - // Use a fetch to get all remote files (as RemoteFile), and merge these + // Use a fetch to get all remote files (as RemoteFile), and merge these // in with the local data files (DataFile) into a MergedFile struct. // Missing remote/local files are None. - // - // Returns: Result with HashMap of directory -> { File -> MergedFile, ... } - pub async fn merge(&mut self, include_remotes: bool) -> Result>> { + // + // Returns: Result with HashMap of directory -> { File -> MergedFile, ... } + pub async fn merge( + &mut self, + include_remotes: bool, + ) -> Result>> { // directory -> {(filename -> MergedFile), ...} let mut result: HashMap> = HashMap::new(); - // Initialize the result with local files // TODO: we need to fix remote_service here, for the // case where we have a local file in a tracked directory @@ -753,13 +828,18 @@ impl DataCollection { let remote_service = self.get_this_files_remote(local_file)?; //info!("local_file: {:?}", local_file); let dir = local_file.directory()?; - result.entry(dir).or_insert_with(HashMap::new) - .insert(name.clone(), - MergedFile { local: Some(local_file.clone()), remote: None, remote_service }); + result.entry(dir).or_default().insert( + name.clone(), + MergedFile { + local: Some(local_file.clone()), + remote: None, + remote_service, + }, + ); } if !include_remotes { - return Ok(result) + return Ok(result); } // iterate through each remote and retrieve remote files @@ -768,32 +848,44 @@ impl DataCollection { // merge remote files with local files for (name, remote_file) in remote_files { // try to get the tracked directory; it doesn't exist make it - let path_key = PathBuf::from(tracked_dir).join(name).to_str().unwrap().to_string(); - if let Some(merged_file) = result.entry(tracked_dir.clone()) - .or_insert_with(HashMap::new).get_mut(&path_key) { - // we have a local and a remote file - // set the joined remote file and the service - merged_file.remote = Some(remote_file.clone()); - merged_file.remote_service = Some(remote_service.to_string()); - } else { - // no local file, but we have a remote - result.entry(tracked_dir.clone()).or_insert_with(HashMap::new).insert(path_key.to_string(), + let path_key = PathBuf::from(tracked_dir) + .join(name) + .to_str() + .unwrap() + .to_string(); + if let Some(merged_file) = result + .entry(tracked_dir.clone()) + .or_default() + .get_mut(&path_key) + { + // we have a local and a remote file + // set the joined remote file and the service + merged_file.remote = Some(remote_file.clone()); + merged_file.remote_service = Some(remote_service.to_string()); + } else { + // no local file, but we have a remote + result.entry(tracked_dir.clone()).or_default().insert( + path_key.to_string(), MergedFile { - local: None, + local: None, remote: Some(remote_file.clone()), - remote_service: Some(remote_service.to_string()) - }); - } + remote_service: Some(remote_service.to_string()), + }, + ); + } } } Ok(result) } - // Get the status of the DataCollection, optionally with remotes. - // + // // Returns Result of BTreeMap of directory -> [ StatusEntry, ...] - pub async fn status(&mut self, path_context: &Path, include_remotes: bool) -> Result>> { + pub async fn status( + &mut self, + path_context: &Path, + include_remotes: bool, + ) -> Result>> { let merged_files = self.merge(include_remotes).await?; let mut statuses_futures = FuturesUnordered::new(); @@ -804,7 +896,10 @@ impl DataCollection { for mf in files { let directory_clone = directory.clone(); statuses_futures.push(async move { - let status_entry = mf.status_entry(path_context, include_remotes).await.map_err(anyhow::Error::from)?; + let status_entry = mf + .status_entry(path_context, include_remotes) + .await + .map_err(anyhow::Error::from)?; Ok::<(String, StatusEntry), anyhow::Error>((directory_clone, status_entry)) }); } @@ -817,7 +912,8 @@ impl DataCollection { // process the futures as they become ready while let Some(result) = statuses_futures.next().await { if let Ok((key, value)) = result { - pb.bar.set_message(format!("Calculating MD5s... {} done.", &value.name)); + pb.bar + .set_message(format!("Calculating MD5s... {} done.", &value.name)); statuses.entry(key).or_insert_with(Vec::new).push(value); pb.bar.inc(1); } else { @@ -847,29 +943,33 @@ impl DataCollection { if let Some(remote) = self.remotes.get(tracked_dir) { for merged_file in files.values() { let name = merged_file.name()?; - let path = PathBuf::from(tracked_dir).join(name).to_str().unwrap().to_string(); + let path = PathBuf::from(tracked_dir) + .join(name) + .to_str() + .unwrap() + .to_string(); let local = merged_file.local.clone(); - // if the file is not tracked or is remote-only, + // if the file is not tracked or is remote-only, // we do not do anything if local.as_ref().map_or(false, |mf| !mf.tracked) { untracked_skipped.push(path); continue; } - // now we need to figure out whether to push the file, + // now we need to figure out whether to push the file, // which depends on the RemoteStatusCode and whether // we should overwrite (TODO) let do_upload = match merged_file.status(path_context).await? { RemoteStatusCode::NoLocal => { - // A file exists on the remote, but not locally: there + // A file exists on the remote, but not locally: there // is nothing to push in this case (or count!) - false - }, + false + } RemoteStatusCode::Current => { current_skipped.push(path); false - }, + } RemoteStatusCode::Exists => { // it exists on the remote, but we cannot // compare MD5s. Push only if overwrite is true. @@ -877,14 +977,14 @@ impl DataCollection { overwrite_skipped.push(path); } overwrite - }, + } RemoteStatusCode::MessyLocal => { messy_skipped.push(path); false - }, + } RemoteStatusCode::Invalid => { return Err(anyhow!("A file ({:}) with RemoteStatusCode::Invalid was encountered. Please report.", path)); - }, + } RemoteStatusCode::Different => { // TODO if remote supports modification times, // could do extra comparison here @@ -893,13 +993,13 @@ impl DataCollection { overwrite_skipped.push(path); } overwrite - }, + } RemoteStatusCode::DeletedLocal => { // there is nothing to upload print_warn!("A file ({:}) was skipped because it was deleted.", path); - false - }, - RemoteStatusCode::NotExists => true + false + } + RemoteStatusCode::NotExists => true, }; if do_upload { @@ -908,38 +1008,48 @@ impl DataCollection { remote.upload(&data_file, path_context, overwrite).await?; num_uploaded += 1; } - } } } println!("Uploaded {}.", pluralize(num_uploaded as u64, "file")); - let num_skipped = overwrite_skipped.len() + current_skipped.len() + - messy_skipped.len() + untracked_skipped.len(); + let num_skipped = overwrite_skipped.len() + + current_skipped.len() + + messy_skipped.len() + + untracked_skipped.len(); let punc = if num_skipped > 0 { "." } else { ":" }; println!("Skipped {}{}", pluralize(num_skipped as u64, "file"), punc); if !untracked_skipped.is_empty() { - println!(" Untracked: {}", pluralize(untracked_skipped.len() as u64, "file")); + println!( + " Untracked: {}", + pluralize(untracked_skipped.len() as u64, "file") + ); for path in untracked_skipped { println!(" - {:}", path); } } if !current_skipped.is_empty() { - println!(" Remote file is indentical to local file: {}", - pluralize(current_skipped.len() as u64, "file")); + println!( + " Remote file is indentical to local file: {}", + pluralize(current_skipped.len() as u64, "file") + ); for path in current_skipped { println!(" - {:}", path); } } if !overwrite_skipped.is_empty() { - println!(" Would overwrite (use --overwrite to push): {}", - pluralize(overwrite_skipped.len() as u64, "file")); + println!( + " Would overwrite (use --overwrite to push): {}", + pluralize(overwrite_skipped.len() as u64, "file") + ); for path in overwrite_skipped { println!(" - {:}", path); } } if !messy_skipped.is_empty() { - println!(" Local is \"messy\" (manifest and file disagree): {}", - pluralize(messy_skipped.len() as u64, "file")); + println!( + " Local is \"messy\" (manifest and file disagree): {}", + pluralize(messy_skipped.len() as u64, "file") + ); println!(" Use 'sdf update ' to add the current version to the manifest."); for path in messy_skipped { println!(" - {:}", path); @@ -957,7 +1067,8 @@ impl DataCollection { for data_file in self.files.values() { if let Some(url) = &data_file.url { let full_path = data_file.full_path(path_context)?; - let download = downloads.add(url.clone(), Some(&full_path.to_string_lossy()), overwrite)?; + let download = + downloads.add(url.clone(), Some(&full_path.to_string_lossy()), overwrite)?; if let Some(dl) = download { let filepath = dl.filename.clone(); filepaths.push(filepath); @@ -975,9 +1086,11 @@ impl DataCollection { downloads.retrieve(Some(" - {}"), None, false).await?; let num_skipped = skipped.len(); - println!("{} files were downloaded.\n\ + println!( + "{} files were downloaded.\n\ {} files were skipped because they existed (and --overwrite was not specified).", - num_downloaded, num_skipped); + num_downloaded, num_skipped + ); Ok(()) } @@ -998,18 +1111,17 @@ impl DataCollection { // can_download() is true only if local and remote are not None. // (local file can be deleted, but will only be None if not in manifest also) for merged_file in merged_files.values().filter(|f| f.can_download()) { - let path = merged_file.name()?; let do_download = match merged_file.status(path_context).await? { RemoteStatusCode::NoLocal => { return Err(anyhow!("Internal error: execution should not have reached this point, please report.\n\ 'sdf pull' filtered by MergedFile.can_download() but found a RemoteStatusCode::NoLocal status.")); - }, + } RemoteStatusCode::Current => { current_skipped.push(path); false - }, + } RemoteStatusCode::Exists => { // it exists on the remote, but we cannot // compare MD5s. Push only if overwrite is true. @@ -1017,14 +1129,14 @@ impl DataCollection { overwrite_skipped.push(path); } overwrite - }, + } RemoteStatusCode::MessyLocal => { messy_skipped.push(path); false - }, + } RemoteStatusCode::Invalid => { return Err(anyhow!("A file ({:}) with RemoteStatusCode::Invalid was encountered. Please report.", path)); - }, + } RemoteStatusCode::Different => { // TODO if remote supports modification times, // could do extra comparison here @@ -1033,16 +1145,15 @@ impl DataCollection { overwrite_skipped.push(path); } overwrite - }, - RemoteStatusCode::DeletedLocal => { - true - }, - RemoteStatusCode::NotExists => true + } + RemoteStatusCode::DeletedLocal => true, + RemoteStatusCode::NotExists => true, }; - if do_download { + if do_download { if let Some(remote) = self.remotes.get(dir) { - let download = remote.get_download_info(merged_file, path_context, overwrite)?; + let download = + remote.get_download_info(merged_file, path_context, overwrite)?; downloads.queue.push(download); } } @@ -1050,28 +1161,35 @@ impl DataCollection { } // now retrieve all the files in the queue. - downloads.retrieve(Some(" - {}"), Some("No files downloaded."), true).await?; + downloads + .retrieve(Some(" - {}"), Some("No files downloaded."), true) + .await?; - let num_skipped = overwrite_skipped.len() + current_skipped.len() + - messy_skipped.len(); + let num_skipped = overwrite_skipped.len() + current_skipped.len() + messy_skipped.len(); println!("Skipped {} files. Reasons:", num_skipped); if !current_skipped.is_empty() { - println!(" Remote file is indentical to local file: {}", - pluralize(current_skipped.len() as u64, "file")); + println!( + " Remote file is indentical to local file: {}", + pluralize(current_skipped.len() as u64, "file") + ); for path in current_skipped { println!(" - {:}", path); } } if !overwrite_skipped.is_empty() { - println!(" Would overwrite (use --overwrite to push): {}", - pluralize(overwrite_skipped.len() as u64, "file")); + println!( + " Would overwrite (use --overwrite to push): {}", + pluralize(overwrite_skipped.len() as u64, "file") + ); for path in overwrite_skipped { println!(" - {:}", path); } } if !messy_skipped.is_empty() { - println!(" Local is \"messy\" (manifest and file disagree): {}", - pluralize(messy_skipped.len() as u64, "file")); + println!( + " Local is \"messy\" (manifest and file disagree): {}", + pluralize(messy_skipped.len() as u64, "file") + ); println!(" Use 'sdf update ' to add the current version to the manifest."); for path in messy_skipped { println!(" - {:}", path); @@ -1080,19 +1198,17 @@ impl DataCollection { Ok(()) } - } - #[cfg(test)] mod tests { - use crate::lib::api::figshare::{FIGSHARE_BASE_URL,FigShareAPI}; + use crate::lib::api::figshare::{FigShareAPI, FIGSHARE_BASE_URL}; use crate::lib::remote::Remote; use crate::lib::test_utilities::check_error; - use super::{DataFile, DataCollection}; - use std::path::Path; + use super::{DataCollection, DataFile}; use std::io::Write; + use std::path::Path; use tempfile::NamedTempFile; fn mock_data_file() -> NamedTempFile { @@ -1109,8 +1225,11 @@ mod tests { match result { Ok(_) => assert!(false, "Expected an error, but got Ok"), Err(err) => { - assert!(err.to_string().contains("does not exist"), - "Unexpected error: {:?}", err); + assert!( + err.to_string().contains("does not exist"), + "Unexpected error: {:?}", + err + ); } }; } @@ -1133,7 +1252,6 @@ mod tests { assert!(observed_md5 == expected_md5, "MD5 mismatch!"); } - #[tokio::test] async fn test_size() { let path_context = Path::new(""); @@ -1147,12 +1265,14 @@ mod tests { let data_file = DataFile::new(path, None, &path_context).await.unwrap(); // Let's also check size - assert!(data_file.size == 11, "Size mismatch {:?} != {:?}!", - data_file.size, 11); + assert!( + data_file.size == 11, + "Size mismatch {:?} != {:?}!", + data_file.size, + 11 + ); } - - #[tokio::test] async fn test_update_md5() { let path_context = Path::new(""); @@ -1179,7 +1299,10 @@ mod tests { // Now update data_file.update_md5(path_context).await.unwrap(); - assert!(data_file.md5 == expected_md5, "DataFile.update_md5() failed!"); + assert!( + data_file.md5 == expected_md5, + "DataFile.update_md5() failed!" + ); } #[tokio::test] @@ -1203,25 +1326,32 @@ mod tests { assert!(data_file.size == 31, "DataFile.update_size() wrong!"); } - #[test] fn test_register_remote_figshare() { let mut dc = DataCollection::new(); let dir = "data/supplement".to_string(); let result = FigShareAPI::new("Test remote", Some(FIGSHARE_BASE_URL.to_string())); - assert!(result.is_ok(), "FigShareAPI::new() resulted in error: {:?}", result); + assert!( + result.is_ok(), + "FigShareAPI::new() resulted in error: {:?}", + result + ); let figshare = result.unwrap(); - assert!(figshare.get_base_url() == FIGSHARE_BASE_URL, "FigShareAPI.base_url is not correct!"); - dc.register_remote(&dir, Remote::FigShareAPI(figshare)).unwrap(); + assert!( + figshare.get_base_url() == FIGSHARE_BASE_URL, + "FigShareAPI.base_url is not correct!" + ); + dc.register_remote(&dir, Remote::FigShareAPI(figshare)) + .unwrap(); // check that it's been inserted assert!(dc.remotes.contains_key(&dir), "Remote not registered!"); // Let's check that validate_remote_directory() is working - let figshare = FigShareAPI::new("Another test remote", Some(FIGSHARE_BASE_URL.to_string())).unwrap(); + let figshare = + FigShareAPI::new("Another test remote", Some(FIGSHARE_BASE_URL.to_string())).unwrap(); let result = dc.register_remote(&dir, Remote::FigShareAPI(figshare)); check_error(result, "already tracked"); } - } diff --git a/src/lib/download.rs b/src/lib/download.rs index 7dd4e7f..42fd17f 100644 --- a/src/lib/download.rs +++ b/src/lib/download.rs @@ -1,12 +1,12 @@ -use anyhow::{anyhow,Result,Context}; +use anyhow::{anyhow, Context, Result}; +use reqwest::Url; use std::fs; use std::path::PathBuf; -use reqwest::Url; -use trauma::downloader::{DownloaderBuilder,StyleOptions,ProgressBarOpts}; use trauma::download::Download; +use trauma::downloader::{DownloaderBuilder, ProgressBarOpts, StyleOptions}; -use crate::lib::progress::{DEFAULT_PROGRESS_STYLE, DEFAULT_PROGRESS_INC}; +use crate::lib::progress::{DEFAULT_PROGRESS_INC, DEFAULT_PROGRESS_STYLE}; use crate::lib::utils::pluralize; pub struct Downloads { @@ -42,19 +42,22 @@ impl Downloads { Downloads { queue } } - pub fn add(&mut self, item: T, filename: Option<&str>, - overwrite: bool) -> Result> { + pub fn add( + &mut self, + item: T, + filename: Option<&str>, + overwrite: bool, + ) -> Result> { let url = item.to_url()?; let resolved_filename = match filename { Some(name) => name.to_string(), - None => { - url.path_segments() - .ok_or_else(|| anyhow::anyhow!("Error parsing URL."))? - .last() - .ok_or_else(|| anyhow::anyhow!("Error getting filename from download URL."))? - .to_string() - } + None => url + .path_segments() + .ok_or_else(|| anyhow::anyhow!("Error parsing URL."))? + .last() + .ok_or_else(|| anyhow::anyhow!("Error getting filename from download URL."))? + .to_string(), }; let file_path = PathBuf::from(&resolved_filename); @@ -62,38 +65,47 @@ impl Downloads { return Ok(None); } - let download = Download { url, filename: resolved_filename }; + let download = Download { + url, + filename: resolved_filename, + }; self.queue.push(download); - Ok(Some(self.queue.last().ok_or(anyhow::anyhow!("Failed to add download"))?)) + Ok(Some( + self.queue + .last() + .ok_or(anyhow::anyhow!("Failed to add download"))?, + )) } pub fn default_style(&self) -> Result { let style = ProgressBarOpts::new( Some(DEFAULT_PROGRESS_STYLE.to_string()), Some(DEFAULT_PROGRESS_INC.to_string()), - true, true); + true, + true, + ); let style_clone = style.clone(); Ok(StyleOptions::new(style, style_clone)) } - - // Retrieve all files in the download queue. + // Retrieve all files in the download queue. // // Note: if the file is in the queue, at this point it is considered *overwrite safe*. // This is because overwrite-safety is checked at Downloads::add(), per-file. - // The trauma crate does not overwrite files; delete must be done manually here + // The trauma crate does not overwrite files; delete must be done manually here // first if it exists. - pub async fn retrieve(&self, - success_status: Option<&str>, - no_downloads_message: Option<&str>, - show_total: bool) -> Result<()> { + pub async fn retrieve( + &self, + success_status: Option<&str>, + no_downloads_message: Option<&str>, + show_total: bool, + ) -> Result<()> { let downloads = &self.queue; let total_files = downloads.len(); - if !downloads.is_empty() { - + if !downloads.is_empty() { // Let's handle the file operations: - // 1) Delete the files if they exist, since if it's in the queue, it's + // 1) Delete the files if they exist, since if it's in the queue, it's // overwrite-safe. // 2) Create the directory structure if it does not exist. for file in downloads { @@ -115,12 +127,18 @@ impl Downloads { downloader.download(downloads).await; if show_total { let punc = if total_files > 0 { "." } else { ":" }; - println!("Downloaded {}{}", pluralize(total_files as u64, "file"), punc); + println!( + "Downloaded {}{}", + pluralize(total_files as u64, "file"), + punc + ); } for download in downloads { if let Some(msg) = success_status { let filename = PathBuf::from(&download.filename); - let name_str = filename.file_name().ok_or(anyhow!("Internal Error: could not extract filename from download"))?; + let name_str = filename.file_name().ok_or(anyhow!( + "Internal Error: could not extract filename from download" + ))?; //println!(" - {}", name_str.to_string_lossy()); println!("{}", msg.replace("{}", &name_str.to_string_lossy())); } diff --git a/src/lib/progress.rs b/src/lib/progress.rs index 46e7b82..bc0a91f 100644 --- a/src/lib/progress.rs +++ b/src/lib/progress.rs @@ -1,8 +1,8 @@ +use anyhow::Result; use indicatif::{ProgressBar, ProgressStyle}; -use std::time::Duration; +use std::sync::mpsc::{self, Receiver, Sender}; use std::thread; -use std::sync::mpsc::{self, Sender, Receiver}; -use anyhow::Result; +use std::time::Duration; // these are separated since some APIs don't overload // indicatif bars, but take the same primitives. @@ -20,7 +20,7 @@ pub struct Progress { pub bar: ProgressBar, stop_spinner: Sender<()>, #[allow(dead_code)] - spinner: Option> + spinner: Option>, } impl Progress { @@ -31,16 +31,18 @@ impl Progress { let (tx, rx): (Sender<()>, Receiver<()>) = mpsc::channel(); let bar_clone = bar.clone(); - let spinner = thread::spawn(move || { - loop { - if rx.try_recv().is_ok() { - break; - } - bar_clone.tick(); - thread::sleep(Duration::from_millis(20)); + let spinner = thread::spawn(move || loop { + if rx.try_recv().is_ok() { + break; } + bar_clone.tick(); + thread::sleep(Duration::from_millis(20)); }); - Ok(Progress { bar, stop_spinner: tx, spinner: Some(spinner) }) + Ok(Progress { + bar, + stop_spinner: tx, + spinner: Some(spinner), + }) } } @@ -52,4 +54,3 @@ impl Drop for Progress { } } } - diff --git a/src/lib/project.rs b/src/lib/project.rs index 89092b6..177940d 100644 --- a/src/lib/project.rs +++ b/src/lib/project.rs @@ -1,33 +1,32 @@ -use std::fs::{File,metadata,canonicalize,rename}; -use anyhow::{anyhow,Result,Context}; +use anyhow::{anyhow, Context, Result}; +use csv::{ReaderBuilder, StringRecord}; +use dirs; +#[allow(unused_imports)] +use log::{debug, info, trace}; +use serde_derive::{Deserialize, Serialize}; use serde_yaml; -use serde_derive::{Serialize,Deserialize}; use std::env; -use std::path::{Path,PathBuf}; +use std::fs::{canonicalize, metadata, rename, File}; use std::io::{Read, Write}; -#[allow(unused_imports)] -use log::{info, trace, debug}; -use csv::{ReaderBuilder, StringRecord}; -use dirs; +use std::path::{Path, PathBuf}; -use crate::lib::download::Downloads; -#[allow(unused_imports)] -use crate::{print_warn,print_info}; -use crate::lib::data::{DataFile,DataCollection}; -use crate::lib::utils::{load_file,print_status, pluralize}; -use crate::lib::remote::{AuthKeys,authenticate_remote}; -use crate::lib::remote::Remote; use crate::lib::api::figshare::FigShareAPI; use crate::lib::api::zenodo::ZenodoAPI; use crate::lib::data::LocalStatusCode; +use crate::lib::data::{DataCollection, DataFile}; +use crate::lib::download::Downloads; +use crate::lib::remote::Remote; +use crate::lib::remote::{authenticate_remote, AuthKeys}; +use crate::lib::utils::{load_file, pluralize, print_status}; +#[allow(unused_imports)] +use crate::{print_info, print_warn}; const MANIFEST: &str = "data_manifest.yml"; - pub fn find_manifest(start_dir: Option<&PathBuf>, filename: &str) -> Option { let mut current_dir = match start_dir { Some(dir) => dir.to_path_buf(), - None => env::current_dir().expect("Failed to get current directory") + None => env::current_dir().expect("Failed to get current directory"), }; loop { @@ -45,8 +44,8 @@ pub fn find_manifest(start_dir: Option<&PathBuf>, filename: &str) -> Option Result { - let mut config_path: PathBuf = dirs::home_dir() - .ok_or_else(|| anyhow!("Cannot load home directory!"))?; + let mut config_path: PathBuf = + dirs::home_dir().ok_or_else(|| anyhow!("Cannot load home directory!"))?; config_path.push(".scidataflow_config"); Ok(config_path) } @@ -58,13 +57,11 @@ pub struct User { pub affiliation: Option, } - #[derive(Debug, Serialize, Deserialize, PartialEq)] pub struct Config { - user: User + user: User, } - // Metadata about *local* project // // The idea of this is to extract the parts of the metadata @@ -77,7 +74,7 @@ pub struct LocalMetadata { pub email: Option, pub affiliation: Option, pub title: Option, - pub description: Option + pub description: Option, } impl LocalMetadata { @@ -87,7 +84,7 @@ impl LocalMetadata { email: project.config.user.email.clone(), affiliation: project.config.user.affiliation.clone(), title: project.data.metadata.title.clone(), - description: project.data.metadata.description.clone() + description: project.data.metadata.description.clone(), } } } @@ -105,15 +102,19 @@ impl Project { pub fn load_config() -> Result { let config_path = config_path()?; - let mut file = File::open(&config_path) - .map_err(|_| anyhow!("No SciDataFlow config found at \ + let mut file = File::open(&config_path).map_err(|_| { + anyhow!( + "No SciDataFlow config found at \ {:?}. Please set with sdf config --name \ - [--email --affiliation ]", &config_path))?; - let mut contents = String::new(); - file.read_to_string(&mut contents)?; + [--email --affiliation ]", + &config_path + ) + })?; + let mut contents = String::new(); + file.read_to_string(&mut contents)?; - let config: Config = serde_yaml::from_str(&contents)?; - Ok(config) + let config: Config = serde_yaml::from_str(&contents)?; + Ok(config) } pub fn save_config(config: Config) -> Result<()> { @@ -129,7 +130,11 @@ impl Project { info!("manifest: {:?}", manifest); let data = Project::load(&manifest).context("Failed to load data from the manifest")?; let config = Project::load_config().context("Failed to load the project configuration")?; - let proj = Project { manifest, data, config }; + let proj = Project { + manifest, + data, + config, + }; Ok(proj) } @@ -138,12 +143,10 @@ impl Project { .and_then(|path| path.file_name()) .map(|os_str| os_str.to_string_lossy().into_owned()) .unwrap_or_else(|| panic!("invalid project location: is it in root?")) - } - - // This tries to figure out a good default name to use, e.g. for - // remote titles or names. + // This tries to figure out a good default name to use, e.g. for + // remote titles or names. // // The precedence is local metadata in manifest > project directory pub fn name(&self) -> String { @@ -157,7 +160,9 @@ impl Project { // the new manifest should be in the present directory let manifest: PathBuf = PathBuf::from(MANIFEST); if manifest.exists() { - return Err(anyhow!("Project already initialized. Manifest file already exists.")); + return Err(anyhow!( + "Project already initialized. Manifest file already exists." + )); } else { // TODO could pass metadata parameters here let mut data = DataCollection::new(); @@ -165,16 +170,23 @@ impl Project { data.metadata.title = Some(name); } let config = Project::load_config()?; - let proj = Project { manifest, data, config }; + let proj = Project { + manifest, + data, + config, + }; // save to create the manifest proj.save()?; - } Ok(()) } // TODO could add support for other metadata here - pub fn set_metadata(&mut self, title: &Option, description: &Option) -> Result<()> { + pub fn set_metadata( + &mut self, + title: &Option, + description: &Option, + ) -> Result<()> { if let Some(new_title) = title { self.data.metadata.title = Some(new_title.to_string()); } @@ -184,13 +196,17 @@ impl Project { self.save() } - pub fn set_config(name: &Option, email: &Option, affiliation: &Option) -> Result<()> { + pub fn set_config( + name: &Option, + email: &Option, + affiliation: &Option, + ) -> Result<()> { let mut config = Project::load_config().unwrap_or_else(|_| Config { user: User { name: "".to_string(), email: None, affiliation: None, - } + }, }); info!("read config: {:?}", config); if let Some(new_name) = name { @@ -230,7 +246,9 @@ impl Project { if contents.trim().is_empty() { // empty manifest, just create a new one - return Err(anyhow!("No 'data_manifest.yml' found, has sdf init been run?")); + return Err(anyhow!( + "No 'data_manifest.yml' found, has sdf init been run?" + )); } let data = serde_yaml::from_str(&contents)?; @@ -252,9 +270,15 @@ impl Project { } pub fn relative_path(&self, path: &Path) -> Result { - let absolute_path = canonicalize(path).context(format!("Failed to canonicalize path '{}'.", path.to_string_lossy()))?; + let absolute_path = canonicalize(path).context(format!( + "Failed to canonicalize path '{}'.", + path.to_string_lossy() + ))?; //ensure_directory(&absolute_path)?; - let path_context = canonicalize(self.path_context()).context(format!("Failed to canonicalize path '{}'.", path.to_string_lossy()))?; + let path_context = canonicalize(self.path_context()).context(format!( + "Failed to canonicalize path '{}'.", + path.to_string_lossy() + ))?; // Compute relative path directly using strip_prefix match absolute_path.strip_prefix(&path_context) { @@ -312,7 +336,7 @@ impl Project { // TODO use different more general struct? // Or print_fixed_width should be a trait? let entry = StatusEntry { - local_status: LocalStatusCode::Invalid, + local_status: LocalStatusCode::Invalid, remote_status: RemoteStatusCode::NotExists, tracked: Some(false), remote_service: None, @@ -323,7 +347,6 @@ impl Project { Ok(()) } */ - pub async fn add(&mut self, files: &Vec) -> Result<()> { let mut num_added = 0; for filepath in files { @@ -343,16 +366,18 @@ impl Project { let filepaths: Result> = match files { None => Ok(self.data.files.keys().cloned().collect::>()), - Some(file_list) => { - file_list.iter() - .map(|f| { - Ok(self.relative_path(Path::new(&f))?.to_string_lossy().to_string()) - }) - .collect() - } + Some(file_list) => file_list + .iter() + .map(|f| { + Ok(self + .relative_path(Path::new(&f))? + .to_string_lossy() + .to_string()) + }) + .collect(), }; - let filepaths = filepaths?; // Use ? here to propagate any errors + let filepaths = filepaths?; // Use ? here to propagate any errors for filepath in filepaths { match self.data.update(Some(&filepath), &path_context).await { @@ -369,9 +394,14 @@ impl Project { self.save() } - - pub async fn link(&mut self, dir: &str, service: &str, - key: &str, name: &Option, link_only: &bool) -> Result<()> { + pub async fn link( + &mut self, + dir: &str, + service: &str, + key: &str, + name: &Option, + link_only: &bool, + ) -> Result<()> { // (0) get the relative directory path let dir = self.relative_path_string(Path::new(dir))?; @@ -382,7 +412,7 @@ impl Project { // (2) create a new remote, with a name // Associate a project (either by creating it, or finding it on FigShare) let name = if let Some(n) = name { - n.to_string() + n.to_string() } else { self.name() }; @@ -391,14 +421,14 @@ impl Project { let mut remote = match service.as_str() { "figshare" => Ok(Remote::FigShareAPI(FigShareAPI::new(&name, None)?)), "zenodo" => Ok(Remote::ZenodoAPI(ZenodoAPI::new(&name, None)?)), - _ => Err(anyhow!("Service '{}' is not supported!", service)) + _ => Err(anyhow!("Service '{}' is not supported!", service)), }?; // (3) authenticate remote authenticate_remote(&mut remote)?; - // (4) validate this a proper remote directory (this is - // also done in register_remote() for caution, + // (4) validate this a proper remote directory (this is + // also done in register_remote() for caution, // but we also want do it here to prevent the situation // where self.data.register_remote() fails, but remote_init() // is already done. @@ -406,7 +436,7 @@ impl Project { // (5) initialize the remote (e.g. for FigShare, this // checks that the article doesn't exist (error if it - // does), creates it, and sets the FigShare.article_id + // does), creates it, and sets the FigShare.article_id // once it is assigned by the remote). // Note: we pass the Project to remote_init let local_metadata = LocalMetadata::from_project(self); @@ -429,10 +459,10 @@ impl Project { } // Move a file within the project. - // - // Note: file moving is done within relatively higher project-level API. + // + // Note: file moving is done within relatively higher project-level API. // The reason why is that we need to access Project::relative_path_string() for - // both the source *and* destination; the latter does not exist until after the file + // both the source *and* destination; the latter does not exist until after the file // has been successfully moved. So the updating is all done on the DataFile // directly, since lower interfaces cannot access the relative path. pub async fn mv(&mut self, source: &str, destination: &str) -> Result<()> { @@ -453,44 +483,60 @@ impl Project { self.save() } else { - Err(anyhow!("Cannot move file '{}' with 'sdf mv' since it is not in the manifest.", source)) + Err(anyhow!( + "Cannot move file '{}' with 'sdf mv' since it is not in the manifest.", + source + )) } } - pub async fn get(&mut self, url: &str, filename: Option<&str>, - overwrite: bool) -> Result<()> { + pub async fn get(&mut self, url: &str, filename: Option<&str>, overwrite: bool) -> Result<()> { let mut downloads = Downloads::new(); let download = downloads.add(url.to_string(), filename, overwrite)?; if let Some(dl) = download { let filepath = dl.filename.clone(); // get the file - downloads.retrieve(Some("Downloaded '{}'."), None, false).await?; + downloads + .retrieve(Some("Downloaded '{}'."), None, false) + .await?; // convert to relative path (based on where we are) let filepath = self.relative_path_string(Path::new(&filepath))?; // TODO: should compare MD5s! if !self.data.contains(&filepath).await? { - let data_file = DataFile::new(filepath.clone(), Some(url), &self.path_context()).await?; + let data_file = + DataFile::new(filepath.clone(), Some(url), &self.path_context()).await?; // Note: we do not use Project::add() since this works off strings. // and we need to pass the URL, etc. self.data.register(data_file)?; self.save()?; } else { - println!("File '{}' already existed in \ - the manifest, so it was not added.", &filepath); + println!( + "File '{}' already existed in \ + the manifest, so it was not added.", + &filepath + ); } Ok(()) } else { - Err(anyhow!("The file at '{}' was not downloaded because it would overwrite a file.\n\ - Use 'sdf get --ovewrite' to overwrite it.", url)) + Err(anyhow!( + "The file at '{}' was not downloaded because it would overwrite a file.\n\ + Use 'sdf get --ovewrite' to overwrite it.", + url + )) } } - pub async fn bulk(&mut self, filename: &str, column: Option, - header: bool, overwrite: bool) -> Result<()> { + pub async fn bulk( + &mut self, + filename: &str, + column: Option, + header: bool, + overwrite: bool, + ) -> Result<()> { let extension = std::path::Path::new(filename) .extension() .and_then(std::ffi::OsStr::to_str); @@ -507,7 +553,7 @@ impl Project { .has_headers(header) .from_reader(file); - // convert 0-indexed to 1; first column is default + // convert 0-indexed to 1; first column is default let column = column.unwrap_or(0) as usize - 1; let mut downloads = Downloads::new(); @@ -539,7 +585,8 @@ impl Project { for (filepath, url) in filepaths.iter().zip(urls.iter()) { let rel_file_path = self.relative_path_string(Path::new(&filepath))?; if !self.data.contains(&rel_file_path).await? { - let data_file = DataFile::new(rel_file_path.clone(), Some(url), &self.path_context()).await?; + let data_file = + DataFile::new(rel_file_path.clone(), Some(url), &self.path_context()).await?; self.data.register(data_file)?; num_added += 1; } else { @@ -547,12 +594,17 @@ impl Project { } } let num_skipped = skipped.len(); - println!("{} URLs found in '{}.'\n\ + println!( + "{} URLs found in '{}.'\n\ {} files were downloaded, {} added to manifest ({} were already registered).\n\ {} files were skipped because they existed (and --overwrite was no specified).", - num_lines, filename, - urls.len(), num_added, num_already_registered, - num_skipped); + num_lines, + filename, + urls.len(), + num_added, + num_already_registered, + num_skipped + ); self.save()?; Ok(()) } @@ -585,4 +637,3 @@ impl Project { self.data.push(&self.path_context(), overwrite).await } } - diff --git a/src/lib/remote.rs b/src/lib/remote.rs index 763c336..c397572 100644 --- a/src/lib/remote.rs +++ b/src/lib/remote.rs @@ -1,21 +1,21 @@ +use anyhow::{anyhow, Result}; +#[allow(unused_imports)] +use log::{debug, info, trace}; +use reqwest::Url; +use serde_derive::{Deserialize, Serialize}; use serde_yaml; -use trauma::download::Download; +use std::collections::HashMap; +use std::env; use std::fs; use std::fs::File; -use std::path::Path; use std::io::Read; -use std::env; -use anyhow::{anyhow,Result}; -#[allow(unused_imports)] -use log::{info, trace, debug}; -use std::collections::HashMap; -use serde_derive::{Serialize,Deserialize}; -use reqwest::Url; +use std::path::Path; +use trauma::download::Download; -use crate::lib::data::{DataFile,MergedFile}; -use crate::lib::api::figshare::FigShareAPI; use crate::lib::api::dryad::DataDryadAPI; +use crate::lib::api::figshare::FigShareAPI; use crate::lib::api::zenodo::ZenodoAPI; +use crate::lib::data::{DataFile, MergedFile}; use crate::lib::project::LocalMetadata; const AUTHKEYS: &str = ".scidataflow_authkeys.yml"; @@ -26,38 +26,37 @@ pub struct RemoteFile { pub md5: Option, pub size: Option, pub remote_service: String, - pub url: Option + pub url: Option, } - // This is the status of the local state with the remote state. // There are huge number of combinations between tracked, untracked -// local files, and whether the manifest and file MD5s agree or +// local files, and whether the manifest and file MD5s agree or // disagree (a "messy" state). However, it's better to handle few // cases well, and error out. // // Note that these states are independent of whether something is -// tracked. Tracking only indicates whether the next pull/push -// should sync the file (if tracked). Tracked files are *always* +// tracked. Tracking only indicates whether the next pull/push +// should sync the file (if tracked). Tracked files are *always* // in the manifest (since that is where that state is stored). // -// NoLocal files will *not* be sync'd, since they are no in the -// manifest, and sciflow will only get pull/push things in the +// NoLocal files will *not* be sync'd, since they are no in the +// manifest, and sciflow will only get pull/push things in the // manifest. // // Clean state: everything on the manifest tracked by the remote is // local, with nothing else. -#[derive(Debug,PartialEq,Clone)] +#[derive(Debug, PartialEq, Clone)] pub enum RemoteStatusCode { - Current, // local and remote files are identical - MessyLocal, // local file is different than remote and manifest, which agree - Different, // the local file is current, but different than the remote - NotExists, // no remote file - Exists, // remote file exists, but remote does not support MD5s - NoLocal, // a file on the remote, but not in manifest or found locally - DeletedLocal, // a file on the remote and in manifest, but not found locally + Current, // local and remote files are identical + MessyLocal, // local file is different than remote and manifest, which agree + Different, // the local file is current, but different than the remote + NotExists, // no remote file + Exists, // remote file exists, but remote does not support MD5s + NoLocal, // a file on the remote, but not in manifest or found locally + DeletedLocal, // a file on the remote and in manifest, but not found locally //OutsideSource, // a file on the remote, but not in manifest but *is* found locally - Invalid + Invalid, } impl RemoteFile { @@ -75,13 +74,12 @@ impl RemoteFile { #[derive(Serialize, Deserialize, Default, PartialEq, Debug)] pub struct AuthKeys { - keys: HashMap + keys: HashMap, } impl AuthKeys { pub fn new() -> Self { - let home_dir = env::var("HOME") - .expect("Could not infer home directory"); + let home_dir = env::var("HOME").expect("Could not infer home directory"); let path = Path::new(&home_dir).join(AUTHKEYS); let keys = match path.exists() { true => { @@ -92,9 +90,9 @@ impl AuthKeys { .unwrap(); serde_yaml::from_str(&contents) .unwrap_or_else(|_| panic!("Cannot load {}!", AUTHKEYS)) - }, + } false => { - let keys: HashMap = HashMap::new(); + let keys: HashMap = HashMap::new(); keys } }; @@ -109,7 +107,7 @@ impl AuthKeys { } pub fn temporary_add(&mut self, service: &str, key: &str) { - // no save, i.e. for testing -- we do *not* want to overwrite the + // no save, i.e. for testing -- we do *not* want to overwrite the // dev's own keys. let service = service.to_lowercase(); self.keys.insert(service, key.to_owned()); @@ -118,15 +116,14 @@ impl AuthKeys { pub fn get(&self, service: String) -> Result { match self.keys.get(&service) { None => Err(anyhow!("no key found for service '{}'", service)), - Some(key) => Ok(key.to_string()) + Some(key) => Ok(key.to_string()), } } pub fn save(&self) { - let serialized_keys = serde_yaml::to_string(&self.keys) - .expect("Cannot serialize authentication keys!"); - let home_dir = env::var("HOME") - .expect("Could not infer home directory"); + let serialized_keys = + serde_yaml::to_string(&self.keys).expect("Cannot serialize authentication keys!"); + let home_dir = env::var("HOME").expect("Could not infer home directory"); let path = Path::new(&home_dir).join(AUTHKEYS); fs::write(path, serialized_keys) .unwrap_or_else(|_| panic!("Cound not write {}!", AUTHKEYS)); @@ -140,7 +137,6 @@ pub enum Remote { ZenodoAPI(ZenodoAPI), } - macro_rules! service_not_implemented { ($service:expr) => { Err(anyhow!("{} not implemented yet.", $service)) @@ -154,11 +150,15 @@ impl Remote { match self { Remote::FigShareAPI(_) => "FigShare", Remote::DataDryadAPI(_) => "Dryad", - Remote::ZenodoAPI(_) => "Zenodo" + Remote::ZenodoAPI(_) => "Zenodo", } } // initialize the remote (i.e. tell it we have a new empty data set) - pub async fn remote_init(&mut self, local_metadata: LocalMetadata, link_only: bool) -> Result<()> { + pub async fn remote_init( + &mut self, + local_metadata: LocalMetadata, + link_only: bool, + ) -> Result<()> { match self { Remote::FigShareAPI(fgsh_api) => fgsh_api.remote_init(local_metadata, link_only).await, Remote::ZenodoAPI(znd_api) => znd_api.remote_init(local_metadata, link_only).await, @@ -172,18 +172,25 @@ impl Remote { Remote::DataDryadAPI(_) => service_not_implemented!("DataDryad"), } } - pub async fn get_files_hashmap(&self) -> Result> { + pub async fn get_files_hashmap(&self) -> Result> { // now we can use the common interface! :) let remote_files = self.get_files().await?; - let mut file_map: HashMap = HashMap::new(); + let mut file_map: HashMap = HashMap::new(); for file in remote_files.into_iter() { file_map.insert(file.name.clone(), file.clone()); } Ok(file_map) } - pub async fn upload(&self, data_file: &DataFile, path_context: &Path, overwrite: bool) -> Result { + pub async fn upload( + &self, + data_file: &DataFile, + path_context: &Path, + overwrite: bool, + ) -> Result { match self { - Remote::FigShareAPI(fgsh_api) => fgsh_api.upload(data_file, path_context, overwrite).await, + Remote::FigShareAPI(fgsh_api) => { + fgsh_api.upload(data_file, path_context, overwrite).await + } Remote::ZenodoAPI(znd_api) => znd_api.upload(data_file, path_context, overwrite).await, Remote::DataDryadAPI(_) => service_not_implemented!("DataDryad"), } @@ -191,23 +198,36 @@ impl Remote { // Get Download info: the URL (with token) and destination // TODO: could be struct, if some APIs require more authentication // Note: requires each API actually *check* overwrite. - pub fn get_download_info(&self, merged_file: &MergedFile, path_context: &Path, overwrite: bool) -> Result { - // if local DataFile is none, not in manifest; + pub fn get_download_info( + &self, + merged_file: &MergedFile, + path_context: &Path, + overwrite: bool, + ) -> Result { + // if local DataFile is none, not in manifest; // do not download let data_file = match &merged_file.local { None => return Err(anyhow!("Cannot download() without local DataFile.")), - Some(file) => file + Some(file) => file, }; // check to make sure we won't overwrite if data_file.is_alive(path_context) && !overwrite { - return Err(anyhow!("Data file '{}' exists locally, and would be \ + return Err(anyhow!( + "Data file '{}' exists locally, and would be \ overwritten by download. Use --overwrite to download.", - data_file.path)); + data_file.path + )); } // if no remote, there is nothing to download, // silently return Ok. Get URL. - let remote = merged_file.remote.as_ref().ok_or(anyhow!("Remote is None"))?; - let url = remote.url.as_ref().ok_or(anyhow!("Cannot download; download URL not set."))?; + let remote = merged_file + .remote + .as_ref() + .ok_or(anyhow!("Remote is None"))?; + let url = remote + .url + .as_ref() + .ok_or(anyhow!("Cannot download; download URL not set."))?; let authenticated_url = match self { Remote::FigShareAPI(fgsh_api) => fgsh_api.authenticate_url(url), @@ -217,7 +237,7 @@ impl Remote { let save_path = &data_file.full_path(path_context)?; let url = Url::parse(&authenticated_url)?; let filename = save_path.to_string_lossy().to_string(); - Ok( Download { url, filename }) + Ok(Download { url, filename }) } } @@ -233,25 +253,32 @@ pub fn authenticate_remote(remote: &mut Remote) -> Result<()> { match remote { Remote::FigShareAPI(ref mut fgsh_api) => { - let token = auth_keys.keys.get("figshare").cloned() + let token = auth_keys + .keys + .get("figshare") + .cloned() .ok_or_else(|| anyhow::anyhow!(error_message("FigShare", "figshare")))?; fgsh_api.set_token(token); - }, + } Remote::ZenodoAPI(ref mut znd_api) => { - let token = auth_keys.keys.get("zenodo").cloned() + let token = auth_keys + .keys + .get("zenodo") + .cloned() .ok_or_else(|| anyhow::anyhow!(error_message("Zenodo", "zenodo")))?; znd_api.set_token(token); - }, + } // handle other Remote variants as necessary - _ => Err(anyhow!("Could not find correct API in authenticate_remote()"))? + _ => Err(anyhow!( + "Could not find correct API in authenticate_remote()" + ))?, } Ok(()) } - // Common enum for issue_request() methods of APIs // -// Notes: Binary() should be used only for small amounts of data, +// Notes: Binary() should be used only for small amounts of data, // that can be read into memory, e.g. FigShare's upload_parts(). #[derive(Debug)] pub enum RequestData { @@ -259,20 +286,18 @@ pub enum RequestData { Binary(Vec), File(tokio::fs::File), Stream(tokio::fs::File), - Empty + Empty, } - /* impl DataDryadAPI { - fn upload(&self) { - } - fn download(&self) { - } - fn ls(&self) { - } - fn get_project(&self) -> Result { - Ok("ID".to_string()) - } - } - */ - +fn upload(&self) { +} +fn download(&self) { +} +fn ls(&self) { +} +fn get_project(&self) -> Result { +Ok("ID".to_string()) +} +} +*/ diff --git a/src/lib/test_utilities.rs b/src/lib/test_utilities.rs index f9af536..c087717 100644 --- a/src/lib/test_utilities.rs +++ b/src/lib/test_utilities.rs @@ -4,9 +4,12 @@ pub fn check_error(result: Result, pattern: &str) { match result { Ok(_) => panic!("Expected an error, but got Ok"), Err(err) => { - assert!(err.to_string().contains(pattern), - "Unexpected error {:?} containing pattern \"{:?}\" ", - err, pattern); + assert!( + err.to_string().contains(pattern), + "Unexpected error {:?} containing pattern \"{:?}\" ", + err, + pattern + ); } } } diff --git a/src/lib/utils.rs b/src/lib/utils.rs index 31221c1..18b07d0 100644 --- a/src/lib/utils.rs +++ b/src/lib/utils.rs @@ -1,15 +1,15 @@ -use std::collections::HashMap; +use anyhow::{anyhow, Result}; +use chrono::{Local, Utc}; +use colored::*; +#[allow(unused_imports)] +use log::{debug, info, trace}; +use md5::Context; use std::collections::BTreeMap; -use anyhow::{anyhow,Result}; -use chrono::{Utc,Local}; -use timeago::Formatter; -use std::path::{Path,PathBuf}; +use std::collections::HashMap; use std::fs::File; use std::io::Read; -use md5::Context; -#[allow(unused_imports)] -use log::{info, trace, debug}; -use colored::*; +use std::path::{Path, PathBuf}; +use timeago::Formatter; use crate::lib::data::StatusEntry; use crate::lib::remote::Remote; @@ -19,7 +19,8 @@ pub const ISSUE_URL: &str = "https://github.com/vsbuffalo/scidataflow/issues"; pub fn load_file(path: &PathBuf) -> String { let mut file = File::open(path).expect("unable to open file"); let mut contents = String::new(); - file.read_to_string(&mut contents).expect("unable to read file"); + file.read_to_string(&mut contents) + .expect("unable to read file"); contents } @@ -28,7 +29,10 @@ pub fn ensure_directory(dir: &Path) -> Result<()> { if path.is_dir() { Ok(()) } else { - Err(anyhow!("'{}' is not a directory or doesn't exist.", dir.to_string_lossy())) + Err(anyhow!( + "'{}' is not a directory or doesn't exist.", + dir.to_string_lossy() + )) } } @@ -120,17 +124,23 @@ println!(); } */ // More specialized version of print_fixed_width() for statuses. -// Handles coloring, manual annotation, etc -pub fn print_fixed_width_status(rows: BTreeMap>, nspaces: Option, - indent: Option, color: bool, all: bool) { +// Handles coloring, manual annotation, etc +pub fn print_fixed_width_status( + rows: BTreeMap>, + nspaces: Option, + indent: Option, + color: bool, + all: bool, +) { //debug!("rows: {:?}", rows); let indent = indent.unwrap_or(0); let nspaces = nspaces.unwrap_or(6); let abbrev = Some(8); - // get the max number of columns (in case ragged) - let max_cols = rows.values() + // get the max number of columns (in case ragged) + let max_cols = rows + .values() .flat_map(|v| v.iter()) .map(|entry| entry.columns(abbrev).len()) .max() @@ -140,7 +150,7 @@ pub fn print_fixed_width_status(rows: BTreeMap>, nspace // compute max lengths across all rows for status in rows.values().flat_map(|v| v.iter()) { - let cols = status.columns(abbrev); + let cols = status.columns(abbrev); for (i, col) in cols.iter().enumerate() { max_lengths[i] = max_lengths[i].max(col.len()); // Assuming col is a string } @@ -152,7 +162,11 @@ pub fn print_fixed_width_status(rows: BTreeMap>, nspace for key in dir_keys { let statuses = &rows[key]; let pretty_key = if key.is_empty() { "." } else { key }; - let prettier_key = if color { pretty_key.bold().to_string() } else { pretty_key.clone().to_string() }; + let prettier_key = if color { + pretty_key.bold().to_string() + } else { + pretty_key.to_string() + }; println!("[{}]", prettier_key); // Print the rows with the correct widths @@ -171,7 +185,11 @@ pub fn print_fixed_width_status(rows: BTreeMap>, nspace } let spacer = " ".repeat(nspaces); let line = fixed_row.join(&spacer); - let status_line = if color { status.color(line) } else { line.to_string() }; + let status_line = if color { + status.color(line) + } else { + line.to_string() + }; println!("{}{}", " ".repeat(indent), status_line); } println!(); @@ -179,22 +197,22 @@ pub fn print_fixed_width_status(rows: BTreeMap>, nspace } /* fn organize_by_dir(rows: Vec) -> BTreeMap> { - let mut dir_map: BTreeMap> = BTreeMap::new(); - - for entry in rows { - if let Some(cols) = &entry.cols { - if let Some(first_elem) = cols.first() { - let path = Path::new(first_elem); - if let Some(parent_path) = path.parent() { - let parent_dir = parent_path.to_string_lossy().into_owned(); - dir_map.entry(parent_dir).or_default().push(entry); - } - } - } - } - dir_map - } - */ +let mut dir_map: BTreeMap> = BTreeMap::new(); + +for entry in rows { +if let Some(cols) = &entry.cols { +if let Some(first_elem) = cols.first() { +let path = Path::new(first_elem); +if let Some(parent_path) = path.parent() { +let parent_dir = parent_path.to_string_lossy().into_owned(); +dir_map.entry(parent_dir).or_default().push(entry); +} +} +} +} +dir_map +} +*/ pub fn pluralize>(count: T, noun: &str) -> String { let count = count.into(); @@ -211,10 +229,10 @@ struct FileCounts { both: u64, total: u64, #[allow(dead_code)] - messy: u64 + messy: u64, } -fn get_counts(rows: &BTreeMap>) -> Result { +fn get_counts(rows: &BTreeMap>) -> Result { let mut local = 0; let mut remote = 0; let mut both = 0; @@ -223,56 +241,67 @@ fn get_counts(rows: &BTreeMap>) -> Result { for files in rows.values() { for file in files { total += 1; - match (&file.local_status, &file.remote_status, &file.tracked) { + match (&file.local_status, &file.remote_status, &file.tracked) { (None, None, _) => { return Err(anyhow!("Internal Error: get_counts found a file with both local/remote set to None.")); - }, + } (None, Some(_), None) => { remote += 1; - }, + } (Some(_), None, Some(false)) => { local += 1; - }, + } (Some(_), None, None) => { local += 1; - }, + } (None, Some(_), Some(true)) => { remote += 1; - }, + } (None, Some(_), Some(false)) => { local += 1; - }, + } (Some(_), Some(_), Some(true)) => { both += 1; - }, + } (Some(_), Some(_), Some(false)) => { messy += 1; - }, + } (Some(_), None, Some(true)) => { remote += 1; - }, + } (Some(_), Some(_), None) => { messy += 1; } } } } - Ok(FileCounts { local, remote, both, total, messy }) + Ok(FileCounts { + local, + remote, + both, + total, + messy, + }) } -pub fn print_status(rows: BTreeMap>, remote: Option<&HashMap>, - all: bool) { +pub fn print_status( + rows: BTreeMap>, + remote: Option<&HashMap>, + all: bool, +) { println!("{}", "Project data status:".bold()); let counts = get_counts(&rows).expect("Internal Error: get_counts() panicked."); - println!("{} local and tracked by a remote ({} only local, {} only remote), {} total.\n", - pluralize(counts.both, "file"), - pluralize(counts.local, "file"), - pluralize(counts.remote, "file"), - //pluralize(counts.messy as u64, "file"), - pluralize(counts.total, "file")); - - // this brings the remote name (if there is a corresponding remote) into - // the key, so the linked remote can be displayed in the status + println!( + "{} local and tracked by a remote ({} only local, {} only remote), {} total.\n", + pluralize(counts.both, "file"), + pluralize(counts.local, "file"), + pluralize(counts.remote, "file"), + //pluralize(counts.messy as u64, "file"), + pluralize(counts.total, "file") + ); + + // this brings the remote name (if there is a corresponding remote) into + // the key, so the linked remote can be displayed in the status let rows_by_dir: BTreeMap> = match remote { Some(remote_map) => { let mut new_map = BTreeMap::new(); @@ -285,7 +314,7 @@ pub fn print_status(rows: BTreeMap>, remote: Option<&Has } } new_map - }, + } None => rows, }; @@ -313,16 +342,12 @@ pub fn format_bytes(size: u64) -> String { } } - pub fn format_mod_time(mod_time: chrono::DateTime) -> String { let now = Utc::now(); let duration_since_mod = now.signed_duration_since(mod_time); // convert chrono::Duration to std::time::Duration - let std_duration = std::time::Duration::new( - duration_since_mod.num_seconds() as u64, - 0 - ); + let std_duration = std::time::Duration::new(duration_since_mod.num_seconds() as u64, 0); let formatter = Formatter::new(); let local_time = mod_time.with_timezone(&Local); @@ -335,7 +360,11 @@ pub fn shorten(hash: &String, abbrev: Option) -> String { hash.chars().take(n).collect() } -pub fn md5_status(new_md5: Option<&String>, old_md5: Option<&String>, abbrev: Option) -> String { +pub fn md5_status( + new_md5: Option<&String>, + old_md5: Option<&String>, + abbrev: Option, +) -> String { match (new_md5, old_md5) { (Some(new), Some(old)) => { if new == old { @@ -343,9 +372,8 @@ pub fn md5_status(new_md5: Option<&String>, old_md5: Option<&String>, abbrev: Op } else { format!("{} → {}", shorten(old, abbrev), shorten(new, abbrev)) } - }, + } (None, Some(old)) => shorten(old, abbrev), _ => "".to_string(), } } - diff --git a/src/main.rs b/src/main.rs index e9b82e2..c3f4200 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,14 +1,13 @@ use std::path::Path; +use anyhow::{anyhow, Result}; use clap::{Parser, Subcommand}; -use anyhow::{Result, anyhow}; #[allow(unused_imports)] -use log::{info, trace, debug}; +use log::{debug, info, trace}; use scidataflow::lib::assets::GitHubRepo; use scidataflow::lib::download::Downloads; use tokio::runtime::Builder; - use scidataflow::lib::project::Project; use scidataflow::logging_setup::setup; @@ -60,7 +59,6 @@ Please submit bugs or feature requests to: https://github.com/vsbuffalo/scidataflow/issues. "; - #[derive(Parser)] #[clap(name = "sdf")] #[clap(about = INFO)] @@ -80,7 +78,7 @@ enum Commands { #[arg(required = true)] filenames: Vec, }, - /// Set local system-wide metadata (e.g. your name, email, etc.), which + /// Set local system-wide metadata (e.g. your name, email, etc.), which /// can be propagated to some APIs. Config { /// Your name. @@ -97,7 +95,7 @@ enum Commands { Init { /// Project name (default: the name of the directory). #[arg(long)] - name: Option + name: Option, }, /// Download a file from a URL. Get { @@ -107,7 +105,7 @@ enum Commands { name: Option, /// Overwrite local files if they exit. #[arg(long)] - overwrite: bool + overwrite: bool, }, /// Download a bunch of files from links stored in a file. Bulk { @@ -127,16 +125,14 @@ enum Commands { Status { /// Show remotes status (requires network). #[arg(long)] - remotes: bool, + remotes: bool, /// Show statuses of all files, including those on remote(s) but not in the manifest. #[arg(long)] - all: bool - + all: bool, }, /// Show file size statistics. - Stats { - }, + Stats {}, /// Update MD5s Update { /// Which file to update (if not set, all tracked files are update). @@ -144,7 +140,7 @@ enum Commands { filenames: Vec, /// Update all files presently registered in the manifest. #[arg(long)] - all: bool + all: bool, }, /// Remove a file from the manifest Rm { @@ -161,7 +157,7 @@ enum Commands { #[arg(long)] url: Option, /// A SciDataFlow Asset name - asset: Option + asset: Option, }, /// Link a directory to a remote storage solution. Link { @@ -171,7 +167,7 @@ enum Commands { service: String, /// The authentication token. key: String, - /// Project name for remote (default: the metadata title in the data + /// Project name for remote (default: the metadata title in the data /// manifest, or if that's not set, the directory name). #[arg(long)] name: Option, @@ -180,24 +176,20 @@ enum Commands { /// the remote information (i.e. the FigShare Article ID or Zenodo /// Depository ID) to add to the manifest. Requires network. #[arg(long)] - link_only: bool - + link_only: bool, }, /// No longer keep track of this file on the remote. Untrack { /// The file to untrack with remote. - filename: String + filename: String, }, /// Keep track of this file on the remote. Track { /// The file to track with remote. - filename: String + filename: String, }, /// Move or rename a file on the file system and in the manifest. - Mv { - source: String, - destination: String - }, + Mv { source: String, destination: String }, /// Push all tracked files to remote. Push { /// Overwrite remote files if they exit. @@ -219,7 +211,6 @@ enum Commands { /// Pull in files from remotes and URLs. #[arg(long)] all: bool, - // multiple optional directories //directories: Vec, }, @@ -236,7 +227,7 @@ enum Commands { pub fn print_errors(response: Result<()>) { match response { - Ok(_) => {}, + Ok(_) => {} Err(err) => eprintln!("Error: {}", err), } } @@ -252,7 +243,6 @@ fn main() { .build() .unwrap(); - runtime.block_on(async { match run().await { Ok(_) => {} @@ -271,25 +261,34 @@ async fn run() -> Result<()> { let mut proj = Project::new()?; proj.add(filenames).await } - Some(Commands::Config { name, email, affiliation }) => { - Project::set_config(name, email, affiliation) - } - Some(Commands::Get { url, name, overwrite }) => { + Some(Commands::Config { + name, + email, + affiliation, + }) => Project::set_config(name, email, affiliation), + Some(Commands::Get { + url, + name, + overwrite, + }) => { let mut proj = Project::new()?; proj.get(url, name.as_deref(), *overwrite).await } - Some(Commands::Bulk { filename, column, header, overwrite }) => { + Some(Commands::Bulk { + filename, + column, + header, + overwrite, + }) => { let mut proj = Project::new()?; proj.bulk(filename, *column, *header, *overwrite).await } - Some(Commands::Init { name }) => { - Project::init(name.clone()) - } + Some(Commands::Init { name }) => Project::init(name.clone()), Some(Commands::Status { remotes, all }) => { let mut proj = Project::new()?; proj.status(*remotes, *all).await } - Some(Commands::Stats { }) => { + Some(Commands::Stats {}) => { //let proj = Project::new()?; //proj.stats() Ok(()) @@ -303,41 +302,50 @@ async fn run() -> Result<()> { if !*all && filenames.is_empty() { return Err(anyhow!("Specify --all or one or more file to update.")); } - let filepaths = if *all { - None - } else { - Some(filenames) - }; + let filepaths = if *all { None } else { Some(filenames) }; proj.update(filepaths).await } - Some(Commands::Link { dir, service, key, name, link_only }) => { + Some(Commands::Link { + dir, + service, + key, + name, + link_only, + }) => { let mut proj = Project::new()?; proj.link(dir, service, key, name, link_only).await } Some(Commands::Track { filename }) => { let mut proj = Project::new()?; proj.track(filename) - }, + } Some(Commands::Untrack { filename }) => { let mut proj = Project::new()?; proj.untrack(filename) - }, - Some(Commands::Mv { source, destination }) => { + } + Some(Commands::Mv { + source, + destination, + }) => { let mut proj = Project::new()?; proj.mv(source, destination).await - }, + } Some(Commands::Push { overwrite }) => { let mut proj = Project::new()?; proj.push(*overwrite).await - }, - Some(Commands::Pull { overwrite, urls, all }) => { + } + Some(Commands::Pull { + overwrite, + urls, + all, + }) => { let mut proj = Project::new()?; proj.pull(*overwrite, *urls, *all).await - }, + } Some(Commands::Metadata { title, description }) => { let mut proj = Project::new()?; proj.set_metadata(title, description) - }, + } Some(Commands::Asset { github, url, asset }) => { if Path::new("data_manifest.yml").exists() { return Err(anyhow!("data_manifest.yml already exists in the current directory; delete it manually first to use sdf asset.")); @@ -345,31 +353,27 @@ async fn run() -> Result<()> { let msg = "Set either --github, --url, or specify an SciDataFlow Asset name."; let url = match (github, url, asset) { (Some(gh), None, None) => { - let gh = GitHubRepo::new(gh).map_err(|e| { - anyhow!("GitHubRepo initialization failed: {}", e) - })?; + let gh = GitHubRepo::new(gh) + .map_err(|e| anyhow!("GitHubRepo initialization failed: {}", e))?; gh.url("data_manifest.yml") - }, + } (None, None, Some(asset)) => { let url = format!("{}/{}", SDF_ASSET_URL, asset); - let gh = GitHubRepo::new(&url).expect("Internal Error: invalid Asset URL; please report."); + let gh = GitHubRepo::new(&url) + .expect("Internal Error: invalid Asset URL; please report."); gh.url("data_manifest.yml") - }, - (None, Some(url), None) => { - url.to_string() - }, - _ => return Err(anyhow!(msg)) + } + (None, Some(url), None) => url.to_string(), + _ => return Err(anyhow!(msg)), }; let mut downloads = Downloads::new(); downloads.add(url.clone(), None, false)?; downloads.retrieve(None, None, false).await?; Ok(()) - }, + } None => { println!("{}\n", INFO); std::process::exit(1); } } - - } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index bfedc37..7f4bf23 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,29 +1,29 @@ ///! Testing Utility Functions #[allow(unused_imports)] -use anyhow::{anyhow,Result}; +use anyhow::{anyhow, Result}; +use flate2::write::GzEncoder; +use flate2::Compression; +use lazy_static::lazy_static; use log::info; -use std::env; +use rand::rngs::StdRng; use rand::Rng; -use std::path::{Path,PathBuf}; -use std::fs::File; -use std::io::Write; -use std::collections::{BTreeMap,HashMap}; -use tempfile::TempDir; +use rand::SeedableRng; use serde_derive::{Deserialize, Serialize}; -use std::io::BufWriter; -use flate2::write::GzEncoder; -use flate2::Compression; +use std::collections::{BTreeMap, HashMap}; +use std::env; use std::fs::create_dir_all; +use std::fs::File; +use std::io::BufWriter; +use std::io::Write; +use std::path::{Path, PathBuf}; use std::sync::Once; -use rand::rngs::StdRng; -use rand::SeedableRng; -use lazy_static::lazy_static; +use tempfile::TempDir; -use scidataflow::lib::project::Project; use scidataflow::lib::data::StatusEntry; +use scidataflow::lib::project::Project; -pub fn make_mock_fixtures() -> Vec { +pub fn make_mock_fixtures() -> Vec { let files = vec![ DataFileFixture { path: "data/data.tsv".to_string(), @@ -49,17 +49,21 @@ pub fn make_mock_fixtures() -> Vec { add: true, track: true, }, - ]; + ]; files } - fn create_seeded_rng() -> StdRng { let seed = 0; StdRng::seed_from_u64(seed) } -pub fn generate_random_tsv(file_path: &Path, size: usize, gzip: bool, rng: &mut StdRng) -> Result<()> { +pub fn generate_random_tsv( + file_path: &Path, + size: usize, + gzip: bool, + rng: &mut StdRng, +) -> Result<()> { let file = File::create(file_path)?; let writer: Box = if gzip { Box::new(GzEncoder::new(file, Compression::default())) @@ -80,9 +84,12 @@ pub fn generate_random_tsv(file_path: &Path, size: usize, gzip: bool, rng: &mut Ok(()) } - -fn generate_directory_structure(data_fixtures: &Vec, base_path: &Path, - cache_dir: &Path, rng: &mut StdRng) -> Result<()> { +fn generate_directory_structure( + data_fixtures: &Vec, + base_path: &Path, + cache_dir: &Path, + rng: &mut StdRng, +) -> Result<()> { for data_file_fixture in data_fixtures { let file_path = base_path.join(&data_file_fixture.path); let directory_path = file_path.parent().unwrap(); @@ -110,7 +117,7 @@ pub struct TestEnvironment { pub main_dir: PathBuf, pub cache_dir: PathBuf, pub files: Option>, - pub rng: StdRng + pub rng: StdRng, } pub struct TestFixture { @@ -126,7 +133,6 @@ pub struct DataFileFixture { pub track: bool, } - impl TestEnvironment { // Create a new TestEnvironment pub fn new(name: &str) -> Result { @@ -147,12 +153,17 @@ impl TestEnvironment { main_dir: pwd, cache_dir, files: None, - rng + rng, }) } pub fn build_project_directories(&mut self, data_fixtures: Vec) -> Result<()> { - generate_directory_structure(&data_fixtures, &self.temp_dir.path(), &self.cache_dir, &mut self.rng)?; + generate_directory_structure( + &data_fixtures, + &self.temp_dir.path(), + &self.cache_dir, + &mut self.rng, + )?; self.files = Some(data_fixtures); Ok(()) } @@ -160,12 +171,11 @@ impl TestEnvironment { pub fn get_file_path(&self, file_path: &str) -> PathBuf { self.temp_dir.path().join(PathBuf::from(file_path)) } - } -#[allow(dead_code)] // will implement later +#[allow(dead_code)] // will implement later pub fn read_keep_temp() -> bool { - return env::var("KEEP_TEMP_DIR").is_ok() + return env::var("KEEP_TEMP_DIR").is_ok(); } impl Drop for TestEnvironment { @@ -185,25 +195,32 @@ pub async fn setup(do_add: bool) -> TestFixture { }); let project_name = "test_project".to_string(); - let mut test_env = TestEnvironment::new(&project_name).expect("Error creating test environment."); + let mut test_env = + TestEnvironment::new(&project_name).expect("Error creating test environment."); let data_fixtures = make_mock_fixtures(); let _ = test_env.build_project_directories(data_fixtures); // initializes sciflow in the test environment let current_dir = env::current_dir().unwrap(); - info!("temp_dir: {:?}, current directory: {:?}", test_env.temp_dir, current_dir); - let _ = Project::set_config(&Some("Joan B. Scientist".to_string()), - &Some("joan@ucberkely.edu".to_string()), - &Some("UC Berkeley".to_string())); + info!( + "temp_dir: {:?}, current directory: {:?}", + test_env.temp_dir, current_dir + ); + let _ = Project::set_config( + &Some("Joan B. Scientist".to_string()), + &Some("joan@ucberkely.edu".to_string()), + &Some("UC Berkeley".to_string()), + ); let _ = Project::init(Some(project_name)); let mut project = Project::new().expect("setting up TestFixture failed"); if do_add { - // add the files that should be added (e.g. a setup further + // add the files that should be added (e.g. a setup further // in setting up the mock project) // get the files to add let files = &test_env.files.as_ref().unwrap(); - let add_files: Vec = files.into_iter() + let add_files: Vec = files + .into_iter() .filter(|f| f.add) .map(|f| f.path.clone()) .collect(); @@ -212,13 +229,15 @@ pub async fn setup(do_add: bool) -> TestFixture { let _ = project.add(&add_files).await; } - TestFixture { env: test_env, project } + TestFixture { + env: test_env, + project, + } } - pub fn iter_status_entries<'a>( statuses: &'a BTreeMap>, - ) -> impl Iterator + 'a { +) -> impl Iterator + 'a { statuses.iter().flat_map(|(dir, entries)| { entries.iter().map(move |status| { let mut path = PathBuf::from(dir); @@ -228,17 +247,32 @@ pub fn iter_status_entries<'a>( }) } -pub async fn get_statuses(fixture: &mut TestFixture, path_context: &Path) -> Vec<(PathBuf,StatusEntry)> { - let statuses = fixture.project.data - .status(&path_context, false).await +pub async fn get_statuses( + fixture: &mut TestFixture, + path_context: &Path, +) -> Vec<(PathBuf, StatusEntry)> { + let statuses = fixture + .project + .data + .status(&path_context, false) + .await .expect("Error in getting statuses."); - iter_status_entries(&statuses).map(|(path, status)| (path, status.clone())).collect() + iter_status_entries(&statuses) + .map(|(path, status)| (path, status.clone())) + .collect() } - -pub async fn get_statuses_map(fixture: &mut TestFixture, path_context: &Path) -> HashMap { - let statuses = fixture.project.data - .status(&path_context, false).await +pub async fn get_statuses_map( + fixture: &mut TestFixture, + path_context: &Path, +) -> HashMap { + let statuses = fixture + .project + .data + .status(&path_context, false) + .await .expect("Error in getting statuses."); - iter_status_entries(&statuses).map(|(path, status)| (path, status.clone())).collect() + iter_status_entries(&statuses) + .map(|(path, status)| (path, status.clone())) + .collect() } diff --git a/tests/test_project.rs b/tests/test_project.rs index da66535..a702700 100644 --- a/tests/test_project.rs +++ b/tests/test_project.rs @@ -1,20 +1,19 @@ #[allow(unused_imports)] -use log::{info, trace, debug}; +use log::{debug, info, trace}; mod common; -use common::{setup,get_statuses,generate_random_tsv}; - +use common::{generate_random_tsv, get_statuses, setup}; #[cfg(test)] mod tests { - use log::info; use crate::common::get_statuses_map; + use log::info; - use super::setup; - use super::get_statuses; use super::generate_random_tsv; - use std::path::PathBuf; + use super::get_statuses; + use super::setup; use scidataflow::lib::data::LocalStatusCode; + use std::path::PathBuf; #[tokio::test] async fn test_fixture() { @@ -34,7 +33,10 @@ mod tests { // test that init() creates the data manifest let data_manifest = fixture.env.get_file_path("data_manifest.yml"); info!("Checking for file at path: {}", data_manifest.display()); // Add this log - assert!(data_manifest.exists(), "Project::init() did not create 'data_manifest.yml'"); + assert!( + data_manifest.exists(), + "Project::init() did not create 'data_manifest.yml'" + ); } #[tokio::test] @@ -48,7 +50,8 @@ mod tests { // get the files to add let files = &fixture.env.files.as_ref().unwrap(); - let add_files: Vec = files.into_iter() + let add_files: Vec = files + .into_iter() .filter(|f| f.add) .map(|f| f.path.clone()) .collect(); @@ -64,13 +67,26 @@ mod tests { for (full_path, status) in statuses { if add_files.contains(&full_path.to_string_lossy().to_string()) { // check that the status is current - assert!(status.local_status.is_some(), "File '{:?}' does not have a local status", full_path); + assert!( + status.local_status.is_some(), + "File '{:?}' does not have a local status", + full_path + ); if let Some(local_status) = &status.local_status { - assert_eq!(*local_status, LocalStatusCode::Current, "Added file '{:?}' does not have 'Current' status", full_path); + assert_eq!( + *local_status, + LocalStatusCode::Current, + "Added file '{:?}' does not have 'Current' status", + full_path + ); } } else { // check that the status is None (not registered) - assert!(status.local_status.is_none(), "File '{:?}' should not have a local status", full_path); + assert!( + status.local_status.is_none(), + "File '{:?}' should not have a local status", + full_path + ); } } } @@ -92,7 +108,11 @@ mod tests { // Now, let's check the status is modified. let updated_statuses = get_statuses_map(&mut fixture, &path_context).await; - let updated_status_option = updated_statuses.get(&file_to_check).unwrap().local_status.clone(); + let updated_status_option = updated_statuses + .get(&file_to_check) + .unwrap() + .local_status + .clone(); let updated_status = updated_status_option.unwrap().clone(); assert_eq!(updated_status, LocalStatusCode::Modified); @@ -104,10 +124,14 @@ mod tests { let result = fixture.project.update(Some(&files)).await; assert!(result.is_ok(), "re-adding raised Error!"); } - + // and make sure the status goes back to current. let readd_statuses = get_statuses_map(&mut fixture, &path_context).await; - let readd_status_option = readd_statuses.get(&file_to_check).unwrap().local_status.clone(); + let readd_status_option = readd_statuses + .get(&file_to_check) + .unwrap() + .local_status + .clone(); let readd_status = readd_status_option.unwrap().clone(); assert_eq!(readd_status, LocalStatusCode::Current); } @@ -122,19 +146,18 @@ mod tests { file_list.push(file.path.clone()); let result = fixture.project.add(&file_list).await; - // check that we get + // check that we get match result { Ok(_) => assert!(false, "Expected an error, but got Ok"), Err(err) => { - assert!(err.to_string().contains("already registered"), - "Unexpected error: {:?}", err); + assert!( + err.to_string().contains("already registered"), + "Unexpected error: {:?}", + err + ); } }; - } } } - - } -