add progress, remove wallet configs, better list (#12)

.gitignore

@@ -5,6 +5,3 @@ bin
 
 .yarn
 
-# wallets and such
-wallet.json
-

README.md

@@ -22,15 +22,15 @@
 
 ## Installation
 
-Dria CLI requires NodeJS & Docker to be installed on your machine, and is available on NPM. It can be installed to your system with:
+Dria CLI requires NodeJS (>= 18.0.0) & Docker to be installed on your machine, and is available on NPM. It can be installed to your system with:
 
 ```sh
 npm i -g dria-cli
 ```
 
 ## Usage
 
-You can see available commands with `dria help`:
+You can see available commands with `dria help`, which outputs:
 
 ```sh
 dria <command>
@@ -41,7 +41,6 @@ Commands:
   dria clear [contract]         Clear local knowledge.
   dria fetch <txid>             Fetch an existing index at the given URL directly.
   dria set-contract <contract>  Set default contract.
-  dria set-wallet <wallet>      Set default wallet.
   dria config                   Show default configurations.
   dria list                     List all local knowledge.
   dria stop                     Stop serving knowledge.
@@ -52,16 +51,19 @@ Options:
   -v, --verbose  Show extra information             [boolean] [default: false]
 ```
 
+> [!WARNING]
+>
+> If you are using Docker with `sudo` only, you must use the Dria CLI with `sudo` as well, since it needs access
+> to Docker in the background. Otherwise, the CLI will not be able to detect your Docker engine.
+> See more [here](https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user).
+
 ### Pull Knowledge
 
 A Dria knowledge is stored on blockchain, and we can pull that knowledge to our local machine with the `pull` command:
 
 ```sh
 dria pull <contract>
 dria pull # use configured contract
-
-# provide a wallet
-dria pull -w <wallet-path>
 ```
 
 ### Serve Knowledge
@@ -93,22 +95,19 @@ Note that the argument here is not the knowledge ID (i.e. the corresponding Arwe
 
 ### Configurations
 
-You can set the default wallet & contract with `set-contract` and `set-wallet` commands respectively. When a contract is set by default, `[contract]` can be omitted such that the CLI will use the default one. To see the defaults:
+You can set the default contract with `set-contract` command. When a contract is set by default, `[contract]` can be omitted such that the CLI will use the default one. To see the defaults:
 
 ```sh
-# view configured wallet & contract
+# view configurations
 dria config
 
 # change contract
 dria set-contract <contract>
-
-# change wallet
-dria set-wallet ./path/to/wallet.json
 ```
 
-### List Pulled Knowledge
+### List Knowledge
 
-You can print out the list of contracts [pulled](#pull-knowledge) so far, along with their last modification date, with the command:
+You can print out the list of contracts [pulled](#pull-knowledge) or [fetched](#fetch-knowledge) so far, along with their last modification date, with the command:
 
 ```sh
 dria list

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dria-cli",
-  "version": "0.0.3",
+  "version": "0.0.4",
   "description": "A command-line tool for Dria",
   "author": "FirstBatch Team <[email protected]>",
   "contributors": [
@@ -23,19 +23,22 @@
     "dria": "yarn build && yarn start",
     "test": "npx jest"
   },
+  "engines": {
+    "node": ">=18.0.0"
+  },
   "dependencies": {
-    "adm-zip": "^0.5.10",
     "axios": "^1.6.7",
     "dockerode": "^4.0.2",
     "loglevel": "^1.9.1",
+    "unzipper": "^0.10.14",
     "yargs": "^17.7.2"
   },
   "packageManager": "[email protected]",
   "devDependencies": {
-    "@types/adm-zip": "^0",
     "@types/dockerode": "^3.3.23",
     "@types/jest": "^29.5.12",
     "@types/node": "^20.11.16",
+    "@types/unzipper": "^0",
     "@types/yargs": "^17.0.32",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^6.21.0",

src/commands/list.ts

@@ -16,8 +16,8 @@ export default function cmdList() {
   } else {
     for (const file of files) {
       const lastModified = statSync(`${constants.DRIA.DATA}/${file}`).mtime.toLocaleString();
-      logger.info(`${file.split(".")[0]}\t(last modified: ${lastModified})`);
+      const name = file.split(".")[0];
+      logger.info(`${name.padEnd(45)}\t(last modified: ${lastModified})`);
     }
   }
-  // logger.info(files.length ? files.map((v) => "  " + v).join("\n") : "  no contracts pulled yet!");
 }

src/commands/pull.ts

@@ -1,5 +1,4 @@
-import { logger, sleep } from "../common";
-import constants from "../constants";
+import { logger } from "../common";
 import { hollowdbContainer, redisContainer } from "../containers";
 
 /**
@@ -12,16 +11,15 @@ import { hollowdbContainer, redisContainer } from "../containers";
  * The saved `.rdb` file has the contract name, which can be used later by
  * Dria HNSW.
  *
- * @param walletPath wallet required for HollowDB
  * @param contractId contract ID to download
  */
-export default async function cmdPull(walletPath: string, contractId: string) {
+export default async function cmdPull(contractId: string) {
   logger.debug("Running Redis.");
   const redis = await redisContainer(contractId);
   await redis.start();
 
   logger.debug("Running HollowDB.");
-  const hollowdb = await hollowdbContainer(walletPath, contractId);
+  const hollowdb = await hollowdbContainer(contractId);
   await hollowdb.start();
 
   logger.info("Pulling the latest contract data.");
@@ -44,14 +42,14 @@ export default async function cmdPull(walletPath: string, contractId: string) {
           } else {
             const idx = str.indexOf(targetStr);
             if (idx != -1) {
-              logger.info(str.slice(idx + targetStr.length - 1));
+              logger.info(str.slice(idx + targetStr.length - 1).replace("\n", ""));
             }
           }
         });
       }
     });
   });
-  logger.info("\nDone! Cleaning up...");
+  logger.info("Done! Cleaning up...");
 
   await hollowdb.stop();
   await redis.stop();

src/common/download.ts

@@ -1,27 +1,60 @@
-import axios from "axios";
-import AdmZip from "adm-zip";
+import Axios from "axios";
+import unzipper from "unzipper";
 import constants from "../constants";
 import { logger } from ".";
+import { createReadStream, createWriteStream, rmSync } from "fs";
 
 /** Download a zipped data from Arweave, unzip & extract it at a given path.
+ *
+ * Uses streaming to write request to tmp disk, and then to target folder due to large size.
  *
  * @param txid txID on Arweave
  * @param outDir output directory for the extraction file
  */
 export async function downloadAndUnzip(contractId: string, outDir: string) {
   const url = `${constants.ARWEAVE.DOWNLOAD_URL}/${contractId}`;
 
-  logger.debug("Downloading...");
-  const response = await axios.get<Buffer>(url, {
+  logger.info("Downloading from", url);
+
+  // download the file using a stream (due to large size)
+  const tmpPath = `/tmp/${contractId}.zip`;
+  const writer = createWriteStream(tmpPath);
+  await Axios({
+    url,
+    method: "get",
     timeout: constants.ARWEAVE.DOWNLOAD_TIMEOUT,
-    responseType: "arraybuffer",
+    responseType: "stream",
+    // show download progress here
+    onDownloadProgress(progressEvent) {
+      if (progressEvent.total) {
+        const percentCompleted = ((progressEvent.loaded / progressEvent.total) * 100).toFixed(2);
+        logger.info(`Progress: ${percentCompleted}%`);
+      }
+    },
+  }).then((response) => {
+    return new Promise((resolve, reject) => {
+      response.data.pipe(writer);
+
+      let error: Error | null = null;
+
+      writer.on("error", (err) => {
+        error = err;
+        writer.close();
+        reject(err);
+      });
+
+      writer.on("close", () => {
+        if (!error) resolve(true);
+        // if error, we've rejected above
+      });
+    });
   });
-  if (response.status !== 200) {
-    throw new Error(`Arweave Download failed with ${response.status}`);
-  }
-  logger.debug(`${response.data.length} bytes downloaded.`);
 
-  const zip = new AdmZip(response.data);
-  zip.extractAllTo(outDir, true); // overwrite existing data at path
+  // unzip to correct folder
+  // TODO: connecting axios stream to unzipper could work, but couldnt solve it yet
+  createReadStream(tmpPath).pipe(unzipper.Extract({ path: outDir }));
   logger.info("Knowledge extracted at", outDir);
+
+  // cleanup temporary zip
+  rmSync(tmpPath);
 }

src/configurations/index.ts

@@ -3,15 +3,12 @@ import constants from "../constants";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
 
 export type DriaCLIConfig = {
-  /** Defualt Contract ID to fall-back if none is provided. */
+  /** Default Contract ID to fall-back if none is provided. */
   contract?: string;
-  /** Defualt absolute path to wallet. */
-  wallet?: string;
 };
 
 const defaultConfig: DriaCLIConfig = {
   contract: undefined,
-  wallet: undefined,
 };
 
 const CONFIG_PATH = constants.DRIA.CONFIG;
@@ -36,7 +33,6 @@ export function setConfig(args: DriaCLIConfig) {
   const cfg = getConfig();
 
   if (args.contract) cfg.contract = args.contract;
-  if (args.wallet) cfg.wallet = args.wallet;
 
   writeFileSync(CONFIG_PATH, JSON.stringify(cfg));
 }

src/containers/hnsw.ts

@@ -4,6 +4,7 @@ import constants from "../constants";
 export async function hnswContainer(contractId: string) {
   const portBinding = `${constants.PORTS.HNSW}/tcp`;
   const redisPortBinding = `${constants.PORTS.REDIS}/tcp`;
+  const guestDataDir = "/data";
 
   await pullImageIfNotExists(constants.IMAGES.HNSW);
   await removeContainerIfExists(constants.CONTAINERS.HNSW);
@@ -14,11 +15,11 @@ export async function hnswContainer(contractId: string) {
     Env: [
       `REDIS_URL=redis://default:redispw@${constants.NETWORK.IPS.REDIS}:6379`,
       `CONTRACT_ID=${contractId}`,
-      `ROCKSDB_PATH=/data/${contractId}`,
+      `ROCKSDB_PATH=${guestDataDir}/${contractId}`,
     ],
     ExposedPorts: { [portBinding]: {} },
     HostConfig: {
-      Binds: [`${constants.DRIA.DATA}:/data`],
+      Binds: [`${constants.DRIA.DATA}:${guestDataDir}`],
       PortBindings: {
         [portBinding]: [{ HostPort: constants.PORTS.HNSW.toString() }],
         [redisPortBinding]: [{ HostPort: constants.PORTS.REDIS.toString() }],

src/containers/hollowdb.ts

@@ -1,9 +1,10 @@
 import { docker, pullImageIfNotExists, removeContainerIfExists } from "../common";
 import constants from "../constants";
 
-export async function hollowdbContainer(walletPath: string, contractId: string) {
+export async function hollowdbContainer(contractId: string) {
   const portBinding = `${constants.PORTS.HOLLOWDB}/tcp`;
   const redisPortBinding = `${constants.PORTS.REDIS}/tcp`;
+  const guestDataDir = "/app/data";
 
   await pullImageIfNotExists(constants.IMAGES.HOLLOWDB);
   await removeContainerIfExists(constants.CONTAINERS.HOLLOWDB);
@@ -13,8 +14,8 @@ export async function hollowdbContainer(walletPath: string, contractId: string)
     name: constants.CONTAINERS.HOLLOWDB,
     Env: [
       `REDIS_URL=redis://default:redispw@${constants.NETWORK.IPS.REDIS}:6379`,
-      `ROCKSDB_PATH=/app/data/${contractId}`,
-      `CONTRACT_TXID=${contractId}`,
+      `ROCKSDB_PATH=${guestDataDir}/${contractId}`,
+      `CONTRACT_ID=${contractId}`,
       "USE_BUNDLR=true", // true if your contract uses Bundlr
       "USE_HTX=true", // true if your contract stores values as `hash.txid`
       "BUNDLR_FBS=80", // batch size for downloading bundled values from Arweave
@@ -23,9 +24,8 @@ export async function hollowdbContainer(walletPath: string, contractId: string)
     ExposedPorts: { [portBinding]: {} },
     HostConfig: {
       // prettier-ignore
-      Binds: [
-        `${walletPath}:/app/config/wallet.json:ro`,
-        `${constants.DRIA.DATA}:/app/data`
+      Binds: [ 
+        `${constants.DRIA.DATA}:${guestDataDir}`
       ],
       PortBindings: {
         [portBinding]: [{ HostPort: constants.PORTS.HOLLOWDB.toString() }],