oJobBasics.yaml

# Define the jobs
# Copyright 2023 Nuno Aguiar

jobs:
  ## ----------------------
  ## oJob Start/Stop basics
  ## ----------------------

  # Start processing
  - name        : oJob Start
    exec        : >
      log("init");
    
  # Stop processing on shutdown
  - name        : oJob Shutdown
    deps        :
       - oJob Start
    type        : shutdown
    exec        : >
      log("done");

  # Stop processing
  - name        : oJob Stop
    deps        :
       - oJob Start
    exec        : >
      log("done");

  # Exit the current script (include openaf-console)
  - name        : oJob Exit
    exec        : exit(0);

  ## -----
  ## Utils
  ## -----
  
  # Sleep for 5 seconds
  - name        : oJob Sleep 5s
    exec        : sleep(5000); 
 
  # Sleep for 1 second
  - name        : oJob Sleep 1s
    exec        : sleep(1000);
    
  # Print an object as a json
  - name        : oJob Stringify
    help        : >
      This job prints to the console the provided javascript expression. Expects:
        name  (String)   A javascript object or expression.
        min   (Boolean)  Minify the json output
    exec        : |
      _$(args.name).isString().$_("Please provide an expression or object.");
      args.min = _$(args.min).isBoolean().default(false);

      if (args.min)
        print(stringify(eval(args.name), void 0, ""));
      else
        sprint(eval(args.name));

  # Assign an object to a variable
  - name : oJob Set
    help : >
      Assigns a value to a variable. Expects:
        
        name  (String)  The global variable name.
        value (Object)  The value to assign to the variable.
    exec : |
      args.value = _$(args.value).default(void 0);
      _$(args.name).isString().$_("Please provide the variable name.");

      eval(args.name + " = args.value");

  # Assign a map with path
  - name : oJob Path Set
    help : >
      Assigns a value to a variable on a specific path. Expects:
        
        name  (String)  The global variable name.
        path  (String)  The object path where the variable will be assigned.
        value (Object)  The value to assign to the variable on the specific object path.
    exec : |
      args.value = _$(args.value).default(void 0);
      args.path = _$(args.path).isString().default(void 0);
      _$(args.name).isString().$_("Please provide the variable name.");

      ow.loadObj();
      if (isUnDef(global[args.name])) global[args.name] = {};
      ow.obj.setPath(global[args.name], args.path, args.value);

  # Print an object as a yaml
  - name        : oJob YAML
    help        : >
      This job prints to the console the provided javascript expression as YAML. Expects:
        name  (String)   A javascript object or expression.
    exec        : |
      _$(args.name).isString().$_("Please provide an expression or object.");

      print(af.toYAML(eval(args.name)).replace(/\n$/mg, ""));

  # Sort an array
  - name        : oJob Sort Array
    help        : >
      This job sorts a variable 'srcName' to 'name'. Expects:

        name     (String)  The global variable name.
        srcName  (String)  The source variable array or expression (defaults to name).
        reverse  (Boolean) Reverse result (defautls to false).
    exec        : |
      _$(args.name).isString().$_("Please provide the variable name.");
      args.srcName = _$(args.srcName).isString().default(args.name);
      args.reverse = _$(args.reverse).isBoolean().default(false);

      var arr = eval(args.srcName);
      _$(arr).isArray("srcName must result into an array.").$_("srcName must result into an array");

      eval(args.name + " = (args.reverse ? arr.sort().reverse() : arr.sort())");

  # Map an array
  - name        : oJob Map Array
    help        : >
      This job sets a variable 'name' from a srcName array of maps remapped with the provided selectors. Expects:

        name      (String)  The global variable name.
        srcName   (String)  The source variable array or expression.
        selectors (Array)   Array of object path selectors.
        limit     (Number)  Optional limit number of array entries to process.
    exec        : |
      _$(args.name).isString().$_("Please provide the variable name.");
      _$(args.srcName).isString().$_("Please provide a source array variable name or expression.");
      args.selectors = _$(args.selectors).isArray().default([]);
      args.limit = _$(args.limit).isNumber().default(void 0);

      eval(args.name + " = mapArray(eval(args.srcName), args.selectors, args.limit)");

  # Apply a JMESPath
  - name        : oJob Map
    help        : >
      This job sets a variable 'name' mapped using $path/JMESPath, defined by path, from the variable/expression 'srcName'. Expects:

        name      (String)  The global variable name.
        srcName   (String)  The source variable array or expression.
        path      (String)  The $path/JMESPath expression (see help of $path).
    exec        : |
      _$(args.name).isString().$_("Please provide the variable name.");
      _$(args.srcName).isString().$_("Please provide a source array variable name or expression.");
      args.path = _$(args.path).isString().default("");

      eval(args.name + " = $path(eval(args.srcName), args.path)");

  # Set arguments from a global variable
  - name        : oJob From global
    help        : |
      This job will reset or merge a global variable map. Expects:
        - args.global (String)  Load arguments from the global variable specified.
    exec        : |
      args.global = _$(args.global).isString().$_("Please provide a global variable name"); 
      args = merge(args, global[args.global]);
  
  # Load args from JSON
  - name        : oJob Args from JSON
    help        : |
      This job will load the args map from a JSON file. Expects:
        - args.file   (String)  The filepath to read the JSON file from.
        - args.global (String)  Alternatively load to the global variable specified.
    exec        : |
      _$(args.file)
       .isString()
       .check(io.fileExists, "The file " + args.file + " doesn't exist.")
       .$_("You need to provide a JSON file to load arguments from.");
      args.global = _$(args.global).isString().default(void 0); 

      if (isDef(args.global))
        global[args.global] = merge(global[args.global], io.readFile(args.file));
      else
        args = merge(args, io.readFile(args.file));

  # Load args from YAML
  - name        : oJob Args from YAML
    help        : |
      This job will load the args map from a YAML file. Expects:
        - args.file   (String)  The filepath to read the YAML file from.
        - args.global (String)  Alternatively load to the global variable specified.
    exec        : |
      _$(args.file)
       .isString()
       .check(io.fileExists, "The file " + args.file + " doesn't exist.")
       .$_("You need to provide a YAML file to load arguments from.");
      args.global = _$(args.global).isString().default(void 0); 

      if (isDef(args.global))
        global[args.global] = merge(global[args.global], io.readFileYAML(args.file));
      else
        args = merge(args, io.readFileYAML(args.file));

  # ---------------------------
  - name: oJob Split into items
    help: |
      Splits an args source into an array of maps (_list). Expects:

        source    (String)  A object path to the string source to split
        separator (String)  The separator for the source string (defaults to \n)

      Example:

        a source string with the value "abc, xyz, 1" 
        + separator = ',' 
        transforms into:

        - item: abc
        - item: xyz
        - item: 1
    exec: |
      _$(args.source, "source").isString().$_();
      args.separator = _$(args.separator, "separator").isString().default("\n");

      ow.loadObj();
      args._list = String(ow.obj.getPath(args, args.source)).split(args.separator).map(r => {
        return { item: r };
      });
  
  # ---------------------
  - name: oJob Print list
    help: |
      Prints the current list (args._list).
    exec: |
      _$(args._list, "_list").$_("No list found.");
      print(af.toYAML(args._list));

  # ---------------------
  - name: oJob Print args
    help: |
      Prints the current args. Optionally expects:

        _path (String)  The object path over args.
    exec: |
      var r = args;
      if (isDef(args._path) && isString(args._path)) {
        ow.loadObj();
        r = ow.obj.getPath(r, args._path);
      }
      print(isMap(r) ? af.toYAML(r) : r);

  # --------------------------
  - name: oJob Get kept result
    help: |
      Retrieves a previously kept result, with OAF Keep result. Optionally expects:

        ojobkeep  (Map)
          name      (String)  The name of the result that was kept.

    exec: |
      args.ojobkeep    = _$(args.ojobkeep, "ojobkeep").isMap().default({});
      global._ojobkeep = _$(global._ojobkeep).isMap().default({});

      args.ojobkeep.name = _$(args.ojobkeep.name).default("default");
      var res = global._ojobkeep[args.ojobkeep.name];

      if (isUnDef(args.ojobkeep.keep)) {
        if (isArray(res)) args._list = res;
        if (isMap(res))   args = merge(args, res);
      } else {
        ow.loadObj();
        ow.obj.getPath(args, args.ojobkeep.keep);
      }

  # ----------------------
  - name: oJob Keep result
    help: |
      Keeps the current args result to later be retrieved with OAF Get kept result or the function oafGetResult(aName). Optionally expects:

        ojobkeep  (Map)
          name     (String)  The name of the result to keep.
          keep     (String)  A object path on args to keep. If not defined it will keep _list or the entire args.

    exec: |
      args.ojobkeep    = _$(args.ojobkeep, "ojobkeep").isMap().default({});
      global._ojobkeep = _$(global._ojobkeep).isMap().default({});

      args.ojobkeep.name = _$(args.ojobkeep.name).default("default");

      if (isUnDef(global.oJobGetKeptResult)) global.oJobGetKeptResult = (name) => {
          name = _$(name).default("default");
          if (isUnDef(global._ojobkeep)) global._ojobkeep = {};
          return global._ojobkeep[name];
      }

      if (isUnDef(args.ojobkeep.keep)) {
        if (isDef(args._list)) 
          global._ojobkeep[args.ojobkeep.name] = args._list;
        else
          global._ojobkeep[args.ojobkeep.name] = args;
      } else {
        ow.loadObj();
        global._ojobkeep[args.ojobkeep.name] = ow.obj.getPath(args, args.ojobkeep.keep);
      }

  # -----------------
  - name: oJob Switch
    help: |
      Adds new "todo" entries depending on the value of a provided args variable. Expects:

        switchOn      (String)  The variable in args that will define which set of "todo"s will be added (trimmed)
        lowerCase     (Boolean) Compare the switchOn in lower case (defaults to false)
        todos.[value] (Map)     Map of "todo"s
        default       (Map)     Default map of "todo"s

        Example:

          switchOn : mode
          lowerCase: true
          todos    :
            mode1:
            - Job 1
            - Job 2
            mode2:
            - Job 2
            - Job 3
          default  :
            - Job 2

    exec: |
      _$(args.switchOn, "switchOn").isString().$_();
      _$(args.todos, "todos").isMap().$_();
      args.lowerCase = _$(args.lowerCase, "lowerCase").isBoolean().default(false);
      args.default = _$(args.default, "default").isArray().default([]);
      
      var todoValue = String(args[args.switchOn]).trim();
      if (args.lowerCase) todoValue = todoValue.toLowerCase();

      if (isDef(args.todos[todoValue]) && isArray(args.todos[todoValue])) {
        args.todos[todoValue].map(j => ow.oJob.add2Todo(j));
      } else {
        args.default.map(j => ow.oJob.add2Todo(j));
      }

  ## ------------
  ## oJob Sec
  ## ------------

  # Get a SBucket secret
  - name: oJob Sec Get
    help: |
      This job will get a SBucket secret and map it to oJob's args:

        - args.secOut      (string)  The args path to be mapped with the secret
        - args.secKey      (string)  The SBucket key
        - args.secRepo     (string)  The SBucket repository
        - args.secBucket   (string)  The SBucket name
        - args.secPass     (string)  The SBucket password
        - args.secMainPass (string)  The SBucket repository password
        - args.secFile     (string)  Optional provide a specific sbucket file
        - args.secDontAsk  (string)  Determine if passwords should be asked from the user (default=false)
        - args.secIgnore   (boolean) If true will ignore errors of sec parameters not being provided (default=false)
    
    exec: |
      ow.loadSec()
      args.secIgnore = _$(args.secIgnore, "secIgnore").isBoolean().default(false);
      args.secOut = _$(args.secOut, "secOut").default(__);
      if (!args.secIgnore) _$(args.secKey, "secKey").$_();
      args.secDontAsk = toBoolean( _$(args.secDontAsk, "secDontAsk").default(false) );

      if (!args.secDontAsk && isDef(args.secKey)) {
         args.secBucket = askDef(args.secBucket, "Sec bucket: ");
         if (isDef(args.secBucket) && isUnDef(args.secFile)) args.secPass = askDef(args.secPass, "Sec bucket '" + args.secBucket + "' pass: ", true);
         if (isDef(args.secRepo))   args.secMainPass = askDef(args.secMainPass, "Sec repo '" + args.secRepo + "' pass: ", true);
      }
      if (isDef(args.secKey)) {
        var res = $sec(args.secRepo, args.secBucket, args.secPass, args.secMainPass, args.secFile).get(args.secKey);
        if (isUnDef(res)) throw "Couldn't retrieve secKey '" + args.secKey + "' from bucket '" + args.secBucket + "'!";
        if (isDef(args.secOut)) {
          $$(args).set(args.secOut, res);
        } else {
          Object.keys(res).forEach(r => $$(args).set(r, res[r]));
        }
      }

  ## ------------
  ## oJob shell
  ## ------------
  
  # Run a shell command
  - name        : oJob sh
    help        : |
      This job runs a local shell command and accepts the following args:
        - args.cmd       (string)  The command to execute (or an array of commands)
        - args.quiet     (boolean) Determines if the stdout should be visible or not (default is false)
        - args.directory (string)  Sets the working directory for the command
        - args.stdin     (string)  Provide any stdin needed
        - args.exitcode  (number)  Determines what exitcode should be consider success (default is 0)
        - args.stderr    (string)  Get's set to the stderr if quiet is true
        - args.stdout    (string)  Get's set to the stdout if quiet is true
        - args.prefix    (string)  A prefix for each output line if quiet is false
        - args.prefixLog (boolean) Determines that the prefix will use log instead of print (default is false)
        
    exec        : |
      var cmdf, iio = !args.quiet;
      if (isUnDef(args.stdin)) args.stdin = true;
      if (isUnDef(args.exitcode)) args.exitcode = 0;
      if (isUnDef(args.prefixLog)) args.prefixLog = false;
      if (isDef(args.prefix) && !args.quiet) {
        iio = false;
        ow.loadFormat();
        if (args.prefixLog) 
          cmdf = ow.format.streamSHLog((f) => { return args.prefix + " | " + f; });
        else
          cmdf = ow.format.streamSH((f) => { return args.prefix + " | " + f; });
      }
      if (isArray(args.cmd)) {
        args.stdout = "";
        args.stderr = "";
        for(var i in args.cmd) {
          var res = sh(templify(args.cmd[i], args), args.stdin, args.timeout, iio, args.directory, true, cmdf);
          args.stdout += res.stdout
          args.stderr += res.stderr;
          if (res.exitcode != args.exitcode) throw "Exit code (for command '" + args.cmd[i] + "'): " + res.exitcode;
        }
      } else {
        var res = sh(templify(args.cmd, args), args.stdin, args.timeout, iio, args.directory, true, cmdf);
        args.stdout = res.stdout;
        args.stderr = res.stderr;
        if (res.exitcode != args.exitcode) throw "Exit code: " + res.exitcode;
      }

  # oJob Launch process
  #
  - name: oJob Process Launch
    help: >
      Launches a process in background.

        cmd     (String)  The command to launch the process (mandatory)
        console (Boolean) Defines if stdout/stderr should be printed or not (defaults to true)
        success (String)  Code to execute as a function in case of success. Receives a "res" map from executing a sh function.
        error   (String)  Code to execute as a function in case of error. Receives a "e" exception and a "cmd" with the original cmd argument.

    exec: |
      global.__ojobProc = _$(global.__ojobProc).isArray("Global __ojobProc needs to be an array.").default([]);

      _$(args.cmd).$_("A args.cmd needs to be defined.");
      args.console  = _$(args.console).isBoolean("args.console needs to be a boolean.").default(true);
      args.success  = _$(args.success).isString("args.success needs to be a string.").default(void 0);
      args.error    = _$(args.error).isString("args.error needs to be a string.").default("logErr(\"Process error '\" + cmd + \"': \" + String(e))");

      global.__ojobProc.push($do((s, f) => {
          var res = sh(templify(args.cmd, args), void 0, void 0, args.console, void 0, true);
          if (isDef(args.success)) (new Function("res", args.success))(res);
      }).catch((e) => {
          if (isDef(args.error)) (new Function("e", "cmd", args.error))(e, args.cmd);
      }));

  # oJob Process Wait
  #
  - name: oJob Process Wait
    help: >
      Waits for the background launched processes.
    exec: |
      $doWait($doAll(global.__ojobProc));

  ## ------------
  ## oJob Logging
  ## ------------
  
  # Display oJob Log
  - name        : oJob Show Log
    exec        : >
      log(stringify(
        (isUnDef(args.name) ? 
         ow.oJob.getLogCh().getAll() 
         : 
         $from(ow.oJob.getLogCh().getAll()).equals("name", args.name).select()
        )
      ));

  # Display report
  - name        : oJob Jobs Report
    help: >
      Outputs a jobs report (e.g. job name, status, number of executions, total time, avg time and last execution)

        format  (String)  Can be json, yaml or table (default)

    exec        : |
      ow.loadFormat(); print("\n");
      var logs = $ch("oJob::log").getAll();
      var report = $from(logs)
        .notEquals("name", "oJob Jobs Report")
        .notEquals("name", "oJob Jobs Final Report")
        .select((r) => {
        return {
          "Job name"  : r.name,
          "Status"    : (r.start ? (r.success ? "OK" : (r.deps ? (r.error ? "NOT OK!" : "executing") : "failed deps")) : "not started"),
          "# execs"   : r.count,
          "Total time": ow.format.elapsedTime4ms(r.totalTime, { abrev: true }),
          "Avg time"  : ow.format.elapsedTime4ms(r.avgTime, { abrev: true }),
          "Last exec" : (r.log.length > 0 ? ow.format.timeago($from(r.log).sort("-endTime").at(0).endTime) : "n/a")
        }
      });

      report.push({
        "Job name"  : "TOTAL",
        "Status"    : "",
        "# execs"   : "",
        "Total time": ow.format.elapsedTime4ms((now() - $from(logs).sort("createDate").at(0).createDate)),
        "Avg time"  : "",
        "Last exec" : ""
      });

      if (isUnDef(args.format)) args.format = "table";
      switch(args.format) {
      case "json": sprint(report); break;
      case "yaml": print(af.toYAML(report)); break;
      default    : print(printTable(report));
      }
  
  # Same as oJob Jobs Report but on shutdown
  # ------------------------------------
  - name        : oJob Jobs Final Report
    type        : shutdown
    from        : oJob Jobs Report

  # Wrapper for $doA2B
  # --------------
  - name: oJob A2B
    help: >
      Will execute A code providing a function (fn) to send data, in batches, to B code which receives it async as values. Expects:

        A          (String) The function A code that should use the provided parameter "fn" as a function to send batches of data to B
        B          (String) The function B code that uses the provided parameter "values" to async handle data sent by A
        numThreads (Number) Optionally provide the number of threads to launch 
        timeout    (Number) Optionally provide a different safe wait timeout (defaults to 2500 ms)
    exec: |
      _$(args.A, "A").$_();
      _$(args.B, "B").$_();

      $doA2B(new Function("fn", args.A), new Function("valuesk", args.B), args.numThreads, args.timeout);

  # -----------------------
  - name: oJob Get settings
    help: 
      text   : Will retrieve the yaml settings stored on the user home folder
      expects:
      - name: settingsFile
        desc: Which file, under the user home folder, will the settings be stored
      returns:
      - name: settings
        desc: Returns a map with the current settings
    exec: |
      _$(args.settingsFile, "settingsFile").$_();
      var hd = String(java.lang.System.getProperty("user.home")).replace(/\\/g, "/") + "/" + args.settingsFile;
      
      if (io.fileExists(hd))
        args.settings = io.readFileYAML(hd);
      else
        args.settings = {}

  # -----------------------
  - name: oJob Set settings 
    help:
      text   : Will set yaml settings stored on the user home folder
      expects:
      - name: settingsFile
        desc: Which file, under the user home folder, will the settings be stored
      - name: settings
        desc: The settings map to merge with the current structure
      - name: reset
        desc: Boolean if true will overwrite any existing settings (default false)
    exec: |
      _$(args.settingsFile, "settingsFile").$_();
      _$(args.settings, "settings").$_();
      args.reset = _$(args.reset, "reset").default(false);

      var hd = String(java.lang.System.getProperty("user.home")).replace(/\\/g, "/") + "/" + args.settingsFile;
      if (args.reset || !io.fileExists(hd)) {
        io.writeFileYAML(hd, args.settings);
      } else {
        var existing = io.readFileYAML(hd);
        io.writeFileYAML(hd, merge(existing, args.settings));
      }