oJobTest: several improvements

OpenAF · Mar 31, 2024 · 771c765 · 771c765
1 parent e1d92ef
commit 771c765
Show file tree

Hide file tree

Showing 4 changed files with 243 additions and 16 deletions.
diff --git a/.package.yaml b/.package.yaml
@@ -35,7 +35,7 @@ name: oJob-common
 main: ''
 mainJob: ''
 license: Apache 2.0
-version: '20231014'
+version: '20240331'
 dependencies:
   openaf: '>=20211229'
 files:
@@ -89,9 +89,9 @@ filesHash:
   oJobSQL.yaml: c7d0ab718f87470f83c07c85b6c4074a810ea3ef
   oJobSSH.yaml: 6dbefb7028a6be855ee4d3cc93e971eeee563d5b
   oJobSVN.yaml: 10920575ac4c337fe99448b8a04a0689c2bde77f
-  oJobTest.yaml: caebc8302482279d936b47210ef579022bc3b167
+  oJobTest.yaml: 8cdead5a3d98bf31929697ae88e7488857888324
   oJobWatchDog.yaml: d1dbeab36784bd6722daa27e572425c84ebbc36b
   oJobXLS.yaml: 00def6b195d7487989d7aa2c774b584a0e0b01a0
   oJobs.yaml: 83d8406c43b4a5325686467ed80911a9c1e26d49
-  ojob.json: 26aba4aebe886c1c744804d43abf7ba90a6f8ead
-  ojob.yaml: 353fee0d3be45701e21559ac16a9b9f25d42251f
+  ojob.json: f80cc7a603150a6abb4427fa1914f71a5c391f8a
+  ojob.yaml: c7615c8ed0f5eeb0f6e64fffcd9fb5545c4f55ea
diff --git a/oJobTest.yaml b/oJobTest.yaml
@@ -38,9 +38,10 @@ jobs:
       name  : test
       keyArg: name
       args  :
-        func : func
-        job  : job
-        count: count
+        func   : func
+        job    : job
+        count  : count
+        asserts: asserts
   help: 
     text   : This job will test an args.func (function) or an args.job provided. The test will use the current oJob job name. Optionally you can specify args.count (defaults to 1) to test more than one time.
     expects:
@@ -52,21 +53,38 @@ jobs:
       desc: The ojob to execute and test.
     - name: count
       desc: The number of test repeats (defaults to 1).
+    - name: asserts
+      desc: Array of asserts maps to test the results (composed of 'path', 'value' and 'msg')
   exec: |
     if (isUnDef(args.func) && isUnDef(args.job)) throw "Please define args.func or an args.job."
     var count = _$(args.count, "count").toNumber().isNumber().default(1)
     var jname = _$(args.name, "name").isString().default(job.name)
 
     if (isDef(args.func)) {
       if (!(isFunction(args.func))) args.func = new Function(args.func)
-      for(var c = 0; c < count; c++) ow.loadTest().test(jname, args.func) 
+      ow.loadTest()
+      args.asserts = _$(args.asserts, "asserts").isArray().default([])
+      for(var c = 0; c < count; c++) {
+        var _r = ow.test.test(jname, args.func) 
+        args.asserts.forEach(a => {
+          a.path = _$(a.path, "path").isString().default("@")
+          a.msg  = _$(a.msg, "msg").isString().default("Assert failed " + af.toSLON(a))
+          ow.test.assert($path(_r, a.path), a.value, a.msg)
+        })
+      }
     } else {
       if (isDef(args.job)) {
         for(var c = 0; c < count; c++) ow.loadTest().test(jname, () => {
           id = "|" + genUUID()
           var _r = $job(args.job, __, id)
 
           if (!isMap(_r)) throw $from( ow.oJob.getLogCh().get({ ojobId: ow.oJob.getID() + id, name: args.job }).log ).last().error
+
+          args.asserts.forEach(a => {
+            a.path = _$(a.path, "path").isString().default("@")
+            a.msg  = _$(a.msg, "msg").isString().default("Assert failed " + af.toSLON(a))
+            ow.test.assert($path(_r, a.path), a.value, a.msg)
+          })
         })
       }
     }
@@ -106,13 +124,16 @@ jobs:
       keyArg: quiet
       args  : 
         noprofile: noprofile
+        key      : key
   help : 
     text   : "Prints or outputs (if args.quiet = true) a args.results with: count of tests, tests passed, tests failed and all profile information."
     expects:
     - name: quiet
       desc: Boolean flag that if true wont output any test results to stdout.
     - name: noprofile
       desc: Boolean flag that if true will not include profile results
+    - name: key
+      desc: If defined will output the results to the provided key.
   check:
     in:
       quiet    : toBoolean.isBoolean.default(false)
@@ -135,6 +156,7 @@ jobs:
       }))
     }
     if (args.noprofile) delete args.results.profile
+    if (isDef(args.key)) $set(args.key, args.results)
     if ((isDef(args.quiet) && !args.quiet) || isUnDef(args.quiet)) ow.oJob.output(args.results, args)
 
 # ----------------------------
@@ -152,11 +174,11 @@ jobs:
       desc: Boolean flag that if true wont output any test results to stdout.
   exec: |
     args.results = {
-      count: $ch("oJob::logs").size(),
-      pass: $from($ch("oJob::logs").getAll()).equals("success", true).count(),
-      fail: $from($ch("oJob::logs").getAll()).equals("error", true).count()
+      count: $ch("oJob::log").size(),
+      pass: $from($ch("oJob::log").getAll()).equals("success", true).count(),
+      fail: $from($ch("oJob::log").getAll()).equals("error", true).count()
     };
-    if ((isDef(args.quiet) && !args.quiet) || isUnDef(args.quiet)) ow.oJob.output(args.results)
+    if ((isDef(args.quiet) && !args.quiet) || isUnDef(args.quiet)) ow.oJob.output(args.results, args)
 
 # -----------------------------
 - name : oJob Generate Markdown
@@ -168,22 +190,32 @@ jobs:
       args  : 
         path: path
         file: file
+        includeLogs: includeLogs
   help : 
-    text   : "This job will generate a Markdown with the current test results."
+    text   : "This job will generate a Markdown with the current test results and optionally logs."
     expects: 
     - name: key
       desc: If a 'file' is not defined will output to provided 'key' (including 'args') and 'path'
     - name: path
       desc: The path withing the provided 'key' to store the markdown output
     - name: file
       desc: If defined will output the markdown content to the provided file.
+    - name: includeLogs
+      desc: If ojob.channels.recordLog is true will include the logs in the markdown.
   check:
     in:
       key : isString.default(__)
       path: isString.default(__)
       file: isString.default(__)
+      includeLogs: isBoolean.default(false)
   exec : |
     var _r = ow.loadTest().toMarkdown()
+    if (args.includeLogs && $ch().list().indexOf("__log") >= 0)
+      _r += "\n## Logs\n\n" + ow.template.md.table($ch("__log").getAll().map(r => ({
+        timestamp: r.d,
+        level    : r.t,
+        message  : r.m
+      })))
     if (isDef(args.file)) {
       io.writeFileString(args.file, _r)
     } else {
@@ -196,6 +228,66 @@ jobs:
       }
     }
 
+# JSON results
+# ------------
+- name: oJob Generate JSON results
+  typeArgs:
+    noLog: false
+    shortcut:
+      name  : testGenJSON
+      keyArg: file
+      args  : 
+        includeLogs: includeLogs
+        noProfile  : noProfile
+  help:
+    text   : "This job will generate a JSON with the current test results and optionally logs." 
+    expects:
+    - name: file
+      desc: If defined will output the JSON content to the provided file (if '-' will output to stdout).
+    - name: noprofile
+      desc: Boolean flag that if true will not include profile results
+    - name: includeLogs
+      desc: If ojob.channels.recordLog is true will include the logs in the markdown.
+  check:
+    in:
+      file: isString.default("-")
+      includeLogs: isBoolean.default(false)
+      noProfile: isBoolean.default(false)
+  exec : |
+    var prof = ow.loadTest().__profile
+    args.results = {
+      count: ow.loadTest().getCountTest(),
+      pass: ow.loadTest().getCountPass(),
+      fail: ow.loadTest().getCountFail(),
+      asserts: ow.loadTest().getCountAssert(),
+      profile: Object.keys(prof).map(r => ({
+        name: r,
+        hits: prof[r].hits,
+        sum : prof[r].sum,
+        last: prof[r].last,
+        max : prof[r].max,
+        min : prof[r].min,
+        avg : prof[r].sum / prof[r].hits
+      }))
+    }
+    if (args.noprofile) delete args.results.profile
+
+    var _r = args.results
+    if (args.includeLogs && $ch().list().indexOf("__log") >= 0) {
+      _r = merge(_r, {
+        logs: $ch("__log").getAll().map(r => ({
+          timestamp: r.d,
+          level    : r.t,
+          message  : r.m
+        }))
+      })
+    }
+    if (args.file == "-") {
+      sprint(_r, "")
+    } else {
+      io.writeFileJSON(args.file, _r, "")
+    }
+
 # JUnit results
 #
 # -----------------------------

diff --git a/ojob.json b/ojob.json
diff --git a/ojob.yaml b/ojob.yaml
@@ -1300,7 +1300,7 @@ jobs:
     var secFile     = _$($$(args).get(secIn + "secFile"), "secFile").default(__);
     var secRepo     = _$($$(args).get(secIn + "secRepo"), "secRepo").default(__);
     var secMainPass = _$($$(args).get(secIn + "secMainPass"), "secMainPass").default(__);
-    var secEnv      = toBoolean( _$($$(args).get(secIn + "secEnv"), "secEnv").isBoolean().default(false) )
+    var secEnv      = _$(toBoolean($$(args).get(secIn + "secEnv")), "secEnv").isBoolean().default(false)
 
     // Set to env variable
     if (secEnv) {
@@ -2167,4 +2167,139 @@ jobs:
     }
 
     // Delete all __ args
-    Object.keys(args).filter(k => k.startsWith("__")).forEach(k => delete args[k])
+    Object.keys(args).filter(k => k.startsWith("__")).forEach(k => delete args[k])
+
+# ---------------
+- name : ojob ask
+  typeArgs:
+    noLog   : true
+    shortcut:
+      name  : ask
+      keyArg: __answers
+      args  :
+        question: __question
+        force   : __force
+  help : 
+    text: |
+      Asks for user input and stores the result into args if the args value is not yet defined.
+    expects:
+    - name: __question
+      desc: The question to ask.
+    - name: __answers
+      desc: "A map of answers to store into args. Each value should on the form of \"type.slonOptionsArray.siz\" (e.g. text, encrypt, password, char.['y'|'n'], option.['op1'|'op2'|'op3'].3)."
+    - name: __force
+      desc: Boolean flag to force asking for an args value even if not defined.
+  check:
+    in:
+      __question: isString.default(__)
+      __answers : isMap.default({})
+      __force   : toBoolean.isBoolean.default(false)
+  exec : |
+    if (isDef(args.__question)) {
+      args.__question = args.__question.replace(/\n/g, "")
+      if (!args.__question.endsWith(" ")) args.__question += " "
+    }
+
+    traverse(args.__answers, (aK, aV, aP, aO) => {
+      if (isString(aV)) {
+        if (args.__force || isUnDef($$(args).get(aP + "." + aK)) ) {
+          var parts = aV.split(".")
+          var options = _$(options, "options").isArray().default([])
+          if (isDef(parts[1]) && parts[1].startsWith("[") && parts[1].endsWith("]")) options = af.fromSLON(parts[1])
+          if (!isArray(options)) options = [ options ]
+
+          if (isString(parts[0])) {
+            if (isUnDef(args.__question)) args.__question = "args" + aP + "." + aK + " value: "
+            switch(parts[0].toLowerCase()) {
+            case "encrypt" :
+            case "password": $$(args).set(aP + "." + aK, askEncrypt(args.__question)); break
+            case "char"    : 
+              var _o = options.join("")
+              $$(args).set(aP + "." + aK, ask1(args.__question), _o)
+              break
+            case "option"  :
+            case "list"    : 
+              if (isArray(options)) {
+                $$(args).set(aP + "." + aK, options[ askChoose(args.__question, options, (isDef(parts[2]) ? Number(parts[2]) : __) )] ); break
+              }
+            case "text"    :
+            default        : $$(args).set(aP + "." + aK, ask(args.__question))
+            }
+          }
+        }
+      }
+    })
+
+# ---------------
+- name : ojob llm
+  typeArgs:
+    noLog   : true
+    shortcut:
+      name  : llm
+      keyArg: __llmPrompt
+      args  :
+        context: __llmContext
+        inPath : __llmInPath
+        inKey  : __llmInKey
+        env    : __llmEnv
+        options: __llmOptions
+        outPath: __llmOutPath
+        debug  : __llmDebug
+  help :
+    text: |
+      "Executes a LLM (Local Language Model) prompt using $llm"
+    expects:
+    - name: __llmContext
+      desc: The context of the input data
+    - name: __llmInPath
+      desc: The path to consider from __llmInKey
+    - name: __llmInKey
+      desc: The key for an existing list or 'args'
+    - name: __llmEnv
+      desc: "The environment variable to consider for $llm options"
+    - name: __llmOptions
+      desc: "The options to consider for $llm"
+    - name: __llmOutPath
+      desc: The path to store the result
+    - name: __llmPrompt
+      desc: The prompt to send to the LLM model
+    - name: __llmDebug
+      desc: Boolean to indicate that the job execution parameters should be printed before executing
+  check: 
+    in:
+      __llmContext: isString.default(__)
+      __llmInKey  : isString.default("args")
+      __llmInPath : isString.default(__)
+      __llmEnv    : isString.default("OJOB_MODEL")
+      __llmOptions: isMap.default(__)
+      __llmOutPath: isString.default(__)
+      __llmPrompt : isString
+      __llmDebug  : isBoolean.default(false)
+  exec : |
+    try {
+    var _lo = isDef(args.__llmOptions) ? args.__llmOptions : __
+    if (isUnDef(_lo) && isDef(args.__llmEnv)) _lo = $sec("system", "envs").get(args.__llmEnv)
+
+    var _l = $llm(_lo)
+    if (isDef(args.__llmInPath)) {
+      if (isUnDef(args.__llmContext)) { args.__llmContext = "a JSON input data" }
+      var _i = args.__llmInKey == "args" ? $$(args).get(args.__llmInPath) : $$($get(args.__llmInKey)).get(args.__llmInPath)
+      if (args.__llmDebug) {
+        print("DEBUG | Input data:")
+        cprint(_i)
+      }
+      if (isDef(_i)) _l = _l.withContext(_i, args.__llmContext) 
+    }
+
+    var _r = _l.promptJSON(args.__llmPrompt)
+    if (args.__llmDebug) {
+      print("DEBUG | Result data:")
+      cprint(_r)
+    }
+    if (isUnDef(args.__llmOutPath)) 
+      $set("res", _r)
+    else
+      $$(args).set(args.__llmOutPath, _r)
+    } catch(e) {
+      throw "Error on LLM: " + e
+    }