oJobRest.yaml

# Copyright 2023 Nuno Aguiar

global:
   __oJobRestDocTemplate: &__oJobRestDocTemplate |
      # {{title}}
      
      ## GET
      
      {{{docGET.desc}}}
      
      {{#with docGET.in}}
      ### Input
      | Field | Type | Description |
      |---|---|---|
      {{#each this}}
      |{{field}}|{{type}}|{{description}}|\n
      {{/each}}
      {{/with}}

      {{#with docGET.out}}
      ### Output
      | Field | Type | Description |
      |---|---|---|
      {{#each this}}
      |{{field}}|{{type}}|{{description}}|
      {{/each}}
      {{/with}}

      {{#with docGET.examples}}
      ### Examples
      {{{this}}}
      {{/with}}

      ## POST
      
      {{{docPOST.desc}}}
      
      {{#with docPOST.in}}
      ### Input
      | Field | Type | Description |
      |---|---|---|
      {{#each this}}
      |{{field}}|{{type}}|{{description}}|\n
      {{/each}}
      {{/with}}
      
      {{#with docPOST.out}}
      ### Output
      | Field | Type | Description |
      |---|---|---|
      {{#each this}}
      |{{field}}|{{type}}|{{description}}|
      {{/each}}
      {{/with}}

      {{#with docPOST.examples}}
      ### Examples
      {{{this}}}
      {{/with}}

      ## PUT
      
      {{{docPUT.desc}}}
      
      {{#with docPUT.in}}
      ### Input
      | Field | Type | Description |
      |---|---|---|
      {{#each this}}
      |{{field}}|{{type}}|{{description}}|\n
      {{/each}}
      {{/with}}
      
      {{#with docPUT.out}}
      ### Output
      | Field | Type | Description |
      |---|---|---|
      {{#each this}}
      |{{field}}|{{type}}|{{description}}|
      {{/each}}
      {{/with}}

      {{#with docPUT.examples}}
      ### Examples
      {{{this}}}
      {{/with}}
      
      ## DELETE
      
      {{{docDELETE.desc}}}
      
      {{#with docDELETE.in}}
      ### Input
      | Field | Type | Description |
      |---|---|---|
      {{#each this}}
      |{{field}}|{{type}}|{{description}}|\n
      {{/each}}
      {{/with}}
      
      {{#with docDELETE.out}}
      ### Output
      | Field | Type | Description |
      |---|---|---|
      {{#each this}}
      |{{field}}|{{type}}|{{description}}|
      {{/each}}
      {{/with}}

      {{#with docDELETE.examples}}
      ### Examples
      {{{this}}}
      {{/with}}

jobs:
   # Prepares a REST server
   #
   - name: REST Start Server
     help: >
        Provides a REST server. The expected arguments are:
           - args.port     (Number)     The port where the server should be made available (default is 8091)
           - args.hs       (HTTPServer) A HTTPServer to use (is not provided one will be created)
           - args.keystore (String)     Optionally to create a https server provide a keystore file
           - args.pass     (String)     The password for the keystore of a https server
           - args.host     (String)     Optionally the interface where the server should be listening (default is all)
           - args.cp       (String)     Optionally providing the classpath URL to include the keystore file
     exec: |
        ow.loadServer();
        $do(() => { ow.loadTemplate(); });
        var ks;
        if (isUnDef(__pmIn.__ojobHttp)) __pmIn.__ojobHttp = {};
        if (isUnDef(__pmIn.__ojobRoutes)) __pmIn.__ojobRoutes = {};
        if (isUnDef(__pmIn.__ojobRoutesMetrics)) __pmIn.__ojobRoutesMetrics = {};
        if (isUnDef(args.port)) args.port = 8091;
        if (isDef(args.cp)) af.externalAddClasspath(args.cp);
        if (isDef(args.keystore)) ks = "/" + args.keystore;
        if (isUnDef(__pmIn.__ojobHttp[args.port])) __pmIn.__ojobHttp[args.port] = (isDef(args.hs) ? hs : ow.server.httpd.start(args.port, args.host, ks, args.pass));
        __pmIn.__ojobRoutes[args.port] = {};
        __pmIn.__ojobRoutesMetrics[args.port] = {};

   # Applies a REST service definition
   #
   - name: REST Apply Service 
     exec: |
        if (isUnDef(args.port)) args.port = 8091;

        if (isDef(global.__oJobRestDoc)) {
           ow.server.httpd.route(__pmIn.__ojobHttp[args.port], ow.server.httpd.mapRoutesWithLibs(__pmIn.__ojobHttp[args.port], __pmIn.__ojobRoutes[args.port]), function(r) { return __pmIn.__ojobHttp[args.port].reply("", "", 401, {}); });
        } else {
           ow.server.httpd.route(__pmIn.__ojobHttp[args.port], __pmIn.__ojobRoutes[args.port], function(r) { return __pmIn.__ojobHttp[args.port].reply("", "", 401, {}); });
        }
        log("READY [" + args.port + "]");

   # Stopping the REST server
   #
   - name: REST Stop Server
     type: shutdown
     deps:
     - REST Start Server
     exec: |
        if (isUnDef(args.port)) args.port = 8091;
        ow.server.httpd.stop(__pmIn.__ojobHttp[args.port]);
        delete __pmIn.__ojobHttp[args.port];
        delete __pmIn.__ojobRoutes[args.port];
        log("STOPPED [" + args.port + "]");

   # ------------------------
   - name: REST List Services
     deps:
     - REST Start Server
     to  :
     - REST Apply Service
     help: |
       Provides a REST endpoint list for a endpoint prefix (aURI). The expected arguments:
         - args.uri         (String)   The URI where the documentation will be available (defaults to "/")
         - args.port        (Number)   The port for the REST documentation service (defaults to 8091)
         - args.list        (String)   The map key that will hold the endpoint list (defaults to "paths")
         - args.stamp       (Map)      The map to stamp the reply with.
         - args.excludeLibs (Boolean)  If true it will not show /js, /css and /fonts (defaults to true)
     exec: |
       args.uri         = _$(args.uri, "uri").isString().default("/");
       args.port        = _$(args.port, "port").isNumber().default(8091);
       args.stamp       = _$(args.stamp, "stamp").isMap().default({});
       args.list        = _$(args.list, "list").isString().default("paths");
       args.excludeLibs = _$(args.excludeLibs, "excludeLibs").isBoolean().default(true);

       var lst = {};
       lst[args.list] = $from(Object.keys(__pmIn.__ojobRoutes[args.port]))
                        .starts(args.uri)
                        .select();

       if (args.excludeLibs) { 
         var t = [];
         for(var ii in lst[args.list]) {
           if (!lst[args.list][ii].startsWith("/fonts/") &&
               !lst[args.list][ii].startsWith("/css/") && 
               !lst[args.list][ii].startsWith("/js/")) 
             t.push(lst[args.list][ii]);
         }
         lst[args.list] = t;
       }
 
       __pmIn.__ojobRoutes[args.port][args.uri] = function(r) {
         return __pmIn.__ojobHttp[args.port].reply(stringify(merge(args.stamp, lst), void 0, ""));
       }

   # Doc endpoint for a REST service
   #
   - name: REST Doc Service
     deps:
     - REST Start Server
     to  : REST Apply Service
     args:
      baseTemplate: *__oJobRestDocTemplate
     help: >
        Provides a REST documentation service defintion to display the markdown documentation of an URI. The expected arguments are:
           - args.uri              (String)   The URI where the documentation will be available (defaults to "/docs")
           - args.port             (Number)   The port for the REST documentation service (defaults to 8091)
           - args.docURI           (String)   The original URI to which the documentation relates to.
           - args.docPort          (Number)   The original endpoint port to which the documentation relates to (default to args.port)
           - args.template         (String)   Use a different ow.template template for documentation (available vars: title, uri, port, docURI, docPort, docGET, docPOST, docPUT and docDELETE (with desc, examples, in and out))
           - args.title            (String)   The documentation page title (defaults to docURI).
           - args.justJson         (Boolean)  Returns just the documentation json (defaults to false).
     exec: |
        args.uri      = _$(args.uri, "uri").isString().default("/docs");
        args.port     = _$(args.port, "port").isNumber().default(8091);
        args.docURI   = _$(args.docURI, "docURI").isString().$_();
        args.docPort  = _$(args.docPort, "docPort").isNumber().default(args.port);
        args.title    = _$(args.title, "title").isString().default(args.docURI);
        args.template = _$(args.template, "template").isString().default(args.baseTemplate);
        args.justJson = _$(args.justJson, "justJson").isBoolean().default(false);

        __pmIn.__ojobRoutes[args.port][args.uri] = function(r) {
          if (args.justJson) {
            var o = { 
              uri   : args.uri,
              get   : global.__oJobRestDoc[args.docPort][args.docURI].get,
              post  : global.__oJobRestDoc[args.docPort][args.docURI].post,
              put   : global.__oJobRestDoc[args.docPort][args.docURI].put,
              delete: global.__oJobRestDoc[args.docPort][args.docURI].delete,
            }; 
            return ow.server.httpd.reply(o);
          } else {
            var templ = templify(args.template, {
               title  : args.title,
               uri    : args.uri,
               port   : args.port,
               docURI : args.docURI,
               docPort: args.docPort,
               docGET    : global.__oJobRestDoc[args.docPort][args.docURI].get,
               docPOST   : global.__oJobRestDoc[args.docPort][args.docURI].post,
               docPUT    : global.__oJobRestDoc[args.docPort][args.docURI].put,
               docDELETE : global.__oJobRestDoc[args.docPort][args.docURI].delete,
            });
            return __pmIn.__ojobHttp[args.port].replyOKHTML(ow.template.parseMD2HTML(templ, true));
          }
        };

   # Defines a REST service
   #
   - name: REST Service
     deps: 
     - REST Start Server
     to  : REST Apply Service
     help: >
        Provides a REST service definition. The expected arguments are:
           - args.uri              (String)   The URI where the REST service will be available.
           - args.port             (Number)   The port for the REST service endpoint (defaults to 8091)
           - args.execGET          (Function) The GET function. Receives a map of idxs.
           - args.execPOST         (Function) The POST function. Receives a map of idxs and a data map.
           - args.execPUT          (Function) The PUT function. Receives a map of idxs and a data map.
           - args.execDELETE       (Function) The DELETE function. Receives a map of idxs.
           - args.returnWithParams (Boolean)  Use a more detailed return map.
           - args.returnHTML       (Boolean)  Returns a partial HTML. 
           - args.error            (Boolean)  If true will return a __error entry in case of exception.
           - args.audit            (Boolean)  Turns request audit logging on (default true)
           - args.auditTemplate    (String)   Provide an audit logging template based on the request argument.
           - args.docGET           (Map)      Markdown documentation for execGET (desc (string), in and out (arrays of field, type, description))
           - args.docPOST          (Map)      Markdown documentation for execPOST (desc (string), in and out (arrays of field, type, description))
           - args.docPUT           (Map)      Markdown documentation for execPUT (desc (string), in and out (arrays of field, type, description))
           - args.docDELETE        (Map)      Markdown documentation for execDELETE (desc (string), in and out (arrays of field, type, description))

        Do ensure that the job 'REST Start Server' gets executed first.
     exec: |
        if (isUnDef(args.port)) args.port = 8091;
        if (isUnDef(args.uri))  args.uri = "/rest";
        args.audit = _$(args.audit).default(true);
        args.auditTemplate = _$(args.auditTemplate).default("AUDIT HTTP | {{method}} {{uri}} {{reply.status}} {{{reply.mimetype}}} ({{header.remote-addr}}; {{header.user-agent}})");
        args.error = _$(args.error).default(true);

        if (isDef(args.docGET) || isDef(args.docPOST) || isDef(args.docPUT) || isDef(args.docDELETE)) {
           if (isUnDef(global.__oJobRestDoc))            global.__oJobRestDoc = {};
           if (isUnDef(global.__oJobRestDoc[args.port])) global.__oJobRestDoc[args.port] = {};

           global.__oJobRestDoc[args.port][args.uri] = {
              get   : _$(args.docGET).isMap().default({ desc: "n/a" }),
              post  : _$(args.docPOST).isMap().default({ desc: "n/a" }),
              put   : _$(args.docPUT).isMap().default({ desc: "n/a" }),
              delete: _$(args.docDELETE).isMap().default({ desc: "n/a" })
           }
        }
  
        var fn = void 0;
        if (args.audit) {
           fn = (req, reply) => {
              var data = merge(req, { 
                  reply: {
                     status  : reply.status,
                     mimetype: reply.mimetype
                  }
               });
               try { 
                  tlog(args.auditTemplate, data);
               } catch(e) {
                  logErr("Error on auditing access: " + String(e));
               }
           }
        }
   
        var _efn = s => {
           if (args.error) {
              if (args.returnWithParams)
                return "try { " + s + " } catch(e) { logErr(String(e)); return { data: { __error: String(e) }, status: 500, mimetype: \"application/json\" } }";
              else
                return "try { " + s + " } catch(e) { logErr(String(e)); return { __error: String(e) } }";
           } else {
              return s;
           }
        };

        __pmIn.__ojobRoutesMetrics[args.port][args.uri] = {
           start: $atomic(),
           err  : $atomic(),
           end  : $atomic(), 
           sum  : $atomic() 
        };

        __pmIn.__ojobRoutes[args.port][args.uri] = function(r) {
           __pmIn.__ojobRoutesMetrics[args.port][args.uri].start.inc();
           var ini = now(); 

           var ress = ow.server.rest.reply(args.uri, r, 
             af.eval("(function(idxs, data, request) { " + _efn(args.execPOST) + " })"),
             af.eval("(function(idxs, request) { " + _efn(args.execGET) + " })"),
             af.eval("(function(idxs, data, request) { " + _efn(args.execPUT) + " })"),
             af.eval("(function(idxs, request) { " + _efn(args.execDELETE) + " })"),
             args.returnWithParams,
             fn,
             args.returnHTML
           );
           
           __pmIn.__ojobRoutesMetrics[args.port][args.uri].sum.getAdd(now() - ini);
           if (isDef(ress.__error) || (isDef(ress.data) && isDef(ress.data.__error)))
              __pmIn.__ojobRoutesMetrics[args.port][args.uri].err.inc();
           else
              __pmIn.__ojobRoutesMetrics[args.port][args.uri].end.inc();
 
           return ress;
        };

   # ------------------
   - name: REST Healthz
     deps:
     - REST Start Server
     to  : REST Service
     help:
       text: Adds a healthz URI
       expects:
       - name: port
         desc: The port of the REST Start Server (default to 8091)
       - name: uri
         desc: The uri to use (default /healthz)
       - name: healthz
         desc: A custom function to return the /healthz result (defaults to ow.metrics.getAll with status 200 or "error" with status 500 if an exception is thrown). Should return a result with params.
       - name: openmetrics
         desc: If openmetrics = true the output will be in openmetrics/prometheus instead of json
     exec: |
         ow.loadMetrics();

         args.port    = _$(args.port).isNumber().default(8091);
         args.uri     = _$(args.uri).isString().default("/healthz");
         
         var s1 = " try { return { data: (isDef(idxs.s) ? ow.metrics.getSome(idxs.s.split(\",\")) : ow.metrics.getAll()), status: 200, mimetype: \"application/json\" } } catch(e) { logErr(e); return { data: \"error\", status: 500, mimetype: \"text/plain\" }; } ";
         var s2 = " try { return String(isDef(idxs.s) ? ow.metrics.fromObj2OpenMetrics(ow.metrics.getSome(idxs.s.split(\",\"))) : ow.metrics.fromObj2OpenMetrics(ow.metrics.getAll())); } catch(e) { logErr(e); return { data: \"error\", status: 500, mimetype: \"text/plain\" }; } ";
         args.healthz = _$(args.healthz).isString().default(args.openmetrics ? s2 : s1);
         args.execGET = args.healthz;
         args.returnWithParams = !args.openmetrics;

         ow.metrics.add("oJobRest", () => {
            var u = [];
            var k = Object.keys(__pmIn.__ojobRoutesMetrics[args.port]);
            k.map(ki => {
               var o = __pmIn.__ojobRoutesMetrics[args.port][ki];
               u.push({ uri: ki, start: o.start.get(), err: o.err.get(), end: o.end.get(), sum: o.sum.get(), avg: o.sum.get() / (o.end.get() + o.err.get()) });
            });
            return u;
         });

   # ------------------
   - name: REST Metrics
     deps:
     - REST Start Server
     to  : REST Service
     help:
       text: Adds a metrics URI with a OpenMetrics/Prometheus output
       expects:
       - name: port
         desc: The port of the REST Start Server (default to 8091)
       - name: uri
         desc: The uri to use (default /metrics)
       - name: metrics
         desc: A custom function to return the /metrics result (defaults to ow.metrics.getAll with status 200 or "error" with status 500 if an exception is thrown). Should return a result with params.
       - name: prefix
         desc: OpenMetrics/Prometheus metrics' name prefix
       - name: helpmap
         desc: OpenMetrics/Prometheus metrics' help map (see help ow.metrics.fromObj2OpenMetrics)
     exec: |
         ow.loadMetrics();

         args.port    = _$(args.port).isNumber().default(8091);
         args.uri     = _$(args.uri).isString().default("/metrics");
         args.prefix  = _$(args.prefix).isString().default("metrics");
         args.helpmap = _$(args.helpmap).isMap().default({});
         
         args.metrics = _$(args.metrics).isString().default("try { var rrr = ow.server.httpd.reply(ow.metrics.fromObj2OpenMetrics((isDef(idxs.s) ? ow.metrics.getSome(idxs.s.split(\",\")) : ow.metrics.getAll()), '" + args.prefix + "', now(), " + stringify(args.helpmap, __, "") + ")); sprint(rrr); rrr.data = String(rrr.data); return rrr; } catch(e) { logErr(e); return { data: \"error\", status: 500, mimetype: \"text/plain\" }; } ");
         args.execGET = args.metrics;
         args.returnWithParams = true;

         ow.metrics.add("oJobRest", () => {
            var u = [];
            var k = Object.keys(__pmIn.__ojobRoutesMetrics[args.port]);
            k.map(ki => {
               var o = __pmIn.__ojobRoutesMetrics[args.port][ki];
               u.push({ uri: ki, start: o.start.get(), err: o.err.get(), end: o.end.get(), sum: o.sum.get(), avg: o.sum.get() / (o.end.get() + o.err.get()) });
            });
            return u;
         });

   # ---------------------
   - name: REST Add Metric
     help: 
       text   : Add a metric to the current daemon execution
       expects:
       - name: metric
         desc: Metric name
       - name: fn 
         desc: Metric function code returning a map
     exec: |
       _$(args.metric, "metric").$_();
       _$(args.fn, "fn").$_();
       ow.loadMetrics();
       ow.metrics.add(args.metric, new Function(args.fn));