Merge branch 'feat-backup'

Refs NethServer/dev#6971 NethServer/dev#6972
Closes #679

core/imageroot/usr/local/agent/bin/module-backup

@@ -129,6 +129,24 @@ if stats_proc.returncode == 0:
     wrdb.hset(f"module/{module_id}/backup_status/{backup_id}", mapping=backup_status)
 wrdb.close()
 
+try:
+    ometa = {}
+    ometa["module_id"] = module_id
+    ometa["module_ui_name"] = rdb.get(f'module/{module_id}/ui_name') or ""
+    ometa["node_fqdn"] = agent.get_hostname()
+    ometa["cluster_uuid"] = rdb.get("cluster/uuid") or ""
+    ometa["uuid"] = os.environ["MODULE_UUID"]
+    ometa["timestamp"] = time_end
+    ometa["success"] = bool(errors == 0)
+    subprocess.run(["rclone-wrapper", str(backup_id), "rcat", f"REMOTE_PATH/{repopath}.json"],
+        stdout=sys.stderr,
+        input='\n' + json.dumps(ometa, separators=(',', ':')) + '\n',
+        text=True,
+        check=True,
+    )
+except subprocess.CalledProcessError as ex:
+    errors += 1
+
 try:
     subprocess.run(["module-cleanup-state"] + sys.argv[1:], check=True)
 except FileNotFoundError:

core/imageroot/var/lib/nethserver/cluster/actions/read-backup-repositories/50read

@@ -26,45 +26,81 @@ import json
 import agent
 import subprocess
 import os
+import time
 from datetime import datetime, timezone
 
 request = json.load(sys.stdin)
-
-popen_args={"encoding": 'utf-8', "stdout": subprocess.PIPE, "stderr": sys.stderr}
-
+popen_args={"encoding": 'utf-8', "stdout": subprocess.PIPE, "stderr": sys.stderr, "text": True}
 rdb = agent.redis_connect(privileged=False)
-
-uuid_regex = re.compile('[0-9a-f]{32}\Z', re.I)
-backups = list()
+cluster_uuid = rdb.get("cluster/uuid") or ""
+backups = {}
 for krepo in rdb.scan_iter('cluster/backup_repository/*'):
     repo_uuid = krepo.removeprefix('cluster/backup_repository/')
-    rclone_cmd = ['rclone-wrapper', repo_uuid, 'lsjson']
-    repo = rdb.hgetall(krepo)
-
-    proot = subprocess.Popen(rclone_cmd + ["REMOTE_PATH"], **popen_args)
+    rclone_lsjson_cmd = ['rclone-wrapper', repo_uuid,
+        '--max-depth=3',
+        '--include=config',
+        'lsjson',
+        '--files-only',
+        'REMOTE_PATH',
+    ]
+    orepo = rdb.hgetall(krepo)
+    proot = subprocess.Popen(rclone_lsjson_cmd, **popen_args)
     for oroot in json.load(proot.stdout):
-        if oroot["IsDir"] is False:
-            continue # skip non-dir entries
-        pchild = subprocess.Popen(rclone_cmd + ["REMOTE_PATH/" + oroot['Path']], **popen_args)
-        for backup in json.load(pchild.stdout):
-            # parse almost-ISO-8601 dates
-            try:
-                sec_index = backup['ModTime'].rfind('.')
-                mod_time = datetime.strptime(backup['ModTime'][0:sec_index],'%Y-%m-%dT%H:%M:%S')
-                mod_time = int(mod_time.replace(tzinfo=timezone.utc).timestamp())
-            except:
-                mod_time = -1
+        restic_prefix, restic_uuid, _ = oroot["Path"].split("/", 2)
+        try:
+            # Obtain from lsjson the repository creation timestamp
+            unix_timestamp = int(time.mktime(datetime.fromisoformat(oroot["ModTime"]).timetuple()))
+        except:
+            unix_timestamp = int(time.time())
+        backups[restic_uuid] = {
+            "module_id": "",
+            "module_ui_name": "",
+            "node_fqdn": "",
+            "path": restic_prefix + '/' + restic_uuid,
+            "name": restic_prefix, # keep "name" attribute for historical reason
+            "uuid": restic_uuid,
+            "timestamp": unix_timestamp,
+            "repository_id" : repo_uuid,
+            "repository_name": orepo["name"],
+            "repository_provider": orepo["provider"],
+            "repository_url": orepo["url"],
+            "installed_instance": "",
+            "installed_instance_ui_name": "",
+            "is_generated_locally": None,
+        }
+
+# Fetch module UUIDs to search destination matches:
+for module_id in rdb.hkeys("cluster/module_node"):
+    omodule = rdb.hgetall(f"module/{module_id}/environment")
+    module_uuid = omodule.get("MODULE_UUID", "")
+    if module_uuid not in backups:
+        continue
+    backups[module_uuid]["is_generated_locally"] = True
+    backups[module_uuid]["installed_instance"] = module_id
+    backups[module_uuid]["installed_instance_ui_name"] = rdb.get(f"module/{module_id}/ui_name") or ""
+    backups[module_uuid]["module_id"] = module_id
+    backups[module_uuid]["module_ui_name"] = backups[module_uuid]["installed_instance_ui_name"]
+
+rclone_cat_cmd = ['rclone-wrapper', repo_uuid,
+    '--include=*.json',
+    '--max-depth=2',
+    'cat',
+    'REMOTE_PATH',
+]
+proc_cat = subprocess.Popen(rclone_cat_cmd, **popen_args)
+for cat_slice in proc_cat.stdout.readlines():
+    try:
+        ometa = json.loads(cat_slice.strip())
+    except:
+        ometa = {}
 
-            if re.match('[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', backup['Path'], re.I) is None or backup["IsDir"] is False:
-                continue # skip non-repo entries
-            installed_instance = ""
-            ui_name = ""
-            for mod in rdb.scan_iter('module/*/environment'):
-                u = rdb.hget(mod, 'MODULE_UUID')
-                if rdb.hget(mod, 'MODULE_UUID') == backup['Path']:
-                    installed_instance = rdb.hget(mod, 'MODULE_ID')
-                    ui_name = rdb.get(f'module/{installed_instance}/ui_name') or ""
-                    break
-            backups.append({'name': oroot['Path'], 'path': f'{oroot["Path"]}/{backup["Path"]}', 'uuid': backup["Path"], 'timestamp': mod_time, 'repository_id' : repo_uuid, 'repository_name': repo['name'], 'repository_provider': repo['provider'], 'repository_url': repo['url'], 'installed_instance': installed_instance, 'installed_instance_ui_name': ui_name})
+    if ometa.get('uuid') in backups:
+        module_uuid = ometa["uuid"]
+        backups[module_uuid]["module_id"] = ometa.get("module_id", "")
+        backups[module_uuid]["module_ui_name"] = ometa.get("module_ui_name", "")
+        backups[module_uuid]["node_fqdn"] = ometa.get("node_fqdn", "")
+        backups[module_uuid]["timestamp"] = ometa.get("timestamp", unix_timestamp)
+    if "cluster_uuid" in ometa and not backups[module_uuid].get("is_generated_locally"):
+        backups[module_uuid]["is_generated_locally"] = cluster_uuid == ometa["cluster_uuid"]
 
-print(json.dumps(backups))
+json.dump(list(backups.values()), fp=sys.stdout)

core/imageroot/var/lib/nethserver/cluster/actions/read-backup-repositories/validate-output.json

@@ -1,22 +1,107 @@
 {
-    "$schema": "http://json-schema.org/draft-07/schema#",
-    "title": "read-backup-repositories output",
-    "$id": "http://schema.nethserver.org/cluster/read-backup-repositories-output.json",
-    "description": "Read the content of all backup repositories",
-    "examples": [
-        [
-            {
-                "name": "dokuwiki",
-                "path": "dokuwiki/dokuwiki1@cc7335d7-4d67-408c-8c35-42257667e51b",
-                "uuid": "cc7335d7-4d67-408c-8c35-42257667e51b",
-                "timestamp": 1644403745,
-                "repository_id": "e181c936-1bc3-5032-a809-d8b0551eebe9",
-                "repository_name": "B2 repo",
-                "repository_provider": "backblaze",
-                "repository_url": "b2:giacomons8",
-                "installed_instance": "dokuwiki4",
-                "installed_instance_ui_name": "My Dokuwiki"
-            }
-        ]
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "read-backup-repositories output",
+  "$id": "http://schema.nethserver.org/cluster/read-backup-repositories-output.json",
+  "description": "Look up Restic repositories inside all backup destinations and return a list of them.",
+  "examples": [
+    [
+      {
+        "module_id": "loki1",
+        "module_ui_name": "My Loki",
+        "node_fqdn": "rl1.dp.nethserver.net",
+        "path": "loki/35f45b73-f81e-467b-b622-96ec3b7fec19",
+        "name": "loki",
+        "uuid": "35f45b73-f81e-467b-b622-96ec3b7fec19",
+        "timestamp": 1721405723,
+        "repository_id": "14030a59-a4e6-54cc-b8ea-cd5f97fe44c8",
+        "repository_name": "BackBlaze repo1",
+        "repository_provider": "backblaze",
+        "repository_url": "b2:ns8-davidep",
+        "installed_instance": "loki1",
+        "installed_instance_ui_name": "My Loki",
+        "is_generated_locally": true
+      }
     ]
+  ],
+  "type": "array",
+  "items": {
+    "type": "object",
+    "properties": {
+      "module_id": {
+        "type": "string",
+        "description": "Original module ID value."
+      },
+      "module_ui_name": {
+        "type": "string",
+        "description": "Original module label, assigned by the user."
+      },
+      "node_fqdn": {
+        "type": "string",
+        "description": "The FQDN of the node where the module of the backup is hosted."
+      },
+      "path": {
+        "type": "string",
+        "description": "Path of the repository, relative to the backup destination."
+      },
+      "name": {
+        "type": "string",
+        "description": "Name of the module. It is equal to the module image name."
+      },
+      "uuid": {
+        "type": "string",
+        "description": "Universal, unique identifier of the module instance."
+      },
+      "timestamp": {
+        "type": "integer",
+        "description": "Unix timestamp of the last backup run."
+      },
+      "repository_id": {
+        "type": "string",
+        "description": "UUID of the backup destination."
+      },
+      "repository_name": {
+        "type": "string",
+        "description": "Human readable name of the backup destination."
+      },
+      "repository_provider": {
+        "type": "string",
+        "description": "Type of backup destination provider, e.g. SMB, S3..."
+      },
+      "repository_url": {
+        "type": "string",
+        "description": "Restic URL of the backup destination."
+      },
+      "installed_instance": {
+        "type": "string",
+        "description": "If the backup belongs to an installed module instance this is its module ID."
+      },
+      "installed_instance_ui_name": {
+        "type": "string",
+        "description": "If the backup belongs to an installed module instance this is its module friendly name."
+      },
+      "is_generated_locally": {
+        "type": [
+          "boolean",
+          "null"
+        ],
+        "description": "Tells if the backup originates from the local cluster or from another cluster. The null value is returned if this information is missing completely, as it happens in old backups."
+      }
+    },
+    "required": [
+      "module_id",
+      "module_ui_name",
+      "node_fqdn",
+      "path",
+      "name",
+      "uuid",
+      "timestamp",
+      "repository_id",
+      "repository_name",
+      "repository_provider",
+      "repository_url",
+      "installed_instance",
+      "installed_instance_ui_name",
+      "is_generated_locally"
+    ]
+  }
 }