feat(gitlab): add support for paginated project list (#681)

BREAKING CHANGE: The response of `gitlab_projects` API endpoint has been
modified to also include the pagination details.

Closes #518

docs/openapi.json

@@ -162,14 +162,73 @@
             "name": "search",
             "required": false,
             "type": "string"
+          },
+          {
+            "description": "Results page number (pagination).",
+            "in": "query",
+            "name": "page",
+            "required": false,
+            "type": "integer"
+          },
+          {
+            "description": "Number of results per page (pagination).",
+            "in": "query",
+            "name": "size",
+            "required": false,
+            "type": "integer"
           }
         ],
         "produces": [
           "application/json"
         ],
         "responses": {
           "200": {
-            "description": "This resource return all projects owned by the user on GitLab in JSON format."
+            "description": "This resource return all projects owned by the user on GitLab in JSON format.",
+            "schema": {
+              "properties": {
+                "has_next": {
+                  "type": "boolean"
+                },
+                "has_prev": {
+                  "type": "boolean"
+                },
+                "items": {
+                  "items": {
+                    "properties": {
+                      "hook_id": {
+                        "type": "integer",
+                        "x-nullable": true
+                      },
+                      "id": {
+                        "type": "integer"
+                      },
+                      "name": {
+                        "type": "string"
+                      },
+                      "path": {
+                        "type": "string"
+                      },
+                      "url": {
+                        "type": "string"
+                      }
+                    },
+                    "type": "object"
+                  },
+                  "type": "array"
+                },
+                "page": {
+                  "type": "integer"
+                },
+                "size": {
+                  "type": "integer"
+                },
+                "total": {
+                  "type": "integer",
+                  "x-nullable": true
+                }
+              },
+              "type": "object"
+            }
           },
           "403": {
             "description": "Request failed. User token not valid.",

reana_server/rest/gitlab.py

@@ -27,7 +27,7 @@
 from itsdangerous import BadData, TimedJSONWebSignatureSerializer
 from reana_commons.k8s.secrets import REANAUserSecretsStore
 from werkzeug.local import LocalProxy
-from webargs import fields
+from webargs import fields, validate
 from webargs.flaskparser import use_kwargs
 
 
@@ -187,9 +187,17 @@ def gitlab_oauth(user):  # noqa
 
 
 @blueprint.route("/gitlab/projects", methods=["GET"])
-@use_kwargs({"search": fields.Str(location="query")})
+@use_kwargs(
+    {
+        "search": fields.Str(location="query"),
+        "page": fields.Int(validate=validate.Range(min=1), location="query"),
+        "size": fields.Int(validate=validate.Range(min=1), location="query"),
+    }
+)
 @signin_required()
-def gitlab_projects(user, search: Optional[str] = None):  # noqa
+def gitlab_projects(
+    user, search: Optional[str] = None, page: int = 1, size: Optional[int] = None
+):  # noqa
     r"""Endpoint to retrieve GitLab projects.
     ---
     get:
@@ -210,11 +218,51 @@ def gitlab_projects(user, search: Optional[str] = None):  # noqa
           description: The search string to filter the project list.
           required: false
           type: string
+        - name: page
+          in: query
+          description: Results page number (pagination).
+          required: false
+          type: integer
+        - name: size
+          in: query
+          description: Number of results per page (pagination).
+          required: false
+          type: integer
       responses:
         200:
           description: >-
             This resource return all projects owned by
             the user on GitLab in JSON format.
+          schema:
+            type: object
+            properties:
+              has_next:
+                type: boolean
+              has_prev:
+                type: boolean
+              page:
+                type: integer
+              size:
+                type: integer
+              total:
+                type: integer
+                x-nullable: true
+              items:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    id:
+                      type: integer
+                    name:
+                      type: string
+                    path:
+                      type: string
+                    url:
+                      type: string
+                    hook_id:
+                      type: integer
+                      x-nullable: true
         403:
           description: >-
             Request failed. User token not valid.
@@ -255,29 +303,47 @@ def gitlab_projects(user, search: Optional[str] = None):  # noqa
             # show projects in which user is at least a `Maintainer`
             # as that's the minimum access level needed to create webhooks
             "min_access_level": 40,
-            "per_page": 100,
+            "page": page,
+            "per_page": size,
             "search": search,
             # include ancestor namespaces when matching search criteria
             "search_namespaces": "true",
             # return only basic information about the projects
             "simple": "true",
         }
 
-        response = requests.get(gitlab_url, params=params)
-        projects = dict()
-        if response.status_code == 200:
-            for gitlab_project in response.json():
+        gitlab_res = requests.get(gitlab_url, params=params)
+        if gitlab_res.status_code == 200:
+            projects = list()
+            for gitlab_project in gitlab_res.json():
                 hook_id = _get_gitlab_hook_id(gitlab_project["id"], gitlab_token)
-                projects[gitlab_project["id"]] = {
-                    "name": gitlab_project["name"],
-                    "path": gitlab_project["path_with_namespace"],
-                    "url": gitlab_project["web_url"],
-                    "hook_id": hook_id,
-                }
-            return jsonify(projects), 200
+                projects.append(
+                    {
+                        "id": gitlab_project["id"],
+                        "name": gitlab_project["name"],
+                        "path": gitlab_project["path_with_namespace"],
+                        "url": gitlab_project["web_url"],
+                        "hook_id": hook_id,
+                    }
+                )
+
+            response = {
+                "has_next": bool(gitlab_res.headers.get("x-next-page")),
+                "has_prev": bool(gitlab_res.headers.get("x-prev-page")),
+                "items": projects,
+                "page": int(gitlab_res.headers.get("x-page")),
+                "size": int(gitlab_res.headers.get("x-per-page")),
+                "total": (
+                    int(gitlab_res.headers.get("x-total"))
+                    if gitlab_res.headers.get("x-total")
+                    else None
+                ),
+            }
+
+            return jsonify(response), 200
         return (
             jsonify({"message": "Project list could not be retrieved"}),
-            response.status_code,
+            gitlab_res.status_code,
         )
     except ValueError:
         return jsonify({"message": "Token is not valid."}), 403

reana_server/utils.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of REANA.
-# Copyright (C) 2018, 2019, 2020, 2021, 2022, 2023 CERN.
+# Copyright (C) 2018, 2019, 2020, 2021, 2022, 2023, 2024 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
@@ -509,23 +509,30 @@ def _get_gitlab_hook_id(project_id, gitlab_token):
     :param project_id: Project id on GitLab.
     :param gitlab_token: GitLab token.
     """
-    reana_hook_id = None
+    # FIXME: handle pagination of results
     gitlab_hooks_url = (
         REANA_GITLAB_URL
         + "/api/v4/projects/{0}/hooks?access_token={1}".format(project_id, gitlab_token)
     )
-    response_json = requests.get(gitlab_hooks_url).json()
-    create_workflow_url = url_for("workflows.create_workflow", _external=True)
-    if response_json:
-        reana_hook_id = next(
-            (
-                hook["id"]
-                for hook in response_json
-                if hook["url"] and hook["url"] == create_workflow_url
-            ),
-            None,
+    response = requests.get(gitlab_hooks_url)
+
+    if not response.ok:
+        logging.warning(
+            f"GitLab hook request failed with status code: {response.status_code}, "
+            f"content: {response.content}"
         )
-    return reana_hook_id
+        return None
+
+    response_json = response.json()
+    create_workflow_url = url_for("workflows.create_workflow", _external=True)
+    return next(
+        (
+            hook["id"]
+            for hook in response_json
+            if hook["url"] and hook["url"] == create_workflow_url
+        ),
+        None,
+    )
 
 
 class RequestStreamWithLen(object):

tests/test_views.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of REANA.
-# Copyright (C) 2018, 2019, 2020, 2021, 2022, 2023 CERN.
+# Copyright (C) 2018, 2019, 2020, 2021, 2022, 2023, 2024 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
@@ -15,7 +15,7 @@
 from uuid import uuid4
 
 import pytest
-from flask import url_for
+from flask import Flask, url_for
 from mock import Mock, patch
 from pytest_reana.test_utils import make_mock_api_client
 from reana_db.models import User, InteractiveSessionType, RunStatus
@@ -784,3 +784,87 @@ def test_prune_workspace(app, default_user, sample_serial_workflow_in_db):
         )
         assert res.status_code == status_code
         assert "The workspace has been correctly pruned." in res.json["message"]
+
+
+def test_gitlab_projects(app: Flask, default_user):
+    """Test fetching of GitLab projects."""
+    with app.test_client() as client:
+        # token not provided
+        res = client.get("/api/gitlab/projects")
+        assert res.status_code == 401
+
+        # invalid REANA token
+        res = client.get(
+            "/api/gitlab/projects", query_string={"access_token": "invalid"}
+        )
+        assert res.status_code == 403
+
+        # missing GitLab token
+        mock_get_secret_value = Mock()
+        mock_get_secret_value.return_value = None
+        with patch(
+            "reana_commons.k8s.secrets.REANAUserSecretsStore.get_secret_value",
+            mock_get_secret_value,
+        ):
+            res = client.get(
+                "/api/gitlab/projects",
+                query_string={"access_token": default_user.access_token},
+            )
+            assert res.status_code == 401
+
+        # normal behaviour
+        mock_response_projects = Mock()
+        mock_response_projects.headers = {
+            "x-prev-page": "3",
+            "x-next-page": "",
+            "x-page": "4",
+            "x-total": "100",
+            "x-per-page": "20",
+        }
+        mock_response_projects.ok = True
+        mock_response_projects.status_code = 200
+        mock_response_projects.json.return_value = [
+            {
+                "id": 123,
+                "path_with_namespace": "abcd",
+                "web_url": "url",
+                "name": "qwerty",
+            }
+        ]
+
+        mock_response_webhook = Mock()
+        mock_response_webhook.ok = True
+        mock_response_webhook.status_code = 200
+        mock_response_webhook.json.return_value = [
+            {"id": 1234, "url": "wrong_url"},
+            {
+                "id": 456,
+                "url": "http://localhost:5000/api/workflows",
+            },
+        ]
+
+        mock_requests_get = Mock()
+        mock_requests_get.side_effect = [mock_response_projects, mock_response_webhook]
+
+        mock_get_secret_value = Mock()
+        mock_get_secret_value.return_value = "gitlab_token"
+
+        with patch("requests.get", mock_requests_get), patch(
+            "reana_commons.k8s.secrets.REANAUserSecretsStore.get_secret_value",
+            mock_get_secret_value,
+        ):
+            res = client.get(
+                "/api/gitlab/projects",
+                query_string={"access_token": default_user.access_token},
+            )
+
+        assert res.status_code == 200
+        assert res.json["has_prev"]
+        assert not res.json["has_next"]
+        assert res.json["total"] == 100
+        assert len(res.json["items"]) == 1
+        assert res.json["items"][0]["id"] == 123
+        assert res.json["items"][0]["name"] == "qwerty"
+        assert res.json["items"][0]["url"] == "url"
+        assert res.json["items"][0]["path"] == "abcd"
+        assert res.json["items"][0]["hook_id"] == 456