Rebuild Dockerfile/docker-compose files

- Using stable python:3.11.5-slim-bullseye image
- Move Dockerfiles to ckan-pycsw folder.
- Create GHCR files (Github Registry Container images).
- Add PYCSW_DEV_PORT to avoid hardcode the dev_port.
- Update README

.env.example

@@ -1,4 +1,5 @@
 PYCSW_PORT=8000
+PYCSW_DEV_PORT=5678
 
 # URLS
 CKAN_URL=http://localhost:5000/

.github/workflows/docker-dev.yaml

@@ -14,24 +14,26 @@ on:
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}-dev
+  CONTEXT: .
+  DOCKERFILE: ./ckan-pycsw/Dockerfile.dev
 
 jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       -
         name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       -
         name: Login to registry
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -44,10 +46,18 @@ jobs:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
       -
         name: Build and push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          context: .
-          file: ./ckan-pycsw/Dockerfile.dev
+          context: ${{ env.CONTEXT }}
+          file: ${{ env.DOCKERFILE }}
+
+      - 
+        name: Scan image
+        uses: ISID/[email protected]
+        with:
+          tag: ${{ env.IMAGE_NAME }}
+          path: ${{ env.CONTEXT }}
+          dockerfile: ${{ env.DOCKERFILE }}

.github/workflows/docker.yaml

@@ -23,17 +23,17 @@ jobs:
     steps:
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       -
         name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       -
         name: Login to registry
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -46,7 +46,7 @@ jobs:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
       -
         name: Build and push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
@@ -56,8 +56,8 @@ jobs:
 
       - 
         name: Scan image
-        uses: ISID/build-and-scan-image@main
+        uses: ISID/build-and-scan-image@v0.4.2
         with:
           tag: ${{ env.IMAGE_NAME }}
           path: ${{ env.CONTEXT }}
-          dockerfile: ${{ env.DOCKERFILE }}
+          dockerfile: ${{ env.DOCKERFILE }}

README.md

@@ -51,14 +51,19 @@ Select the CKAN Schema (`PYCSW_CKAN_SCHEMA`), and the pycsw output schema (`PYCS
 >**Note**<br>
 > The output pycsw schema (`iso19139_inspire`), to comply with INSPIRE ISO 19139 is WIP. The validation of the dataset/series is complete and conforms to the [INSPIRE reference validator](https://inspire.ec.europa.eu/validator/home/index.html) datasets and dataset series (Conformance Class 1, 2, 2b and 2c). In contrast, spatial data services still fail in only 1 dimension [WIP]. 
 
-To deploy the environment, `docker compose` will build the latest image ([`ghcr.io/mjanez/ckan-pycsw:latest`](https://github.com/mjanez/ckan-pycsw/pkgs/container/ckan-pycsw)).
+To deploy the environment, `docker compose` will build the latest source in the repo.
+
+If you can deploy a `5 minutes` image, use the latest ([`ghcr.io/mjanez/ckan-pycsw:latest`](https://github.com/mjanez/ckan-pycsw/pkgs/container/ckan-pycsw)) with [`docker-compose.ghcr.yml`](/docker-compose.ghcr.yml)
 
 ```bash
 git clone https://github.com/mjanez/ckan-pycsw
 cd ckan-pycsw
 
 docker compose up --build
 
+# Github latest registry image
+docker compose -f docker-compose.ghcr.yml --build
+
 # Or detached mode
 docker compose up -d --build
 ```
@@ -89,7 +94,7 @@ pdm install --no-self
 
 Configuration:
 ```bash
-PYCSW_URL=http://localhost:8000 envsubst < pycsw/conf/pycsw.conf.template > pycsw.conf
+PYCSW_URL=http://localhost:8000 envsubst < ckan-pycsw/conf/pycsw.conf.template > pycsw.conf
 ```
 
 Generate database:
@@ -218,9 +223,18 @@ Perform a `GetRecords` request and return all:
 
 ## Debug
 ### VSCode
+#### Python debugger with Docker
 1. Build and run container.
-2. Attach Visual Studio Code to container
-3. Start debugging on `ckan2pycsw.py` Python file (`Debug the currently active Python file`).
+2. Attach Visual Studio Code to container.
+3. Start debugging on `ckan2pycsw.py` Python file (`Debug the currently active Python file`) in the container.
+
+#### Python debugger without Docker
+1. Update the previously created `.env` file in the root of the `ckan-ogc` repo and move it to: [`/ckan2pycsw`](/ckan2pycsw)
+2. Open [`ckan2pycsw.py`](/ckan2pycsw/ckan2pycsw.py).
+3. Start debugging on `ckan2pycsw.py` Python file (`Debug the currently active Python file`). 
+
+>**Note**<br
+> By default, the Python extension looks for and loads a file named `.env` in the current workspace folder. More info about Python debugger and [Enviromental variables use](https://code.visualstudio.com/docs/python/environments#_environment-variables).
 
 ## Containers
 List of *containers*:
@@ -236,9 +250,10 @@ List of *containers*:
 ### Built images
 | Repository | Type | Docker tag | Size | Notes |
 | --- | --- | --- | --- | --- |
-| mjanez/ckan-pycsw| custom image | `mjanez/ckan-pycsw:v*.*.*` | 175 MB |  Tag version. |
+| mjanez/ckan-pycsw| custom image | `mjanez/ckan-pycsw:latest-dev` | 175 MB |  Latest stable version from Registry. |
+| mjanez/ckan-pycsw| custom image | `mjanez/ckan-pycsw:main-dev` | 175 MB |  Dev version from Registry.  |
 | mjanez/ckan-pycsw| custom image | `mjanez/ckan-pycsw:latest` | 175 MB |  Latest stable version. |
-| mjanez/ckan-pycsw| custom image | `mjanez/ckan-pycsw:main` | 175 MB |  Dev version.  |
+| mjanez/ckan-pycsw| custom image | `mjanez/ckan-pycsw:main` | 175 MB |  Development branch version. |
 
 ### Network ports settings
 | Ports | Container |

Dockerfile → ckan-pycsw/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11-slim-bullseye
+FROM python:3.11.5-slim-bullseye
 LABEL maintainer="[email protected]"
 
 ENV APP_DIR=/app
@@ -25,7 +25,7 @@ COPY pyproject.toml pdm.lock .
 
 RUN pdm install --no-self --group prod
 
-COPY pycsw/conf/pycsw.conf.template pycsw/entrypoint.sh .
+COPY ckan-pycsw/conf/pycsw.conf.template ckan-pycsw/docker-entrypoint.d/entrypoint.sh .
 COPY ckan2pycsw ckan2pycsw
 
 EXPOSE ${PYCSW_PORT}/TCP

Dockerfile.dev → ckan-pycsw/Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM python:3.11-slim-bullseye
+FROM python:3.11.5-slim-bullseye
 LABEL maintainer="[email protected]"
 
 ENV APP_DIR=/app
@@ -10,12 +10,13 @@ ENV CKAN_URL=http://localhost:5000/
 ENV PYCSW_PORT=8880
 ENV PYCSW_URL=http://localhost:${PYCSW_PORT}/
 ENV DEV_MODE=True
+ENV PYCSW_DEV_PORT=5678
 ENV TIMEOUT=300
 
 RUN apt-get -q -y update && \
     apt-get install -y wget procps && \
     DEBIAN_FRONTEND=noninteractive apt-get -yq install gettext-base && \
-    wget -O /wait-for https://raw.githubusercontent.com/eficode/wait-for/v2.2.3/wait-for && \
+    wget -O /wait-for https://raw.githubusercontent.com/eficode/wait-for/v2.2.4/wait-for && \
     chmod +x /wait-for && \
     python3 -m pip install pdm debugpy
 
@@ -24,12 +25,12 @@ COPY pyproject.toml pdm.lock .
 
 RUN pdm install --no-self --group prod
 
-COPY pycsw/conf/pycsw.conf.template pycsw/entrypoint_dev.sh .
+COPY ckan-pycsw/conf/pycsw.conf.template ckan-pycsw/docker-entrypoint.d/entrypoint_dev.sh .
 COPY ckan2pycsw ckan2pycsw
 
 EXPOSE ${PYCSW_PORT}/TCP
-EXPOSE 5678/TCP
+EXPOSE ${PYCSW_DEV_PORT}/TCP
 
 # Set entrypoint with debugpy
-ENTRYPOINT ["python3", "-m", "debugpy", "--listen", "0.0.0.0:5678", "--wait-for-client", "./entrypoint_dev.sh"]
+ENTRYPOINT ["/bin/bash", "-c", "python3 -m debugpy --listen 0.0.0.0:${PYCSW_DEV_PORT} --wait-for-client ./entrypoint_dev.sh"]
 CMD ["pdm", "run", "python3", "-m", "gunicorn", "pycsw.wsgi:application", "-b", "0.0.0.0:${PYCSW_PORT}"]

ckan-pycsw/Dockerfile.ghcr

@@ -0,0 +1,20 @@
+FROM ghcr.io/mjanez/ckan-pycsw:latest
+LABEL maintainer="[email protected]"
+
+ENV APP_DIR=/app
+ENV TZ=UTC
+RUN echo ${TZ} > /etc/timezone
+ENV PYCSW_CKAN_SCHEMA=iso19139_inspire
+ENV PYCSW_CONFIG=${APP_DIR}/pycsw.conf
+ENV CKAN_URL=http://localhost:5000/
+ENV PYCSW_PORT=8000
+ENV PYCSW_URL=http://localhost:${PYCSW_PORT}/
+ENV DEV_MODE=False
+ENV TIMEOUT=300
+ENV PYCSW_CRON_DAYS_INTERVAL=2
+
+WORKDIR ${APP_DIR}
+
+COPY ckan-pycsw/conf/pycsw.conf.template ckan-pycsw/docker-entrypoint.d/entrypoint.sh .
+
+ENTRYPOINT ["/bin/bash", "./entrypoint.sh"]

ckan-pycsw/Dockerfile.ghcr.dev

@@ -0,0 +1,25 @@
+FROM ghcr.io/mjanez/ckan-pycsw-dev:latest
+LABEL maintainer="[email protected]"
+
+ENV APP_DIR=/app
+ENV TZ=UTC
+RUN echo ${TZ} > /etc/timezone
+ENV PYCSW_CKAN_SCHEMA=iso19139_inspire
+ENV PYCSW_CONFIG=${APP_DIR}/pycsw.conf
+ENV CKAN_URL=http://localhost:5000/
+ENV PYCSW_PORT=8880
+ENV PYCSW_URL=http://localhost:${PYCSW_PORT}/
+ENV DEV_MODE=True
+ENV PYCSW_DEV_PORT=5678
+ENV TIMEOUT=300
+
+WORKDIR ${APP_DIR}
+
+COPY ckan-pycsw/conf/pycsw.conf.template ckan-pycsw/docker-entrypoint.d/entrypoint_dev.sh .
+
+EXPOSE ${PYCSW_PORT}/TCP
+EXPOSE ${PYCSW_DEV_PORT}/TCP
+
+# Set entrypoint with debugpy
+ENTRYPOINT ["/bin/bash", "-c", "python3 -m debugpy --listen 0.0.0.0:${PYCSW_DEV_PORT} --wait-for-client ./entrypoint_dev.sh"]
+CMD ["pdm", "run", "python3", "-m", "gunicorn", "pycsw.wsgi:application", "-b", "0.0.0.0:${PYCSW_PORT}"]

pycsw/README.md → ckan-pycsw/README.md

@@ -1,4 +1,4 @@
-# pycsw
+# ckan-pycsw
 
 This is a [base container for pycsw](https://github.com/mjanez/ckan-pycsw), an OGC CSW server implementation written in Python. For more information about pycsw got to [pycsw.org](https://pycsw.org). For the source code of pycsw got to [gepython/pycsw](https://github.com/geopython/pycsw) on GitHub.
 
@@ -17,10 +17,4 @@ This is a [base container for pycsw](https://github.com/mjanez/ckan-pycsw), an O
 
 ## Environment
 
-* `PYCSW_CONFIG` --- path to `pycsw.cfg` config file, default `/etc/pycsw/pycsw.cfg`
-
-## Usage
-
-To run the container as is
-
-    docker run -d -p 8000:8000 -v my_pycsw.cfg:/etc/pycsw/pycsw.cfg:ro pycsw
+* `PYCSW_CONFIG` --- path to `pycsw.cfg` config file, default `/etc/pycsw/pycsw.cfg`

pycsw/entrypoint_dev.sh → ckan-pycsw/docker-entrypoint.d/entrypoint_dev.sh

@@ -3,6 +3,6 @@
 set -xeuo pipefail
 
 envsubst < pycsw.conf.template > pycsw.conf
-/wait-for --timeout "$TIMEOUT" "$CKAN_URL" -- pdm run python3 -m ptvsd --host 0.0.0.0 --port 5678 --wait ckan2pycsw/ckan2pycsw.py
+/wait-for --timeout "$TIMEOUT" "$CKAN_URL" -- pdm run python3 -m ptvsd --host 0.0.0.0 --port "$PYCSW_DEV_PORT" --wait ckan2pycsw/ckan2pycsw.py
 
 exec "$@"

ckan2pycsw/ckan2pycsw.py

@@ -41,6 +41,7 @@
 URL = os.environ["CKAN_URL"]
 PYCSW_URL = os.environ["PYCSW_URL"]
 PYCSW_PORT = os.environ["PYCSW_PORT"]
+PYCSW_DEV_PORT = os.environ["PYCSW_DEV_PORT"]
 APP_DIR = os.environ.get("APP_DIR", "/app")
 CKAN_API = "api/3/action/package_search"
 PYCSW_CKAN_SCHEMA = os.environ.get("PYCSW_CKAN_SCHEMA", "iso19139_geodcatap")
@@ -214,9 +215,9 @@ def run_tasks():
         logging.error(f"{log_module}:ckan2pycsw | Error starting gunicorn: {e}")
 
 if __name__ == "__main__":
-    if DEV_MODE == True or DEV_MODE == "True":
+    if DEV_MODE == True or DEV_MODE.lower() == "true":
         # Allow other computers to attach to ptvsd at this IP address and port.
-        ptvsd.enable_attach(address=("0.0.0.0", 5678), redirect_output=True)
+        ptvsd.enable_attach(address=("0.0.0.0", PYCSW_DEV_PORT), redirect_output=True)
 
         # Pause the program until a remote debugger is attached
         ptvsd.wait_for_attach()

docker-compose.dev.yml

@@ -1,9 +1,11 @@
-name: ckan2pycsw
+version: "3"
+
 services:
-  pycsw:
+  ckan2pycsw:
+    container_name: ckan-pycsw-dev
     build:
       context: .
-      dockerfile: Dockerfile.dev
+      dockerfile: ckan-pycsw/Dockerfile.dev
     env_file:
       - .env
     logging:
@@ -14,10 +16,11 @@ services:
     ports:
       - "0.0.0.0:${PYCSW_PORT}:${PYCSW_PORT}"
       # debugpy
-      - "5678:5678"
+      - "${PYCSW_DEV_PORT}:${PYCSW_DEV_PORT}"
     volumes:
         - ./log:${APP_DIR}/log
         - ./metadata:${APP_DIR}/metadata
+        - ./ckan2pycsw:${APP_DIR}/ckan2pycsw
 networks:
   default:
     name: ckan2pycsw_default

docker-compose.ghcr.dev.yml

@@ -0,0 +1,26 @@
+version: "3"
+
+services:
+  ckan2pycsw:
+    container_name: ckan-pycsw-dev
+    build:
+      context: .
+      dockerfile: ckan-pycsw/Dockerfile.ghcr.dev
+    env_file:
+      - .env
+    logging:
+      driver: "json-file"
+      options:
+          max-size: "100m"
+          max-file: "10"
+    ports:
+      - "0.0.0.0:${PYCSW_PORT}:${PYCSW_PORT}"
+      # debugpy
+      - "${PYCSW_DEV_PORT}:${PYCSW_DEV_PORT}"
+    volumes:
+        - ./log:${APP_DIR}/log
+        - ./metadata:${APP_DIR}/metadata
+        - ./ckan2pycsw:${APP_DIR}/ckan2pycsw
+networks:
+  default:
+    name: ckan2pycsw_default