Merge pull request #51 from mjanez/latest

Add dockers to test "without Docker Compose" config

.env.example

@@ -18,4 +18,16 @@ PYCSW_CRON_DAYS_INTERVAL=2
 # ckan2pycsw hour of start of the scheduler job (0-23)
 PYCSW_CRON_HOUR_START=4
 ## Timezone
-TZ=UTC
+TZ=UTC
+
+# Testing ckan-pycsw: docker/README.md
+## Containers
+CONTAINER_OS_NAME=rhel-test
+### Docker Compose OS container
+CONTAINER_OS=rhel
+CONTAINER_OS_VERSION=9
+PYCSW_CONFIG=${APP_DIR}/pycsw.conf
+## Ports
+PYCSW_HOST_PORT=8000
+SSH_HOST_PORT=2222
+SSH_PORT=22

.gitignore

@@ -9,4 +9,5 @@ log/*
 metadata/*
 !metadata/README.md
 samples
-.env
+.env
+.pdm-python

README.md

@@ -85,27 +85,38 @@ docker compose up -d --build
 
 
 ### Without Docker
+Requirements:
+- `>=` [Python 3.9](./pyproject.toml)
+
 Dependencies:
 ```bash
 python3 -m pip install --user pipx
+python3 -m pipx ensurepath --force
+
+# You will need to open a new terminal or re-login for the PATH changes to take effect.
 pipx install pdm
-pdm install --no-self
+pdm install --no-self --group prod
 ```
 
 Configuration:
 ```bash
 PYCSW_URL=http://localhost:8000 envsubst < ckan-pycsw/conf/pycsw.conf.template > pycsw.conf
+
+# Or update pycsw.conf vars manually
+vi pycsw.conf
 ```
 
-Generate database:
+Generate database and add:
 ```bash
 rm -f cite.db
-CKAN_URL=http://localhost:5000 pdm run python3 ckan2pycsw/ckan2pycsw.py
+
+# Remember create and update .env vars. Next add to .env environment:
+bash doc/scripts/00_ennvars.sh
 ```
 
-Run:
+Run ckan2pycsw:
 ```bash
-PYCSW_CONFIG=pycsw.conf pdm run python -m pycsw.wsgi
+PYCSW_CONFIG=pycsw.conf pdm run python3 ckan2pycsw/ckan2pycsw.py
 ```
 
 ## Schema development

ckan2pycsw/ckan2pycsw.py

@@ -46,7 +46,7 @@
 CKAN_API = "api/3/action/package_search"
 PYCSW_CKAN_SCHEMA = os.environ.get("PYCSW_CKAN_SCHEMA", "iso19139_geodcatap")
 PYCSW_OUPUT_SCHEMA = os.environ.get("PYCSW_OUPUT_SCHEMA", "iso19139_inspire")
-DEV_MODE = os.environ.get("DEV_MODE", None)
+DEV_MODE = os.environ.get("DEV_MODE", False)
 PYCSW_CONF = f"{APP_DIR}/pycsw.conf.template" if DEV_MODE == "True" else "pycsw.conf"
 MAPPINGS_FOLDER = "ckan2pycsw/mappings"
 log_module = "[ckan2pycsw]"
@@ -215,7 +215,7 @@ def run_tasks():
         logging.error(f"{log_module}:ckan2pycsw | Error starting gunicorn: {e}")
 
 if __name__ == "__main__":
-    if DEV_MODE == True or DEV_MODE.lower() == "true":
+    if str(DEV_MODE).lower() == "true":
         # Allow other computers to attach to ptvsd at this IP address and port.
         ptvsd.enable_attach(address=("0.0.0.0", PYCSW_DEV_PORT), redirect_output=True)
 

doc/scripts/00_ennvars.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+echo "[00_load_envvars] Loading environment variables..."
+set -o allexport
+# Convert .env line endings to Unix style
+sed -i 's/\r$//' .env
+source <(grep -v '^#' .env | xargs -0)
+set +o allexport
+
+# Add environment variables to .bashrc
+while IFS= read -r line
+do
+    if [[ ! $line =~ ^# && $line = *[!\ ]* ]]; then
+        echo "export $line" >> $HOME/.bashrc
+    fi
+done < .env
+
+echo "[00_load_envvars] Environment variables loaded."

docker-compose.test-docker.yml

@@ -0,0 +1,24 @@
+version: '3'
+
+services:
+  ckan2pycsw_test:
+    container_name: ${CONTAINER_OS_NAME}
+    build:
+      context: .
+      dockerfile: docker/${CONTAINER_OS}/${CONTAINER_OS}-${CONTAINER_OS_VERSION}/Dockerfile
+    env_file:
+      - .env
+    logging:
+      driver: "json-file"
+      options:
+          max-size: "100m"
+          max-file: "10"
+    ports:
+      - ${PYCSW_HOST_PORT}:${PYCSW_PORT}
+      - ${SSH_HOST_PORT}:${SSH_PORT}
+    volumes:
+        - ./log:${APP_DIR}/ckan-pycsw/log
+        - ./metadata:${APP_DIR}/ckan-pycsw/metadata
+networks:
+  default:
+    name: ckan2pycsw-test

docker/README.md

@@ -0,0 +1,24 @@
+# ckan-pycsw testing Environments
+This repository contains a `docker` directory with various Dockerfiles for different environments such as RHEL, SUSE, Debian, etc. These Dockerfiles are designed to test the `ckan-pycsw` software in a clean, isolated, and volatile environment.
+
+## Why Docker?
+Docker allows us to create lightweight and isolated environments, known as containers, where we can run our software with all its dependencies. This makes it easy to test our software in different environments without having to install and configure each environment manually.
+
+## What is ckan-pycsw?
+`ckan-pycsw` is a software that allows CKAN data portals to publish metadata to CSW catalogs. It is written in Python 3.
+
+## How to use these Dockerfiles?
+To use these Dockerfiles, you need to have Docker installed on your machine. Once you have Docker installed, you can build a Docker image for a specific environment and run a container from that image.
+
+Here is an example of how to build a Docker image for Debian and run a container from that image:
+
+```bash
+# Edit .env vars and select OS
+vi .env
+
+# To build the images:
+docker compose -f docker-compose.test-docker.yml build
+
+# To start the container:
+docker compose -f docker-compose.test-docker.yml up
+```

docker/rhel/rhel-8/Dockerfile

@@ -0,0 +1,59 @@
+FROM registry.access.redhat.com/ubi8/ubi:latest
+LABEL maintainer="mjanez" \
+      name="ckan-pycsw" \
+      version="rhel-8"
+
+# Set up environment variables
+ENV PYCSW_PORT=8000
+ENV CKAN_USER=ckan
+ENV CKAN_GROUP=ckan
+ENV CKAN_USER_SSH_PWD=ckan
+ENV SSH_PORT=22
+ENV APP_DIR=/app
+
+# Install necessary packages and install Ansible
+RUN dnf update -y && \
+    # Basic packages
+    dnf -y install sudo nano git wget gcc openssh-server openssl-devel libffi-devel make automake cmake \
+    # Python dependencies
+    python3 python3-pip libpq && \
+    dnf clean all && \
+    # Update pip
+    pip3 install --upgrade pip && \
+    # Install Ansible
+    pip3 install --no-cache-dir virtualenv && \
+    pip3 install --no-cache-dir ansible && \
+    # Create ckan user/group with sudo access
+    groupadd -g 92 ckan && \
+    useradd -u 92 -m -d /home/ckan -s /bin/bash -g ckan ckan && \
+    echo "ckan:${CKAN_USER_SSH_PWD}" | chpasswd && \
+    echo "ckan ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/ckan && \
+    chmod 0440 /etc/sudoers.d/ckan && \
+    # Clean up
+    dnf clean all
+
+# SSH setup
+RUN mkdir /var/run/sshd && \
+    # Configure SSH for non-root public key authentication
+    sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \
+    sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd && \
+    # Generate host keys
+    ssh-keygen -A
+
+# Copy the public key
+RUN if [ -f .ssh/keys/ssh.pub ]; then \
+    cp .ssh/keys/ssh.pub /home/ckan/.ssh/authorized_keys && \
+    chown ckan:ckan /home/ckan/.ssh/authorized_keys && \
+    chmod 600 /home/ckan/.ssh/authorized_keys; \
+fi
+
+# Clone the git repository
+RUN git clone https://github.com/mjanez/ckan-pycsw ${APP_DIR}
+
+# Assign all of app_dir to the ckan user
+RUN chown -R ${CKAN_USER}:${CKAN_GROUP} ${APP_DIR}
+
+EXPOSE ${PYCSW_PORT} ${SSH_PORT} 
+
+# Keep container running starts SSHD
+CMD ["/usr/sbin/sshd", "-D"]

docker/rhel/rhel-9/Dockerfile

@@ -0,0 +1,59 @@
+FROM registry.access.redhat.com/ubi9/ubi:latest
+LABEL maintainer="mjanez" \
+      name="ckan-pycsw" \
+      version="rhel-9"
+
+# Set up environment variables
+ENV PYCSW_PORT=8000
+ENV CKAN_USER=ckan
+ENV CKAN_GROUP=ckan
+ENV CKAN_USER_SSH_PWD=ckan
+ENV SSH_PORT=22
+ENV APP_DIR=/app
+
+# Install necessary packages and install Ansible
+RUN dnf update -y && \
+    # Basic packages
+    dnf -y install sudo nano git wget gcc openssh-server openssl-devel libffi-devel make automake cmake \
+    # Python dependencies
+    python3 python3-pip libpq && \
+    dnf clean all && \
+    # Update pip
+    pip3 install --upgrade pip && \
+    # Install Ansible
+    pip3 install --no-cache-dir virtualenv && \
+    pip3 install --no-cache-dir ansible && \
+    # Create ckan user/group with sudo access
+    groupadd -g 92 ckan && \
+    useradd -u 92 -m -d /home/ckan -s /bin/bash -g ckan ckan && \
+    echo "ckan:${CKAN_USER_SSH_PWD}" | chpasswd && \
+    echo "ckan ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/ckan && \
+    chmod 0440 /etc/sudoers.d/ckan && \
+    # Clean up
+    dnf clean all
+
+# SSH setup
+RUN mkdir /var/run/sshd && \
+    # Configure SSH for non-root public key authentication
+    sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \
+    sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd && \
+    # Generate host keys
+    ssh-keygen -A
+
+# Copy the public key
+RUN if [ -f .ssh/keys/ssh.pub ]; then \
+    cp .ssh/keys/ssh.pub /home/ckan/.ssh/authorized_keys && \
+    chown ckan:ckan /home/ckan/.ssh/authorized_keys && \
+    chmod 600 /home/ckan/.ssh/authorized_keys; \
+fi
+
+# Clone the git repository
+RUN git clone https://github.com/mjanez/ckan-pycsw ${APP_DIR}
+
+# Assign all of app_dir to the ckan user
+RUN chown -R ${CKAN_USER}:${CKAN_GROUP} ${APP_DIR}
+
+EXPOSE ${PYCSW_PORT} ${SSH_PORT} 
+
+# Keep container running starts SSHD
+CMD ["/usr/sbin/sshd", "-D"]

pyproject.toml

@@ -19,7 +19,7 @@ dependencies = [
     "apscheduler>=3.10.1",
     "psutil>=5.9.5",
 ]
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 license = {text = "GPLv3"}
 [project.optional-dependencies]
 prod = [