From 03db6c59c7de076fde43d640f1578b7233541c32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Bari=C5=A1i=C4=87?= Date: Thu, 28 Mar 2024 16:55:44 +0100 Subject: [PATCH 1/6] Changes to prepare for debian-12 base image and toolchain v5 --- .github/workflows/docker_publish.yml | 4 ++-- .github/workflows/test.yml | 15 ++++++++++----- .github/workflows/test_no_ml.yml | 15 ++++++++++----- Dockerfile | 13 +++++++------ Dockerfile.no_ML | 9 +++++---- Dockerfile.release | 11 ++++++----- python/requirements.txt | 2 +- 7 files changed, 41 insertions(+), 28 deletions(-) diff --git a/.github/workflows/docker_publish.yml b/.github/workflows/docker_publish.yml index 1bddb988c..831d0838a 100644 --- a/.github/workflows/docker_publish.yml +++ b/.github/workflows/docker_publish.yml @@ -89,8 +89,8 @@ jobs: - name: Download memgraph binary run: | - curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.memgraph_version }}/debian-11/memgraph_${{ github.event.inputs.memgraph_version }}-1_amd64.deb > memgraph-amd64.deb - curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.memgraph_version }}/debian-11-aarch64/memgraph_${{ github.event.inputs.memgraph_version }}-1_arm64.deb > memgraph-arm64.deb + curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.memgraph_version }}/debian-12/memgraph_${{ github.event.inputs.memgraph_version }}-1_amd64.deb > memgraph-amd64.deb + curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.memgraph_version }}/debian-12-aarch64/memgraph_${{ github.event.inputs.memgraph_version }}-1_arm64.deb > memgraph-arm64.deb - name: Build & push prod docker images run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5b61332e2..0b837dfef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,7 +1,7 @@ name: Build and Test env: - PY_VERSION: "3.9" + PY_VERSION: "3.11" CORE_COUNT: "8" MAGE_CONTAINER: "mage" MEMGRAPH_PORT: 7687 @@ -39,12 +39,17 @@ jobs: - name: Download memgraph binaries run: | if [ "${{ env.OFFICIAL }}" = "true" ]; then - curl -L "https://download.memgraph.com/memgraph/v${MEMGRAPH_VERSION}/debian-11/memgraph_${MEMGRAPH_VERSION}-1_amd64.deb" > memgraph-amd64.deb - curl -L "https://download.memgraph.com/memgraph/v${MEMGRAPH_VERSION}/debian-11-aarch64/memgraph_${MEMGRAPH_VERSION}-1_arm64.deb" > memgraph-arm64.deb + curl -L "https://download.memgraph.com/memgraph/v${MEMGRAPH_VERSION}/debian-12/memgraph_${MEMGRAPH_VERSION}-1_amd64.deb" > memgraph-amd64.deb + curl -L "https://download.memgraph.com/memgraph/v${MEMGRAPH_VERSION}/debian-12-aarch64/memgraph_${MEMGRAPH_VERSION}-1_arm64.deb" > memgraph-arm64.deb else sudo apt update && sudo apt install -y ca-certificates wget git - wget -q https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/memgraph-unofficial/memgraph_${MEMGRAPH_VERSION}-1_amd64.deb -O memgraph-amd64.deb - wget -q https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/memgraph-unofficial/memgraph_${MEMGRAPH_VERSION}-1_arm64.deb -O memgraph-arm64.deb + wget -q \ + https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/memgraph-unofficial/v${MEMGRAPH_VERSION}/debian-12/memgraph_${MEMGRAPH_VERSION}-1_amd64.deb \ + -O memgraph-amd64.deb + wget -q \ + https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/memgraph-unofficial/v${MEMGRAPH_VERSION}/debian-12-aarch64/memgraph_${MEMGRAPH_VERSION}-1_arm64.deb \ + -O memgraph-arm64.deb + fi fi - name: Disk status before cleaning diff --git a/.github/workflows/test_no_ml.yml b/.github/workflows/test_no_ml.yml index b3d71b4dd..0b99fb31f 100644 --- a/.github/workflows/test_no_ml.yml +++ b/.github/workflows/test_no_ml.yml @@ -1,7 +1,7 @@ name: Build and Test no env: - PY_VERSION: "3.9" + PY_VERSION: "3.11" CORE_COUNT: "8" MAGE_CONTAINER: "mage" MEMGRAPH_PORT: 7687 @@ -39,12 +39,17 @@ jobs: - name: Download memgraph binaries run: | if [ "${{ env.OFFICIAL }}" = "true" ]; then - curl -L "https://download.memgraph.com/memgraph/v${MEMGRAPH_VERSION}/debian-11/memgraph_${MEMGRAPH_VERSION}-1_amd64.deb" > memgraph-amd64.deb - curl -L "https://download.memgraph.com/memgraph/v${MEMGRAPH_VERSION}/debian-11-aarch64/memgraph_${MEMGRAPH_VERSION}-1_arm64.deb" > memgraph-arm64.deb + curl -L "https://download.memgraph.com/memgraph/v${MEMGRAPH_VERSION}/debian-12/memgraph_${MEMGRAPH_VERSION}-1_amd64.deb" > memgraph-amd64.deb + curl -L "https://download.memgraph.com/memgraph/v${MEMGRAPH_VERSION}/debian-12-aarch64/memgraph_${MEMGRAPH_VERSION}-1_arm64.deb" > memgraph-arm64.deb else sudo apt update && sudo apt install -y ca-certificates wget git - wget -q https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/memgraph-unofficial/memgraph_${MEMGRAPH_VERSION}-1_amd64.deb -O memgraph-amd64.deb - wget -q https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/memgraph-unofficial/memgraph_${MEMGRAPH_VERSION}-1_arm64.deb -O memgraph-arm64.deb + wget -q \ + https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/memgraph-unofficial/v${MEMGRAPH_VERSION}/debian-12/memgraph_${MEMGRAPH_VERSION}-1_amd64.deb \ + -O memgraph-amd64.deb + wget -q \ + https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/memgraph-unofficial/v${MEMGRAPH_VERSION}/debian-12-aarch64/memgraph_${MEMGRAPH_VERSION}-1_arm64.deb \ + -O memgraph-arm64.deb + fi fi - name: Disk status before cleaning diff --git a/Dockerfile b/Dockerfile index 8b42d9200..1d1e532fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -ARG PY_VERSION_DEFAULT=3.9 +ARG PY_VERSION_DEFAULT=3.11 -FROM debian:bullseye as base +FROM debian:bookworm as base USER root @@ -13,6 +13,7 @@ ENV PY_VERSION ${PY_VERSION_DEFAULT} RUN apt-get update && apt-get install -y \ libcurl4 `memgraph` \ libpython${PY_VERSION} `memgraph` \ + libssl3 `memgraph` \ libssl-dev `memgraph` \ openssl `memgraph` \ build-essential `mage-memgraph` \ @@ -30,7 +31,7 @@ RUN apt-get update && apt-get install -y \ libboost-all-dev `mage-memgraph` \ --no-install-recommends \ # Download and install Memgraph - && curl https://download.memgraph.com/memgraph/v${MG_VERSION}/debian-11/memgraph_${MG_VERSION}-1_amd64.deb --output memgraph.deb \ + && curl https://download.memgraph.com/memgraph/v${MG_VERSION}/debian-12/memgraph_${MG_VERSION}-1_amd64.deb --output memgraph.deb \ && dpkg -i memgraph.deb \ && rm memgraph.deb \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -48,9 +49,9 @@ COPY . /mage #MAGE RUN curl https://sh.rustup.rs -sSf | sh -s -- -y \ && export PATH="/root/.cargo/bin:${PATH}" \ - && python3 -m pip install -r /mage/python/requirements.txt \ - && python3 -m pip install -r /mage/python/tests/requirements.txt \ - && python3 -m pip install torch-sparse torch-cluster torch-spline-conv torch-geometric torch-scatter -f https://data.pyg.org/whl/torch-1.12.0+cu102.html\ + && python3 -m pip install --break-system-packages -r /mage/python/requirements.txt \ + && python3 -m pip install --break-system-packages -r /mage/python/tests/requirements.txt \ + && python3 -m pip install --break-system-packages torch-sparse torch-cluster torch-spline-conv torch-geometric torch-scatter -f https://data.pyg.org/whl/torch-1.12.0+cu102.html\ && python3 /mage/setup build -p /usr/lib/memgraph/query_modules/ #DGL build from source diff --git a/Dockerfile.no_ML b/Dockerfile.no_ML index 344767542..6e7f33d5f 100644 --- a/Dockerfile.no_ML +++ b/Dockerfile.no_ML @@ -1,6 +1,6 @@ -ARG PY_VERSION_DEFAULT=3.9 +ARG PY_VERSION_DEFAULT=3.11 -FROM debian:bullseye as base +FROM debian:bookworm as base USER root @@ -12,6 +12,7 @@ ENV PY_VERSION ${PY_VERSION_DEFAULT} RUN apt-get update && apt-get install -y \ libcurl4 `memgraph` \ libpython${PY_VERSION} `memgraph` \ + libssl3 `memgraph` \ libssl-dev `memgraph` \ openssl `memgraph` \ build-essential `mage-memgraph` \ @@ -47,8 +48,8 @@ COPY . /mage #MAGE RUN curl https://sh.rustup.rs -sSf | sh -s -- -y \ && export PATH="/root/.cargo/bin:${PATH}" \ - && python3 -m pip install -r /mage/python/tests/requirements.txt \ - && python3 -m pip install -r /mage/python/requirements_no_ml.txt \ + && python3 -m pip install --break-system-packages -r /mage/python/tests/requirements.txt \ + && python3 -m pip install --break-system-packages -r /mage/python/requirements_no_ml.txt \ && python3 /mage/setup build -p /usr/lib/memgraph/query_modules/ --no-ml \ && chown -R memgraph: /mage/e2e diff --git a/Dockerfile.release b/Dockerfile.release index fc3f35eb5..9f1acace6 100644 --- a/Dockerfile.release +++ b/Dockerfile.release @@ -1,6 +1,6 @@ -ARG PY_VERSION_DEFAULT=3.9 +ARG PY_VERSION_DEFAULT=3.11 -FROM debian:bullseye as base +FROM debian:bookworm as base USER root @@ -12,6 +12,7 @@ ENV PY_VERSION ${PY_VERSION_DEFAULT} RUN apt-get update && apt-get install -y \ libcurl4 `memgraph` \ libpython${PY_VERSION} `memgraph` \ + libssl3 `memgraph` \ libssl-dev `memgraph` \ openssl `memgraph` \ build-essential `mage-memgraph` \ @@ -49,9 +50,9 @@ COPY . /mage #MAGE RUN curl https://sh.rustup.rs -sSf | sh -s -- -y \ && export PATH="/root/.cargo/bin:${PATH}" \ - && python3 -m pip install -r /mage/python/requirements.txt \ - && python3 -m pip install -r /mage/python/tests/requirements.txt \ - && python3 -m pip install torch-sparse torch-cluster torch-spline-conv torch-geometric torch-scatter -f https://data.pyg.org/whl/torch-1.12.0+cu102.html \ + && python3 -m pip install --break-system-packages -r /mage/python/requirements.txt \ + && python3 -m pip install --break-system-packages -r /mage/python/tests/requirements.txt \ + && python3 -m pip install --break-system-packages torch-sparse torch-cluster torch-spline-conv torch-geometric torch-scatter -f https://data.pyg.org/whl/torch-1.12.0+cu102.html \ && python3 /mage/setup build -p /usr/lib/memgraph/query_modules/ \ && chown -R memgraph: /mage/e2e diff --git a/python/requirements.txt b/python/requirements.txt index ceecc9dfe..24baee05e 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,5 +1,5 @@ gekko==0.2.8 -networkx==2.6.2 +networkx==3.2.1 python-Levenshtein==0.12.1 elasticsearch==8.4.3 gensim==4.0.0 From 17894a07dd73fc3c1c2d1896daae006a3f599d03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Bari=C5=A1i=C4=87?= Date: Fri, 29 Mar 2024 10:26:26 +0100 Subject: [PATCH 2/6] Fix dependecies --- Dockerfile.release | 2 +- python/requirements.txt | 2 +- python/tests/requirements.txt | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile.release b/Dockerfile.release index 9f1acace6..2e984a97e 100644 --- a/Dockerfile.release +++ b/Dockerfile.release @@ -52,7 +52,7 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- -y \ && export PATH="/root/.cargo/bin:${PATH}" \ && python3 -m pip install --break-system-packages -r /mage/python/requirements.txt \ && python3 -m pip install --break-system-packages -r /mage/python/tests/requirements.txt \ - && python3 -m pip install --break-system-packages torch-sparse torch-cluster torch-spline-conv torch-geometric torch-scatter -f https://data.pyg.org/whl/torch-1.12.0+cu102.html \ + && python3 -m pip install --break-system-packages torch-sparse torch-cluster torch-spline-conv torch-geometric torch-scatter -f https://data.pyg.org/whl/torch-1.13.0+cu102.html \ && python3 /mage/setup build -p /usr/lib/memgraph/query_modules/ \ && chown -R memgraph: /mage/e2e diff --git a/python/requirements.txt b/python/requirements.txt index 24baee05e..26a6a5239 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -3,7 +3,7 @@ networkx==3.2.1 python-Levenshtein==0.12.1 elasticsearch==8.4.3 gensim==4.0.0 -torch==1.12.0 +torch==1.13.0 six==1.16.0 torchmetrics==0.9.3 igraph==0.10.2 diff --git a/python/tests/requirements.txt b/python/tests/requirements.txt index 42f6a1190..38705d7ee 100644 --- a/python/tests/requirements.txt +++ b/python/tests/requirements.txt @@ -1,5 +1,5 @@ pytest==7.0 -PyYAML==5.4.1 +PyYAML==6.0.1 black==22.3.0 flake8==3.9.2 pymgclient==1.3.1 @@ -9,4 +9,4 @@ pytest-cov==2.12.1 pytest-benchmark==3.4.1 pytest-flake8==1.0.7 pytest-black==0.3.12 -gqlalchemy==1.4.1 \ No newline at end of file +gqlalchemy==1.4.1 From 3ca658e5cd72a6b238e4633981c9e12bda4a1f53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Bari=C5=A1i=C4=87?= Date: Fri, 29 Mar 2024 13:55:04 +0100 Subject: [PATCH 3/6] Remove versions from requirement files --- python/requirements.txt | 30 +++++++++++++++--------------- python/requirements_no_ml.txt | 26 +++++++++++++------------- python/tests/requirements.txt | 24 ++++++++++++------------ 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/python/requirements.txt b/python/requirements.txt index 26a6a5239..a5fe8f166 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,15 +1,15 @@ -gekko==0.2.8 -networkx==3.2.1 -python-Levenshtein==0.12.1 -elasticsearch==8.4.3 -gensim==4.0.0 -torch==1.13.0 -six==1.16.0 -torchmetrics==0.9.3 -igraph==0.10.2 -scikit-learn==0.24.2 -gqlalchemy==1.4.1 -mysql-connector-python==8.0.32 -oracledb==1.2.2 -pyodbc==4.0.35 -defusedxml==0.7.1 +gekko +networkx +python-Levenshtein +elasticsearch +gensim +torch +six +torchmetrics +igraph +scikit-learn +gqlalchemy +mysql-connector-python +oracledb +pyodbc +defusedxml diff --git a/python/requirements_no_ml.txt b/python/requirements_no_ml.txt index 65f2dde40..ded6ab0c1 100644 --- a/python/requirements_no_ml.txt +++ b/python/requirements_no_ml.txt @@ -1,13 +1,13 @@ -gekko==0.2.8 -networkx==2.6.2 -python-Levenshtein==0.12.1 -elasticsearch==8.4.3 -gensim==4.0.0 -six==1.16.0 -igraph==0.10.2 -scikit-learn==0.24.2 -gqlalchemy==1.4.1 -mysql-connector-python==8.0.32 -oracledb==1.2.2 -pyodbc==4.0.35 -defusedxml==0.7.1 +gekko +networkx +python-Levenshtein +elasticsearch +gensim +six +igraph +scikit-learn +gqlalchemy +mysql-connector-python +oracledb +pyodbc +defusedxml diff --git a/python/tests/requirements.txt b/python/tests/requirements.txt index 38705d7ee..9f433feb8 100644 --- a/python/tests/requirements.txt +++ b/python/tests/requirements.txt @@ -1,12 +1,12 @@ -pytest==7.0 -PyYAML==6.0.1 -black==22.3.0 -flake8==3.9.2 -pymgclient==1.3.1 -pytest-pylint==0.21.0 -pytest-timeout==1.4.2 -pytest-cov==2.12.1 -pytest-benchmark==3.4.1 -pytest-flake8==1.0.7 -pytest-black==0.3.12 -gqlalchemy==1.4.1 +pytest +PyYAML +black +flake8 +pymgclient +pytest-pylint +pytest-timeout +pytest-cov +pytest-benchmark +pytest-flake8 +pytest-black +gqlalchemy From efadf6acf7d5ea80c28dca2c13f60777628c33e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Bari=C5=A1i=C4=87?= Date: Fri, 29 Mar 2024 16:38:31 +0100 Subject: [PATCH 4/6] Bump memgraph version --- .github/workflows/test.yml | 2 +- .github/workflows/test_no_ml.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0b837dfef..2d3cec325 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest continue-on-error: True env: - MEMGRAPH_VERSION: 2.14.1 + MEMGRAPH_VERSION: 2.15.1 strategy: matrix: architecture: ["amd64", "arm64"] diff --git a/.github/workflows/test_no_ml.yml b/.github/workflows/test_no_ml.yml index 0b99fb31f..40fdb6aa5 100644 --- a/.github/workflows/test_no_ml.yml +++ b/.github/workflows/test_no_ml.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest continue-on-error: True env: - MEMGRAPH_VERSION: 2.14.1 + MEMGRAPH_VERSION: 2.15.1 strategy: matrix: architecture: ["amd64", "arm64"] From 05434647dc7aa3b6321ca918eee04845ee847efe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Bari=C5=A1i=C4=87?= Date: Fri, 29 Mar 2024 16:50:00 +0100 Subject: [PATCH 5/6] Fix fi typo --- .github/workflows/test.yml | 3 +-- .github/workflows/test_no_ml.yml | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2d3cec325..ebd8d0d3d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest continue-on-error: True env: - MEMGRAPH_VERSION: 2.15.1 + MEMGRAPH_VERSION: 2.14.1 strategy: matrix: architecture: ["amd64", "arm64"] @@ -51,7 +51,6 @@ jobs: -O memgraph-arm64.deb fi - fi - name: Disk status before cleaning run: | df -h diff --git a/.github/workflows/test_no_ml.yml b/.github/workflows/test_no_ml.yml index 40fdb6aa5..b9637b8a4 100644 --- a/.github/workflows/test_no_ml.yml +++ b/.github/workflows/test_no_ml.yml @@ -51,7 +51,6 @@ jobs: -O memgraph-arm64.deb fi - fi - name: Disk status before cleaning run: | df -h From f9455c43550b2d44784ae793be0089ced8ea4799 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Bari=C5=A1i=C4=87?= Date: Fri, 29 Mar 2024 16:53:51 +0100 Subject: [PATCH 6/6] Bump memgraph version in test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ebd8d0d3d..3591a01e3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest continue-on-error: True env: - MEMGRAPH_VERSION: 2.14.1 + MEMGRAPH_VERSION: 2.15.1 strategy: matrix: architecture: ["amd64", "arm64"]