From 4e6e076abb848ef6edba0c7762418883a02b018b Mon Sep 17 00:00:00 2001 From: CahidArda Date: Wed, 9 Oct 2024 21:21:26 +0300 Subject: [PATCH 1/5] Revert "fix: rm retention script & workflow" This reverts commit e508e6e3e0b9a35202a0c677aa25eb6c2a09b9b2. --- .github/scripts/npm_retention.py | 96 ++++++++++++++++++++++++++++ .github/workflows/npm_retention.yaml | 40 ++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 .github/scripts/npm_retention.py create mode 100644 .github/workflows/npm_retention.yaml diff --git a/.github/scripts/npm_retention.py b/.github/scripts/npm_retention.py new file mode 100644 index 0000000..ad73123 --- /dev/null +++ b/.github/scripts/npm_retention.py @@ -0,0 +1,96 @@ +import os +import requests +from datetime import datetime, timedelta +import json +import subprocess +import re + +PACKAGE_NAME = "@upstash/vector" +DAYS_TO_KEEP = 14 +CI_VERSIONS_TO_KEEP = 5 +NPM_TOKEN = os.environ.get("NPM_TOKEN") + + +def run_npm_command(command): + try: + result = subprocess.run(command, capture_output=True, text=True, check=True) + return result.stdout.strip() + except subprocess.CalledProcessError as e: + print(f"Error running command: {e}") + print(e.stderr) + return None + + +def get_package_versions(): + output = run_npm_command(["npm", "view", PACKAGE_NAME, "versions", "--json"]) + if output: + return json.loads(output) + return [] + + +def parse_ci_version_date(version): + match = re.search(r"-(\d{14})$", version) + if match: + date_str = match.group(1) + return datetime.strptime(date_str, "%Y%m%d%H%M%S") + return None + + +def is_ci_version(version): + return bool(re.search(r"-ci\.", version)) + + +def deprecate_package_version(version): + result = run_npm_command( + [ + "npm", + "deprecate", + f"{PACKAGE_NAME}@{version}", + "This CI version has been deprecated due to retention policy", + ] + ) + if result is not None: + print(f"Successfully deprecated version: {version}") + return True + else: + print(f"Failed to deprecate version: {version}") + return False + + +def apply_retention_policy(): + versions = get_package_versions() + now = datetime.utcnow() + retention_date = now - timedelta(days=DAYS_TO_KEEP) + + ci_versions = [] + + for version in versions: + if is_ci_version(version): + ci_date = parse_ci_version_date(version) + if ci_date: + ci_versions.append((version, ci_date)) + else: + print(f"Warning: Could not parse date from CI version: {version}") + + ci_versions.sort(key=lambda x: x[1], reverse=True) + + versions_to_keep = [] + versions_to_deprecate = [] + + for version, date in ci_versions: + if len(versions_to_keep) < CI_VERSIONS_TO_KEEP or date > retention_date: + versions_to_keep.append(version) + else: + versions_to_deprecate.append(version) + + version_deprecated = deprecate_package_version(version) + if not version_deprecated: + print(f"Failed to delete or deprecate version: {version}") + + print(f"Keeping {len(versions_to_keep)} CI versions:") + for version in versions_to_keep: + print(f" {version}") + + +if __name__ == "__main__": + apply_retention_policy() diff --git a/.github/workflows/npm_retention.yaml b/.github/workflows/npm_retention.yaml new file mode 100644 index 0000000..30841ee --- /dev/null +++ b/.github/workflows/npm_retention.yaml @@ -0,0 +1,40 @@ +name: NPM Package Retention + +on: + # schedule: + # - cron: '0 0 * * 0' # Run weekly on Sunday at midnight + workflow_dispatch: # Allow manual trigger only + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Node.js + uses: actions/setup-node@v2 + with: + node-version: "14" + + - name: Install npm + run: | + npm install -g npm@latest + + - name: Configure npm + run: | + echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}" > ~/.npmrc + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install requests + + # - name: Run retention script + # env: + # NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + # run: python .github/scripts/npm_retention.py From 33996e539cc0f3bfbbc8d635d2428f0f9e132fd9 Mon Sep 17 00:00:00 2001 From: fahreddinozcan Date: Fri, 18 Oct 2024 10:55:51 +0300 Subject: [PATCH 2/5] finalize script --- .github/scripts/npm_retention.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/scripts/npm_retention.py b/.github/scripts/npm_retention.py index ad73123..e6ebbdf 100644 --- a/.github/scripts/npm_retention.py +++ b/.github/scripts/npm_retention.py @@ -6,7 +6,7 @@ import re PACKAGE_NAME = "@upstash/vector" -DAYS_TO_KEEP = 14 +DAYS_TO_KEEP = 9 CI_VERSIONS_TO_KEEP = 5 NPM_TOKEN = os.environ.get("NPM_TOKEN") @@ -28,6 +28,13 @@ def get_package_versions(): return [] +def get_version_details(version): + output = run_npm_command(["npm", "view", f"{PACKAGE_NAME}@{version}", "--json"]) + if output: + return json.loads(output) + return {} + + def parse_ci_version_date(version): match = re.search(r"-(\d{14})$", version) if match: @@ -59,6 +66,7 @@ def deprecate_package_version(version): def apply_retention_policy(): versions = get_package_versions() + now = datetime.utcnow() retention_date = now - timedelta(days=DAYS_TO_KEEP) @@ -68,6 +76,10 @@ def apply_retention_policy(): if is_ci_version(version): ci_date = parse_ci_version_date(version) if ci_date: + version_details = get_version_details(version) + if version_details.get("deprecated"): + print(f"Skipping deprecated version: {version}") + continue ci_versions.append((version, ci_date)) else: print(f"Warning: Could not parse date from CI version: {version}") @@ -82,10 +94,12 @@ def apply_retention_policy(): versions_to_keep.append(version) else: versions_to_deprecate.append(version) + print(f"Deprecating version: {version}") - version_deprecated = deprecate_package_version(version) - if not version_deprecated: - print(f"Failed to delete or deprecate version: {version}") + for version in versions_to_deprecate: + version_deprecated = deprecate_package_version(version) + if not version_deprecated: + print(f"Failed to delete or deprecate version: {version}") print(f"Keeping {len(versions_to_keep)} CI versions:") for version in versions_to_keep: From 2df3b83bf6f839d9728960088ab290935602c24a Mon Sep 17 00:00:00 2001 From: fahreddinozcan Date: Fri, 18 Oct 2024 10:56:44 +0300 Subject: [PATCH 3/5] enable workflow --- .github/workflows/npm_retention.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/npm_retention.yaml b/.github/workflows/npm_retention.yaml index 30841ee..8c30404 100644 --- a/.github/workflows/npm_retention.yaml +++ b/.github/workflows/npm_retention.yaml @@ -1,9 +1,8 @@ name: NPM Package Retention on: - # schedule: - # - cron: '0 0 * * 0' # Run weekly on Sunday at midnight - workflow_dispatch: # Allow manual trigger only + schedule: + - cron: "0 0 * * 0" # Run weekly on Sunday at midnight jobs: cleanup: @@ -34,7 +33,7 @@ jobs: python -m pip install --upgrade pip pip install requests - # - name: Run retention script - # env: - # NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - # run: python .github/scripts/npm_retention.py + - name: Run retention script + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + run: python .github/scripts/npm_retention.py From 64107769bcf5ba04fd345dfbfed14883e747f4a4 Mon Sep 17 00:00:00 2001 From: fahreddinozcan Date: Fri, 18 Oct 2024 10:58:05 +0300 Subject: [PATCH 4/5] change days_to_keep to 14 from 9 --- .github/scripts/npm_retention.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/npm_retention.py b/.github/scripts/npm_retention.py index e6ebbdf..ac5b9c3 100644 --- a/.github/scripts/npm_retention.py +++ b/.github/scripts/npm_retention.py @@ -6,7 +6,7 @@ import re PACKAGE_NAME = "@upstash/vector" -DAYS_TO_KEEP = 9 +DAYS_TO_KEEP = 7 CI_VERSIONS_TO_KEEP = 5 NPM_TOKEN = os.environ.get("NPM_TOKEN") From 59ed3b60c0960be09c998b7bc694dea60199d730 Mon Sep 17 00:00:00 2001 From: CahidArda Date: Fri, 18 Oct 2024 13:59:01 +0300 Subject: [PATCH 5/5] fix: add warnings --- .github/scripts/npm_retention.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/npm_retention.py b/.github/scripts/npm_retention.py index ac5b9c3..d087c13 100644 --- a/.github/scripts/npm_retention.py +++ b/.github/scripts/npm_retention.py @@ -25,6 +25,7 @@ def get_package_versions(): output = run_npm_command(["npm", "view", PACKAGE_NAME, "versions", "--json"]) if output: return json.loads(output) + print("Warning: No package version returned.") return [] @@ -32,6 +33,7 @@ def get_version_details(version): output = run_npm_command(["npm", "view", f"{PACKAGE_NAME}@{version}", "--json"]) if output: return json.loads(output) + print("Warning: No version detail returned.") return {}