refactor: simplify, move frameworks, etc. (#63)

ibis-project · Aug 24, 2024 · 6e9d667 · 6e9d667
1 parent 7b555a7
commit 6e9d667
Show file tree

Hide file tree

Showing 62 changed files with 1,849 additions and 2,613 deletions.
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
@@ -0,0 +1,49 @@
+name: docs
+
+permissions:
+  contents: write
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'justfile'
+      - 'website/**'
+      - 'dev-requirements.txt'
+      - '.github/workflows/docs.yaml'
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - uses: extractions/setup-just@v1
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.12
+
+      - uses: quarto-dev/quarto-actions/setup@v2
+
+      - name: install requirements
+        run: |
+          pip install uv
+          just setup
+
+      - name: build site
+        run: |
+          . .venv/bin/activate
+          just docs-build
+
+      - name: deploy
+        uses: peaceiris/actions-gh-pages@v3
+        if: github.ref == 'refs/heads/main'
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: website/_output
diff --git a/.github/workflows/etl.yaml b/.github/workflows/etl.yaml
@@ -0,0 +1,44 @@
+name: etl
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 9 * * *"
+
+jobs:
+
+  ingest-etl:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - uses: google-github-actions/auth@v1
+        with:
+          credentials_json: ${{ secrets.GCLOUD_JSON }}
+
+      - uses: extractions/setup-just@v1
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.12
+
+      - uses: quarto-dev/quarto-actions/setup@v2
+
+      - name: install requirements
+        run: |
+          pip install uv
+          just setup
+
+      - name: ingest and etl
+        run: |
+          . .venv/bin/activate
+          ia ingest
+          ia etl
+        env:
+          BQ_PROJECT_ID: voltrondata-demo
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ZULIP_KEY: ${{ secrets.ZULIP_KEY }}
+          GOAT_TOKEN: ${{ secrets.GOAT_TOKEN }}
diff --git a/.gitignore b/.gitignore
@@ -1,7 +1,14 @@
-.env
 .venv
-venv
-data
-target
-.streamlit/secrets.toml
-*.ddb*
+.env
+datalake
+
+.quarto
+pres.html
+pres.ipynb
+pres_files
+
+_output
+
+dist
+
+tmp*
diff --git a/.streamlit/config.toml b/.streamlit/config.toml
diff --git a/config.toml b/config.toml
diff --git a/dag/__init__.py b/dag/__init__.py
diff --git a/dag/assets/__init__.py b/dag/assets/__init__.py
diff --git a/dag/assets/extract/backends.py b/dag/assets/extract/backends.py
diff --git a/dag/assets/extract/docs.py b/dag/assets/extract/docs.py
diff --git a/dag/assets/extract/github.py b/dag/assets/extract/github.py
diff --git a/dag/assets/extract/pypi.py b/dag/assets/extract/pypi.py
diff --git a/dag/assets/extract/zulip.py b/dag/assets/extract/zulip.py
diff --git a/dag/assets/load/__init__.py b/dag/assets/load/__init__.py