Merge pull request #6 from adidoesnt/development

merge development branch

.github/workflows/deploy.yml

@@ -0,0 +1,185 @@
+name: Deploy to Heroku
+
+on:
+    push:
+        branches:
+            - main
+
+jobs:
+    deploy-database:
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Checkout code
+              uses: actions/checkout@v2
+
+            - name: Install Heroku CLI
+              run: |
+                  curl https://cli-assets.heroku.com/install.sh | sh
+
+            - name: Login to Heroku container registry
+              run: heroku container:login
+              env:
+                  HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+
+            - name: Check if Heroku app exists
+              id: check_app
+              run: |
+                  if heroku apps:info echofinder-database > /dev/null 2>&1; then
+                      echo "App exists"
+                      echo "::set-output name=exists::true"
+                  else
+                      echo "App does not exist"
+                      echo "::set-output name=exists::false"
+                  fi
+              env:
+                  HEROKU_APP_NAME: echofinder-database
+                  HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+
+            - name: Create Heroku app
+              if: steps.check_app.outputs.exists == 'false'
+              run: heroku create echofinder-database
+              env:
+                  HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+
+            - name: Build Docker image
+              run: cd database && docker build -t registry.heroku.com/echofinder-database/web .
+
+            - name: Push Docker image to Heroku container registry
+              run: cd database && docker push registry.heroku.com/echofinder-database/web
+
+            - name: Set environment variables
+              run: |
+                  cd database && heroku config:set \
+                  CHROMA_SERVER_AUTH_CREDENTIALS=${{ secrets.CHROMA_SERVER_AUTH_CREDENTIALS }} \
+                  CHROMA_SERVER_AUTH_CREDENTIALS_PROVIDER=${{ secrets.CHROMA_SERVER_AUTH_CREDENTIALS_PROVIDER }} \
+                  CHROMA_SERVER_AUTH_PROVIDER=${{ secrets.CHROMA_SERVER_AUTH_PROVIDER }} \
+                  CHROMA_SERVER_AUTH_TOKEN_TRANSPORT_HEADER=${{ secrets.CHROMA_SERVER_AUTH_TOKEN_TRANSPORT_HEADER }} \
+                  PERSIST_DIRECTORY=${{ secrets.PERSIST_DIRECTORY }} \
+                  --app echofinder-database
+
+            - name: Release Docker image to Heroku
+              run: cd database && heroku container:release web --app echofinder-database
+              env:
+                  HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+
+    deploy-backend:
+        needs: deploy-database
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Checkout code
+              uses: actions@checkout@v2
+
+            - name: Install Heroku CLI
+              run: |
+                  curl https://cli-assets.heroku.com/install.sh | sh
+
+            - name: Login to Heroku container registry
+              run: heroku container:login
+              env:
+                  HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+
+            - name: Check if Heroku app exists
+              id: check_app
+              run: |
+                  if heroku apps:info echofinder-backend > /dev/null 2>&1; then
+                      echo "App exists"
+                      echo "::set-output name=exists::true"
+                  else
+                      echo "App does not exist"
+                      echo "::set-output name=exists::false"
+                  fi
+              env:
+                  HEROKU_APP_NAME: echofinder-backend
+                  HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+
+            - name: Create Heroku app
+              if: steps.check_app.outputs.exists == 'false'
+              run: heroku create echofinder-backend
+              env:
+                  HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+
+            - name: Build Docker image
+              run: cd backend && docker build -t registry.heroku.com/echofinder-backend/web .
+
+            - name: Push Docker image to Heroku container registry
+              run: cd backend && docker push registry.heroku.com/echofinder-backend/web
+
+            - name: Set environment variables
+              run: |
+                  cd database && heroku config:set \
+                  CHROMA_HOST=${{ secrets.CHROMA_HOST }} \
+                  CHROMA_PORT=${{ secrets.CHROMA_PORT }} \
+                  API_KEY=${{ secrets.BACKEND_API_KEY }} \
+                  CHROMA_API_KEY=${{ secrets.CHROMA_SERVER_AUTH_CREDENTIALS }} \
+                  --app echofinder-database
+
+            - name: Release Docker image to Heroku
+              run: cd backend && heroku container:release web --app echofinder-backend
+              env:
+                  HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+
+    deploy-telegram-bot:
+        needs: deploy-backend
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Checkout code
+              uses: actions/checkout@v2
+
+            - name: Install Heroku CLI
+              run: |
+                  curl https://cli-assets.heroku.com/install-ubuntu.sh | sh
+
+            - name: Login to Heroku Container Registry
+              run: heroku container:login
+              env:
+                  HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+
+            - name: Check if Heroku app exists
+              id: check_app
+              run: |
+                  if heroku apps:info echofinder-telegram > /dev/null 2>&1; then
+                    echo "App exists"
+                    echo "::set-output name=exists::true"
+                  else
+                    echo "App does not exist"
+                    echo "::set-output name=exists::false"
+                  fi
+              env:
+                  HEROKU_APP_NAME: echofinder-telegram
+                  HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+
+            - name: Create Heroku app
+              if: steps.check_app.outputs.exists == 'false'
+              run: heroku apps:create echofinder-telegram
+              env:
+                  HEROKU_APP_NAME: echofinder-telegram
+                  HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+
+            - name: Build Docker image
+              run: cd telegram-bot && docker build -t registry.heroku.com/echofinder-telegram/web .
+
+            - name: Push Docker image to Heroku Container Registry
+              run: cd telegram-bot && docker push registry.heroku.com/echofinder-telegram/web
+
+            - name: Set environment variables
+              run: |
+                  cd telegram-bot && heroku config:set \
+                  TELEGRAM_BOT_TOKEN=${{ secrets.TELEGRAM_BOT_TOKEN }} \
+                  TELEGRAM_BOT_WEBHOOK_URL=${{ secrets.TELEGRAM_BOT_WEBHOOK_URL }} \
+                  NODE_ENV=${{ secrets.NODE_ENV }} \
+                  BACKEND_HOST=${{ secrets.BACKEND_HOST }} \
+                  BACKEND_PORT=${{ secrets.BACKEND_PORT }} \
+                  BACKEND_API_KEY=${{ secrets.BACKEND_API_KEY }} \
+                  API_CLIENT_TIMEOUT=${{ secrets.API_CLIENT_TIMEOUT }} \
+                  PLATFORM=${{ secrets.PLATFORM }} \
+                  --app echofinder-telegram
+              env:
+                  HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+
+            - name: Release Docker image on Heroku
+              run: cd telegram-bot && heroku container:release web --app echofinder-telegram
+              env:
+                  HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}

.gitignore

@@ -0,0 +1 @@
+.DS_Store

backend/.gitignore

@@ -1,3 +1,5 @@
 venv/
 __pycache__/
-**/*.log
+**/*.log
+**/*.env
+**/*.dockerignore

backend/Dockerfile

@@ -4,5 +4,9 @@ WORKDIR /app
 
 COPY . /app
 
+EXPOSE 8080
+
 RUN pip install -r requirements.txt
 
+CMD ["uvicorn", "main:app", "--port", "8080"]
+

backend/README.md

@@ -1,6 +1,20 @@
 # Backend Setup
 
-## Steps to Run:
+## Environment variables
+
+### FastAPI (chroma client)
+* CHROMA_HOST
+* CHROMA_PORT
+* API_KEY
+* CHROMA_API_KEY
+
+### Chroma server
+* CHROMA_SERVER_AUTH_CREDENTIALS
+* CHROMA_SERVER_AUTH_CREDENTIALS_PROVIDER="chromadb.auth.token.TokenConfigServerAuthCredentialsProvider"
+* CHROMA_SERVER_AUTH_TOKEN_TRANSPORT_HEADER="X_CHROMA_TOKEN"
+* CHROMA_SERVER_AUTH_PROVIDER="chromadb.auth.token.TokenAuthServerProvider"
+
+## Steps to Run (development):
 
 1. Navigate to the backend directory:
     ```
@@ -19,7 +33,12 @@
     uvicorn main:app --port 8080 --reload --reload-exclude venv/
     ```
 
-4. Once the server is running, open your web browser and go to:
+4. Run chroma server locally
+    ```
+    chroma run --path /tmp
+    ```
+
+5. Once the server is running, open your web browser and go to:
     ```
     http://127.0.0.1:8080/docs
     ```

backend/docker-compose.yml

@@ -2,11 +2,9 @@ version: '3'
 
 services:
   chroma:
-    image: chromadb/chroma
+    build: ../database/.
     ports:
       - 8000:8000
-    volumes:
-      - /tmp/rando_data:/chroma/chroma
 
   web:
     build: .

backend/main.py

@@ -1,14 +1,34 @@
+import os
+from dotenv import load_dotenv
 from datetime import datetime
 from typing import Any, Dict, List, Literal, Optional
 import chromadb
-from fastapi import FastAPI, HTTPException, Query
+from fastapi import Depends, FastAPI, HTTPException, Query, Security, status
+from fastapi.security import APIKeyHeader
 from pydantic import BaseModel, Field
 from custom_logger import setup_custom_logger
 
+load_dotenv()
+
+
 # Initialize FastAPI app
 app = FastAPI()
 
-client = chromadb.HttpClient(host="chroma", port = 8000)
+
+api_key = os.environ.get('API_KEY')
+api_key_header = APIKeyHeader(name="X-Api-Key")
+
+def get_api_key(api_key_header: str = Security(api_key_header)) -> str:
+    if api_key_header == api_key:
+        return api_key_header
+    raise HTTPException(
+        status_code=status.HTTP_401_UNAUTHORIZED,
+        detail="Invalid or missing API Key",
+    )
+
+client = chromadb.HttpClient(host=os.environ.get('CHROMA_HOST'),
+                             port = os.environ.get('CHROMA_PORT'),
+                             headers={"X-Chroma-Token": os.environ.get('CHROMA_API_KEY')})
 collection = client.get_or_create_collection(name="messages")
 
 # Initialize the custom logger
@@ -29,7 +49,7 @@ class MessageInfo(BaseModel):
 
 
 @app.post("/messages/", response_model=List[MessageInfo])
-async def upsert_messages(messages: List[MessageInfo]):
+async def upsert_messages(messages: List[MessageInfo], api_key: str = Security(get_api_key)):
 
     ids = [msg.message_id for msg in messages]
     documents = [msg.message_content for msg in messages]
@@ -42,14 +62,14 @@ async def upsert_messages(messages: List[MessageInfo]):
 
 
 @app.get("/message/{message_id}") # response_model=TelegramMessage)
-async def get_message_by_id(message_id: str):
+async def get_message_by_id(message_id: str, api_key: str = Security(get_api_key)):
     message = collection.get(message_id)
     return message
 
 
 
 @app.get("/messages/search/") # TODO - Add response model
-async def search_messages(search_string: str = Query(..., min_length=1)):
+async def search_messages(search_string: str = Query(..., min_length=1), api_key: str = Security(get_api_key)):
     query_text = f"Who says this - {search_string}"
 
     results = collection.query(
@@ -63,12 +83,12 @@ async def search_messages(search_string: str = Query(..., min_length=1)):
 
 
 @app.delete("/messages/delete/", response_model=List[str])
-async def delete_messages(message_ids: List[str]):
+async def delete_messages(message_ids: List[str], api_key: str = Security(get_api_key)):
     collection.delete(ids=message_ids)
     return message_ids
 
 
 @app.delete("/messages/delete_all/")
-async def delete_messages():
+async def delete_messages(api_key: str = Security(get_api_key)):
     client.delete_collection("messages")
     return {"Deleted": "RESTART API NOW!!"}

database/.gitignore

@@ -0,0 +1 @@
+.env