perf(workers): enables max worker variable to be set from env variable

docs/src/installation/configurations.md

@@ -126,6 +126,7 @@ API Tokens have expiry date, It is `important to update API Tokens manually each
 | `ALLOW_BIND_ZIP_FILTER` | `[API_CONFIG]` | Yes | Yes |
 | `EXTRA_README_TXT` | `[API_CONFIG]` | No | Yes |
 | `INDEX_THRESHOLD` | `[API_CONFIG]` | No | Yes |
+| `MAX_WORKERS` | `[API_CONFIG]` | No | Yes |
 | `DEFAULT_QUEUE_NAME` | `[API_CONFIG]` | Yes | No |
 | `ONDEMAND_QUEUE_NAME` | `[API_CONFIG]` | Yes | No |
 | `ENABLE_POLYGON_STATISTICS_ENDPOINTS` | `[API_CONFIG]` | Yes | Yes |

src/app.py

@@ -63,6 +63,7 @@
 from src.config import EXPORT_PATH as export_path
 from src.config import INDEX_THRESHOLD as index_threshold
 from src.config import (
+    MAX_WORKERS,
     PARALLEL_PROCESSING_CATEGORIES,
     POLYGON_STATISTICS_API_URL,
     PROCESS_SINGLE_CATEGORY_IN_POSTGRES,
@@ -1467,10 +1468,10 @@ def process_export_format(export_format):
             logging.info(
                 "Using Parallel Processing for %s Export formats with total %s workers",
                 category_name.lower(),
-                os.cpu_count(),
+                MAX_WORKERS,
             )
             with concurrent.futures.ThreadPoolExecutor(
-                max_workers=os.cpu_count()
+                max_workers=int(MAX_WORKERS)
             ) as executor:
                 futures = [
                     executor.submit(process_export_format, export_format)

src/config.py

@@ -76,6 +76,10 @@ def get_bool_env_var(key, default=False):
     config.get("API_CONFIG", "INDEX_THRESHOLD", fallback=5000)
 )
 
+MAX_WORKERS = os.environ.get("MAX_WORKERS") or config.get(
+    "API_CONFIG", "MAX_WORKERS", fallback=os.cpu_count()
+)
+
 # get log level from config
 LOG_LEVEL = os.environ.get("LOG_LEVEL") or config.get(
     "API_CONFIG", "LOG_LEVEL", fallback="debug"