Merge branch 'main' into gio/fix-app-selection

celery_config.py

@@ -106,9 +106,10 @@ def init_celery_tracing(*args, **kwargs):
 hourly_check_task_name = "app.cron.hourly_check.HourlyCheckTask"
 daily_plan_manager_task_name = "app.cron.daily.PlanManagerTask"
 
-backfill_gh_app_installations_name = (
- "app.tasks.backfill_gh_app_installations.BackfillGHAppInstallationsTask"
-)
+# Backfill GH Apps
+backfill_existing_gh_app_installations_name = "app.tasks.backfill_existing_gh_app_installations.BackfillExistingGHAppInstallationsTask"
+backfill_owners_without_gh_app_installations_name = "app.tasks.backfill_owners_without_gh_app_installations.BackfillOwnersWithoutGHAppInstallationsTask"
+
 trial_expiration_task_name = "app.tasks.plan.TrialExpirationTask"
 trial_expiration_cron_task_name = "app.cron.plan.TrialExpirationCronTask"
 

conftest.py

@@ -325,7 +325,7 @@ def mock_feature(mocker, request):
 
  from shared.rollouts import Feature
 
- def check_value(self, *, owner_id=None, repo_id=None, default=False):
+ def check_value(self, identifier, default=False):
  return default
 
  return mocker.patch.object(Feature, "check_value", check_value)

django_scaffold/settings.py

@@ -15,33 +15,56 @@
 IS_DEV = os.getenv("RUN_ENV") == "DEV"
 
 # Application definition
-
 INSTALLED_APPS = [
- "django_scaffold", # must be first to override migrate command
+ "shared.django_apps.legacy_migrations",
+ "shared.django_apps.codecov_auth",
+ "shared.django_apps.core",
+ "shared.django_apps.reports",
  "shared.django_apps.pg_telemetry",
  "shared.django_apps.ts_telemetry",
  "shared.django_apps.rollouts",
+ "shared.django_apps.user_measurements",
+ "psqlextra",
+ # Needed to install legacy migrations
+ "django.contrib.admin",
+ "django.contrib.contenttypes",
+ "django.contrib.auth",
+ "django.contrib.messages",
+ "django.contrib.sessions",
 ]
 
 TELEMETRY_VANILLA_DB = "default"
 TELEMETRY_TIMESCALE_DB = "timeseries"
 
-DATABASE_ROUTERS = [
- "shared.django_apps.db_routers.TelemetryDatabaseRouter",
- "shared.django_apps.db_routers.MultiDatabaseRouter",
-]
-
 SKIP_RISKY_MIGRATION_STEPS = get_config("migrations", "skip_risky_steps", default=False)
 
-MIDDLEWARE = []
+MIDDLEWARE = [
+ "django.contrib.auth.middleware.AuthenticationMiddleware",
+ "django.contrib.messages.middleware.MessageMiddleware",
+ "django.contrib.sessions.middleware.SessionMiddleware",
+ "django.middleware.csrf.CsrfViewMiddleware",
+]
 
-TEMPLATES = []
+TEMPLATES = [
+ {
+ "BACKEND": "django.template.backends.django.DjangoTemplates",
+ "DIRS": [],
+ "APP_DIRS": True,
+ "OPTIONS": {
+ "context_processors": [
+ "django.template.context_processors.debug",
+ "django.contrib.auth.context_processors.auth",
+ "django.template.context_processors.request",
+ "django.contrib.messages.context_processors.messages",
+ ]
+ },
+ }
+]
 
 # Password validation
 # https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators
 AUTH_PASSWORD_VALIDATORS = []
 
-
 # Internationalization
 # https://docs.djangoproject.com/en/4.2/topics/i18n/
 

helpers/backfills.py

@@ -0,0 +1,67 @@
+import logging
+
+import shared.torngit as torngit
+from asgiref.sync import async_to_sync
+from sqlalchemy.orm.session import Session
+
+from database.models.core import GithubAppInstallation, Repository
+
+log = logging.getLogger(__name__)
+
+# GH App Backfills
+# Looping and adding all repositories in the installation app
+def add_repos_service_ids_from_provider(
+ db_session: Session,
+ ownerid: int,
+ owner_service: torngit.base.TorngitBaseAdapter,
+ gh_app_installation: GithubAppInstallation,
+):
+ repos = async_to_sync(owner_service.list_repos_using_installation)()
+
+ if repos:
+ # Fetching all repos service ids we have for that owner in the DB
+ repo_service_ids_in_db = [
+ repo.service_id
+ for repo in db_session.query(Repository.service_id)
+ .filter_by(ownerid=ownerid)
+ .all()
+ ]
+
+ # Add service ids from provider that we have DB records for to a list
+ new_repo_service_ids = []
+ for repo in repos:
+ repo_data = repo["repo"]
+ service_id = repo_data["service_id"]
+ if service_id and service_id in repo_service_ids_in_db:
+ new_repo_service_ids.append(service_id)
+ log.info(
+ "Added the following repo service ids to this gh app installation",
+ extra=dict(
+ ownerid=ownerid,
+ installation_id=gh_app_installation.installation_id,
+ new_repo_service_ids=new_repo_service_ids,
+ ),
+ )
+ gh_app_installation.repository_service_ids = new_repo_service_ids
+ db_session.commit()
+
+
+# Check if gh selection is set to all and act accordingly
+def maybe_set_installation_to_all_repos(
+ db_session: Session,
+ owner_service,
+ gh_app_installation: GithubAppInstallation,
+):
+ remote_gh_app_installation = async_to_sync(owner_service.get_gh_app_installation)(
+ installation_id=gh_app_installation.installation_id
+ )
+ repository_selection = remote_gh_app_installation.get("repository_selection", "")
+ if repository_selection == "all":
+ gh_app_installation.repository_service_ids = None
+ db_session.commit()
+ log.info(
+ "Selection is set to all, no installation is needed",
+ extra=dict(ownerid=gh_app_installation.ownerid),
+ )
+ return True
+ return False

helpers/sentry.py

@@ -41,3 +41,5 @@ def initialize_sentry() -> None:
  ],
  release=os.getenv("SENTRY_RELEASE", version_str),
  )
+ if os.getenv("CLUSTER_ENV"):
+ sentry_sdk.set_tag("cluster", os.getenv("CLUSTER_ENV"))

helpers/tests/unit/test_sentry.py

@@ -6,8 +6,13 @@
 class TestSentry(object):
  def test_initialize_sentry(self, mocker, mock_configuration):
  mock_configuration._params["services"] = {"sentry": {"server_dsn": "this_dsn"}}
- mocker.patch.dict(os.environ, {"RELEASE_VERSION": "FAKE_VERSION_FOR_YOU"})
+ cluster = "test_env"
+ mocker.patch.dict(
+ os.environ,
+ {"RELEASE_VERSION": "FAKE_VERSION_FOR_YOU", "CLUSTER_ENV": cluster},
+ )
  mocked_init = mocker.patch("helpers.sentry.sentry_sdk.init")
+ mocked_set_tag = mocker.patch("helpers.sentry.sentry_sdk.set_tag")
  assert initialize_sentry() is None
  mocked_init.assert_called_with(
  "this_dsn",
@@ -18,3 +23,4 @@ def test_initialize_sentry(self, mocker, mock_configuration):
  environment="production",
  integrations=[mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY],
  )
+ mocked_set_tag.assert_called_with("cluster", cluster)

migrate-timeseries.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+echo "Running Timeseries Django migrations"
+prefix=""
+if [ -f "/usr/local/bin/berglas" ]; then
+ prefix="berglas exec --"
+fi
+
+$prefix python manage.py migrate --database timeseries rollouts
+$prefix python manage.py migrate --database timeseries pg_telemetry
+$prefix python manage.py migrate --database timeseries ts_telemetry

requirements.in

@@ -1,11 +1,12 @@
-https://github.com/codecov/shared/archive/5df0e67bbe5093750a4bb48871e825b694d8f338.tar.gz#egg=shared
+https://github.com/codecov/shared/archive/148b7ae3a6d4cdfc554ba9ca8b911c13e82d77b8.tar.gz#egg=shared
 https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem
 https://github.com/codecov/test-results-parser/archive/5515e960d5d38881036e9127f86320efca649f13.tar.gz#egg=test-results-parser
 boto3>=1.34
 celery>=5.3.6
 click
 codecov-ribs
 coverage
+django-postgres-extra>=2.0.8
 factory-boy
 google-cloud-storage>=2.10.0
 httpx
@@ -20,7 +21,6 @@ pytest-asyncio
 pytest-cov
 pytest-celery
 pytest-django
-django-postgres-extra>=2.0.8
 pytest-mock
 pytest-sqlalchemy
 pytest-freezegun

requirements.txt

@@ -1,8 +1,8 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
+# This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
-# pip-compile
+# pip-compile requirements.in
 #
 amqp==5.2.0
  # via kombu
@@ -361,9 +361,7 @@ requests==2.31.0
 respx==0.20.2
  # via -r requirements.in
 rfc3986[idna2008]==1.4.0
- # via
- # httpx
- # rfc3986
+ # via httpx
 rsa==4.7.2
  # via google-auth
 s3transfer==0.10.1
@@ -374,7 +372,7 @@ sentry-sdk==1.40.0
  # via
  # -r requirements.in
  # shared
-shared @ https://github.com/codecov/shared/archive/5df0e67bbe5093750a4bb48871e825b694d8f338.tar.gz
+shared @ https://github.com/codecov/shared/archive/c0901f806b0af8c7d0ce19bbb78d5f5a541753d3.tar.gz
  # via -r requirements.in
 six==1.16.0
  # via
@@ -429,6 +427,7 @@ typing==3.7.4.3
 typing-extensions==4.6.3
  # via
  # asgiref
+ # kombu
  # openai
  # opentelemetry-sdk
  # pydantic

services/report/__init__.py

@@ -295,7 +295,7 @@ async def initialize_and_save_report(
  # finisher can build off of it later. Makes the assumption that the CFFs occupy the first
  # j to i session ids where i is the max id of the CFFs and j is some integer less than i.
  if await PARALLEL_UPLOAD_PROCESSING_BY_REPO.check_value_async(
- repo_id=commit.repository.repoid
+ identifier=commit.repository.repoid
  ):
  await self.save_parallel_report_to_archive(
  commit, report, report_code
@@ -755,7 +755,7 @@ async def create_new_report_for_commit(self, commit: Commit) -> Report:
  )
  max_parenthood_deepness = (
  await CARRYFORWARD_BASE_SEARCH_RANGE_BY_OWNER.check_value_async(
- owner_id=repo.ownerid, default=10
+ identifier=repo.ownerid, default=10
  )
  )
 
@@ -856,7 +856,7 @@ def parse_raw_report_from_storage(
  # so that the parallel pipeline can use those to parse. The serial pipeline rewrites the raw uploaded
  # reports to a human readable version that doesn't include file fixes, so that's why copying is necessary.
  if PARALLEL_UPLOAD_PROCESSING_BY_REPO.check_value(
- repo_id=repo.repoid, default=False
+ identifier=repo.repoid, default=False
  ) and (not is_error_case):
  parallel_url = archive_url.removesuffix(".txt") + "_PARALLEL.txt"
  log.info(

services/report/raw_upload_processor.py

@@ -114,7 +114,7 @@ def process_raw_upload(
  should_use_encoded_labels = (
  upload
  and USE_LABEL_INDEX_IN_REPORT_PROCESSING_BY_REPO_ID.check_value(
- repo_id=upload.report.commit.repository.repoid, default=False
+ identifier=upload.report.commit.repository.repoid, default=False
  )
  )
  # [javascript] check for both coverage.json and coverage/coverage.lcov
@@ -357,7 +357,7 @@ def _adjust_sessions(
  if (
  upload
  and USE_LABEL_INDEX_IN_REPORT_PROCESSING_BY_REPO_ID.check_value(
- repo_id=upload.report.commit.repository.repoid, default=False
+ identifier=upload.report.commit.repository.repoid, default=False
  )
  and to_partially_overwrite_flags
  ):