From a76a670cac2b3bbd36b45bce43a534ddd308d646 Mon Sep 17 00:00:00 2001 From: Ovv Date: Fri, 5 Jul 2019 23:18:21 +0200 Subject: [PATCH] Init tests --- .circleci/config.yml | 8 +- docker-compose.test.yml | 15 +++ pyslackersweb/settings.py | 2 +- pyslackersweb/website/models.py | 6 +- pyslackersweb/website/tasks.py | 32 +++--- pyslackersweb/website/views.py | 28 ++--- requirements/base.txt | 2 +- requirements/testing.txt | 1 + tests/__init__.py | 0 tests/conftest.py | 76 +++++++++++++ tests/data.py | 196 ++++++++++++++++++++++++++++++++ tests/test_nothing.py | 2 - tests/test_website.py | 115 +++++++++++++++++++ tox.ini | 18 +-- 14 files changed, 461 insertions(+), 40 deletions(-) create mode 100644 docker-compose.test.yml create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/data.py delete mode 100644 tests/test_nothing.py create mode 100644 tests/test_website.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 165578e2..9c70ab17 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,6 +6,12 @@ jobs: working_directory: ~/app docker: - image: circleci/python:3.7.3-node-browsers + - image: circleci/redis:5.0.5-alpine + - image: circleci/postgres:11.4-alpine + environment: + POSTGRES_USER: main + POSTGRES_DB: main + POSTGRES_PASSWORD: main steps: - checkout - restore_cache: @@ -24,7 +30,7 @@ jobs: - .venv - run: | source .venv/bin/activate - tox + tox -e ci -e lint - save_cache: key: 'tox-{{ checksum "requirements/base.txt" }}-{{ checksum "requirements/testing.txt" }}-{{ checksum "requirements/development.txt" }}-{{ checksum "requirements/production.txt" }}' paths: diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 00000000..cbba61b1 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,15 @@ +version: "3.4" + +services: + redis: + image: redis:5-alpine + ports: + - "6379:6379" + + postgresql: + image: postgres:11 + environment: + POSTGRES_PASSWORD: main + POSTGRES_USER: main + ports: + - "5432:5432" diff --git a/pyslackersweb/settings.py b/pyslackersweb/settings.py index 358604a8..1351e272 100644 --- a/pyslackersweb/settings.py +++ b/pyslackersweb/settings.py @@ -3,7 +3,7 @@ REDIS_URL = os.getenv("REDIS_URL", "redis://127.0.0.1:6379/0") -POSTGRESQL_DSN = os.getenv("POSTGRESQL_DSN") +POSTGRESQL_DSN = os.getenv("POSTGRESQL_DSN", "postgresql://main:main@127.0.0.1:5432/main") SENTRY_DSN = os.getenv("SENTRY_DSN") diff --git a/pyslackersweb/website/models.py b/pyslackersweb/website/models.py index c5fb73ad..c218c687 100644 --- a/pyslackersweb/website/models.py +++ b/pyslackersweb/website/models.py @@ -1,6 +1,10 @@ from marshmallow import Schema, fields +def validate_true(field): + return bool(field) + + class InviteSchema(Schema): email = fields.Email(required=True) - agree_tos = fields.Boolean(required=True) + agree_tos = fields.Boolean(required=True, validate=validate_true) diff --git a/pyslackersweb/website/tasks.py b/pyslackersweb/website/tasks.py index 58465aa8..a810206e 100644 --- a/pyslackersweb/website/tasks.py +++ b/pyslackersweb/website/tasks.py @@ -44,8 +44,9 @@ class Channel: async def sync_github_repositories( session: ClientSession, redis: RedisConnection, *, cache_key: str = GITHUB_REPO_CACHE_KEY -) -> None: +) -> List[Repository]: logger.debug("Refreshing GitHub cache") + repositories = [] try: async with session.get( "https://api.github.com/orgs/pyslackers/repos", @@ -53,7 +54,6 @@ async def sync_github_repositories( ) as r: repos = await r.json() - repositories = [] for repo in repos: if repo["archived"]: continue @@ -72,13 +72,15 @@ async def sync_github_repositories( repositories.sort(key=lambda r: r.stars, reverse=True) - await redis.set(cache_key, json.dumps([x.__dict__ for x in repositories[:6]])) - + await redis.set( + cache_key, json.dumps([dataclasses.asdict(repo) for repo in repositories[:6]]) + ) except asyncio.CancelledError: logger.debug("Github cache refresh canceled") - except Exception: + except Exception: # pylint: disable=broad-except logger.exception("Error refreshing GitHub cache") - raise + + return repositories async def sync_slack_users( @@ -87,11 +89,11 @@ async def sync_slack_users( *, cache_key_tz: str = SLACK_TZ_CACHE_KEY, cache_key_count: str = SLACK_COUNT_CACHE_KEY, -): +) -> Counter: logger.debug("Refreshing slack users cache.") + counter: Counter = Counter() try: - counter: Counter = Counter() async for user in slack_client.iter(slack.methods.USERS_LIST, minimum_time=3): if user["deleted"] or user["is_bot"] or not user["tz"]: continue @@ -112,16 +114,17 @@ async def sync_slack_users( logger.debug("Slack users cache refresh canceled") except Exception: # pylint: disable=broad-except logger.exception("Error refreshing slack users cache") - return + + return counter async def sync_slack_channels( slack_client: SlackAPI, redis: RedisConnection, *, cache_key: str = SLACK_CHANNEL_CACHE_KEY -) -> None: +) -> List[Channel]: logger.debug("Refreshing slack channels cache.") + channels = [] try: - channels = [] async for channel in slack_client.iter(slack.methods.CHANNELS_LIST): channels.append( Channel( @@ -137,10 +140,13 @@ async def sync_slack_channels( logger.debug("Found %s slack channels", len(channels)) - await redis.set(cache_key, json.dumps([x.__dict__ for x in channels])) + await redis.set( + cache_key, json.dumps([dataclasses.asdict(channel) for channel in channels]) + ) except asyncio.CancelledError: logger.debug("Slack channels cache refresh canceled") except Exception: # pylint: disable=broad-except logger.exception("Error refreshing slack channels cache") - return + + return channels diff --git a/pyslackersweb/website/views.py b/pyslackersweb/website/views.py index e887d531..7a41f3c2 100644 --- a/pyslackersweb/website/views.py +++ b/pyslackersweb/website/views.py @@ -1,6 +1,8 @@ import json import logging +import slack.exceptions + from aiohttp import web from aiohttp_jinja2 import template from marshmallow.exceptions import ValidationError @@ -24,7 +26,9 @@ async def get(self): return { "member_count": int((await redis.get(SLACK_COUNT_CACHE_KEY, encoding="utf-8")) or 0), - "projects": json.loads(await redis.get(GITHUB_REPO_CACHE_KEY, encoding="utf-8")), + "projects": json.loads( + await redis.get(GITHUB_REPO_CACHE_KEY, encoding="utf-8") or "{}" + ), "sponsors": [ { "image": self.request.app.router["static"].url_for( @@ -65,19 +69,17 @@ async def post(self): try: invite = self.schema.load(await self.request.post()) - async with self.request.app["client_session"].post( - "https://slack.com/api/users.admin.invite", - headers={"Authorization": f"Bearer {self.request.app['slack_invite_token']}"}, - data={"email": invite["email"], "resend": True}, - ) as r: - body = await r.json() - - if body["ok"]: - context["success"] = True - else: - logger.warning("Error sending slack invite: %s", body["error"], extra=body) - context["errors"].update(non_field=[body["error"]]) + await self.request.app["slack_client"].query( + url="users.admin.invite", data={"email": invite["email"], "resend": True} + ) + context["success"] = True except ValidationError as e: context["errors"] = e.normalized_messages() + except slack.exceptions.SlackAPIError as e: + logger.warning("Error sending slack invite: %s", e.error, extra=e.data) + context["errors"].update(non_field=[e.error]) + except slack.exceptions.HTTPException: + logger.exception("Error contacting slack API") + context["errors"].update(non_field=["Error contacting slack API"]) return context diff --git a/requirements/base.txt b/requirements/base.txt index 1a20a890..b7021c3f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -7,6 +7,6 @@ apscheduler==3.6.1 gunicorn==19.9.0 marshmallow==3.0.0rc8 sentry-sdk==0.10.2 -slack-sansio==1.0.0 +slack-sansio==1.1.0 asyncpg==0.18.3 pyyaml==5.1.1 diff --git a/requirements/testing.txt b/requirements/testing.txt index a6a882c4..7e17927e 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -7,3 +7,4 @@ pytest-cov==2.7.1 pytest-aiohttp==0.3.0 tox==3.13.2 mypy==0.720 +asynctest==0.13.0 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..3624a826 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,76 @@ +import json +import pytest +import aioredis +import asynctest + +import pyslackersweb +import pyslackersweb.website.tasks + +pytest_plugins = ("slack.tests.plugin",) + + +def pytest_addoption(parser): + parser.addoption("--redis", action="store_true", default=False) + parser.addoption("--postgresql", action="store_true", default=False) + + +@pytest.fixture +async def app(pytestconfig): + application = await pyslackersweb.app_factory() + + if not pytestconfig.getoption("postgresql"): + application.cleanup_ctx.remove(pyslackersweb.contexts.postgresql_pool) + + if not pytestconfig.getoption("redis"): + application.cleanup_ctx.remove(pyslackersweb.contexts.redis_pool) + + return application + + +@pytest.fixture +async def redis(loop, pytestconfig): + + if pytestconfig.getoption("redis"): + redis = await aioredis.create_redis_pool(pyslackersweb.settings.REDIS_URL) + else: + redis = FakeRedis() + + yield redis + + if pytestconfig.getoption("redis"): + redis.close() + await redis.wait_closed() + + +class FakeRedis: + async def get(self, key, *args, **kwargs): + if key == pyslackersweb.website.tasks.GITHUB_REPO_CACHE_KEY: + return json.dumps( + { + "name": "foo", + "descriptions": "bar", + "href": "https://github.com/pyslackers/website", + "stars": 20, + "topics": ["foo", "bar", "baz"], + } + ) + elif key == pyslackersweb.website.tasks.SLACK_COUNT_CACHE_KEY: + return 10 + elif key == pyslackersweb.website.tasks.SLACK_TZ_CACHE_KEY: + return {"foo": 10, "bar": 20} + else: + raise RuntimeError(f"Mock redis key not found: {key}") + + async def hgetall(self, key, *args, **kwargs): + if key == pyslackersweb.website.tasks.SLACK_TZ_CACHE_KEY: + return {"foo": 10, "bar": 20} + else: + raise RuntimeError(f"Mock redis key not found: {key}") + + async def set(self, key, *args, **kwargs): + pass + + def multi_exec(self): + tx = asynctest.Mock() + tx.execute = asynctest.CoroutineMock() + return tx diff --git a/tests/data.py b/tests/data.py new file mode 100644 index 00000000..ef11f536 --- /dev/null +++ b/tests/data.py @@ -0,0 +1,196 @@ +SLACK_USERS = [ + { + "id": "W012A3CDE", + "team_id": "T012AB3C4", + "name": "spengler", + "deleted": False, + "color": "9f69e7", + "real_name": "spengler", + "tz": "America/Los_Angeles", + "tz_label": "Pacific Daylight Time", + "tz_offset": -25200, + "profile": { + "avatar_hash": "ge3b51ca72de", + "status_text": "Print is dead", + "status_emoji": ":books:", + "real_name": "Egon Spengler", + "display_name": "spengler", + "real_name_normalized": "Egon Spengler", + "display_name_normalized": "spengler", + "email": "spengler@ghostbusters.example.com", + "image_24": "https://.../avatar/e3b51ca72dee4ef87916ae2b9240df50.jpg", + "image_32": "https://.../avatar/e3b51ca72dee4ef87916ae2b9240df50.jpg", + "image_48": "https://.../avatar/e3b51ca72dee4ef87916ae2b9240df50.jpg", + "image_72": "https://.../avatar/e3b51ca72dee4ef87916ae2b9240df50.jpg", + "image_192": "https://.../avatar/e3b51ca72dee4ef87916ae2b9240df50.jpg", + "image_512": "https://.../avatar/e3b51ca72dee4ef87916ae2b9240df50.jpg", + "team": "T012AB3C4", + }, + "is_admin": True, + "is_owner": False, + "is_primary_owner": False, + "is_restricted": False, + "is_ultra_restricted": False, + "is_bot": False, + "updated": 1502138686, + "is_app_user": False, + "has_2fa": False, + }, + { + "id": "W07QCRPA4", + "team_id": "T0G9PQBBK", + "name": "glinda", + "deleted": False, + "color": "9f69e7", + "real_name": "Glinda Southgood", + "tz": "America/Los_Angeles", + "tz_label": "Pacific Daylight Time", + "tz_offset": -25200, + "profile": { + "avatar_hash": "8fbdd10b41c6", + "image_24": "https://a.slack-edge.com...png", + "image_32": "https://a.slack-edge.com...png", + "image_48": "https://a.slack-edge.com...png", + "image_72": "https://a.slack-edge.com...png", + "image_192": "https://a.slack-edge.com...png", + "image_512": "https://a.slack-edge.com...png", + "image_1024": "https://a.slack-edge.com...png", + "image_original": "https://a.slack-edge.com...png", + "first_name": "Glinda", + "last_name": "Southgood", + "title": "Glinda the Good", + "phone": "", + "skype": "", + "real_name": "Glinda Southgood", + "real_name_normalized": "Glinda Southgood", + "display_name": "Glinda the Fairly Good", + "display_name_normalized": "Glinda the Fairly Good", + "email": "glenda@south.oz.coven", + }, + "is_admin": True, + "is_owner": False, + "is_primary_owner": False, + "is_restricted": False, + "is_ultra_restricted": False, + "is_bot": False, + "updated": 1480527098, + "has_2fa": False, + }, + { + "id": "W012A3CDE", + "team_id": "T012AB3C4", + "name": "spengler", + "deleted": True, + "color": "9f69e7", + "real_name": "spengler", + "tz": "America/Los_Angeles", + "tz_label": "Pacific Daylight Time", + "tz_offset": -25200, + "profile": { + "avatar_hash": "ge3b51ca72de", + "status_text": "Print is dead", + "status_emoji": ":books:", + "real_name": "Egon Spengler", + "display_name": "spengler", + "real_name_normalized": "Egon Spengler", + "display_name_normalized": "spengler", + "email": "spengler@ghostbusters.example.com", + "image_24": "https://.../avatar/e3b51ca72dee4ef87916ae2b9240df50.jpg", + "image_32": "https://.../avatar/e3b51ca72dee4ef87916ae2b9240df50.jpg", + "image_48": "https://.../avatar/e3b51ca72dee4ef87916ae2b9240df50.jpg", + "image_72": "https://.../avatar/e3b51ca72dee4ef87916ae2b9240df50.jpg", + "image_192": "https://.../avatar/e3b51ca72dee4ef87916ae2b9240df50.jpg", + "image_512": "https://.../avatar/e3b51ca72dee4ef87916ae2b9240df50.jpg", + "team": "T012AB3C4", + }, + "is_admin": True, + "is_owner": False, + "is_primary_owner": False, + "is_restricted": False, + "is_ultra_restricted": False, + "is_bot": False, + "updated": 1502138686, + "is_app_user": False, + "has_2fa": False, + }, + { + "id": "W07QCRPA4", + "team_id": "T0G9PQBBK", + "name": "glinda", + "deleted": False, + "color": "9f69e7", + "real_name": "Glinda Southgood", + "tz": None, + "tz_label": "Pacific Daylight Time", + "tz_offset": -25200, + "profile": { + "avatar_hash": "8fbdd10b41c6", + "image_24": "https://a.slack-edge.com...png", + "image_32": "https://a.slack-edge.com...png", + "image_48": "https://a.slack-edge.com...png", + "image_72": "https://a.slack-edge.com...png", + "image_192": "https://a.slack-edge.com...png", + "image_512": "https://a.slack-edge.com...png", + "image_1024": "https://a.slack-edge.com...png", + "image_original": "https://a.slack-edge.com...png", + "first_name": "Glinda", + "last_name": "Southgood", + "title": "Glinda the Good", + "phone": "", + "skype": "", + "real_name": "Glinda Southgood", + "real_name_normalized": "Glinda Southgood", + "display_name": "Glinda the Fairly Good", + "display_name_normalized": "Glinda the Fairly Good", + "email": "glenda@south.oz.coven", + }, + "is_admin": True, + "is_owner": False, + "is_primary_owner": False, + "is_restricted": False, + "is_ultra_restricted": False, + "is_bot": False, + "updated": 1480527098, + "has_2fa": False, + }, + { + "id": "W07QCRPA4", + "team_id": "T0G9PQBBK", + "name": "glinda", + "deleted": False, + "color": "9f69e7", + "real_name": "Glinda Southgood", + "tz": "America/Los_Angeles", + "tz_label": "Pacific Daylight Time", + "tz_offset": -25200, + "profile": { + "avatar_hash": "8fbdd10b41c6", + "image_24": "https://a.slack-edge.com...png", + "image_32": "https://a.slack-edge.com...png", + "image_48": "https://a.slack-edge.com...png", + "image_72": "https://a.slack-edge.com...png", + "image_192": "https://a.slack-edge.com...png", + "image_512": "https://a.slack-edge.com...png", + "image_1024": "https://a.slack-edge.com...png", + "image_original": "https://a.slack-edge.com...png", + "first_name": "Glinda", + "last_name": "Southgood", + "title": "Glinda the Good", + "phone": "", + "skype": "", + "real_name": "Glinda Southgood", + "real_name_normalized": "Glinda Southgood", + "display_name": "Glinda the Fairly Good", + "display_name_normalized": "Glinda the Fairly Good", + "email": "glenda@south.oz.coven", + }, + "is_admin": True, + "is_owner": False, + "is_primary_owner": False, + "is_restricted": False, + "is_ultra_restricted": False, + "is_bot": True, + "updated": 1480527098, + "has_2fa": False, + }, +] diff --git a/tests/test_nothing.py b/tests/test_nothing.py deleted file mode 100644 index a3e981b5..00000000 --- a/tests/test_nothing.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_nothing(): - pass diff --git a/tests/test_website.py b/tests/test_website.py new file mode 100644 index 00000000..3698ec1a --- /dev/null +++ b/tests/test_website.py @@ -0,0 +1,115 @@ +import pytest +import aiohttp.web +import logging + +from collections import namedtuple +from pyslackersweb.website import tasks + +SlackInviteTestParam = namedtuple("Param", "response data expected") + + +@pytest.fixture +async def website_client(app: aiohttp.web.Application, aiohttp_client, slack_client, redis): + app["redis"] = app["website_app"]["redis"] = redis + + client = await aiohttp_client(app) + + client.app["website_app"]["slack_client"] = slack_client + return client + + +async def test_endpoint_index(website_client): + r = await website_client.get("/") + + assert r.history[0].url.path == "/" + assert r.history[0].status == 302 + + assert r.status == 200 + assert r.url.path == "/web" + + +async def test_endpoint_slack(website_client): + r = await website_client.get("/web/slack") + assert r.status == 200 + + +@pytest.mark.parametrize( + "slack_client,data,expected", + ( + SlackInviteTestParam( + response={}, + data={"email": "error@example.com", "agree_tos": True}, + expected="successAlert", + ), + SlackInviteTestParam( + response={}, data={"agree_tos": True}, expected="Missing data for required field" + ), + SlackInviteTestParam( + response={}, + data={"email": "error@example.com", "agree_tos": False}, + expected="There was an error processing your invite", + ), + SlackInviteTestParam( + response={}, + data={"email": "foobar", "agree_tos": True}, + expected="Not a valid email address", + ), + SlackInviteTestParam( + response={"body": {"ok": False, "error": "already_in_team"}}, + data={"email": "error@example.com", "agree_tos": True}, + expected="Reason: already_in_team", + ), + SlackInviteTestParam( + response={"body": {"ok": False, "error": "not_authed"}}, + data={"email": "error@example.com", "agree_tos": True}, + expected="Reason: not_authed", + ), + SlackInviteTestParam( + response={"status": 500}, + data={"email": "error@example.com", "agree_tos": True}, + expected="Reason: Error contacting slack API", + ), + ), + indirect=["slack_client"], +) +async def test_endpoint_slack_invite(website_client, data, expected): + r = await website_client.post(path="/web/slack", data=data) + html = await r.text() + + assert r.status == 200 + assert expected in html + + +async def test_task_sync_github_repositories(caplog, redis): + + async with aiohttp.ClientSession() as session: + result = await tasks.sync_github_repositories(session, redis) + + assert result + + for record in caplog.records: + assert record.levelno <= logging.INFO + + +@pytest.mark.parametrize("slack_client", ({"body": ["users_iter", "users"]},), indirect=True) +async def test_task_sync_slack_users(caplog, slack_client, redis): + + result = await tasks.sync_slack_users(slack_client, redis) + + assert result + assert len(result) == 1 + assert result["America/Los_Angeles"] == 2 + + for record in caplog.records: + assert record.levelno <= logging.INFO + + +@pytest.mark.parametrize("slack_client", ({"body": ["channels_iter", "channels"]},), indirect=True) +async def test_task_sync_slack_channels(caplog, slack_client, redis): + + result = await tasks.sync_slack_channels(slack_client, redis) + + assert result + + for record in caplog.records: + assert record.levelno <= logging.INFO diff --git a/tox.ini b/tox.ini index 8858642e..e1400dbc 100644 --- a/tox.ini +++ b/tox.ini @@ -1,18 +1,20 @@ [tox] -envlist = py37,lint,fmt +envlist = py37,lint skipsdist = true [testenv] -deps = -r requirements/testing.txt -commands = python -m pytest --verbose --cov=pyslackersweb/ --cov-report=term-missing --junit-xml={envdir}/artifacts/test-results.xml +commands = + pip install -r requirements/testing.txt + python -m pytest --verbose --cov=pyslackersweb/ --cov-report=term-missing --junit-xml={envdir}/artifacts/test-results.xml [testenv:lint] -deps = - -r requirements/testing.txt commands = + pip install -r requirements/testing.txt + black --check . pylint pyslackersweb mypy . --ignore-missing-imports -[testenv:fmt] -deps = black -commands = black --check . +[testenv:ci] +commands = + pip install -r requirements/testing.txt + python -m pytest --verbose --cov=pyslackersweb/ --cov-report=term-missing --junit-xml={envdir}/artifacts/test-results.xml --postgresql --redis