diff --git a/.circleci/config.yml b/.circleci/config.yml index 165578e2..2379a5c8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,6 +6,13 @@ jobs: working_directory: ~/app docker: - image: circleci/python:3.7.3-node-browsers + - image: circleci/redis:5.0.5-alpine + - image: circleci/postgres:11.4-alpine + environment: + POSTGRES_USER: main + POSTGRES_DB: main + POSTGRES_PASSWORD: main + steps: - checkout - restore_cache: @@ -22,9 +29,12 @@ jobs: key: 'venv-{{ checksum "requirements/base.txt" }}-{{ checksum "requirements/testing.txt" }}-{{ checksum "requirements/development.txt" }}-{{ checksum "requirements/production.txt" }}' paths: - .venv - - run: | - source .venv/bin/activate - tox + - run: + environment: + POSTGRESQL_DSN: postgresql://main:main@127.0.0.1:5432/main + command: | + source .venv/bin/activate + tox - save_cache: key: 'tox-{{ checksum "requirements/base.txt" }}-{{ checksum "requirements/testing.txt" }}-{{ checksum "requirements/development.txt" }}-{{ checksum "requirements/production.txt" }}' paths: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5771eb96..fcae2ce7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,7 +8,9 @@ $ docker-compose up --build This will build the docker container for you which uses the correct [python version](.python-version), installs the dependencies, and binds the ports. This also "volume mounts" your local directory into the container, meaning that any changes you make on your host machine will be available in the docker container. The exception to these changes being reflected immediately will be if/when a dependency is added or updated, in which case you'll need to run the above command again (basically just ctrl-c, up arrow, enter, and wait for the rebuild). -For testing, you can run: +## Testing + +While you can allow for CircleCI to run tests/checks, running locally simply uses `docker` and `tox`: ```bash # if you need to rebuild first, `docker-compose build` @@ -16,6 +18,18 @@ For testing, you can run: $ docker-compose run web tox ``` +Tox forwards positional arguments to pytest, that way you can use all standard pytest arguments. For example, only running a specific test can be done like this: + +```bash +$ docker-compose run web tox -e py37 tests/test_website.py::test_endpoint_index +``` + +To run the black auto-formatter on the code you can use: + +```bash +$ docker-compose run web tox -e autoformat +``` + # The Involved Path If instead you'd prefer to set-up your project on the host machine, you are free to do so. This is a non-exhaustive primer on the steps required, if you need help directly please ask in [#community_projects](slack://open?team=T07EFKXHR&id=C2FMLUBEU). @@ -86,14 +100,6 @@ Now you should be good to run the application: Once that launches you can visit [localhost:8000](http://localhost:8000) in your browser and be in business. -### 7. Testing - -While you can allow for CircleCI to run tests/checks, running locally simply uses `tox`: - -```bash -(.venv) $ tox -``` - ## Windows Systems TODO: see [#330](https://github.com/pyslackers/website/issues/330) diff --git a/docker-compose.yml b/docker-compose.yml index c98f4794..d3155c90 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ services: build: . depends_on: - redis + - postgresql environment: REDIS_URL: "redis://redis:6379/0" POSTGRESQL_DSN: "postgresql://main:main@postgresql:5432/main" @@ -15,7 +16,7 @@ services: - "8000:8000" volumes: - "${PWD}:/app" - - /app/.tox + - tox-data:/app/.tox command: gunicorn pyslackersweb:app_factory --access-logfile - --bind=0.0.0.0:8000 --worker-class=aiohttp.GunicornWebWorker --reload redis: @@ -26,3 +27,6 @@ services: environment: POSTGRES_PASSWORD: main POSTGRES_USER: main + +volumes: + tox-data: {} diff --git a/pyslackersweb/contexts.py b/pyslackersweb/contexts.py index d5a7342c..8a0da17e 100644 --- a/pyslackersweb/contexts.py +++ b/pyslackersweb/contexts.py @@ -9,8 +9,11 @@ async def apscheduler(app: web.Application) -> AsyncGenerator[None, None]: app["scheduler"] = app["website_app"]["scheduler"] = AsyncIOScheduler() app["scheduler"].start() + yield - app["scheduler"].shutdown() + + if app["scheduler"].running: + app["scheduler"].shutdown() async def client_session(app: web.Application) -> AsyncGenerator[None, None]: diff --git a/pyslackersweb/website/models.py b/pyslackersweb/website/models.py index c5fb73ad..29833cc3 100644 --- a/pyslackersweb/website/models.py +++ b/pyslackersweb/website/models.py @@ -3,4 +3,4 @@ class InviteSchema(Schema): email = fields.Email(required=True) - agree_tos = fields.Boolean(required=True) + agree_tos = fields.Boolean(required=True, validate=bool) diff --git a/pyslackersweb/website/tasks.py b/pyslackersweb/website/tasks.py index 58465aa8..a810206e 100644 --- a/pyslackersweb/website/tasks.py +++ b/pyslackersweb/website/tasks.py @@ -44,8 +44,9 @@ class Channel: async def sync_github_repositories( session: ClientSession, redis: RedisConnection, *, cache_key: str = GITHUB_REPO_CACHE_KEY -) -> None: +) -> List[Repository]: logger.debug("Refreshing GitHub cache") + repositories = [] try: async with session.get( "https://api.github.com/orgs/pyslackers/repos", @@ -53,7 +54,6 @@ async def sync_github_repositories( ) as r: repos = await r.json() - repositories = [] for repo in repos: if repo["archived"]: continue @@ -72,13 +72,15 @@ async def sync_github_repositories( repositories.sort(key=lambda r: r.stars, reverse=True) - await redis.set(cache_key, json.dumps([x.__dict__ for x in repositories[:6]])) - + await redis.set( + cache_key, json.dumps([dataclasses.asdict(repo) for repo in repositories[:6]]) + ) except asyncio.CancelledError: logger.debug("Github cache refresh canceled") - except Exception: + except Exception: # pylint: disable=broad-except logger.exception("Error refreshing GitHub cache") - raise + + return repositories async def sync_slack_users( @@ -87,11 +89,11 @@ async def sync_slack_users( *, cache_key_tz: str = SLACK_TZ_CACHE_KEY, cache_key_count: str = SLACK_COUNT_CACHE_KEY, -): +) -> Counter: logger.debug("Refreshing slack users cache.") + counter: Counter = Counter() try: - counter: Counter = Counter() async for user in slack_client.iter(slack.methods.USERS_LIST, minimum_time=3): if user["deleted"] or user["is_bot"] or not user["tz"]: continue @@ -112,16 +114,17 @@ async def sync_slack_users( logger.debug("Slack users cache refresh canceled") except Exception: # pylint: disable=broad-except logger.exception("Error refreshing slack users cache") - return + + return counter async def sync_slack_channels( slack_client: SlackAPI, redis: RedisConnection, *, cache_key: str = SLACK_CHANNEL_CACHE_KEY -) -> None: +) -> List[Channel]: logger.debug("Refreshing slack channels cache.") + channels = [] try: - channels = [] async for channel in slack_client.iter(slack.methods.CHANNELS_LIST): channels.append( Channel( @@ -137,10 +140,13 @@ async def sync_slack_channels( logger.debug("Found %s slack channels", len(channels)) - await redis.set(cache_key, json.dumps([x.__dict__ for x in channels])) + await redis.set( + cache_key, json.dumps([dataclasses.asdict(channel) for channel in channels]) + ) except asyncio.CancelledError: logger.debug("Slack channels cache refresh canceled") except Exception: # pylint: disable=broad-except logger.exception("Error refreshing slack channels cache") - return + + return channels diff --git a/pyslackersweb/website/views.py b/pyslackersweb/website/views.py index e887d531..7a41f3c2 100644 --- a/pyslackersweb/website/views.py +++ b/pyslackersweb/website/views.py @@ -1,6 +1,8 @@ import json import logging +import slack.exceptions + from aiohttp import web from aiohttp_jinja2 import template from marshmallow.exceptions import ValidationError @@ -24,7 +26,9 @@ async def get(self): return { "member_count": int((await redis.get(SLACK_COUNT_CACHE_KEY, encoding="utf-8")) or 0), - "projects": json.loads(await redis.get(GITHUB_REPO_CACHE_KEY, encoding="utf-8")), + "projects": json.loads( + await redis.get(GITHUB_REPO_CACHE_KEY, encoding="utf-8") or "{}" + ), "sponsors": [ { "image": self.request.app.router["static"].url_for( @@ -65,19 +69,17 @@ async def post(self): try: invite = self.schema.load(await self.request.post()) - async with self.request.app["client_session"].post( - "https://slack.com/api/users.admin.invite", - headers={"Authorization": f"Bearer {self.request.app['slack_invite_token']}"}, - data={"email": invite["email"], "resend": True}, - ) as r: - body = await r.json() - - if body["ok"]: - context["success"] = True - else: - logger.warning("Error sending slack invite: %s", body["error"], extra=body) - context["errors"].update(non_field=[body["error"]]) + await self.request.app["slack_client"].query( + url="users.admin.invite", data={"email": invite["email"], "resend": True} + ) + context["success"] = True except ValidationError as e: context["errors"] = e.normalized_messages() + except slack.exceptions.SlackAPIError as e: + logger.warning("Error sending slack invite: %s", e.error, extra=e.data) + context["errors"].update(non_field=[e.error]) + except slack.exceptions.HTTPException: + logger.exception("Error contacting slack API") + context["errors"].update(non_field=["Error contacting slack API"]) return context diff --git a/requirements/testing.txt b/requirements/testing.txt index a6a882c4..7e17927e 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -7,3 +7,4 @@ pytest-cov==2.7.1 pytest-aiohttp==0.3.0 tox==3.13.2 mypy==0.720 +asynctest==0.13.0 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..48cad4ae --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,18 @@ +import pytest + +import pyslackersweb +import pyslackersweb.website.tasks + +pytest_plugins = ("slack.tests.plugin",) + + +@pytest.fixture +async def client(aiohttp_client, slack_client): + + application = await pyslackersweb.app_factory() + + app_client = await aiohttp_client(application) + app_client.app["scheduler"].shutdown() + app_client.app["website_app"]["slack_client"] = slack_client + + return app_client diff --git a/tests/test_nothing.py b/tests/test_nothing.py deleted file mode 100644 index a3e981b5..00000000 --- a/tests/test_nothing.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_nothing(): - pass diff --git a/tests/test_website.py b/tests/test_website.py new file mode 100644 index 00000000..538f9b44 --- /dev/null +++ b/tests/test_website.py @@ -0,0 +1,111 @@ +import logging + +from collections import namedtuple + +import pytest +import aiohttp.web + +from pyslackersweb.website import tasks + +SlackInviteTestParam = namedtuple("Param", "response data expected") + + +async def test_endpoint_index(client): + r = await client.get("/") + + assert r.history[0].url.path == "/" + assert r.history[0].status == 302 + + assert r.status == 200 + assert r.url.path == "/web" + + +async def test_endpoint_slack(client): + r = await client.get("/web/slack") + assert r.status == 200 + + +@pytest.mark.parametrize( + "slack_client,data,expected", + ( + SlackInviteTestParam( + response={}, + data={"email": "error@example.com", "agree_tos": True}, + expected="successAlert", + ), + SlackInviteTestParam( + response={}, data={"agree_tos": True}, expected="Missing data for required field" + ), + SlackInviteTestParam( + response={}, + data={"email": "error@example.com", "agree_tos": False}, + expected="There was an error processing your invite", + ), + SlackInviteTestParam( + response={}, + data={"email": "foobar", "agree_tos": True}, + expected="Not a valid email address", + ), + SlackInviteTestParam( + response={"body": {"ok": False, "error": "already_in_team"}}, + data={"email": "error@example.com", "agree_tos": True}, + expected="Reason: already_in_team", + ), + SlackInviteTestParam( + response={"body": {"ok": False, "error": "not_authed"}}, + data={"email": "error@example.com", "agree_tos": True}, + expected="Reason: not_authed", + ), + SlackInviteTestParam( + response={"status": 500}, + data={"email": "error@example.com", "agree_tos": True}, + expected="Reason: Error contacting slack API", + ), + ), + indirect=["slack_client"], +) +async def test_endpoint_slack_invite(client, data, expected): + r = await client.post(path="/web/slack", data=data) + html = await r.text() + + assert r.status == 200 + assert expected in html + + +async def test_task_sync_github_repositories(client, caplog): + + async with aiohttp.ClientSession() as session: + result = await tasks.sync_github_repositories(session, client.app["redis"]) + + assert result + + for record in caplog.records: + assert record.levelno <= logging.INFO + + +@pytest.mark.parametrize("slack_client", ({"body": ["users_iter", "users"]},), indirect=True) +async def test_task_sync_slack_users(client, caplog): + + result = await tasks.sync_slack_users( + client.app["website_app"]["slack_client"], client.app["redis"] + ) + + assert result + assert len(result) == 1 + assert result["America/Los_Angeles"] == 2 + + for record in caplog.records: + assert record.levelno <= logging.INFO + + +@pytest.mark.parametrize("slack_client", ({"body": ["channels_iter", "channels"]},), indirect=True) +async def test_task_sync_slack_channels(client, caplog): + + result = await tasks.sync_slack_channels( + client.app["website_app"]["slack_client"], client.app["redis"] + ) + + assert result + + for record in caplog.records: + assert record.levelno <= logging.INFO diff --git a/tox.ini b/tox.ini index 32390469..f2a5bfb4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,18 +1,23 @@ [tox] -envlist = py37,lint,fmt +envlist = py37,lint skipsdist = true [testenv] -deps = -r requirements/testing.txt -commands = python -m pytest --verbose --cov=pyslackersweb/ --cov-report=term-missing --junit-xml={envdir}/artifacts/test-results.xml tests/ +passenv = + REDIS_URL + POSTGRESQL_DSN +commands = + pip install -r requirements/testing.txt + python -m pytest --verbose --cov=pyslackersweb/ --cov-report=term-missing --junit-xml={envdir}/artifacts/test-results.xml {posargs:tests/} [testenv:lint] -deps = - -r requirements/testing.txt commands = - pylint ./pyslackersweb - mypy ./pyslackersweb --ignore-missing-imports + pip install -r requirements/testing.txt + black --check . + pylint pyslackersweb tests + mypy . --ignore-missing-imports -[testenv:fmt] -deps = black -commands = black --check pyslackersweb/ tests/ +[testenv:autoformat] +commands = + pip install -r requirements/testing.txt + black .