diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index e9102a995..a577ee09f 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -18,6 +18,8 @@ pygeoapi configuration contains the following core sections: - ``metadata``: server-wide metadata (contact, licensing, etc.) - ``resources``: dataset collections, processes and stac-collections offered by the server +The full configuration schema with descriptions of all available properties can be found `here `_. + .. note:: `Standard YAML mechanisms `_ can be used (anchors, references, etc.) for reuse and compactness. @@ -30,6 +32,7 @@ Reference ^^^^^^^^^^ The ``server`` section provides directives on binding and high level tuning. +Please find more information related to API design rules (the property at the bottom of the example below) :ref:`further down`. .. code-block:: yaml @@ -61,6 +64,12 @@ The ``server`` section provides directives on binding and high level tuning. connection: /tmp/pygeoapi-process-manager.db # connection info to store jobs (e.g. filepath) output_dir: /tmp/ # temporary file area for storing job results (files) + api_rules: # optional API design rules to which pygeoapi should adhere + api_version: 1.2.3 # omit to use pygeoapi's software version + strict_slashes: true # trailing slashes will not be allowed and result in a 404 + url_prefix: 'v{api_major}' # adds a /v1 prefix to all URL paths + version_header: X-API-Version # add a response header of this name with the API version + ``logging`` ^^^^^^^^^^^ @@ -303,6 +312,58 @@ Examples: curl https://example.org/collections/foo # user can access resource normally +API Design Rules +---------------- + +Some pygeoapi setups may wish to adhere to specific API design rules that apply at an organization. +The ``api_rules`` object in the ``server`` section of the configuration can be used for this purpose. + +Note that the entire ``api_rules`` object is optional. No rules will be applied if the object is omitted. + +The following properties can be set: + +``api_version`` +^^^^^^^^^^^^^^^ + +If specified, this property is a string that defines the semantic version number of the API. +Note that this number should reflect the state of the *API data model* (request and response object structure, API endpoints, etc.) +and does not necessarily correspond to the *software* version of pygeoapi. For example, the software could have been +completely rewritten (which changes the software version number), but the API data model might still be the same as before. + +Unfortunately, pygeoapi currently does not offer a way to keep track of the API version. +This means that you need to set (and maintain) your own version here or leave it empty or unset. +In the latter case, the software version of pygeoapi will be used instead. + +``strict_slashes`` +^^^^^^^^^^^^^^^^^^ + +Some API rules state that trailing slashes at the end of a URL are not allowed if they point to a specific resource item. +In that case, you may wish to set this property to ``true``. Doing so will result in a ``404 Not Found`` if a user adds a ``/`` to the end of a URL. +If omitted or ``false`` (default), it does not matter whether the user omits or adds the ``/`` to the end of the URL. + +``url_prefix`` +^^^^^^^^^^^^^^ + +Set this property to include a prefix in the URL path (e.g. `https://base.com//endpoint`). +Note that you do not need to include slashes (either at the start or the end) here: they will be added automatically. + +If you wish to include the API version number (depending on the `api_version`_ property) in the prefix, you can use the following variables: + +- ``{api_version}``: full semantic version number +- ``{api_major}``: major version number +- ``{api_minor}``: minor version number +- ``{api_build}``: build number + +For example, if the API version is *1.2.3*, then a URL prefix template of ``v{api_major}`` will result in *v1* as the actual prefix. + +``version_header`` +^^^^^^^^^^^^^^^^^^ + +Set this property to add a header to each pygeoapi response that includes the semantic API version (see `api_version`_). +If omitted, no header will be added. Common names for this header are ``API-Version`` or ``X-API-Version``. +Note that pygeoapi already adds a ``X-Powered-By`` header by default that includes the software version number. + + Validating the configuration ---------------------------- diff --git a/pygeoapi/api.py b/pygeoapi/api.py index 5dcdffa9f..e0d4806ab 100644 --- a/pygeoapi/api.py +++ b/pygeoapi/api.py @@ -45,7 +45,6 @@ from http import HTTPStatus import json import logging -import os import re from typing import Any, Tuple, Union, Optional import urllib.parse @@ -76,18 +75,16 @@ ProviderTileQueryError, ProviderTilesetIdNotFoundError) from pygeoapi.models.cql import CQLModel - from pygeoapi.util import (dategetter, DATETIME_FORMAT, UrlPrefetcher, filter_dict_by_key_value, get_crs_from_uri, get_supported_crs_list, CrsTransformSpec, get_provider_by_type, get_provider_default, get_typed_value, JobStatus, json_serial, render_j2_template, str2bool, - transform_bbox, TEMPLATES, to_json) + transform_bbox, TEMPLATES, to_json, get_api_rules, get_base_url) from pygeoapi.models.provider.base import TilesMetadataFormat - LOGGER = logging.getLogger(__name__) #: Return headers for requests (e.g:X-Powered-By) @@ -567,7 +564,8 @@ def is_valid(self, additional_formats=None) -> bool: def get_response_headers(self, force_lang: l10n.Locale = None, force_type: str = None, - force_encoding: str = None) -> dict: + force_encoding: str = None, + **custom_headers) -> dict: """ Prepares and returns a dictionary with Response object headers. @@ -595,6 +593,7 @@ def get_response_headers(self, force_lang: l10n.Locale = None, """ headers = HEADERS.copy() + headers.update(**custom_headers) l10n.set_response_language(headers, force_lang or self._locale) if force_type: # Set custom MIME type if specified @@ -638,7 +637,8 @@ def __init__(self, config): """ self.config = config - self.config['server']['url'] = self.config['server']['url'].rstrip('/') + self.api_headers = get_api_rules(self.config).response_headers + self.base_url = get_base_url(self.config) self.prefetcher = UrlPrefetcher() CHARSET[0] = config['server'].get('encoding', 'utf-8') @@ -660,6 +660,10 @@ def __init__(self, config): setup_logger(self.config['logging']) + # Create config clone for HTML templating with modified base URL + self.tpl_config = deepcopy(self.config) + self.tpl_config['server']['url'] = self.base_url + # TODO: add as decorator if 'manager' in self.config['server']: manager_def = self.config['server']['manager'] @@ -708,34 +712,34 @@ def landing_page(self, 'rel': request.get_linkrel(F_JSON), 'type': FORMAT_TYPES[F_JSON], 'title': 'This document as JSON', - 'href': f"{self.config['server']['url']}?f={F_JSON}" + 'href': f"{self.base_url}?f={F_JSON}" }, { 'rel': request.get_linkrel(F_JSONLD), 'type': FORMAT_TYPES[F_JSONLD], 'title': 'This document as RDF (JSON-LD)', - 'href': f"{self.config['server']['url']}?f={F_JSONLD}" + 'href': f"{self.base_url}?f={F_JSONLD}" }, { 'rel': request.get_linkrel(F_HTML), 'type': FORMAT_TYPES[F_HTML], 'title': 'This document as HTML', - 'href': f"{self.config['server']['url']}?f={F_HTML}", + 'href': f"{self.base_url}?f={F_HTML}", 'hreflang': self.default_locale }, { 'rel': 'service-desc', 'type': 'application/vnd.oai.openapi+json;version=3.0', 'title': 'The OpenAPI definition as JSON', - 'href': f"{self.config['server']['url']}/openapi" + 'href': f"{self.base_url}/openapi" }, { 'rel': 'service-doc', 'type': FORMAT_TYPES[F_HTML], 'title': 'The OpenAPI definition as HTML', - 'href': f"{self.config['server']['url']}/openapi?f={F_HTML}", + 'href': f"{self.base_url}/openapi?f={F_HTML}", 'hreflang': self.default_locale }, { 'rel': 'conformance', 'type': FORMAT_TYPES[F_JSON], 'title': 'Conformance', - 'href': f"{self.config['server']['url']}/conformance" + 'href': f"{self.base_url}/conformance" }, { 'rel': 'data', 'type': FORMAT_TYPES[F_JSON], @@ -745,15 +749,15 @@ def landing_page(self, 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/processes', 'type': FORMAT_TYPES[F_JSON], 'title': 'Processes', - 'href': f"{self.config['server']['url']}/processes" + 'href': f"{self.base_url}/processes" }, { 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', 'type': FORMAT_TYPES[F_JSON], 'title': 'Jobs', - 'href': f"{self.config['server']['url']}/jobs" + 'href': f"{self.base_url}/jobs" }] - headers = request.get_response_headers() + headers = request.get_response_headers(**self.api_headers) if request.format == F_HTML: # render fcm['processes'] = False @@ -767,8 +771,8 @@ def landing_page(self, 'type', 'stac-collection'): fcm['stac'] = True - content = render_j2_template(self.config, 'landing_page.html', fcm, - request.locale) + content = render_j2_template(self.tpl_config, 'landing_page.html', + fcm, request.locale) return headers, HTTPStatus.OK, content if request.format == F_JSONLD: @@ -793,19 +797,18 @@ def openapi(self, request: Union[APIRequest, Any], if not request.is_valid(): return self.get_format_exception(request) - headers = request.get_response_headers() + headers = request.get_response_headers(**self.api_headers) if request.format == F_HTML: template = 'openapi/swagger.html' if request._args.get('ui') == 'redoc': template = 'openapi/redoc.html' - path = '/'.join([self.config['server']['url'].rstrip('/'), - 'openapi']) + path = f'{self.base_url}/openapi' data = { 'openapi-document-path': path } - content = render_j2_template(self.config, template, data, + content = render_j2_template(self.tpl_config, template, data, request.locale) return headers, HTTPStatus.OK, content @@ -845,9 +848,9 @@ def conformance(self, 'conformsTo': list(set(conformance_list)) } - headers = request.get_response_headers() + headers = request.get_response_headers(**self.api_headers) if request.format == F_HTML: # render - content = render_j2_template(self.config, 'conformance.html', + content = render_j2_template(self.tpl_config, 'conformance.html', conformance, request.locale) return headers, HTTPStatus.OK, content @@ -869,7 +872,7 @@ def describe_collections(self, request: Union[APIRequest, Any], if not request.is_valid(): return self.get_format_exception(request) - headers = request.get_response_headers() + headers = request.get_response_headers(**self.api_headers) fcm = { 'collections': [], @@ -975,13 +978,13 @@ def describe_collections(self, request: Union[APIRequest, Any], 'type': FORMAT_TYPES[F_JSON], 'rel': 'root', 'title': 'The landing page of this server as JSON', - 'href': f"{self.config['server']['url']}?f={F_JSON}" + 'href': f"{self.base_url}?f={F_JSON}" }) collection['links'].append({ 'type': FORMAT_TYPES[F_HTML], 'rel': 'root', 'title': 'The landing page of this server as HTML', - 'href': f"{self.config['server']['url']}?f={F_HTML}" + 'href': f"{self.base_url}?f={F_HTML}" }) collection['links'].append({ 'type': FORMAT_TYPES[F_JSON], @@ -1154,7 +1157,7 @@ def describe_collections(self, request: Union[APIRequest, Any], 'type': map_mimetype, 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/map', 'title': f'Map as {map_format}', - 'href': f"{self.config['server']['url']}/collections/{k}/map?f={map_format}" # noqa + 'href': f"{self.get_collections_url()}/{k}/map?f={map_format}" # noqa }) try: @@ -1225,11 +1228,11 @@ def describe_collections(self, request: Union[APIRequest, Any], if request.format == F_HTML: # render fcm['collections_path'] = self.get_collections_url() if dataset is not None: - content = render_j2_template(self.config, + content = render_j2_template(self.tpl_config, 'collections/collection.html', fcm, request.locale) else: - content = render_j2_template(self.config, + content = render_j2_template(self.tpl_config, 'collections/index.html', fcm, request.locale) @@ -1265,7 +1268,7 @@ def get_collection_queryables(self, request: Union[APIRequest, Any], if not request.is_valid(): return self.get_format_exception(request) - headers = request.get_response_headers() + headers = request.get_response_headers(**self.api_headers) if any([dataset is None, dataset not in self.config['resources'].keys()]): @@ -1330,7 +1333,7 @@ def get_collection_queryables(self, request: Union[APIRequest, Any], queryables['collections_path'] = self.get_collections_url() - content = render_j2_template(self.config, + content = render_j2_template(self.tpl_config, 'collections/queryables.html', queryables, request.locale) @@ -1359,7 +1362,8 @@ def get_collection_items( # Set Content-Language to system locale until provider locale # has been determined - headers = request.get_response_headers(SYSTEM_LOCALE) + headers = request.get_response_headers(SYSTEM_LOCALE, + **self.api_headers) properties = [] reserved_fieldnames = ['bbox', 'bbox-crs', 'crs', 'f', 'lang', 'limit', @@ -2201,7 +2205,8 @@ def get_collection_item(self, request: Union[APIRequest, Any], # Set Content-Language to system locale until provider locale # has been determined - headers = request.get_response_headers(SYSTEM_LOCALE) + headers = request.get_response_headers(SYSTEM_LOCALE, + **self.api_headers) LOGGER.debug('Processing query parameters') diff --git a/pygeoapi/config.py b/pygeoapi/config.py index d6eb75763..61e47a1ee 100644 --- a/pygeoapi/config.py +++ b/pygeoapi/config.py @@ -40,6 +40,14 @@ THISDIR = Path(__file__).parent.resolve() +def load_schema() -> dict: + """ Reads the JSON schema YAML file. """ + schema_file = THISDIR / 'schemas' / 'config' / 'pygeoapi-config-0.x.yml' + + with schema_file.open() as fh2: + return yaml_load(fh2) + + def validate_config(instance_dict: dict) -> bool: """ Validate pygeoapi configuration against pygeoapi schema @@ -48,14 +56,8 @@ def validate_config(instance_dict: dict) -> bool: :returns: `bool` of validation """ - - schema_file = THISDIR / 'schemas' / 'config' / 'pygeoapi-config-0.x.yml' - - with schema_file.open() as fh2: - schema_dict = yaml_load(fh2) - jsonschema_validate(json.loads(to_json(instance_dict)), schema_dict) - - return True + jsonschema_validate(json.loads(to_json(instance_dict)), load_schema()) + return True @click.group() diff --git a/pygeoapi/django_/settings.py b/pygeoapi/django_/settings.py index 5a09742c2..1185c7c0d 100644 --- a/pygeoapi/django_/settings.py +++ b/pygeoapi/django_/settings.py @@ -48,6 +48,7 @@ import os # pygeoapi specific from pygeoapi.django_app import config +from pygeoapi.util import get_api_rules # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -166,3 +167,9 @@ # pygeoapi specific PYGEOAPI_CONFIG = config() + +API_RULES = get_api_rules(PYGEOAPI_CONFIG) + +# Defaults to True in Django +# https://docs.djangoproject.com/en/3.2/ref/settings/#append-slash +APPEND_SLASH = not API_RULES.strict_slashes diff --git a/pygeoapi/django_/urls.py b/pygeoapi/django_/urls.py index 8fed37be9..85c99662b 100644 --- a/pygeoapi/django_/urls.py +++ b/pygeoapi/django_/urls.py @@ -53,27 +53,46 @@ path, ) from django.conf import settings +from django.conf.urls import include from django.conf.urls.static import static from . import views + +def apply_slash_rule(url: str): + """ Strip trailing slashes if the API rules are strict about it. + This works in conjunction with Django's APPEND_SLASH setting. + """ + if settings.API_RULES.strict_slashes: + url = url.rstrip('/') + return url + + urlpatterns = [ path('', views.landing_page, name='landing-page'), - path('openapi/', views.openapi, name='openapi'), - path('conformance/', views.conformance, name='conformance'), - path('collections/', views.collections, name='collections'), + path(apply_slash_rule('openapi/'), views.openapi, name='openapi'), + path( + apply_slash_rule('conformance/'), + views.conformance, + name='conformance' + ), + path( + apply_slash_rule('collections/'), + views.collections, + name='collections' + ), path( 'collections/', views.collections, name='collection-detail', ), path( - 'collections//queryables/', + apply_slash_rule('collections//queryables/'), views.collection_queryables, name='collection-queryables', ), path( - 'collections//items/', + apply_slash_rule('collections//items/'), views.collection_items, name='collection-items', ), @@ -83,17 +102,17 @@ name='collection-item', ), path( - 'collections//coverage/', + apply_slash_rule('collections//coverage/'), views.collection_coverage, name='collection-coverage', ), path( - 'collections//coverage/domainset/', + apply_slash_rule('collections//coverage/domainset/'), # noqa views.collection_coverage_domainset, name='collection-coverage-domainset', ), path( - 'collections//coverage/rangetype/', + apply_slash_rule('collections//coverage/rangetype/'), # noqa views.collection_coverage_rangetype, name='collection-coverage-rangetype', ), @@ -103,12 +122,12 @@ name='collection-map', ), path( - 'collections//styles//styles//map', views.collection_style_map, name='collection-style-map', ), path( - 'collections//tiles/', + apply_slash_rule('collections//tiles/'), views.collection_tiles, name='collection-tiles', ), @@ -188,12 +207,12 @@ views.get_collection_edr_query, name='collection-edr-instance-corridor', ), - path('processes/', views.processes, name='processes'), + path(apply_slash_rule('processes/'), views.processes, name='processes'), path('processes/', views.processes, name='process-detail'), - path('jobs/', views.jobs, name='jobs'), + path(apply_slash_rule('jobs/'), views.jobs, name='jobs'), path('jobs/', views.jobs, name='job'), path( - 'jobs//results/', + apply_slash_rule('jobs//results/'), views.job_results, name='job-results', ), @@ -202,9 +221,31 @@ views.job_results_resource, name='job-results-resource', ), - path('stac/', views.stac_catalog_root, name='stac-catalog-root'), + path( + apply_slash_rule('stac/'), + views.stac_catalog_root, + name='stac-catalog-root' + ), path('stac/', views.stac_catalog_path, name='stac-catalog-path'), path( - 'stac/search/', views.stac_catalog_search, name='stac-catalog-search' + apply_slash_rule('stac/search/'), + views.stac_catalog_search, + name='stac-catalog-search' ), -] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) +] + +url_route_prefix = settings.API_RULES.get_url_prefix('django') +if url_route_prefix: + # Add a URL prefix to all routes if configured + urlpatterns = [ + path(url_route_prefix, include(urlpatterns)) + ] + +# Add static URL and optionally add prefix (note: do NOT use django style here) +url_static_prefix = settings.API_RULES.get_url_prefix() +urlpatterns.append( + static( + f"{url_static_prefix}{settings.STATIC_URL}", + document_root=settings.STATIC_ROOT + ) +) diff --git a/pygeoapi/django_/views.py b/pygeoapi/django_/views.py index f4038e407..9c6ced8d1 100644 --- a/pygeoapi/django_/views.py +++ b/pygeoapi/django_/views.py @@ -147,6 +147,9 @@ def collection_items(request: HttpRequest, collection_id: str) -> HttpResponse: else: response_ = _feed_response(request, 'post_collection_items', request, collection_id) + elif request.method == 'OPTIONS': + response_ = _feed_response(request, 'manage_collection_item', + request, 'options', collection_id) response = _to_django_response(*response_) @@ -214,6 +217,10 @@ def collection_item(request: HttpRequest, request, 'manage_collection_item', request, 'delete', collection_id, item_id ) + elif request.method == 'OPTIONS': + response_ = _feed_response( + request, 'manage_collection_item', request, 'options', + collection_id, item_id) response = _to_django_response(*response_) diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index 759acf579..74be94dd1 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -37,25 +37,30 @@ from flask import Flask, Blueprint, make_response, request, send_from_directory from pygeoapi.api import API -from pygeoapi.util import get_mimetype, yaml_load +from pygeoapi.util import get_mimetype, yaml_load, get_api_rules -CONFIG = None - if 'PYGEOAPI_CONFIG' not in os.environ: raise RuntimeError('PYGEOAPI_CONFIG environment variable not set') with open(os.environ.get('PYGEOAPI_CONFIG'), encoding='utf8') as fh: CONFIG = yaml_load(fh) +API_RULES = get_api_rules(CONFIG) + STATIC_FOLDER = 'static' if 'templates' in CONFIG['server']: STATIC_FOLDER = CONFIG['server']['templates'].get('static', 'static') APP = Flask(__name__, static_folder=STATIC_FOLDER, static_url_path='/static') -APP.url_map.strict_slashes = False - -BLUEPRINT = Blueprint('pygeoapi', __name__, static_folder=STATIC_FOLDER) +APP.url_map.strict_slashes = API_RULES.strict_slashes + +BLUEPRINT = Blueprint( + 'pygeoapi', + __name__, + static_folder=STATIC_FOLDER, + url_prefix=API_RULES.get_url_prefix('flask') +) # CORS: optionally enable from config. if CONFIG['server'].get('cors', False): diff --git a/pygeoapi/linked_data.py b/pygeoapi/linked_data.py index 17643c2b8..c502922b2 100644 --- a/pygeoapi/linked_data.py +++ b/pygeoapi/linked_data.py @@ -139,7 +139,7 @@ def jsonldify_collection(cls, collection: dict, locale_: str) -> dict: dataset = { "@type": "Dataset", - "@id": f"{cls.config['server']['url']}/collections/{collection['id']}", + "@id": f"{cls.base_url}/collections/{collection['id']}", "name": l10n.translate(collection['title'], locale_), "description": l10n.translate(collection['description'], locale_), "license": cls.fcmld['license'], @@ -173,14 +173,14 @@ def jsonldify_collection(cls, collection: dict, locale_: str) -> dict: return dataset -def geojson2jsonld(config: dict, data: dict, dataset: str, +def geojson2jsonld(cls, data: dict, dataset: str, identifier: str = None, id_field: str = 'id') -> str: """ Render GeoJSON-LD from a GeoJSON base. Inserts a @context that can be read from, and extended by, the pygeoapi configuration for a particular dataset. - :param config: dict of configuration + :param cls: API object :param data: dict of data: :param dataset: dataset identifier :param identifier: item identifier (optional) @@ -190,7 +190,8 @@ def geojson2jsonld(config: dict, data: dict, dataset: str, """ LOGGER.debug('Fetching context and template from resource configuration') - jsonld = config['resources'][dataset].get('linked-data', {}) + jsonld = cls.config['resources'][dataset].get('linked-data', {}) + ds_url = f"{cls.get_collections_url()}/{dataset}" context = jsonld.get('context', []).copy() template = jsonld.get('item_template', None) @@ -221,14 +222,14 @@ def geojson2jsonld(config: dict, data: dict, dataset: str, 'FeatureCollection': 'schema:itemList' }) - data['@id'] = f"{config['server']['url']}/collections/{dataset}" + data['@id'] = ds_url for i, feature in enumerate(data['features']): # Get URI for each feature identifier_ = feature.get(id_field, feature['properties'].get(id_field, '')) if not is_url(str(identifier_)): - identifier_ = f"{config['server']['url']}/collections/{dataset}/items/{feature['id']}" # noqa + identifier_ = f"{ds_url}/items/{feature['id']}" # noqa data['features'][i] = { '@id': identifier_, @@ -250,8 +251,7 @@ def geojson2jsonld(config: dict, data: dict, dataset: str, else: # Render jsonld template for single item with template configured LOGGER.debug(f'Rendering JSON-LD template: {template}') - content = render_j2_template( - config, template, ldjsonData) + content = render_j2_template(cls.config, template, ldjsonData) ldjsonData = json.loads(content) return ldjsonData diff --git a/pygeoapi/models/config.py b/pygeoapi/models/config.py new file mode 100644 index 000000000..148f78d77 --- /dev/null +++ b/pygeoapi/models/config.py @@ -0,0 +1,103 @@ +# ****************************** -*- +# flake8: noqa +# ================================================================= +# +# Authors: Sander Schaminee +# +# Copyright (c) 2023 Sander Schaminee +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +from pydantic import BaseModel, Field + + +class APIRules(BaseModel): + """ Pydantic model for API design rules that must be adhered to. """ + api_version: str = Field(regex=r'^\d+\.\d+\..+$', + description="Semantic API version number.") + url_prefix: str = Field( + "", + description="If set, pygeoapi routes will be prepended with the " + "given URL path prefix (e.g. '/v1'). " + "Defaults to an empty string (no prefix)." + ) + version_header: str = Field( + "", + description="If set, pygeoapi will set a response header with this " + "name and its value will hold the API version. " + "Defaults to an empty string (i.e. no header). " + "Often 'API-Version' or 'X-API-Version' are used here." + ) + strict_slashes: bool = Field( + False, + description="If False (default), URL trailing slashes are allowed. " + "If True, pygeoapi will return a 404." + ) + + @staticmethod + def create(**rules_config) -> 'APIRules': + """ Returns a new APIRules instance for the current API version + and configured rules. """ + obj = { + k: v for k, v in rules_config.items() if k in APIRules.__fields__ + } + # Validation will fail if required `api_version` is missing + # or if `api_version` is not a semantic version number + return APIRules.parse_obj(obj) + + @property + def response_headers(self) -> dict: + """ Gets a dictionary of additional response headers for the current + API rules. Returns an empty dict if no rules apply. """ + headers = {} + if self.version_header: + headers[self.version_header] = self.api_version + return headers + + def get_url_prefix(self, style: str = None) -> str: + """ + Returns an API URL prefix to use in all paths. + May include a (partial) API version. See docs for syntax. + :param style: Set to 'django', 'flask' or 'starlette' to return a + specific prefix formatted for those frameworks. + If not set, only the prefix itself will be returned. + """ + if not self.url_prefix: + return "" + major, minor, build = self.api_version.split('.') + prefix = self.url_prefix.format( + api_version=self.api_version, + api_major=major, + api_minor=minor, + api_build=build + ).strip('/') + style = (style or '').lower() + if style == 'django': + # Django requires the slash at the end + return rf"^{prefix}/" + elif style in ('flask', 'starlette'): + # Flask and Starlette need the slash in front + return f"/{prefix}" + # If no format is specified, return only the bare prefix + return prefix diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index cea581ca3..af1563030 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -41,13 +41,13 @@ from jsonschema import validate as jsonschema_validate import yaml -from pygeoapi import __version__ from pygeoapi import l10n from pygeoapi.plugin import load_plugin from pygeoapi.models.openapi import OAPIFormat from pygeoapi.provider.base import ProviderTypeError, SchemaType from pygeoapi.util import (filter_dict_by_key_value, get_provider_by_type, - filter_providers_by_type, to_json, yaml_load) + filter_providers_by_type, to_json, yaml_load, + get_api_rules, get_base_url) LOGGER = logging.getLogger(__name__) @@ -76,7 +76,8 @@ def get_ogc_schemas_location(server_config): if osl.startswith('http'): value = osl elif osl.startswith('/'): - value = os.path.join(server_config['url'], 'schemas') + base_url = get_base_url({'server': server_config}) + value = f'{base_url}/schemas' return value @@ -141,6 +142,8 @@ def get_oas_30(cfg): server_locales = l10n.get_locales(cfg) locale_ = server_locales[0] + api_rules = get_api_rules(cfg) + osl = get_ogc_schemas_location(cfg['server']) OPENAPI_YAML['oapif-1'] = os.path.join(osl, 'ogcapi/features/part1/1.0/openapi/ogcapi-features-1.yaml') # noqa OPENAPI_YAML['oapif-2'] = os.path.join(osl, 'ogcapi/features/part2/1.0/openapi/ogcapi-features-2.yaml') # noqa @@ -165,12 +168,12 @@ def get_oas_30(cfg): 'name': cfg['metadata']['license']['name'], 'url': cfg['metadata']['license']['url'] }, - 'version': __version__ + 'version': api_rules.api_version } oas['info'] = info oas['servers'] = [{ - 'url': cfg['server']['url'], + 'url': get_base_url(cfg), 'description': l10n.translate(cfg['metadata']['identification']['description'], locale_) # noqa }] @@ -1103,8 +1106,6 @@ def get_oas_30(cfg): processes = filter_dict_by_key_value(cfg['resources'], 'type', 'process') - has_manager = 'manager' in cfg['server'] - if processes: paths['/processes'] = { 'get': { @@ -1202,70 +1203,69 @@ def get_oas_30(cfg): } } - if has_manager: - paths['/jobs'] = { - 'get': { - 'summary': 'Retrieve jobs list', - 'description': 'Retrieve a list of jobs', - 'tags': ['server'], - 'operationId': 'getJobs', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} - } + paths['/jobs'] = { + 'get': { + 'summary': 'Retrieve jobs list', + 'description': 'Retrieve a list of jobs', + 'tags': ['server'], + 'operationId': 'getJobs', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} } } + } - paths['/jobs/{jobId}'] = { - 'get': { - 'summary': 'Retrieve job details', - 'description': 'Retrieve job details', - 'tags': ['server'], - 'parameters': [ - name_in_path, - {'$ref': '#/components/parameters/f'} - ], - 'operationId': 'getJob', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} # noqa - } - }, - 'delete': { - 'summary': 'Cancel / delete job', - 'description': 'Cancel / delete job', - 'tags': ['server'], - 'parameters': [ - name_in_path - ], - 'operationId': 'deleteJob', - 'responses': { - '204': {'$ref': '#/components/responses/204'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} # noqa - } - }, - } + paths['/jobs/{jobId}'] = { + 'get': { + 'summary': 'Retrieve job details', + 'description': 'Retrieve job details', + 'tags': ['server'], + 'parameters': [ + name_in_path, + {'$ref': '#/components/parameters/f'} + ], + 'operationId': 'getJob', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} # noqa + } + }, + 'delete': { + 'summary': 'Cancel / delete job', + 'description': 'Cancel / delete job', + 'tags': ['server'], + 'parameters': [ + name_in_path + ], + 'operationId': 'deleteJob', + 'responses': { + '204': {'$ref': '#/components/responses/204'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} # noqa + } + }, + } - paths['/jobs/{jobId}/results'] = { - 'get': { - 'summary': 'Retrieve job results', - 'description': 'Retrive job resiults', - 'tags': ['server'], - 'parameters': [ - name_in_path, - {'$ref': '#/components/parameters/f'} - ], - 'operationId': 'getJobResults', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} # noqa - } + paths['/jobs/{jobId}/results'] = { + 'get': { + 'summary': 'Retrieve job results', + 'description': 'Retrive job resiults', + 'tags': ['server'], + 'parameters': [ + name_in_path, + {'$ref': '#/components/parameters/f'} + ], + 'operationId': 'getJobResults', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} # noqa } } + } oas['paths'] = paths diff --git a/pygeoapi/process/manager/tinydb_.py b/pygeoapi/process/manager/tinydb_.py index 37b0f6234..822997dd5 100644 --- a/pygeoapi/process/manager/tinydb_.py +++ b/pygeoapi/process/manager/tinydb_.py @@ -27,7 +27,12 @@ # # ================================================================= -import fcntl +try: + import fcntl +except ModuleNotFoundError: + # When on Windows, fcntl does not exist and file locking is automatic + fcntl = None + import json import logging from pathlib import Path @@ -65,7 +70,7 @@ def _connect(self, mode: str = 'r') -> bool: self.db = tinydb.TinyDB(self.connection) - if mode == 'w': + if mode == 'w' and fcntl is not None: fcntl.lockf(self.db.storage._handle, fcntl.LOCK_EX) return True @@ -110,7 +115,7 @@ def add_job(self, job_metadata: dict) -> str: doc_id = self.db.insert(job_metadata) self.db.close() - return doc_id + return doc_id # noqa def update_job(self, job_id: str, update_dict: dict) -> bool: """ @@ -170,7 +175,7 @@ def get_job_result(self, job_id: str) -> Tuple[str, Any]: """ Get a job's status, and actual output of executing the process - :param jobid: job identifier + :param job_id: job identifier :returns: `tuple` of mimetype and raw output """ diff --git a/pygeoapi/provider/sensorthings.py b/pygeoapi/provider/sensorthings.py index 833041c65..1aa75e34e 100644 --- a/pygeoapi/provider/sensorthings.py +++ b/pygeoapi/provider/sensorthings.py @@ -37,7 +37,7 @@ from pygeoapi.provider.base import ( BaseProvider, ProviderQueryError, ProviderConnectionError) from pygeoapi.util import ( - yaml_load, url_join, get_provider_default, crs_transform) + yaml_load, url_join, get_provider_default, crs_transform, get_base_url) LOGGER = logging.getLogger(__name__) @@ -115,7 +115,7 @@ def __init__(self, provider_def): # Read from pygeoapi config with open(os.getenv('PYGEOAPI_CONFIG'), encoding='utf8') as fh: CONFIG = yaml_load(fh) - self.rel_link = CONFIG['server']['url'] + self.rel_link = get_base_url(CONFIG) for (name, rs) in CONFIG['resources'].items(): pvs = rs.get('providers') diff --git a/pygeoapi/schemas/config/pygeoapi-config-0.x.yml b/pygeoapi/schemas/config/pygeoapi-config-0.x.yml index d2e38bc91..cbe664b4b 100644 --- a/pygeoapi/schemas/config/pygeoapi-config-0.x.yml +++ b/pygeoapi/schemas/config/pygeoapi-config-0.x.yml @@ -101,6 +101,24 @@ properties: - name - connection - output_dir + api_rules: + type: object + description: optional API design rules to which pygeoapi should adhere + properties: + api_version: + type: string + description: optional semantic API version number override + strict_slashes: + type: boolean + description: whether trailing slashes are allowed in URLs (disallow = True) + url_prefix: + type: string + description: |- + Set to include a prefix in the URL path (e.g. https://base.com/my_prefix/endpoint). + Please refer to the configuration section of the documentation for more info. + version_header: + type: string + description: API version response header (leave empty or unset to omit this header) required: - bind - url diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index d52438621..afcc3541c 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -38,16 +38,19 @@ import click +from starlette.routing import Route, Mount from starlette.staticfiles import StaticFiles from starlette.applications import Starlette from starlette.requests import Request -from starlette.responses import Response, JSONResponse, HTMLResponse +from starlette.datastructures import URL +from starlette.types import ASGIApp, Scope, Send, Receive +from starlette.responses import ( + Response, JSONResponse, HTMLResponse, RedirectResponse +) import uvicorn from pygeoapi.api import API -from pygeoapi.util import yaml_load - -CONFIG = None +from pygeoapi.util import yaml_load, get_api_rules if 'PYGEOAPI_CONFIG' not in os.environ: raise RuntimeError('PYGEOAPI_CONFIG environment variable not set') @@ -57,7 +60,7 @@ p = Path(__file__) -app = Starlette(debug=True) +APP = Starlette(debug=True) STATIC_DIR = Path(p).parent.resolve() / 'static' try: @@ -65,24 +68,7 @@ except KeyError: pass -app = Starlette() -app.mount('/static', StaticFiles(directory=STATIC_DIR)) - -# CORS: optionally enable from config. -if CONFIG['server'].get('cors', False): - from starlette.middleware.cors import CORSMiddleware - app.add_middleware(CORSMiddleware, allow_origins=['*']) - -try: - OGC_SCHEMAS_LOCATION = Path(CONFIG['server']['ogc_schemas_location']) -except KeyError: - OGC_SCHEMAS_LOCATION = None - -if (OGC_SCHEMAS_LOCATION is not None and - not OGC_SCHEMAS_LOCATION.name.startswith('http')): - if not OGC_SCHEMAS_LOCATION.exists(): - raise RuntimeError('OGC schemas misconfigured') - app.mount('/schemas', StaticFiles(directory=OGC_SCHEMAS_LOCATION)) +API_RULES = get_api_rules(CONFIG) api_ = API(CONFIG) @@ -111,7 +97,6 @@ def get_response(result: tuple) -> Union[Response, JSONResponse, HTMLResponse]: return response -@app.route('/') async def landing_page(request: Request): """ OGC API landing page endpoint @@ -123,8 +108,6 @@ async def landing_page(request: Request): return get_response(api_.landing_page(request)) -@app.route('/openapi') -@app.route('/openapi/') async def openapi(request: Request): """ OpenAPI endpoint @@ -142,8 +125,6 @@ async def openapi(request: Request): return get_response(api_.openapi(request, openapi_)) -@app.route('/conformance') -@app.route('/conformance/') async def conformance(request: Request): """ OGC API conformance endpoint @@ -155,8 +136,6 @@ async def conformance(request: Request): return get_response(api_.conformance(request)) -@app.route('/collections/{collection_id:path}/queryables') -@app.route('/collections/{collection_id:path}/queryables/') async def collection_queryables(request: Request, collection_id=None): """ OGC API collections queryables endpoint @@ -171,8 +150,6 @@ async def collection_queryables(request: Request, collection_id=None): return get_response(api_.get_collection_queryables(request, collection_id)) -@app.route('/collections/{collection_id:path}/tiles') -@app.route('/collections/{collection_id:path}/tiles/') async def get_collection_tiles(request: Request, collection_id=None): """ OGC open api collections tiles access point @@ -188,10 +165,6 @@ async def get_collection_tiles(request: Request, collection_id=None): request, collection_id)) -@app.route('/collections/{collection_id:path}/tiles/{tileMatrixSetId}') -@app.route('/collections/{collection_id:path}/tiles/{tileMatrixSetId}/') -@app.route('/collections/{collection_id:path}/tiles/{tileMatrixSetId}/metadata') # noqa -@app.route('/collections/{collection_id:path}/tiles/{tileMatrixSetId}/metadata/') # noqa async def get_collection_tiles_metadata(request: Request, collection_id=None, tileMatrixSetId=None): """ @@ -210,8 +183,6 @@ async def get_collection_tiles_metadata(request: Request, collection_id=None, request, collection_id, tileMatrixSetId)) -@app.route('/collections/{collection_id:path}/tiles/{tileMatrixSetId}/{tile_matrix}/{tileRow}/{tileCol}') # noqa -@app.route('/collections/{collection_id:path}/tiles/{tileMatrixSetId}/{tile_matrix}/{tileRow}/{tileCol}/') # noqa async def get_collection_items_tiles(request: Request, collection_id=None, tileMatrixSetId=None, tile_matrix=None, tileRow=None, tileCol=None): @@ -242,14 +213,6 @@ async def get_collection_items_tiles(request: Request, collection_id=None, tile_matrix, tileRow, tileCol)) -@app.route('/collections/{collection_id:path}/items', - methods=['GET', 'POST', 'OPTIONS']) -@app.route('/collections/{collection_id:path}/items/', - methods=['GET', 'POST', 'OPTIONS']) -@app.route('/collections/{collection_id:path}/items/{item_id:path}', - methods=['GET', 'PUT', 'DELETE', 'OPTIONS']) -@app.route('/collections/{collection_id:path}/items/{item_id:path}/', - methods=['GET', 'PUT', 'DELETE', 'OPTIONS']) async def collection_items(request: Request, collection_id=None, item_id=None): """ OGC API collections items endpoint @@ -301,7 +264,6 @@ async def collection_items(request: Request, collection_id=None, item_id=None): request, collection_id, item_id)) -@app.route('/collections/{collection_id:path}/coverage') async def collection_coverage(request: Request, collection_id=None): """ OGC API - Coverages coverage endpoint @@ -317,7 +279,6 @@ async def collection_coverage(request: Request, collection_id=None): return get_response(api_.get_collection_coverage(request, collection_id)) -@app.route('/collections/{collection_id:path}/coverage/domainset') async def collection_coverage_domainset(request: Request, collection_id=None): """ OGC API - Coverages coverage domainset endpoint @@ -334,7 +295,6 @@ async def collection_coverage_domainset(request: Request, collection_id=None): request, collection_id)) -@app.route('/collections/{collection_id:path}/coverage/rangetype') async def collection_coverage_rangetype(request: Request, collection_id=None): """ OGC API - Coverages coverage rangetype endpoint @@ -352,8 +312,6 @@ async def collection_coverage_rangetype(request: Request, collection_id=None): request, collection_id)) -@app.route('/collections/{collection_id:path}/map') -@app.route('/collections/{collection_id:path}/styles/{style_id:path}/map') async def collection_map(request: Request, collection_id, style_id=None): """ OGC API - Maps map render endpoint @@ -373,10 +331,6 @@ async def collection_map(request: Request, collection_id, style_id=None): request, collection_id, style_id)) -@app.route('/processes') -@app.route('/processes/') -@app.route('/processes/{process_id}') -@app.route('/processes/{process_id}/') async def get_processes(request: Request, process_id=None): """ OGC API - Processes description endpoint @@ -392,9 +346,6 @@ async def get_processes(request: Request, process_id=None): return get_response(api_.describe_processes(request, process_id)) -@app.route('/jobs') -@app.route('/jobs/{job_id}', methods=['GET', 'DELETE']) -@app.route('/jobs/{job_id}/', methods=['GET', 'DELETE']) async def get_jobs(request: Request, job_id=None): """ OGC API - Processes jobs endpoint @@ -417,8 +368,6 @@ async def get_jobs(request: Request, job_id=None): return get_response(api_.get_jobs(request, job_id)) -@app.route('/processes/{process_id}/execution', methods=['POST']) -@app.route('/processes/{process_id}/execution/', methods=['POST']) async def execute_process_jobs(request: Request, process_id=None): """ OGC API - Processes jobs endpoint @@ -435,8 +384,6 @@ async def execute_process_jobs(request: Request, process_id=None): return get_response(api_.execute_process(request, process_id)) -@app.route('/jobs/{job_id}/results', methods=['GET']) -@app.route('/jobs/{job_id}/results/', methods=['GET']) async def get_job_result(request: Request, job_id=None): """ OGC API - Processes job result endpoint @@ -453,10 +400,6 @@ async def get_job_result(request: Request, job_id=None): return get_response(api_.get_job_result(request, job_id)) -@app.route('/jobs/{job_id}/results/{resource}', - methods=['GET']) -@app.route('/jobs/{job_id}/results/{resource}/', - methods=['GET']) async def get_job_result_resource(request: Request, job_id=None, resource=None): """ @@ -478,18 +421,6 @@ async def get_job_result_resource(request: Request, request, job_id, resource)) -@app.route('/collections/{collection_id:path}/position') -@app.route('/collections/{collection_id:path}/area') -@app.route('/collections/{collection_id:path}/cube') -@app.route('/collections/{collection_id:path}/radius') -@app.route('/collections/{collection_id:path}/trajectory') -@app.route('/collections/{collection_id:path}/corridor') -@app.route('/collections/{collection_id:path}/instances/{instance_id}/position') # noqa -@app.route('/collections/{collection_id:path}/instances/{instance_id}/area') -@app.route('/collections/{collection_id:path}/instances/{instance_id}/cube') -@app.route('/collections/{collection_id:path}/instances/{instance_id}/radius') -@app.route('/collections/{collection_id:path}/instances/{instance_id}/trajectory') # noqa -@app.route('/collections/{collection_id:path}/instances/{instance_id}/corridor') # noqa async def get_collection_edr_query(request: Request, collection_id=None, instance_id=None): # noqa """ OGC EDR API endpoints @@ -511,10 +442,6 @@ async def get_collection_edr_query(request: Request, collection_id=None, instanc instance_id, query_type)) -@app.route('/collections') -@app.route('/collections/') -@app.route('/collections/{collection_id:path}') -@app.route('/collections/{collection_id:path}/') async def collections(request: Request, collection_id=None): """ OGC API collections endpoint @@ -529,7 +456,6 @@ async def collections(request: Request, collection_id=None): return get_response(api_.describe_collections(request, collection_id)) -@app.route('/stac') async def stac_catalog_root(request: Request): """ STAC root endpoint @@ -541,7 +467,6 @@ async def stac_catalog_root(request: Request): return get_response(api_.get_stac_root(request)) -@app.route('/stac/{path:path}') async def stac_catalog_path(request: Request): """ STAC endpoint @@ -554,6 +479,121 @@ async def stac_catalog_path(request: Request): return get_response(api_.get_stac_path(request, path)) +class ApiRulesMiddleware: + """ Custom middleware to properly deal with trailing slashes. + See https://github.com/encode/starlette/issues/869. + """ + def __init__( + self, + app: ASGIApp + ) -> None: + self.app = app + self.prefix = API_RULES.get_url_prefix('starlette') + + async def __call__(self, scope: Scope, + receive: Receive, send: Send) -> None: + if scope['type'] == "http" and API_RULES.strict_slashes: + path = scope['path'] + if path == self.prefix: + # If the root (landing page) is requested without a trailing + # slash, redirect to landing page with trailing slash. + # Starlette will otherwise throw a 404, as it does not like + # empty Route paths. + url = URL(scope=scope).replace(path=f"{path}/") + response = RedirectResponse(url) + await response(scope, receive, send) + return + elif path != f"{self.prefix}/" and path.endswith('/'): + # Resource paths should NOT have trailing slashes + response = Response(status_code=404) + await response(scope, receive, send) + return + + await self.app(scope, receive, send) + + +api_routes = [ + Route('/', landing_page), + Route('/openapi', openapi), + Route('/conformance', conformance), + Route('/collections/{collection_id:path}/queryables', collection_queryables), # noqa + Route('/collections/{collection_id:path}/tiles', get_collection_tiles), + Route('/collections/{collection_id:path}/tiles/{tileMatrixSetId}', get_collection_tiles_metadata), # noqa + Route('/collections/{collection_id:path}/tiles/{tileMatrixSetId}/metadata', get_collection_tiles_metadata), # noqa + Route('/collections/{collection_id:path}/tiles/{tileMatrixSetId}/{tile_matrix}/{tileRow}/{tileCol}', get_collection_items_tiles), # noqa + Route('/collections/{collection_id:path}/items', collection_items, methods=['GET', 'POST', 'OPTIONS']), # noqa + Route('/collections/{collection_id:path}/items/{item_id:path}', collection_items, methods=['GET', 'PUT', 'DELETE', 'OPTIONS']), # noqa + Route('/collections/{collection_id:path}/coverage', collection_coverage), # noqa + Route('/collections/{collection_id:path}/coverage/domainset', collection_coverage_domainset), # noqa + Route('/collections/{collection_id:path}/coverage/rangetype', collection_coverage_rangetype), # noqa + Route('/collections/{collection_id:path}/map', collection_map), + Route('/collections/{collection_id:path}/styles/{style_id:path}/map', collection_map), # noqa + Route('/processes', get_processes), + Route('/processes/{process_id}', get_processes), + Route('/jobs', get_jobs), + Route('/jobs/{job_id}', get_jobs, methods=['GET', 'DELETE']), + Route('/processes/{process_id}/execution', execute_process_jobs, methods=['POST']), # noqa + Route('/jobs/{job_id}/results', get_job_result), + Route('/jobs/{job_id}/results/{resource}', get_job_result_resource), + Route('/collections/{collection_id:path}/position', get_collection_edr_query), # noqa + Route('/collections/{collection_id:path}/area', get_collection_edr_query), + Route('/collections/{collection_id:path}/cube', get_collection_edr_query), + Route('/collections/{collection_id:path}/radius', get_collection_edr_query), # noqa + Route('/collections/{collection_id:path}/trajectory', get_collection_edr_query), # noqa + Route('/collections/{collection_id:path}/corridor', get_collection_edr_query), # noqa + Route('/collections/{collection_id:path}/instances/{instance_id}/position', get_collection_edr_query), # noqa + Route('/collections/{collection_id:path}/instances/{instance_id}/area', get_collection_edr_query), # noqa + Route('/collections/{collection_id:path}/instances/{instance_id}/cube', get_collection_edr_query), # noqa + Route('/collections/{collection_id:path}/instances/{instance_id}/radius', get_collection_edr_query), # noqa + Route('/collections/{collection_id:path}/instances/{instance_id}/trajectory', get_collection_edr_query), # noqa + Route('/collections/{collection_id:path}/instances/{instance_id}/corridor', get_collection_edr_query), # noqa + Route('/collections', collections), + Route('/collections/{collection_id:path}', collections), + Route('/stac', stac_catalog_root), + Route('/stac/{path:path}', stac_catalog_path), +] + +url_prefix = API_RULES.get_url_prefix('starlette') +APP = Starlette( + routes=[ + Mount(f'{url_prefix}/static', StaticFiles(directory=STATIC_DIR)), + Mount(url_prefix or '/', routes=api_routes) + ] +) + +if url_prefix: + # If a URL prefix is in effect, Flask allows the static resource URLs + # to be written both with or without that prefix (200 in both cases). + # Starlette does not allow this, so for consistency we'll add a static + # mount here WITHOUT the URL prefix (due to router order). + APP.mount( + '/static', StaticFiles(directory=STATIC_DIR), + ) + +# If API rules require strict slashes, do not redirect +if API_RULES.strict_slashes: + APP.router.redirect_slashes = False + APP.add_middleware(ApiRulesMiddleware) + +# CORS: optionally enable from config. +if CONFIG['server'].get('cors', False): + from starlette.middleware.cors import CORSMiddleware + APP.add_middleware(CORSMiddleware, allow_origins=['*']) + +try: + OGC_SCHEMAS_LOCATION = Path(CONFIG['server']['ogc_schemas_location']) +except KeyError: + OGC_SCHEMAS_LOCATION = None + +if (OGC_SCHEMAS_LOCATION is not None and + not OGC_SCHEMAS_LOCATION.name.startswith('http')): + if not OGC_SCHEMAS_LOCATION.exists(): + raise RuntimeError('OGC schemas misconfigured') + APP.mount( + f'{url_prefix}/schemas', StaticFiles(directory=OGC_SCHEMAS_LOCATION) + ) + + @click.command() @click.pass_context @click.option('--debug', '-d', default=False, is_flag=True, help='debug') diff --git a/pygeoapi/util.py b/pygeoapi/util.py index 4c1cb8dd7..e5282db4e 100644 --- a/pygeoapi/util.py +++ b/pygeoapi/util.py @@ -41,10 +41,12 @@ import logging import mimetypes import os -from pathlib import Path import re -from typing import Any, IO, Union -from urllib.request import urlopen +from datetime import date, datetime, time +from decimal import Decimal +from enum import Enum +from pathlib import Path +from typing import Any, IO, Union, List from urllib.parse import urlparse import shapely.ops from shapely.geometry import ( @@ -59,17 +61,23 @@ shape as geojson_to_geom, mapping as geom_to_geojson, ) +from urllib.request import urlopen + import dateutil.parser from jinja2 import Environment, FileSystemLoader, select_autoescape from babel.support import Translations import pyproj from pyproj.exceptions import CRSError import yaml +from babel.support import Translations +from jinja2 import Environment, FileSystemLoader, select_autoescape +from shapely.geometry import Polygon from requests import Session from requests.structures import CaseInsensitiveDict from pygeoapi import __version__ from pygeoapi import l10n +from pygeoapi.models import config as config_models from pygeoapi.provider.base import ProviderTypeError @@ -177,6 +185,23 @@ class EnvVarLoader(yaml.SafeLoader): return yaml.load(fh, Loader=EnvVarLoader) +def get_api_rules(config: dict) -> config_models.APIRules: + """ Extracts the default API design rules from the given configuration. + + :param config: Current pygeoapi configuration (dictionary). + :returns: An APIRules instance. + """ + rules = config['server'].get('api_rules') or {} + rules.setdefault('api_version', __version__) + return config_models.APIRules.create(**rules) + + +def get_base_url(config: dict) -> str: + """ Returns the full pygeoapi base URL. """ + rules = get_api_rules(config) + return url_join(config['server']['url'], rules.get_url_prefix()) + + def str2bool(value: Union[bool, str]) -> bool: """ helper function to return Python boolean @@ -212,8 +237,7 @@ def to_json(dict_: dict, pretty: bool = False) -> str: else: indent = None - return json.dumps(dict_, default=json_serial, - indent=indent) + return json.dumps(dict_, default=json_serial, indent=indent) def format_datetime(value: str, format_: str = DATETIME_FORMAT) -> str: @@ -533,7 +557,7 @@ class JobStatus(Enum): def read_data(path: Union[Path, str]) -> Union[bytes, str]: """ - helper function to read data (file or networrk) + helper function to read data (file or network) """ LOGGER.debug(f'Attempting to read {path}') @@ -548,7 +572,7 @@ def read_data(path: Union[Path, str]) -> Union[bytes, str]: return r.read() -def url_join(*parts: list) -> str: +def url_join(*parts: str) -> str: """ helper function to join a URL from a number of parts/fragments. Implemented because urllib.parse.urljoin strips subpaths from @@ -561,7 +585,7 @@ def url_join(*parts: list) -> str: :returns: str of resulting URL """ - return '/'.join([p.strip().strip('/') for p in parts]) + return '/'.join([p.strip().strip('/') for p in parts]).rstrip('/') def get_envelope(coords_list: List[List[float]]) -> list: diff --git a/requirements-dev.txt b/requirements-dev.txt index 097590852..fcbb6d6c9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,6 +11,11 @@ pytest-env coverage pyld +# Testing with mock Starlette client +starlette +uvicorn[standard] +httpx + # PEP8 flake8 diff --git a/tests/pygeoapi-test-config-apirules.yml b/tests/pygeoapi-test-config-apirules.yml new file mode 100644 index 000000000..96c68b84b --- /dev/null +++ b/tests/pygeoapi-test-config-apirules.yml @@ -0,0 +1,347 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# Sander Schaminee +# +# Copyright (c) 2019 Tom Kralidis +# Copyright (c) 2023 Sander Schaminee +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +server: + bind: + host: 0.0.0.0 + port: 5000 + url: http://localhost:5000/api + mimetype: application/json; charset=UTF-8 + encoding: utf-8 + gzip: false + languages: + # First language is the default language + - en-US + - fr-CA + cors: true + pretty_print: true + limit: 10 + # templates: /path/to/templates + map: + url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png + attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + manager: + name: TinyDB + connection: /tmp/pygeoapi-test-process-manager.db + output_dir: /tmp + api_rules: + strict_slashes: true + url_prefix: 'v{api_major}' + version_header: 'X-API-Version' + +logging: + level: DEBUG + #logfile: /tmp/pygeoapi.log + +metadata: + identification: + title: + en: pygeoapi default instance + fr: instance par défaut de pygeoapi + description: + en: pygeoapi provides an API to geospatial data + fr: pygeoapi fournit une API aux données géospatiales + keywords: + en: + - geospatial + - data + - api + fr: + - géospatiale + - données + - api + keywords_type: theme + terms_of_service: https://creativecommons.org/licenses/by/4.0/ + url: http://example.org + license: + name: CC-BY 4.0 license + url: https://creativecommons.org/licenses/by/4.0/ + provider: + name: Organization Name + url: https://pygeoapi.io + contact: + name: Lastname, Firstname + position: Position Title + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Country + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: you@example.org + url: Contact URL + hours: Hours of Service + instructions: During hours of service. Off on weekends. + role: pointOfContact + +resources: + obs: + type: collection + title: + en: Observations + fr: Observations + description: + en: My cool observations + fr: Mes belles observations + keywords: + - observations + - monitoring + links: + - type: text/csv + rel: canonical + title: data + href: https://github.com/mapserver/mapserver/blob/branch-7-0/msautotest/wxs/data/obs.csv + hreflang: en-US + - type: text/csv + rel: alternate + title: data + href: https://raw.githubusercontent.com/mapserver/mapserver/branch-7-0/msautotest/wxs/data/obs.csv + hreflang: en-US + linked-data: + context: + - schema: https://schema.org/ + stn_id: + "@id": schema:identifier + "@type": schema:Text + datetime: + "@type": schema:DateTime + "@id": schema:observationDate + value: + "@type": schema:Number + "@id": schema:QuantitativeValue + extents: + spatial: + bbox: [-180,-90,180,90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 2000-10-30T18:24:39Z + end: 2007-10-30T08:57:29Z + trs: http://www.opengis.net/def/uom/ISO-8601/0/Gregorian + providers: + - type: feature + name: CSV + data: tests/data/obs.csv + id_field: id + geometry: + x_field: long + y_field: lat + + cmip5: + type: collection + title: CMIP5 sample + description: CMIP5 sample + keywords: + - cmip5 + - climate + extents: + spatial: + bbox: [-150,40,-45,90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + links: + - type: text/html + rel: canonical + title: information + href: https://open.canada.ca/data/en/dataset/eddd6eaf-34d7-4452-a994-3d928115a68b + hreflang: en-CA + providers: + - type: coverage + name: xarray + data: tests/data/CMIP5_rcp8.5_annual_abs_latlon1x1_PCP_pctl25_P1Y.nc + x_field: lon + y_field: lat + time_field: time + format: + name: NetCDF + mimetype: application/x-netcdf + + naturalearth/lakes: + type: collection + title: + en: Large Lakes + fr: Grands Lacs + description: + en: lakes of the world, public domain + fr: lacs du monde, domaine public + keywords: + - lakes + links: + - type: text/html + rel: canonical + title: information + href: http://www.naturalearthdata.com/ + hreflang: en-US + extents: + spatial: + bbox: [-180,-90,180,90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 2011-11-11T11:11:11Z + end: null # or empty (either means open ended) + providers: + - type: feature + name: GeoJSON + data: tests/data/ne_110m_lakes.geojson + id_field: id + - type: tile + name: MVT + # data: http://localhost:9000/ne_110m_lakes/{z}/{x}/{y} + data: tests/data/tiles/ne_110m_lakes + options: + metadata_format: raw # default | tilejson + bounds: [[-124.953634,-16.536406],[109.929807,66.969298]] + zoom: + min: 0 + max: 11 + schemes: + - WorldCRS84Quad + format: + name: pbf + mimetype: application/vnd.mapbox-vector-tile + + gdps-temperature: + type: collection + title: Global Deterministic Prediction System sample + description: Global Deterministic Prediction System sample + keywords: + - gdps + - global + extents: + spatial: + bbox: [-180,-90,180,90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + links: + - type: text/html + rel: canonical + title: information + href: https://eccc-msc.github.io/open-data/msc-data/nwp_gdps/readme_gdps_en + hreflang: en-CA + providers: + - type: coverage + name: rasterio + data: tests/data/CMC_glb_TMP_TGL_2_latlon.15x.15_2020081000_P000.grib2 + options: + DATA_ENCODING: COMPLEX_PACKING + format: + name: GRIB + mimetype: application/x-grib2 + + icoads-sst: + type: collection + title: International Comprehensive Ocean-Atmosphere Data Set (ICOADS) + description: International Comprehensive Ocean-Atmosphere Data Set (ICOADS) + keywords: + - icoads + - sst + - air temperature + extents: + spatial: + bbox: [-180,-90,180,90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + links: + - type: text/html + rel: canonical + title: information + href: https://psl.noaa.gov/data/gridded/data.coads.1deg.html + hreflang: en-US + providers: + - type: edr + name: xarray-edr + data: tests/data/coads_sst.nc + format: + name: NetCDF + mimetype: application/x-netcdf + + objects: + type: collection + title: GeoJSON objects + description: GeoJSON geometry types for GeoSparql and Schema Geometry conversion. + keywords: + - shapes + links: + - type: text/html + rel: canonical + title: data source + href: https://en.wikipedia.org/wiki/GeoJSON + hreflang: en-US + linked-data: + item_template: tests/data/base.jsonld + extents: + spatial: + bbox: [-180,-90,180,90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: null + end: null # or empty (either means open ended) + providers: + - type: feature + name: GeoJSON + data: tests/data/items.geojson + id_field: fid + uri_field: uri + + mapserver_world_map: + type: collection + title: MapServer demo WMS world map + description: MapServer demo WMS world map + keywords: + - MapServer + - world map + links: + - type: text/html + rel: canonical + title: information + href: https://demo.mapserver.org + hreflang: en-US + extents: + spatial: + bbox: [-180,-90,180,90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: map + name: WMSFacade + data: https://demo.mapserver.org/cgi-bin/msautotest + options: + layer: world_latlong + style: default + format: + name: png + mimetype: image/png + + hello-world: + type: process + processor: + name: HelloWorld + + pygeometa-metadata-validate: + type: process + processor: + name: pygeometa.pygeoapi_plugin.PygeometaMetadataValidateProcessor diff --git a/tests/test_api.py b/tests/test_api.py index 5efd1e27f..5d6a56e6c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -44,9 +44,9 @@ from pygeoapi.api import ( API, APIRequest, FORMAT_TYPES, validate_bbox, validate_datetime, - validate_subset, F_HTML, F_JSON, F_JSONLD, F_GZIP + validate_subset, F_HTML, F_JSON, F_JSONLD, F_GZIP, __version__ ) -from pygeoapi.util import yaml_load, get_crs_from_uri +from pygeoapi.util import yaml_load, get_crs_from_uri, get_api_rules, get_base_url from .util import get_test_file_path, mock_request @@ -59,6 +59,13 @@ def config(): return yaml_load(fh) +@pytest.fixture() +def config_with_rules() -> dict: + """ Returns a pygeoapi configuration with default API rules. """ + with open(get_test_file_path('pygeoapi-test-config-apirules.yml')) as fh: + return yaml_load(fh) + + @pytest.fixture() def config_enclosure() -> dict: """ Returns a pygeoapi configuration with enclosure links. """ @@ -90,6 +97,14 @@ def enclosure_api(config_enclosure): return API(config_enclosure) +@pytest.fixture() +def rules_api(config_with_rules): + """ Returns an API instance with URL prefix and strict slashes policy. + The API version is extracted from the current version here. + """ + return API(config_with_rules) + + @pytest.fixture() def api_hidden_resources(config_hidden_resources): return API(config_hidden_resources) @@ -235,6 +250,125 @@ def test_apirequest(api_): assert apireq.get_response_headers()['Content-Language'] == 'de' +def test_apirules_active(config_with_rules, rules_api): + assert rules_api.config == config_with_rules + rules = get_api_rules(config_with_rules) + base_url = get_base_url(config_with_rules) + + # Test Flask + flask_prefix = rules.get_url_prefix('flask') + with mock_flask('pygeoapi-test-config-apirules.yml') as flask_client: + # Test happy path + response = flask_client.get(f'{flask_prefix}/conformance') + assert response.status_code == 200 + assert response.headers['X-API-Version'] == __version__ + assert response.request.url == \ + flask_client.application.url_for('pygeoapi.conformance') + response = flask_client.get(f'{flask_prefix}/static/img/pygeoapi.png') + assert response.status_code == 200 + # Test that static resources also work without URL prefix + response = flask_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test strict slashes + response = flask_client.get(f'{flask_prefix}/conformance/') + assert response.status_code == 404 + # For the landing page ONLY, trailing slashes are actually preferred. + # See https://docs.opengeospatial.org/is/17-069r4/17-069r4.html#_api_landing_page # noqa + # Omitting the trailing slash should lead to a redirect. + response = flask_client.get(f'{flask_prefix}/') + assert response.status_code == 200 + response = flask_client.get(flask_prefix) + assert response.status_code in (307, 308) + + # Test links on landing page for correct URLs + response = flask_client.get(flask_prefix, follow_redirects=True) + assert response.status_code == 200 + assert response.is_json + links = response.json['links'] + assert all( + href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa + ) + + # Test Starlette + starlette_prefix = rules.get_url_prefix('starlette') + with mock_starlette('pygeoapi-test-config-apirules.yml') as starlette_client: # noqa + # Test happy path + response = starlette_client.get(f'{starlette_prefix}/conformance') + assert response.status_code == 200 + assert response.headers['X-API-Version'] == __version__ + response = starlette_client.get(f'{starlette_prefix}/static/img/pygeoapi.png') # noqa + assert response.status_code == 200 + # Test that static resources also work without URL prefix + response = starlette_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test strict slashes + response = starlette_client.get(f'{starlette_prefix}/conformance/') + assert response.status_code == 404 + # For the landing page ONLY, trailing slashes are actually preferred. + # See https://docs.opengeospatial.org/is/17-069r4/17-069r4.html#_api_landing_page # noqa + # Omitting the trailing slash should lead to a redirect. + response = starlette_client.get(f'{starlette_prefix}/') + assert response.status_code == 200 + response = starlette_client.get(starlette_prefix) + assert response.status_code in (307, 308) + + # Test links on landing page for correct URLs + response = starlette_client.get(starlette_prefix, follow_redirects=True) # noqa + assert response.status_code == 200 + links = response.json()['links'] + assert all( + href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa + ) + + +def test_apirules_inactive(config, api_): + assert api_.config == config + rules = get_api_rules(config) + + # Test Flask + flask_prefix = rules.get_url_prefix('flask') + assert flask_prefix == '' + with mock_flask('pygeoapi-test-config.yml') as flask_client: + response = flask_client.get('') + assert response.status_code == 200 + response = flask_client.get('/conformance') + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + assert response.request.url == \ + flask_client.application.url_for('pygeoapi.conformance') + response = flask_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test trailing slashes + response = flask_client.get('/') + assert response.status_code == 200 + response = flask_client.get('/conformance/') + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + + # Test Starlette + starlette_prefix = rules.get_url_prefix('starlette') + assert starlette_prefix == '' + with mock_starlette('pygeoapi-test-config.yml') as starlette_client: + response = starlette_client.get('') + assert response.status_code == 200 + response = starlette_client.get('/conformance') + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + assert str(response.url) == f"{starlette_client.base_url}/conformance" + response = starlette_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test trailing slashes + response = starlette_client.get('/') + assert response.status_code == 200 + response = starlette_client.get('/conformance/', follow_redirects=True) + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + + def test_api(config, api_, openapi): assert api_.config == config assert isinstance(api_.config, dict) diff --git a/tests/test_config.py b/tests/test_config.py index 623afb279..394a2bb98 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -28,6 +28,7 @@ # ================================================================= import os +from copy import deepcopy from jsonschema.exceptions import ValidationError import pytest @@ -60,7 +61,7 @@ def test_config_envvars(): with pytest.raises(EnvironmentError): with open(get_test_file_path('pygeoapi-test-config-envvars.yml')) as fh: # noqa - config = yaml_load(fh) + yaml_load(fh) def test_validate_config(config): @@ -68,4 +69,22 @@ def test_validate_config(config): assert is_valid with pytest.raises(ValidationError): - is_valid = validate_config({'foo': 'bar'}) + validate_config({'foo': 'bar'}) + + # Test API rules + cfg_copy = deepcopy(config) + cfg_copy['server']['api_rules'] = { + 'api_version': '1.2.3', + 'strict_slashes': True, + 'url_prefix': 'v{major_version}', + 'version_header': 'API-Version' + } + assert validate_config(cfg_copy) + + cfg_copy['server']['api_rules'] = { + 'api_version': 123, + 'url_prefix': 0, + 'strict_slashes': 'bad_value' + } + with pytest.raises(ValidationError): + validate_config(cfg_copy) diff --git a/tests/test_util.py b/tests/test_util.py index 22e51ce6c..d1c44cbea 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -29,17 +29,32 @@ from datetime import datetime, date, time from decimal import Decimal +from copy import deepcopy import pytest from pyproj.exceptions import CRSError from shapely.geometry import Point from pygeoapi import util +from pygeoapi.api import __version__ from pygeoapi.provider.base import ProviderTypeError from .util import get_test_file_path +@pytest.fixture() +def config(): + with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: + return util.yaml_load(fh) + + +@pytest.fixture() +def config_with_rules() -> dict: + """ Returns a pygeoapi configuration with default API rules. """ + with open(get_test_file_path('pygeoapi-test-config-apirules.yml')) as fh: + return util.yaml_load(fh) + + def test_get_typed_value(): value = util.get_typed_value('2') assert isinstance(value, int) @@ -51,13 +66,11 @@ def test_get_typed_value(): assert isinstance(value, str) -def test_yaml_load(): - with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: - d = util.yaml_load(fh) - assert isinstance(d, dict) +def test_yaml_load(config): + assert isinstance(config, dict) with pytest.raises(FileNotFoundError): with open(get_test_file_path('404.yml')) as fh: - d = util.yaml_load(fh) + util.yaml_load(fh) def test_str2bool(): @@ -112,25 +125,19 @@ def test_path_basename(): assert util.get_path_basename('/path/to/dir') == 'dir' -def test_filter_dict_by_key_value(): - with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: - d = util.yaml_load(fh) - - collections = util.filter_dict_by_key_value(d['resources'], +def test_filter_dict_by_key_value(config): + collections = util.filter_dict_by_key_value(config['resources'], 'type', 'collection') assert len(collections) == 8 - notfound = util.filter_dict_by_key_value(d['resources'], + notfound = util.filter_dict_by_key_value(config['resources'], 'type', 'foo') assert len(notfound) == 0 -def test_get_provider_by_type(): - with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: - d = util.yaml_load(fh) - - p = util.get_provider_by_type(d['resources']['obs']['providers'], +def test_get_provider_by_type(config): + p = util.get_provider_by_type(config['resources']['obs']['providers'], 'feature') assert isinstance(p, dict) @@ -138,20 +145,17 @@ def test_get_provider_by_type(): assert p['name'] == 'CSV' with pytest.raises(ProviderTypeError): - p = util.get_provider_by_type(d['resources']['obs']['providers'], + p = util.get_provider_by_type(config['resources']['obs']['providers'], 'something-else') -def test_get_provider_default(): - with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: - d = util.yaml_load(fh) - - pd = util.get_provider_default(d['resources']['obs']['providers']) +def test_get_provider_default(config): + pd = util.get_provider_default(config['resources']['obs']['providers']) assert pd['type'] == 'feature' assert pd['name'] == 'CSV' - pd = util.get_provider_default(d['resources']['obs']['providers']) + pd = util.get_provider_default(config['resources']['obs']['providers']) def test_read_data(): @@ -160,6 +164,62 @@ def test_read_data(): assert isinstance(data, bytes) +def test_url_join(): + f = util.url_join + assert f('http://localhost:5000') == 'http://localhost:5000' + assert f('http://localhost:5000/') == 'http://localhost:5000' + assert f('http://localhost:5000', '') == 'http://localhost:5000' + assert f('http://localhost:5000/', '') == 'http://localhost:5000' + assert f('http://localhost:5000/', '/') == 'http://localhost:5000' + assert f('http://localhost:5000/api', '/') == 'http://localhost:5000/api' + assert f('http://localhost:5000/api', '/v0') == 'http://localhost:5000/api/v0' # noqa + assert f('http://localhost:5000/api', '/v0/') == 'http://localhost:5000/api/v0' # noqa + assert f('http://localhost:5000', 'api', 'v0') == 'http://localhost:5000/api/v0' # noqa + + +def test_get_base_url(config, config_with_rules): + assert util.get_base_url(config) == 'http://localhost:5000' + assert util.get_base_url(config_with_rules) == 'http://localhost:5000/api/v0' # noqa + + +def test_get_api_rules(config, config_with_rules): + # Test unset/default rules + rules = util.get_api_rules(config) + assert not rules.strict_slashes + assert not rules.url_prefix + assert rules.api_version == __version__ + assert rules.version_header == '' + assert rules.get_url_prefix() == '' + assert rules.response_headers == {} + + # Test configured rules + rules = util.get_api_rules(config_with_rules) + assert rules.strict_slashes + assert rules.url_prefix + assert rules.api_version == __version__ + assert rules.version_header == 'X-API-Version' + assert rules.response_headers == {'X-API-Version': __version__} + + # Test specific version override + config_changed = deepcopy(config_with_rules) + config_changed['server']['api_rules']['api_version'] = '1.2.3' + rules = util.get_api_rules(config_changed) + assert rules.api_version == '1.2.3' + assert rules.get_url_prefix() == 'v1' + assert rules.get_url_prefix('flask') == '/v1' + assert rules.get_url_prefix('starlette') == '/v1' + assert rules.get_url_prefix('django') == r'^v1/' + + # Test prefix without version + config_changed = deepcopy(config_with_rules) + config_changed['server']['api_rules']['url_prefix'] = 'test' + rules = util.get_api_rules(config_changed) + assert rules.get_url_prefix() == 'test' + assert rules.get_url_prefix('flask') == '/test' + assert rules.get_url_prefix('starlette') == '/test' + assert rules.get_url_prefix('django') == r'^test/' + + def test_get_transform_from_crs(): crs_in = util.get_crs_from_uri( 'http://www.opengis.net/def/crs/EPSG/0/4258' diff --git a/tests/util.py b/tests/util.py index b43c9731e..eb67cc509 100644 --- a/tests/util.py +++ b/tests/util.py @@ -27,9 +27,15 @@ # # ================================================================= +import sys import logging import os.path +from urllib.parse import urlsplit +from importlib import reload +from contextlib import contextmanager +from flask.testing import FlaskClient +from starlette.testclient import TestClient as StarletteClient from werkzeug.test import create_environ from werkzeug.wrappers import Request from werkzeug.datastructures import ImmutableMultiDict @@ -67,5 +73,105 @@ def mock_request(params: dict = None, data=None, **headers) -> Request: environ = create_environ(base_url='http://localhost:5000/', data=data) environ.update(headers) request = Request(environ) - request.args = ImmutableMultiDict(params.items()) + request.args = ImmutableMultiDict(params.items()) # noqa return request + + +@contextmanager +def mock_flask(config_file: str = 'pygeoapi-test-config.yml', **kwargs) -> FlaskClient: # noqa + """ + Mocks a Flask client so we can test the API routing with applied API rules. + Does not follow redirects by default. Set `follow_redirects=True` option + on individual requests to enable. + + :param config_file: Optional configuration YAML file to use. + If not set, the default test configuration is used. + """ + flask_app = None + env_conf = os.getenv('PYGEOAPI_CONFIG') + try: + # Temporarily override environment variable so we can import Flask app + os.environ['PYGEOAPI_CONFIG'] = get_test_file_path(config_file) + + # Import current pygeoapi Flask app module + from pygeoapi import flask_app + + # Force a module reload to make sure we really use another config + reload(flask_app) + + # Set server root path + url_parts = urlsplit(flask_app.CONFIG['server']['url']) + app_root = url_parts.path.rstrip('/') or '/' + flask_app.APP.config['SERVER_NAME'] = url_parts.netloc + flask_app.APP.config['APPLICATION_ROOT'] = app_root + + # Create and return test client + client = flask_app.APP.test_client(**kwargs) + yield client + + finally: + if env_conf is None: + # Remove env variable again if it was not set initially + del os.environ['PYGEOAPI_CONFIG'] + # Unload Flask app module + del sys.modules['pygeoapi.flask_app'] + else: + # Restore env variable to its original value and reload Flask app + os.environ['PYGEOAPI_CONFIG'] = env_conf + if flask_app: + reload(flask_app) + del client + + +@contextmanager +def mock_starlette(config_file: str = 'pygeoapi-test-config.yml', **kwargs) -> StarletteClient: # noqa + """ + Mocks a Starlette client so we can test the API routing with applied + API rules. + Does not follow redirects by default. Set `follow_redirects=True` option + on individual requests to enable. + + :param config_file: Optional configuration YAML file to use. + If not set, the default test configuration is used. + """ + starlette_app = None + env_conf = os.getenv('PYGEOAPI_CONFIG') + try: + # Temporarily override environment variable to import Starlette app + os.environ['PYGEOAPI_CONFIG'] = get_test_file_path(config_file) + + # Import current pygeoapi Starlette app module + from pygeoapi import starlette_app + + # Force a module reload to make sure we really use another config + reload(starlette_app) + + # Get server root path + base_url = starlette_app.CONFIG['server']['url'].rstrip('/') + root_path = urlsplit(base_url).path.rstrip('/') or '' + + # Create and return test client + # Note: setting the 'root_path' does NOT really work and + # does not have the same effect as Flask's APPLICATION_ROOT + client = StarletteClient( + starlette_app.APP, + base_url, + root_path=root_path, + **kwargs + ) + # Override follow_redirects so behavior is the same as Flask mock + client.follow_redirects = False + yield client + + finally: + if env_conf is None: + # Remove env variable again if it was not set initially + del os.environ['PYGEOAPI_CONFIG'] + # Unload Starlette app module + del sys.modules['pygeoapi.starlette_app'] + else: + # Restore env variable to original value and reload Starlette app + os.environ['PYGEOAPI_CONFIG'] = env_conf + if starlette_app: + reload(starlette_app) + del client