diff --git a/API/api_worker.py b/API/api_worker.py index 304cd538..7ee1bd0d 100644 --- a/API/api_worker.py +++ b/API/api_worker.py @@ -18,6 +18,7 @@ from src.config import CELERY_RESULT_BACKEND as celery_backend from src.config import ( DEFAULT_HARD_TASK_LIMIT, + DEFAULT_README_TEXT, DEFAULT_SOFT_TASK_LIMIT, ENABLE_TILES, HDX_HARD_TASK_LIMIT, @@ -121,8 +122,7 @@ def process_raw_data(self, params, user=None): utc_offset = utc_now.strftime("%z") # Adding metadata readme.txt readme_content = f"Exported Timestamp (UTC{utc_offset}): {utc_now.strftime('%Y-%m-%d %H:%M:%S')}\n" - readme_content += "Exported through Raw-data-api (https://github.com/hotosm/raw-data-api) using OpenStreetMap data.\n" - readme_content += "Learn more about OpenStreetMap and its data usage policy : https://www.openstreetmap.org/about \n" + readme_content += DEFAULT_README_TEXT if polygon_stats: readme_content += f'{polygon_stats["summary"]["buildings"]}\n' readme_content += f'{polygon_stats["summary"]["roads"]}\n' diff --git a/docs/src/installation/configurations.md b/docs/src/installation/configurations.md index 92160df4..b9bc70ab 100644 --- a/docs/src/installation/configurations.md +++ b/docs/src/installation/configurations.md @@ -55,6 +55,7 @@ The following are the different configuration options that are accepted. | `EXPORT_MAX_AREA_SQKM` | `EXPORT_MAX_AREA_SQKM` | `[API_CONFIG]` | `100000` | max area in sq. km. to support for rawdata input | OPTIONAL | | `USE_CONNECTION_POOLING` | `USE_CONNECTION_POOLING` | `[API_CONFIG]` | `false` | Enable psycopg2 connection pooling | OPTIONAL | | `ALLOW_BIND_ZIP_FILTER` | `ALLOW_BIND_ZIP_FILTER` | `[API_CONFIG]` | `true` | Enable zip compression for exports | OPTIONAL | +| `EXTRA_README_TXT` | `EXTRA_README_TXT` | `[API_CONFIG]` | `` | Append extra string to export readme.txt | OPTIONAL | | `ENABLE_TILES` | `ENABLE_TILES` | `[API_CONFIG]` | `false` | Enable Tile Output (Pmtiles and Mbtiles) | OPTIONAL | | `DEFAULT_QUEUE_NAME` | `DEFAULT_QUEUE_NAME` | `[API_CONFIG]` | `raw_ondemand` | Option to define default queue name| OPTIONAL | | `DAEMON_QUEUE_NAME` | `DAEMON_QUEUE_NAME` | `[API_CONFIG]` | `raw_daemon` | Option to define daemon queue name for scheduled and long exports | OPTIONAL | @@ -116,6 +117,7 @@ API Tokens have expiry date, It is `important to update API Tokens manually each | `USE_CONNECTION_POOLING` | `[API_CONFIG]` | Yes | Yes | | `ENABLE_TILES` | `[API_CONFIG]` | Yes | Yes | | `ALLOW_BIND_ZIP_FILTER` | `[API_CONFIG]` | Yes | Yes | +| `EXTRA_README_TXT` | `[API_CONFIG]` | No | Yes | | `INDEX_THRESHOLD` | `[API_CONFIG]` | No | Yes | | `DEFAULT_QUEUE_NAME` | `[API_CONFIG]` | Yes | No | | `DAEMON_QUEUE_NAME` | `[API_CONFIG]` | Yes | No | diff --git a/src/app.py b/src/app.py index 0b40042d..1cd2cea4 100644 --- a/src/app.py +++ b/src/app.py @@ -49,6 +49,7 @@ AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME, + DEFAULT_README_TEXT, ENABLE_CUSTOM_EXPORTS, ENABLE_HDX_EXPORTS, ENABLE_POLYGON_STATISTICS_ENDPOINTS, @@ -1355,8 +1356,7 @@ def file_to_zip(self, working_dir, zip_path): utc_offset = utc_now.strftime("%z") # Adding metadata readme.txt readme_content = f"Exported Timestamp (UTC{utc_offset}): {utc_now.strftime('%Y-%m-%d %H:%M:%S')}\n" - readme_content += "Exported through Raw-data-api (https://github.com/hotosm/raw-data-api) using OpenStreetMap data.\n" - readme_content += "Learn more about OpenStreetMap and its data usage policy : https://www.openstreetmap.org/about \n" + readme_content += DEFAULT_README_TEXT zf.writestr("Readme.txt", readme_content) if self.params.geometry: zf.writestr("clipping_boundary.geojson", self.params.geometry.json()) diff --git a/src/config.py b/src/config.py index c432ff9d..dc46d09f 100644 --- a/src/config.py +++ b/src/config.py @@ -149,6 +149,9 @@ def not_raises(func, *args, **kwargs): logging.getLogger("boto").propagate = False # disable boto3 logging +## Readme txt + + logger = logging.getLogger("raw_data_api") EXPORT_PATH = os.environ.get("EXPORT_PATH") or config.get( @@ -159,6 +162,14 @@ def not_raises(func, *args, **kwargs): # Create a exports directory because it does not exist os.makedirs(EXPORT_PATH) + +DEFAULT_README_TEXT = """Exported through Raw-data-api (https://github.com/hotosm/raw-data-api) using OpenStreetMap data.\n Exports are made available under the Open Database License: http://opendatacommons.org/licenses/odbl/1.0/. Any rights in individual contents of the database are licensed under the Database Contents License: http://opendatacommons.org/licenses/dbcl/1.0/. \n Learn more about OpenStreetMap and its data usage policy : https://www.openstreetmap.org/about \n""" + +EXTRA_README_TXT = os.environ.get("EXTRA_README_TXT") or config.get( + "API_CONFIG", "EXTRA_README_TXT", fallback="" +) +DEFAULT_README_TEXT += EXTRA_README_TXT + ALLOW_BIND_ZIP_FILTER = get_bool_env_var( "ALLOW_BIND_ZIP_FILTER", config.getboolean("API_CONFIG", "ALLOW_BIND_ZIP_FILTER", fallback=False),