Ajout de données provenant de SEAMIS en JSON

.circleci/config.yml

@@ -1,30 +1,5 @@
 version: 2
 jobs:
-  python:
-    docker:
-      - image: circleci/python:2.7.13
-    working_directory: ~/repo
-    steps:
-      - checkout
-
-      - run:
-          name: Install Ubuntu dependencies
-          command: sudo apt install -y  software-properties-common python-dev libsqlite3-dev zlib1g-dev
-
-      - restore_cache:
-          key: requirements-{{ checksum "requirements.txt" }}
-      - run:
-          name: Install Python deps in a venv
-          command: |
-            virtualenv -p /usr/bin/python2.7 venv
-            source venv/bin/activate
-            pip install -r requirements.txt
-            echo "source $(pwd)/venv/bin/activate" >> $BASH_ENV
-      - save_cache:
-          key: requirements-{{ checksum "requirements.txt" }}
-          paths:
-            - "venv"
-
   opendata_build:
     docker:
       - image: antoineaugusti/secmar-opendata-build:0.0.4
@@ -57,9 +32,6 @@ jobs:
           key: yarn-packages-{{ checksum "/tmp/secmar-documentation/yarn.lock" }}
           paths:
             - ~/.cache/yarn
-      - run:
-          name: Pull CHANGELOG
-          command: wget https://raw.githubusercontent.com/snosan-tools/secmar-data/master/CHANGELOG.md -O doc/CHANGELOG.md
       - run:
           name: Build Markdown documentation for opendata
           command: |
@@ -101,7 +73,6 @@ jobs:
       - restore_cache:
           keys:
           - v1-dependencies-{{ checksum "requirements.txt" }}
-      - run: sudo apt install python-dev
       - run:
           name: Install dependencies
           command: |
@@ -160,7 +131,6 @@ workflows:
   version: 2
   all_jobs:
     jobs:
-      - python
       - opendata_test
       - opendata_build:
           requires:
@@ -170,4 +140,6 @@ workflows:
             - opendata_build
           filters:
             branches:
-              only: master
+              only:
+                - master
+                - snosan_json

.gitignore

@@ -10,3 +10,4 @@ app/plugins
 app/resources/signing
 
 snosan_csv
+snosan_json

airflow/dags/download_secmar_csv_ftp.py

@@ -11,7 +11,7 @@
 from airflow import DAG
 from airflow.models import Variable
 from airflow.operators.dummy_operator import DummyOperator
-from airflow.operators.python_operator import PythonOperator, BranchPythonOperator
+from airflow.operators.python_operator import BranchPythonOperator, PythonOperator
 from transformers import secmar_csv
 
 default_args = helpers.default_args({"start_date": datetime(2022, 6, 22, 10, 0)})
@@ -22,16 +22,16 @@
     max_active_runs=1,
     concurrency=1,
     catchup=True,
-    schedule_interval="0 10 * * *",
+    schedule_interval=None,
 )
 dag.doc_md = __doc__
 
 
 def setup_ftp_env():
     os.environ["FTP_PROXY"] = "false"
-    os.environ["FTP_HOST"] = Variable.get("SECMAR_CSV_FTP_HOST")
-    os.environ["FTP_USER"] = Variable.get("SECMAR_CSV_FTP_USER")
-    os.environ["FTP_PASSWORD"] = Variable.get("SECMAR_CSV_FTP_PASSWORD")
+    os.environ["FTP_HOST"] = Variable.get("SECMAR_FTP_HOST")
+    os.environ["FTP_USER"] = Variable.get("SECMAR_FTP_USER")
+    os.environ["FTP_PASSWORD"] = Variable.get("SECMAR_FTP_PASSWORD")
 
 
 def ftp_download_fn(**kwargs):