Add CI, Dockerfile, rename config key, sort imports

.env.example

@@ -10,5 +10,5 @@ SHIELDS_IO_BASE_URL="https://img.shields.io/"
 # Validata API endpoint
 API_VALIDATE_ENDPOINT=http://127.0.0.1:5600/validate
 
-# UI config file path
-UI_CONFIG_FILE=config.json
\ No newline at end of file
+# Homepage sections and blocks config file path
+# HOMEPAGE_CONFIG_FILE=homepage_config.json
\ No newline at end of file

.gitlab-ci.yml

+Run tests:
+  stage: test
+  image: python:3.7
+  script:
+    - python setup.py test
+
+Build Docker image:
+  stage: deploy
+  only:
+    changes:
+      - Dockerfile
+    refs:
+      - tags
+  image: docker:stable
+  services:
+    - docker:dind
+  variables:
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_DRIVER: overlay2
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+  script:
+    - docker build -t $CI_REGISTRY_IMAGE:latest .
+    - docker push $CI_REGISTRY_IMAGE:latest
+  tags:
+    - docker-privileged
+
+Publish on PyPI:
+  stage: deploy
+  image: python:3.7
+  only:
+    - tags
+  before_script:
+    - pip install twine
+    - python setup.py sdist bdist_wheel
+  variables:
+    TWINE_USERNAME: cbenz
+    # TWINE_PASSWORD: # Secret variable, see project CI settings.
+  script:
+    - twine upload dist/*
+  environment:
+    name: PyPI
+    url: https://pypi.org/project/validata-ui/$CI_COMMIT_TAG

CHANGELOG.md

-## 0.1.0 -> next
+## 0.2.0
+
+New features for users:
+
+- validate a tabular file (e.g. CSV) against a schema URL
+- allow configuring homepage sections and blocks with a JSON config file (see `HOMEPAGE_CONFIG` environment variable in `.env`)
 
 Non-breaking changes:
 
-- New feature: validate a CSV against a schema URL
-- UI now depends on validata-api, no more on validata-core
+- UI now requests `validata-api` service to do the validation, and does not depend on `validata-core` anymore
+- a Dockerfile has been added
+- a Continuous Integration pipeline has been added
+  - the Docker image is rebuilt for each release
+  - the Python package is uploaded to [PyPI](https://pypi.org/) for each release
 
-## 0.0.1 -> 0.1.0
+## 0.1.0
 
 Non-breaking changes:
 

Dockerfile

+FROM python:3.7
+LABEL maintainer="admin-validata@jailbreak.paris"
+
+EXPOSE 5000
+
+RUN pip install gunicorn
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --requirement requirements.txt
+
+COPY . .
+RUN pip install --editable .
+
+CMD gunicorn --bind 0.0.0.0:5000 validata_ui:app
\ No newline at end of file

README.md

@@ -2,15 +2,22 @@
 
 Validata user interface
 
-## Requirements
+## Usage
 
-PDF report uses [Headless Chromium](https://chromium.googlesource.com/chromium/src/+/lkgr/headless/README.md).
-Please install:
-```bash
-apt install -y chromium
-```
+You can use the online instance of Validata:
+- user interface: https://go.validata.fr/
+- API: https://go.validata.fr/api/v1/
+- API docs: https://go.validata.fr/api/v1/apidocs
+
+Several software services compose the Validata stack. The recommended way to run it on your computer is to use Docker. Otherwise you can install each component of this stack manually, for example if you want to contribute by developing a new feature or fixing a bug.
+
+## Run with Docker
+
+Read instructions at https://git.opendatafrance.net/validata/validata-docker
+
+## Develop
 
-## Install
+### Install
 
 We recommend using [virtualenv](https://virtualenv.pypa.io/en/stable/).
 
@@ -20,7 +27,15 @@ Install the project dependencies:
 pip install -e .
 ```
 
-## Configuration
+Validata UI depends on [Validata API](https://git.opendatafrance.net/validata/validata-api/), so you must install it also.
+
+PDF report generation uses [Headless Chromium](https://chromium.googlesource.com/chromium/src/+/lkgr/headless/README.md):
+
+```bash
+apt install -y chromium
+```
+
+### Configure
 
 ```bash
 cp .env.example .env
@@ -30,9 +45,7 @@ Customize the configuration variables in `.env` file.
 
 Do not commit `.env`.
 
-See also: https://github.com/theskumar/python-dotenv
-
-## Development
+### Serve
 
 Start the web server...
 

requirements.txt

+backports-datetime-fromisoformat==1.0.0
+commonmark==0.8.1
+ezodf==0.3.2
+Flask==1.0.2
+lxml==4.2.5
+python-dotenv==0.10.1
+requests==2.22.0
+toml==0.10.0
+tabulator==1.21.0
+opendataschema==0.2.0
+validata_core==0.3.4

setup.cfg

+[isort]
+line_length = 120
+
+[pycodestyle]
+max_line_length = 120
+
+[pylint]
+max_line_length = 120
+
+[aliases]
+test=pytest
+
+[tool:pytest]
+addopts = --doctest-modules
\ No newline at end of file

setup.py

 #!/usr/bin/env python3
-"""Run validata ui"""
+
+from pathlib import Path
+
 from setuptools import setup
 
-classifiers = """\
-Development Status :: 4 - Beta
-Intended Audience :: Developers
-Operating System :: OS Independent
-Programming Language :: Python
-Topic :: Software Development :: Libraries :: Python Modules
-License :: OSI Approved :: GNU Affero General Public License v3
-"""
+# Gets the long description from the README.md file
+readme_filepath = Path(__file__).parent / 'README.md'
+with readme_filepath.open('rt', encoding='utf-8') as fd_in:
+    LONG_DESCRIPTION = fd_in.read()
 
 
 setup(
     name='validata_ui',
-    version='0.1.0',
+    version='0.2.0',
+
+    description='Validata Web UI',
+    long_description=LONG_DESCRIPTION,
+    long_description_content_type="text/markdown",
+
+    url='https://git.opendatafrance.net/validata/validata-ui',
 
     author='Validata team',
-    classifiers=[classifier for classifier in classifiers.split('\n') if classifier],
-    description=__doc__,
+    author_email='admin-validata@jailbreak.paris',
+
+    license='AGPLv3',
+
+    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
+    classifiers=[
+        # How mature is this project? Common values are
+        #   3 - Alpha
+        #   4 - Beta
+        #   5 - Production/Stable
+        'Development Status :: 5 - Production/Stable',
+
+        # Indicate who your project is intended for
+        'Intended Audience :: Developers',
+
+        'Topic :: Software Development :: Libraries :: Python Modules',
+        'Operating System :: OS Independent',
+
+        # Pick your license as you wish (should match "license" above)
+        'License :: OSI Approved :: GNU Affero General Public License v3',
+
+        # Specify the Python versions you support here. In particular, ensure
+        # that you indicate whether you support Python 2, Python 3 or both.
+        'Programming Language :: Python :: 3',
+    ],
 
     packages=['validata_ui'],
-    include_package_data=True,
+    zip_safe=True,
+
     install_requires=[
         'backports-datetime-fromisoformat',
 
@@ -33,9 +61,9 @@ setup(
         'requests',
         'toml',
 
-        'goodtables',
         'tabulator',
 
-        'validata_core >= 0.2.1, < 0.3',
-    ]
+        'opendataschema >= 0.2.0, < 0.3',
+        'validata_core >= 0.3.0, < 0.4',
+    ],
 )

validata_ui/__init__.py

@@ -3,13 +3,13 @@ import os
 from pathlib import Path
 from urllib.parse import quote_plus
 
-import opendataschema
 import flask
 import jinja2
 import requests
 import tableschema
 from cachetools.func import ttl_cache
 
+import opendataschema
 
 # Let this import after app initialisation
 from . import config
@@ -25,16 +25,14 @@ def schema_from_url(url):
     return tableschema.Schema(url)
 
 
-# load config.json
-ui_config = json.load(config.UI_CONFIG_FILE.open('rt', encoding='utf-8')) if config.UI_CONFIG_FILE else []
-
 # And load schema catalogs which urls are found in config.json
 schema_catalog_map = {}
-for section in ui_config['sections']:
-    if isinstance(section['catalog'], str) and section['catalog'].startswith('http'):
-        code = section['code']
-        url = section['catalog']
-        schema_catalog_map[code] = opendataschema.SchemaCatalog(url, download_func=download_with_cache)
+if config.HOMEPAGE_CONFIG:
+    for section in config.HOMEPAGE_CONFIG['sections']:
+        if isinstance(section['catalog'], str) and section['catalog'].startswith('http'):
+            code = section['code']
+            url = section['catalog']
+            schema_catalog_map[code] = opendataschema.SchemaCatalog(url, download_func=download_with_cache)
 
 # Flask things
 app = flask.Flask(__name__)
@@ -53,4 +51,4 @@ def urlencode(context, value):
 
 
 # Keep this import after app initialisation (to avoid cyclic imports)
-from . import views  # isort:skip
+from . import views  # noqa isort:skip

validata_ui/config.py

+import json
 import logging
 import os
 from pathlib import Path
@@ -33,6 +34,9 @@ SHIELDS_IO_BASE_URL = os.environ.get("SHIELDS_IO_BASE_URL") or None
 if SHIELDS_IO_BASE_URL and not SHIELDS_IO_BASE_URL.endswith('/'):
     SHIELDS_IO_BASE_URL += '/'
 
-UI_CONFIG_FILE = os.environ.get("UI_CONFIG_FILE") or None
-if UI_CONFIG_FILE:
-    UI_CONFIG_FILE = Path(UI_CONFIG_FILE)
+HOMEPAGE_CONFIG_FILE = os.environ.get("HOMEPAGE_CONFIG_FILE") or None
+HOMEPAGE_CONFIG = None
+if HOMEPAGE_CONFIG_FILE:
+    HOMEPAGE_CONFIG_FILE = Path(HOMEPAGE_CONFIG_FILE)
+    with HOMEPAGE_CONFIG_FILE.open() as fd:
+        HOMEPAGE_CONFIG = json.load(fd)

validata_ui/views.py

@@ -15,16 +15,16 @@ from urllib.parse import quote_plus, urlencode
 
 import requests
 import tableschema
+import tabulator
 from backports.datetime_fromisoformat import MonkeyPatch
 from commonmark import commonmark
 from flask import make_response, redirect, render_template, request, url_for
-from validata_core import compute_badge, messages
 
-import tabulator
+from validata_core import compute_badge, messages
 
-from . import app, config, ui_config, schema_catalog_map, schema_from_url
+from . import app, config, schema_catalog_map, schema_from_url
 from .ui_util import flash_error, flash_warning
-from .validata_util import ValidataResource, URLValidataResource, UploadedFileValidataResource
+from .validata_util import UploadedFileValidataResource, URLValidataResource, ValidataResource
 
 MonkeyPatch.patch_fromisoformat()
 
@@ -395,7 +395,7 @@ def bytes_data(f):
     return iob.getvalue()
 
 
-def ui_config_with_schema_metadata(ui_config, schema_catalog_map):
+def homepage_config_with_schema_metadata(ui_config, schema_catalog_map):
     """Replace catalog url within ui_config by schema references
     containing schema metadata properties"""
 
@@ -424,7 +424,7 @@ def ui_config_with_schema_metadata(ui_config, schema_catalog_map):
 def home():
     """ Home page """
     flash_warning('Ce service est fourni en mode beta - certains problèmes peuvent subsister - nous mettons tout en œuvre pour améliorer son fonctionnement en continu.')
-    home_config = ui_config_with_schema_metadata(ui_config, schema_catalog_map)
+    home_config = homepage_config_with_schema_metadata(config.HOMEPAGE_CONFIG, schema_catalog_map)
     return render_template('home.html', title='Accueil', config=home_config)
 
 
@@ -507,7 +507,7 @@ def compute_validation_form_url(schema_instance: SchemaInstance):
     return "{}?{}".format(url, '&'.join(param_list))
 
 
-@app.route('/table_schema', methods=['GET', 'POST'])
+@app.route('/table-schema', methods=['GET', 'POST'])
 def custom_validator():
     """Validator form"""