Replace cache_control by requests_cache

.env.example

@@ -24,3 +24,6 @@ CONFIG=config.example.yaml
 #
 # BROWSERLESS_API_URL="https://chrome.browserless.io/pdf"
 # BROWSERLESS_API_TOKEN="XXX"
+
+# Number of minutes to cache downloaded schemas
+# CACHING_EXPIRE_AFTER=0
\ No newline at end of file

.gitignore

 __pycache__/
 validata_ui.egg-info/
+validata_ui_cache.sqlite
 .env
 .mypy_cache/
 .pytest_cache/

requirements.in

 -e git+https://git.opendatafrance.net/validata/validata-core.git@frictionless_py_migration#egg=validata_core
-cachecontrol
 commonmark
 ezodf
 flask
@@ -9,6 +8,7 @@ pydantic
 python-dotenv
 pyyaml
 requests
+requests_cache
 sentry-sdk[flask] == 0.14.3
 toml
 opendataschema >= 0.5.5, < 0.6

requirements.txt

@@ -4,65 +4,121 @@
 #
 #    pip-compile
 #
--e git+https://git.opendatafrance.net/validata/validata-core.git@frictionless_py_migration#egg=validata_core  # via -r requirements.in
-attrs==20.3.0             # via jsonschema
-blinker==1.4              # via sentry-sdk
-cachecontrol==0.12.6      # via -r requirements.in
-certifi==2020.12.5        # via requests, sentry-sdk
-chardet==3.0.4            # via frictionless, requests
-click==7.1.2              # via flask, opendataschema, typer
-colorama==0.4.4           # via typer
-commonmark==0.9.1         # via -r requirements.in
-decorator==4.4.2          # via validators
-deprecated==1.2.10        # via pygithub
-et-xmlfile==1.0.1         # via openpyxl
-ezodf==0.3.2              # via -r requirements.in, validata-core
-flask==1.1.2              # via -r requirements.in, sentry-sdk
-frictionless==3.48.0      # via -r requirements.in, validata-core
-idna==2.10                # via requests
-importlib-metadata==3.3.0  # via jsonschema
-importlib-resources==3.3.0  # via validata-core
-isodate==0.6.0            # via frictionless
-itsdangerous==1.1.0       # via flask
-jdcal==1.4.1              # via openpyxl
-jinja2==2.11.2            # via flask
-jsonschema==3.2.0         # via frictionless, opendataschema
-lxml==4.6.2               # via -r requirements.in, validata-core
-markupsafe==1.1.1         # via jinja2
-msgpack==1.0.2            # via cachecontrol
-numpy==1.19.4             # via pandas
-opendataschema==0.5.5     # via -r requirements.in
-openpyxl==3.0.5           # via validata-core
-pandas==1.1.5             # via tablib
-petl==1.6.8               # via frictionless
-pydantic==1.7.3           # via -r requirements.in
-pygithub==1.54            # via opendataschema
-pyjwt==1.7.1              # via pygithub
-pyrsistent==0.17.3        # via jsonschema
-python-dateutil==2.8.1    # via frictionless, pandas
-python-dotenv==0.15.0     # via -r requirements.in
-python-gitlab==2.5.0      # via opendataschema
-python-slugify==4.0.1     # via frictionless
-python-stdnum==1.14       # via validata-core
-pytz==2020.4              # via pandas
-pyyaml==5.3.1             # via -r requirements.in, frictionless
-requests==2.24.0          # via -r requirements.in, cachecontrol, frictionless, opendataschema, pygithub, python-gitlab, validata-core
-sentry-sdk[flask]==0.14.3  # via -r requirements.in
-shellingham==1.3.2        # via typer
-simpleeval==0.9.10        # via frictionless
-six==1.15.0               # via isodate, jsonschema, python-dateutil, validators
-stringcase==1.2.0         # via frictionless
-tablib[pandas]==3.0.0     # via validata-core
-text-unidecode==1.3       # via python-slugify
-toml==0.10.2              # via -r requirements.in
-toolz==0.11.1             # via validata-core
-typer[all]==0.3.2         # via frictionless
-typing-extensions==3.7.4.3  # via importlib-metadata
-urllib3==1.25.11          # via requests, sentry-sdk
-validators==0.18.1        # via frictionless
-werkzeug==1.0.1           # via flask
-wrapt==1.12.1             # via deprecated
-zipp==3.4.0               # via importlib-metadata, importlib-resources
+-e git+https://git.opendatafrance.net/validata/validata-core.git@frictionless_py_migration#egg=validata_core
+    # via -r requirements.in
+attrs==20.3.0
+    # via jsonschema
+blinker==1.4
+    # via sentry-sdk
+certifi==2020.12.5
+    # via
+    #   requests
+    #   sentry-sdk
+chardet==3.0.4
+    # via
+    #   frictionless
+    #   requests
+click==7.1.2
+    # via
+    #   flask
+    #   opendataschema
+    #   typer
+colorama==0.4.4
+    # via typer
+commonmark==0.9.1
+    # via -r requirements.in
+decorator==4.4.2
+    # via validators
+deprecated==1.2.10
+    # via pygithub
+ezodf==0.3.2
+    # via -r requirements.in
+flask==1.1.2
+    # via
+    #   -r requirements.in
+    #   sentry-sdk
+frictionless==3.48.0
+    # via -r requirements.in
+idna==2.10
+    # via requests
+isodate==0.6.0
+    # via frictionless
+itsdangerous==1.1.0
+    # via flask
+jinja2==2.11.2
+    # via flask
+jsonschema==3.2.0
+    # via
+    #   frictionless
+    #   opendataschema
+lxml==4.6.2
+    # via -r requirements.in
+markupsafe==1.1.1
+    # via jinja2
+opendataschema==0.5.5
+    # via -r requirements.in
+petl==1.6.8
+    # via frictionless
+pydantic==1.7.3
+    # via -r requirements.in
+pygithub==1.54
+    # via opendataschema
+pyjwt==1.7.1
+    # via pygithub
+pyrsistent==0.17.3
+    # via jsonschema
+python-dateutil==2.8.1
+    # via frictionless
+python-dotenv==0.15.0
+    # via -r requirements.in
+python-gitlab==2.5.0
+    # via opendataschema
+python-slugify==4.0.1
+    # via frictionless
+pyyaml==5.3.1
+    # via
+    #   -r requirements.in
+    #   frictionless
+requests-cache==0.5.2
+    # via -r requirements.in
+requests==2.24.0
+    # via
+    #   -r requirements.in
+    #   frictionless
+    #   opendataschema
+    #   pygithub
+    #   python-gitlab
+    #   requests-cache
+sentry-sdk[flask]==0.14.3
+    # via -r requirements.in
+shellingham==1.3.2
+    # via typer
+simpleeval==0.9.10
+    # via frictionless
+six==1.15.0
+    # via
+    #   isodate
+    #   jsonschema
+    #   python-dateutil
+    #   validators
+stringcase==1.2.0
+    # via frictionless
+text-unidecode==1.3
+    # via python-slugify
+toml==0.10.2
+    # via -r requirements.in
+typer[all]==0.3.2
+    # via frictionless
+urllib3==1.25.11
+    # via
+    #   requests
+    #   sentry-sdk
+validators==0.18.1
+    # via frictionless
+werkzeug==1.0.1
+    # via flask
+wrapt==1.12.1
+    # via deprecated
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

validata_ui/__init__.py

 import logging
+from datetime import timedelta
 
-import cachecontrol
 import flask
 import frictionless
 import jinja2
 import opendataschema
 import pkg_resources
-import requests
+import requests_cache
 from commonmark import commonmark
 from pydantic import HttpUrl
 
@@ -45,7 +45,10 @@ class SchemaCatalogRegistry:
         return opendataschema.SchemaCatalog(ref, session=self.session)
 
 
-caching_session = cachecontrol.CacheControl(requests.Session())
+expire_after = timedelta(minutes=config.CACHING_EXPIRE_AFTER)
+caching_session = requests_cache.CachedSession(
+    backend="sqlite", cache_name="validata_ui_cache", expire_after=expire_after
+)
 fetch_schema = generate_schema_from_url_func(caching_session)
 
 # And load schema catalogs which URLs are found in 'homepage' key of config.yaml

validata_ui/config.py

 """Configuration stuff."""
 
-import yaml
 import logging
 import os
 import sys
@@ -8,6 +7,7 @@ from pathlib import Path
 
 import requests
 import toml
+import yaml
 from dotenv import load_dotenv
 from pydantic.error_wrappers import ValidationError
 
@@ -84,3 +84,18 @@ SENTRY_DSN = os.environ.get("SENTRY_DSN")
 # PDF generation service
 BROWSERLESS_API_URL = os.getenv("BROWSERLESS_API_URL") or None
 BROWSERLESS_API_TOKEN = os.getenv("BROWSERLESS_API_TOKEN") or None
+
+# Caching time for schema requests in minutes
+CACHING_EXPIRE_AFTER = os.getenv("CACHING_EXPIRE_AFTER") or None
+if CACHING_EXPIRE_AFTER is None:
+    log.info(
+        "CACHING_EXPIRE_AFTER environment variable not set, using default 0 (no cache)"
+    )
+    CACHING_EXPIRE_AFTER = "0"
+try:
+    CACHING_EXPIRE_AFTER = int(CACHING_EXPIRE_AFTER)
+    log.info("Caching is set to %sm", CACHING_EXPIRE_AFTER)
+except ValueError as exc:
+    raise ValueError(
+        f"Invalid number of minutes for caching: {CACHING_EXPIRE_AFTER}"
+    ) from exc