Commit e7a8d6f5 authored by Christophe Benz's avatar Christophe Benz
Browse files

Update CI and doc generation script

parent 732c513a
Pipeline #1052 passed with stages
in 1 minute and 25 seconds
variables:
CI_REPOSITORY_SSH_USER: git
CI_REPOSITORY_DOMAIN: git.opendatafrance.net
CI_REPOSITORY_SSH_URL: ${CI_REPOSITORY_SSH_USER}@${CI_REPOSITORY_DOMAIN}:${CI_PROJECT_PATH}.git
CI_DOCKER_IMAGE: ${CI_REPOSITORY_DOMAIN}:4567/scdl/documentation:latest
SCHEMA_JSON: schema.json
SCHEMA_MD: schema.md
Generate schema Markdown:
stage: build
image: $CI_DOCKER_IMAGE
only:
changes:
- $SCHEMA_JSON
variables:
LC_ALL: fr_FR.utf8
script:
- table-schema-to-markdown $SCHEMA_JSON > $SCHEMA_MD
artifacts:
paths:
- $SCHEMA_MD
cache:
paths:
- node_modules/
tags:
- scdl
Commit schema Markdown:
stage: deploy
image: $CI_DOCKER_IMAGE
only:
changes:
- $SCHEMA_JSON
before_script:
- eval $(ssh-agent -s)
- ssh-add <(echo "$SSH_PRIVATE_KEY")
- mkdir -p ~/.ssh
- ssh-keyscan -t rsa git.opendatafrance.net >> ~/.ssh/known_hosts
- git config --global user.name "GitLab Continuous Integration"
- git config --global user.email "<>"
script:
- git clone --branch $CI_COMMIT_REF_NAME $CI_REPOSITORY_SSH_URL
- mv $SCHEMA_MD $CI_PROJECT_NAME
- cd $CI_PROJECT_NAME
- git add $SCHEMA_MD
- git commit -m "Update $SCHEMA_MD" || true
- git push
tags:
- scdl
stages:
- build_docker_image
- generate
- commit
- build
- deploy
variables:
CI_DOCKER_IMAGE: $CI_REGISTRY_IMAGE:latest
CI_REPOSITORY_SSH_USER: git
CI_REPOSITORY_DOMAIN: git.opendatafrance.net
CI_REPOSITORY_SSH_URL: ${CI_REPOSITORY_SSH_USER}@${CI_REPOSITORY_DOMAIN}:${CI_PROJECT_PATH}.git
LC_ALL: fr_FR.UTF-8
PDF_FILE: Socle Commun des Données Locales.pdf
IMAGE_TAG: $CI_REGISTRY_IMAGE:latest
TZ: Europe/Paris
build_docker_image:
Build Docker image:
stage: build_docker_image
only:
changes:
- Dockerfile.ci
refs:
- master
image: docker:stable
services:
- docker:dind
......@@ -18,37 +30,105 @@ build_docker_image:
before_script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
script:
- docker build -t $IMAGE_TAG -f Dockerfile.ci .
- docker push $IMAGE_TAG
only:
changes:
- Dockerfile.ci
- docker build -t $CI_DOCKER_IMAGE -f Dockerfile.ci .
- docker push $CI_DOCKER_IMAGE
tags:
- docker-privileged
build_gitbook:
image: $IMAGE_TAG
Generate assets:
stage: generate
only:
variables:
- $GENERATE_ASSETS
image: $CI_DOCKER_IMAGE
variables:
CATALOG_URL: https://git.opendatafrance.net/scdl/catalog/raw/master/catalog.json
before_script:
- pip3 install --requirement requirements.txt
script:
- mkdir schemas templates
- python3 ./scripts/generate_assets_from_schema_catalog.py "$CATALOG_URL" schemas templates
docs.template/SUMMARY.j2 SUMMARY.md --ref "$CI_COMMIT_REF_NAME"
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
paths:
- SUMMARY.md
- schemas/
- templates/
Commit generated assets:
stage: commit
only:
variables:
- $GENERATE_ASSETS
image: $CI_DOCKER_IMAGE
before_script:
- eval $(ssh-agent -s)
- ssh-add <(echo "$COMMIT_SSH_PRIVATE_KEY")
- mkdir -p ~/.ssh
- ssh-keyscan -t rsa $CI_REPOSITORY_DOMAIN >> ~/.ssh/known_hosts
- git config --global user.name "GitLab Continuous Integration"
- git config --global user.email "<>"
script:
- git clone --branch $CI_COMMIT_REF_NAME $CI_REPOSITORY_SSH_URL
- rm -rf $CI_PROJECT_NAME/docs/schemas $CI_PROJECT_NAME/docs/templates
- mv SUMMARY.md schemas templates $CI_PROJECT_NAME/docs/
- cp $CI_PROJECT_NAME/docs.template/schemas/README.md $CI_PROJECT_NAME/docs/schemas
- cd $CI_PROJECT_NAME
- git add -A
- git commit -m "Update generated assets" || true
- git push
Build Gitbook:
stage: build
except:
variables:
- $GENERATE_ASSETS
image: $CI_DOCKER_IMAGE
script:
- ./build.sh
- ./scripts/build_gitbook.sh
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
paths:
- _book/
- "$PDF_FILE"
deploy_gitbook:
image: $IMAGE_TAG
.Deploy Gitbook template:
stage: deploy
except:
variables:
- $GENERATE_ASSETS
only:
- master
variables:
- $SERVER_DIRECTORY
image: $CI_DOCKER_IMAGE
variables:
SERVER_HOST: go.validata.fr
SERVER_USER: validata
before_script:
# Load SSH agent
- eval $(ssh-agent -s)
- ssh-add <(echo "$SSH_PRIVATE_KEY")
- ssh-add <(echo "$DEPLOY_SSH_PRIVATE_KEY")
- mkdir -p ~/.ssh
- ssh-keyscan -t rsa go.validata.fr >> ~/.ssh/known_hosts
- ssh-keyscan -t rsa $SERVER_HOST >> ~/.ssh/known_hosts
script:
- rsync -avz --delete _book/ "$PDF_FILE" validata@go.validata.fr:scdl-documentation-static/
- rsync -avz --delete _book/ "$PDF_FILE" ${SERVER_USER}@${SERVER_HOST}:${SERVER_DIRECTORY}/
Deploy Gitbook prod:
extends: .Deploy Gitbook template
only:
- master
variables:
SERVER_DIRECTORY: scdl-documentation-static
environment:
name: documentation
name: production
url: https://scdl.opendatafrance.net/docs/
Deploy Gitbook next:
extends: .Deploy Gitbook template
only:
- next
variables:
SERVER_DIRECTORY: scdl-documentation-static-next
environment:
name: next
url: https://scdl.opendatafrance.net/docs-next/
- [Socle Commun des Données Locales](README.md)
- [Recommandations relatives aux jeux de données](recommandations-relatives-aux-jeux-de-donnees.md)
- [Recommandations relatives aux schémas de validation](recommandations-relatives-aux-schemas-de-validation.md)
- [Schémas du SCDL](schemas/README.md)
{%- for schema in schemas %}
- [{{ schema.title }}](schemas/{{ schema.name }}.md)
{%- endfor %}
- [Télécharger en PDF](télécharger.md)
- [Badge Validata](badge.md)
\ No newline at end of file
#!/bin/bash
DATE=$(TZ='Europe/Paris' LC_ALL=fr_FR.UTF-8 date "+%d/%m/%Y à %R")
DATE=$(date "+%d/%m/%Y à %R")
sed -i -e "s~UPDATED_AT~${DATE}~" docs/README.md
gitbook build
......
#!/usr/bin/env python3
"""
Generate assets from a catalog of schemas.
"""
import argparse
import logging
import subprocess
import sys
from pathlib import Path
from urllib.parse import urljoin
import requests
from jinja2 import Template
from opendataschema import DEFAULT_GIT_REF, GitSchemaReference, SchemaCatalog
log = logging.getLogger(__name__)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('catalog_path_or_url', help='URL of catalog file')
parser.add_argument('schemas', type=Path, help='schemas output directory')
parser.add_argument('templates', type=Path, help='templates output directory')
parser.add_argument('summary_j2', type=Path, help='Jinja2 template for SUMMARY.md')
parser.add_argument('summary_md', type=Path, help='output file path for SUMMARY.md')
parser.add_argument('--log', default='WARNING', help='level of logging messages')
parser.add_argument('--ref', help='Git reference to try to load with Git schema references')
args = parser.parse_args()
numeric_level = getattr(logging, args.log.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: {}'.format(args.log))
logging.basicConfig(
format="%(levelname)s:%(name)s:%(message)s",
level=numeric_level,
stream=sys.stderr, # script outputs data
)
if not args.schemas.is_dir():
parser.error("schemas directory does not exist")
if not args.templates.is_dir():
parser.error("templates directory does not exist")
if not args.summary_j2.is_file():
parser.error("summary_j2 file does not exist")
if not args.summary_md.parent.is_dir():
parser.error("summary_md parent directory ({}) does not exist".format(str(args.summary_md.parent)))
log.info("Loading schema catalog metadata...")
schema_catalog = SchemaCatalog(source=args.catalog_path_or_url)
schema_json_by_name = {}
for reference in schema_catalog.references:
name = reference.name
ref = None
if isinstance(reference, GitSchemaReference):
refs = reference.get_refs()
ref_names = [ref.name for ref in refs]
ref = args.ref if args.ref in ref_names else DEFAULT_GIT_REF
schema_url = reference.get_schema_url(ref=ref)
log.info("{}: downloading schema file from {}".format(name, schema_url))
response = requests.get(schema_url)
response.raise_for_status()
schema_json = response.json()
schema_json_by_name[name] = schema_json
log.info("{}: generating Markdown file...".format(name))
with (args.schemas / "{}.md".format(name)).open("w") as fd:
readme_url = urljoin(schema_url, "README.md")
response = requests.get(readme_url)
if response.ok:
log.info("{}: found README.md".format(name))
readme_md = response.text
fd.write(readme_md)
fd.write("\n## Carte d'identité du schéma\n")
fd.flush()
log.info("{}: calling table-schema-to-markdown...".format(name))
index_hbs = Path(__file__).parent / "index.hbs"
subprocess.check_call(["table-schema-to-markdown", "--template", str(index_hbs), schema_url], stdout=fd)
log.info("{}: generating XLSX template file...".format(name))
subprocess.check_output(["table-schema-resource-template", "--format", "xlsx",
schema_url, str(args.templates / "{}.xlsx".format(name))])
log.info("Generating SUMMARY.md for GitBook...")
summary_template = Template(args.summary_j2.read_text())
summary_md = summary_template.render(schemas=[
{
"name": name,
"title": schema_json_by_name.get(name, {}).get("title", name)
}
for name in sorted(schema_json_by_name.keys())
])
with args.summary_md.open("w") as fd:
fd.write(summary_md)
if __name__ == '__main__':
sys.exit(main())
{{#if image}}
![{{formatMessage (intlGet "messages.schemaImageAlt")}}]({{{image}}})
{{/if}}
{{> properties }}
## {{ intlGet "messages.dataModel" }}
{{ formatMessage (intlGet "messages.dataModelIntro") count=(formatNumber fields.length) }}
{{> schemaFields }}
\ No newline at end of file
[isort]
line_length = 120
[pycodestyle]
max_line_length = 120
[pylint]
max_line_length = 120
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment