Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
7
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
Validata
Validata Core
Commits
56192cfe
Commit
56192cfe
authored
Dec 17, 2020
by
Pierre Dittgen
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
make badge great again
parent
49830fce
Pipeline
#2414
failed with stage
in 1 minute and 47 seconds
Changes
1
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
49 additions
and
0 deletions
+49
-0
validata_core/__init__.py
validata_core/__init__.py
+49
-0
No files found.
validata_core/__init__.py
View file @
56192cfe
import
logging
from
collections
import
Counter
from
datetime
import
datetime
,
timezone
import
frictionless
...
...
@@ -19,6 +20,54 @@ def extract_required_field_names(schema: frictionless.schema.Schema):
return
field_names
def
compute_badge_metrics
(
report
,
config
)
->
dict
:
"""Compute badge metrics from report statistics and badge configuration."""
def
build_badge
(
structure_status
,
body_status
=
None
,
error_ratio
=
None
):
"""Badge info creation"""
if
structure_status
==
'KO'
:
return
{
"structure"
:
'KO'
}
return
{
"structure"
:
structure_status
,
"body"
:
body_status
,
"error-ratio"
:
error_ratio
}
table_report
=
report
[
"tables"
][
0
]
table_stats
=
table_report
[
"stats"
]
# Compute number of cells
cell_total_number
=
table_stats
[
'fields'
]
*
table_stats
[
'rows'
]
# No errors
if
table_stats
[
"errors"
]
==
0
and
len
(
table_report
[
"structure_warnings"
])
==
0
:
return
build_badge
(
'OK'
,
'OK'
,
0.0
)
# Structure part
structure_status
=
None
structure_errors_count
=
len
([
err
for
err
in
table_report
[
"errors"
]
if
'#head'
in
err
.
tags
or
'#structure'
in
err
.
tags
])
if
structure_errors_count
==
0
:
structure_status
=
'OK'
if
len
(
table_report
[
"structure_warnings"
])
==
0
else
'WARN'
else
:
return
build_badge
(
'KO'
)
# body part
value_errors
=
[
err
for
err
in
table_report
[
"errors"
]
if
"#body"
in
err
.
tags
]
if
len
(
value_errors
)
==
0
:
return
build_badge
(
structure_status
,
'OK'
,
0.0
)
# Computes error ratio
weight_dict
=
config
[
'body'
][
'errors-weight'
]
err_code_counter
=
Counter
([
err
.
code
for
err
in
value_errors
])
ratio
=
sum
([
nb
*
weight_dict
.
get
(
err_code
,
1.0
)
for
err_code
,
nb
in
err_code_counter
.
items
()])
\
/
cell_total_number
body_status
=
'WARN'
if
ratio
<
config
[
'body'
][
'acceptability-threshold'
]
else
'KO'
return
build_badge
(
structure_status
,
body_status
,
ratio
)
def
validate
(
source
,
schema
,
**
options
):
"""Validate a `source` using a `schema`."""
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment