Commit 3866e779 authored by Florian Pagnoux's avatar Florian Pagnoux
Browse files

Generate CSV files

parent 0406d47a
import {toCSV, cleanDatum, generateTables} from '../../server/csv'
import rsaMajoration from '../services/rsa_majoration.json'
const dirtyDatum = { date: '2009-06-20',
'prestations.minima_sociaux.rsa.majoration_rsa.taux_deuxieme_personne': { value: 0.5, unit: undefined },
'prestations.minima_sociaux.rsa.majoration_rsa.taux_troisieme_personne': { value: 0.3, unit: undefined },
'prestations.minima_sociaux.rsa.majoration_rsa.taux_personne_supp': { value: 0.4, unit: undefined },
'prestations.minima_sociaux.rsa.majoration_parent_isole.femmes_enceintes': { value: 1.28412, unit: undefined },
'prestations.minima_sociaux.rsa.majoration_parent_isole.par_enfant_a_charge': { value: 0.42804, unit: undefined },
'reference': 'Article X du 1er janvier 2019' }
it('should clean a datum', () => {
const result = cleanDatum(dirtyDatum, rsaMajoration.id)
expect(result['majoration_rsa.taux_personne_supp']).toEqual(0.4)
});
it('should convert to CSV', () => {
const result = toCSV(rsaMajoration)
// console.log(result)
});
it('should create a CSV', () => {
const result = toCSV(generateTables, '')
// console.log(result)
});
import extractData from '../../services/dataPreprocesser'
import {extractValues} from '../../services/dataPreprocesser'
import {extractValues, extractData} from '../../services/dataPreprocesser'
import rsaForFaitLogement from './rsa_forfait_logement.json'
import rsaMajoration from './rsa_majoration.json'
import bareme from './bareme.json'
......
{
"Majoration montant maximal (en % de la base RSA)": {
"Couples ou célibataire avec un enfant": {
"description": "Taux pour une 2nde personne",
"id": "prestations.minima_sociaux.rsa.majoration_rsa.taux_deuxieme_personne",
"values": {
"2009-06-01": 0.5
"id": "prestations.minima_sociaux.rsa",
"subparams": {
"Majoration montant maximal (en % de la base RSA)": {
"Couples ou célibataire avec un enfant": {
"description": "Taux pour une 2nde personne",
"id": "prestations.minima_sociaux.rsa.majoration_rsa.taux_deuxieme_personne",
"values": {
"2009-06-01": 0.5
}
},
"Couple, 1 enfant - ou pour le deuxième enfant": {
"description": "Taux pour une 3e personne",
"id": "prestations.minima_sociaux.rsa.majoration_rsa.taux_troisieme_personne",
"values": {
"2009-06-01": 0.3
}
},
"Par enfant supplémentaire": {
"description": "Taux par personne supplémentaire",
"id": "prestations.minima_sociaux.rsa.majoration_rsa.taux_personne_supp",
"values": {
"2009-06-01": 0.4
}
}
},
"Couple, 1 enfant - ou pour le deuxième enfant": {
"description": "Taux pour une 3e personne",
"id": "prestations.minima_sociaux.rsa.majoration_rsa.taux_troisieme_personne",
"values": {
"2009-06-01": 0.3
}
},
"Par enfant supplémentaire": {
"description": "Taux par personne supplémentaire",
"id": "prestations.minima_sociaux.rsa.majoration_rsa.taux_personne_supp",
"values": {
"2009-06-01": 0.4
}
}
},
"Majoration isolement (en % de la base RSA) ": {
"Femmes enceintes": {
"description": "Taux de la majoration du RSA socle pour une femme enceinte sans enfant (en pourcentage du rsa de base pour un célibataire sans personne à charge)",
"id": "prestations.minima_sociaux.rsa.majoration_parent_isole.femmes_enceintes",
"values": {
"1997-01-01": 0,
"2009-06-01": 1.284,
"2009-06-20": 1.28412
}
},
"Par enfant à charge": {
"description": "Majoration du RSA socle par enfant supplémentaire (en pourcentage du rsa de base)",
"id": "prestations.minima_sociaux.rsa.majoration_parent_isole.par_enfant_a_charge",
"values": {
"1997-01-01": 0,
"2009-06-01": 0.428,
"2009-06-20": 0.42804
"Majoration isolement (en % de la base RSA) ": {
"Femmes enceintes": {
"description": "Taux de la majoration du RSA socle pour une femme enceinte sans enfant (en pourcentage du rsa de base pour un célibataire sans personne à charge)",
"id": "prestations.minima_sociaux.rsa.majoration_parent_isole.femmes_enceintes",
"values": {
"1997-01-01": 0,
"2009-06-01": 1.284,
"2009-06-20": 1.28412
}
},
"Par enfant à charge": {
"description": "Majoration du RSA socle par enfant supplémentaire (en pourcentage du rsa de base)",
"id": "prestations.minima_sociaux.rsa.majoration_parent_isole.par_enfant_a_charge",
"values": {
"1997-01-01": 0,
"2009-06-01": 0.428,
"2009-06-20": 0.42804
}
}
}
}
......
......@@ -6,7 +6,7 @@ import isString from 'lodash.isstring'
import sortBy from 'lodash.sortby'
import flow from 'lodash.flow'
import extractData from '../services/dataPreprocesser'
import {extractData} from '../services/dataPreprocesser'
import {formatNumber, formatDate} from '../services/formatter'
import Table from '../components/Table'
......
......@@ -7,8 +7,10 @@
"@babel/plugin-proposal-optional-chaining": "^7.0.0-rc.2",
"@zeit/next-css": "^0.2.0",
"bluebird": "^3.5.1",
"d3-dsv": "^1.1.1",
"dotenv": "^6.1.0",
"express": "^4.16.3",
"fs-extra": "^7.0.1",
"intl": "^1.2.5",
"isomorphic-unfetch": "^2.0.0",
"js-yaml-loader": "^0.1.0",
......@@ -24,6 +26,7 @@
"lodash.keys": "^4.2.0",
"lodash.last": "^3.0.0",
"lodash.map": "^4.6.0",
"lodash.mapkeys": "^4.6.0",
"lodash.mapvalues": "^4.6.0",
"lodash.max": "^4.0.1",
"lodash.merge": "^4.6.1",
......@@ -50,7 +53,7 @@
"scripts": {
"dev": "node server/index.js",
"build": "next build",
"export": "next export",
"export": "next export && rsync --recursive --remove-source-files csv-out/ out/",
"start": "next start",
"test": "NODE_ENV=test jest"
}
......
const getDirName = require('path').dirname;
const fs = require('fs-extra');
const d3 = require('d3-dsv')
const mapValues = require('lodash.mapvalues')
const mapKeys = require('lodash.mapkeys')
const isPlainObject = require('lodash.isplainobject')
const last = require('lodash.last')
const extractData = require('../services/dataPreprocesser').extractData
function writeFile(path, contents) {
fs.ensureDirSync(getDirName(path))
fs.writeFileSync(path, contents)
}
function cleanValues(datum) {
return mapValues(datum, value => {
if (isPlainObject(value)) {
return value.value
}
return value
})
}
function cleanKeys(datum, path) {
return mapKeys(datum, (value, key) => {
return key.replace(new RegExp('^'+ path + '\\.'), '');
})
}
function cleanDatum(datum, path) {
return cleanKeys(cleanValues(datum), path)
}
function toCSV(parameter) {
const data = extractData(parameter).map(datum => cleanDatum(datum, parameter.id))
return d3.csvFormat(data)
}
function generateTables(parameter, path) {
const csv = toCSV(parameter, path)
const tableName = last(parameter.id.split('.'))
const filePath = `csv-out/${path}/${tableName}.csv`
writeFile(filePath, csv)
}
module.exports = {generateTables, toCSV, cleanDatum}
......@@ -6,7 +6,7 @@ const map = require('lodash.map')
const flatten = require('lodash.flatten')
const fromPairs = require('lodash.frompairs')
const {generateTables} = require('./csv')
const resolver = require('./resolver')
async function loadSectionFile(file) {
......@@ -36,6 +36,7 @@ function addLeadingSlash(string) {
function extractRoutes(desc, path, parametersTree, parents = []) {
const isPage = ! parents.length
if (desc.table) {
generateTables(desc.table, path)
return [{
route: addLeadingSlash(path),
page: '/table',
......
/** Process a parameterNode so that the data it contains are easily exploitable by React-Table */
import includes from 'lodash.includes'
import flatten from 'lodash.flatten';
import flow from 'lodash.flow';
import fromPairs from 'lodash.frompairs'
import isPlainObject from 'lodash.isplainobject'
import keys from 'lodash.keys'
import last from 'lodash.last'
import map from 'lodash.map';
import mapValues from 'lodash.mapvalues';
import merge from 'lodash.merge';
import pick from 'lodash.pick'
import pickBy from 'lodash.pickby'
import union from 'lodash.union';
const includes = require('lodash.includes')
const flatten = require('lodash.flatten')
const flow = require('lodash.flow')
const fromPairs = require('lodash.frompairs')
const isPlainObject = require('lodash.isplainobject')
const keys = require('lodash.keys')
const last = require('lodash.last')
const map = require('lodash.map')
const mapValues = require('lodash.mapvalues')
const merge = require('lodash.merge')
const pick = require('lodash.pick')
const pickBy = require('lodash.pickby')
const union = require('lodash.union')
const METADATA_FIELD_NAMES = ['date_parution_jo', 'reference', 'notes']
export default function extractData(parameterNode) {
function extractData(parameterNode) {
const values = extractValues(parameterNode)
const dates = union(...map(values, keys)).sort()
return dates.reduce((data, date) => {
......@@ -34,7 +34,7 @@ export default function extractData(parameterNode) {
}, []).reverse()
}
export function extractValuesFromScale(scale) {
function extractValuesFromScale(scale) {
const bracketsValues = flow([
x => map(x, (scaleAtInstant, date) => {
const thresolds = keys(scaleAtInstant).sort((x, y) => Number(x) - Number(y))
......@@ -76,7 +76,7 @@ function getUnitAtDate(units, date) {
]
}
export function extractValues(parameterNode) {
function extractValues(parameterNode) {
if (parameterNode.values) {
const data = {}
const unit = parameterNode.metadata && parameterNode.metadata.unit
......@@ -99,3 +99,9 @@ export function extractValues(parameterNode) {
return merge({}, data, metadata)
}
module.exports = {
extractData,
extractValuesFromScale,
extractValues
}
......@@ -1548,6 +1548,10 @@ combined-stream@1.0.6, combined-stream@~1.0.5:
dependencies:
delayed-stream "~1.0.0"
commander@2:
version "2.19.0"
resolved "https://registry.yarnpkg.com/commander/-/commander-2.19.0.tgz#f6198aa84e5b83c46054b94ddedbfed5ee9ff12a"
commander@~2.13.0:
version "2.13.0"
resolved "https://registry.yarnpkg.com/commander/-/commander-2.13.0.tgz#6964bca67685df7c1f1430c584f07d7597885b9c"
......@@ -1808,6 +1812,14 @@ cyclist@~0.2.2:
version "0.2.2"
resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-0.2.2.tgz#1b33792e11e914a2fd6d6ed6447464444e5fa640"
d3-dsv@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/d3-dsv/-/d3-dsv-1.1.1.tgz#aaa830ecb76c4b5015572c647cc6441e3c7bb701"
dependencies:
commander "2"
iconv-lite "0.4"
rw "1"
d@1:
version "1.0.0"
resolved "https://registry.yarnpkg.com/d/-/d-1.0.0.tgz#754bb5bfe55451da69a58b94d45f4c5b0462d58f"
......@@ -2625,6 +2637,14 @@ from2@^2.1.0:
inherits "^2.0.1"
readable-stream "^2.0.0"
fs-extra@^7.0.1:
version "7.0.1"
resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-7.0.1.tgz#4f189c44aa123b895f722804f55ea23eadc348e9"
dependencies:
graceful-fs "^4.1.2"
jsonfile "^4.0.0"
universalify "^0.1.0"
fs-minipass@^1.2.5:
version "1.2.5"
resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-1.2.5.tgz#06c277218454ec288df77ada54a03b8702aacb9d"
......@@ -2758,6 +2778,10 @@ graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.4:
version "4.1.11"
resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658"
graceful-fs@^4.1.6:
version "4.1.15"
resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.15.tgz#ffb703e1066e8a0eeaa4c8b80ba9253eeefbfb00"
growly@^1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/growly/-/growly-1.3.0.tgz#f10748cbe76af964b7c96c93c6bcc28af120c081"
......@@ -2942,6 +2966,12 @@ https-browserify@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73"
iconv-lite@0.4:
version "0.4.24"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b"
dependencies:
safer-buffer ">= 2.1.2 < 3"
iconv-lite@0.4.19:
version "0.4.19"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b"
......@@ -3022,10 +3052,6 @@ interpret@^1.0.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.1.0.tgz#7ed1b1410c6a0e0f78cf95d3b8440c63f78b8614"
intl-locales-supported@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/intl-locales-supported/-/intl-locales-supported-1.0.0.tgz#9a9d94dbf104a87818881952dcb782053f0aeefa"
intl@^1.2.5:
version "1.2.5"
resolved "https://registry.yarnpkg.com/intl/-/intl-1.2.5.tgz#82244a2190c4e419f8371f5aa34daa3420e2abde"
......@@ -3755,6 +3781,12 @@ json5@^0.5.0, json5@^0.5.1:
version "0.5.1"
resolved "https://registry.yarnpkg.com/json5/-/json5-0.5.1.tgz#1eade7acc012034ad84e2396767ead9fa5495821"
jsonfile@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb"
optionalDependencies:
graceful-fs "^4.1.6"
jsonify@~0.0.0:
version "0.0.0"
resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73"
......@@ -3915,6 +3947,10 @@ lodash.map@^4.6.0:
version "4.6.0"
resolved "https://registry.yarnpkg.com/lodash.map/-/lodash.map-4.6.0.tgz#771ec7839e3473d9c4cde28b19394c3562f4f6d3"
lodash.mapkeys@^4.6.0:
version "4.6.0"
resolved "https://registry.yarnpkg.com/lodash.mapkeys/-/lodash.mapkeys-4.6.0.tgz#df2cfa231d7c57c7a8ad003abdad5d73d3ea5195"
lodash.mapvalues@^4.6.0:
version "4.6.0"
resolved "https://registry.yarnpkg.com/lodash.mapvalues/-/lodash.mapvalues-4.6.0.tgz#1bafa5005de9dd6f4f26668c30ca37230cc9689c"
......@@ -5662,6 +5698,10 @@ run-queue@^1.0.0, run-queue@^1.0.3:
dependencies:
aproba "^1.1.1"
rw@1:
version "1.3.3"
resolved "https://registry.yarnpkg.com/rw/-/rw-1.3.3.tgz#3f862dfa91ab766b14885ef4d01124bfda074fb4"
safe-buffer@5.1.1:
version "5.1.1"
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853"
......@@ -6402,6 +6442,10 @@ unique-slug@^2.0.0:
dependencies:
imurmurhash "^0.1.4"
universalify@^0.1.0:
version "0.1.2"
resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66"
unpipe@1.0.0, unpipe@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment