Commit 2b363257 authored by Phyks (Lucas Verney)'s avatar Phyks (Lucas Verney)
Browse files

Do not run migrations at each init

Also do not load all the data at every migration, but only once for all
migrations.

This should close #573.
parent b6f90606
Pipeline #20846 passed with stage
in 2 minutes and 50 seconds
......@@ -69,8 +69,6 @@
- 2
use-isnan:
- 2
valid-jsdoc:
- 2
valid-typeof:
- 2
......
......@@ -322,10 +322,23 @@ export async function import_(req, res) {
log.info('Done.');
log.info('Import settings...');
let shouldResetMigration = false;
for (let setting of world.settings) {
if (setting.name === 'weboob-log' || setting.name === 'weboob-installed')
continue;
if (setting.name === 'migration-version') {
// Overwrite previous value of migration-version setting.
let found = await Config.byName('migration-version');
if (found) {
found.value = setting.value;
shouldResetMigration = true;
log.debug(`Updating migration-version index to ${setting.value}.`);
await found.save();
continue;
}
}
if (setting.name === 'defaultAccountId') {
if (typeof accountMap[setting.value] === 'undefined') {
log.warn(`unknown default account id: ${setting.value}, skipping.`);
......@@ -345,6 +358,19 @@ export async function import_(req, res) {
// Note that former existing values are not overwritten!
await Config.findOrCreateByName(setting.name, setting.value);
}
if (!shouldResetMigration) {
// If no migration-version has been set, just reset
// migration-version value to 0, to force all the migrations to be
// run again.
log.info(
'The imported file did not provide a migration-version value.' +
'Resetting it to 0 to run all migrations again.'
);
let migrationVersion = await Config.byName('migration-version');
migrationVersion.value = '0';
await migrationVersion.save();
}
log.info('Done.');
log.info('Import alerts...');
......
......@@ -48,6 +48,15 @@ function reduceOperationsDate(oldest, operation) {
return Math.min(oldest, +new Date(operation.dateImport));
}
/**
* This is an array of all the migrations to apply on the database, in order to
* automatically keep database schema in sync with Kresus code.
*
* _Note_: As only the necessary migrations are run at each startup, you should
* NEVER update a given migration, but instead add a new migration to reflect
* the changes you want to apply on the db. Updating an existing migration
* might not update the database as expected.
*/
let migrations = [
async function m1() {
......@@ -65,18 +74,19 @@ let migrations = [
}
},
async function m2() {
async function m2(cache) {
log.info('Checking that operations with categories are consistent...');
let ops = await Operation.all();
let categories = await Category.all();
cache.operations = cache.operations || await Operation.all();
cache.categories = cache.categories || await Category.all();
let categorySet = new Set;
for (let c of categories) {
for (let c of cache.categories) {
categorySet.add(c.id);
}
let catNum = 0;
for (let op of ops) {
for (let op of cache.operations) {
let needsSave = false;
if (typeof op.categoryId !== 'undefined' && !categorySet.has(op.categoryId)) {
......@@ -94,12 +104,13 @@ let migrations = [
log.info(`\t${catNum} operations had an inconsistent category.`);
},
async function m3() {
async function m3(cache) {
log.info('Replacing NONE_CATEGORY_ID by undefined...');
let ops = await Operation.all();
cache.operations = cache.operations || await Operation.all();
let num = 0;
for (let o of ops) {
for (let o of cache.operations) {
if (typeof o.categoryId !== 'undefined' && o.categoryId.toString() === '-1') {
delete o.categoryId;
await o.save();
......@@ -111,10 +122,11 @@ let migrations = [
log.info(`\t${num} operations had -1 as categoryId.`);
},
async function m4() {
async function m4(cache) {
log.info('Migrating websites to the customFields format...');
let accesses = await Access.all();
cache.accesses = cache.accesses || await Access.all();
let num = 0;
let updateFields = website => customFields => {
......@@ -129,7 +141,7 @@ let migrations = [
return customFields;
};
for (let a of accesses) {
for (let a of cache.accesses) {
if (typeof a.website === 'undefined' || !a.website.length)
continue;
......@@ -146,9 +158,10 @@ let migrations = [
log.info(`\t${num} accesses updated to the customFields format.`);
},
async function m5() {
async function m5(cache) {
log.info('Migrating HelloBank users to BNP and BNP users to the new website format.');
let accesses = await Access.all();
cache.accesses = cache.accesses || await Access.all();
let updateFieldsBnp = customFields => {
if (customFields.filter(field => field.name === 'website').length)
......@@ -171,7 +184,7 @@ let migrations = [
return customFields;
};
for (let a of accesses) {
for (let a of cache.accesses) {
if (a.bank === 'bnporc') {
await updateCustomFields(a, updateFieldsBnp);
......@@ -196,10 +209,12 @@ let migrations = [
},
async function m6() {
async function m6(cache) {
log.info('Ensure "importDate" field is present in accounts.');
let accounts = await Account.all();
for (let a of accounts) {
cache.accounts = cache.accounts || await Account.all();
for (let a of cache.accounts) {
if (typeof a.importDate !== 'undefined')
continue;
......@@ -219,16 +234,16 @@ let migrations = [
}
},
async function m7() {
async function m7(cache) {
log.info('Migrate operationTypeId to type field...');
let types = [];
try {
types = await Type.all();
if (types.length) {
cache.types = cache.types || await Type.all();
if (cache.types.length) {
let operations = await Operation.allWithOperationTypesId();
log.info(`${operations.length} operations to migrate`);
let typeMap = new Map();
for (let { id, name } of types) {
for (let { id, name } of cache.types) {
typeMap.set(id, name);
}
......@@ -243,34 +258,40 @@ let migrations = [
}
// Delete operation types
for (let type of types) {
for (let type of cache.types) {
await type.destroy();
}
// Purge operation types from cache
delete cache.types;
}
} catch (e) {
log.error(`Error while updating operation type: ${e}`);
}
},
async function m8() {
async function m8(cache) {
log.info('Ensuring consistency of accounts with alerts...');
try {
let accountSet = new Set;
let accounts = await Account.all();
for (let account of accounts) {
cache.accounts = cache.accounts || await Account.all();
cache.alerts = cache.alerts || await Alert.all();
for (let account of cache.accounts) {
accountSet.add(account.accountNumber);
}
let alerts = await Alert.all();
let numOrphans = 0;
for (let al of alerts) {
for (let al of cache.alerts) {
if (!accountSet.has(al.bankAccount)) {
numOrphans++;
await al.destroy();
}
}
// Purge the alerts cache, next migration requiring it will rebuild
// an updated cache.
delete cache.alerts;
if (numOrphans)
log.info(`\tfound and removed ${numOrphans} orphan alerts`);
......@@ -279,13 +300,14 @@ let migrations = [
}
},
async function m9() {
async function m9(cache) {
log.info('Deleting banks from database');
try {
let banks = await Bank.all();
for (let bank of banks) {
cache.banks = cache.banks || await Bank.all();
for (let bank of cache.banks) {
await bank.destroy();
}
delete cache.banks;
} catch (e) {
log.error(`Error while deleting banks: ${e.toString()}`);
}
......@@ -346,11 +368,12 @@ let migrations = [
}
},
async function m12() {
async function m12(cache) {
log.info('Searching accounts with IBAN value set to None');
try {
let accounts = await Account.all();
for (let account of accounts.filter(acc => acc.iban === 'None')) {
cache.accounts = cache.accounts || await Account.all();
for (let account of cache.accounts.filter(acc => acc.iban === 'None')) {
log.info(`\tDeleting iban for ${account.title} of bank ${account.bank}`);
delete account.iban;
await account.save();
......@@ -361,8 +384,22 @@ let migrations = [
}
];
/**
* Run all the required migrations.
*
* To determine whether a migration has to be run or not, we are comparing its
* index in the migrations Array above with the `migration-version` config value.
*/
export async function run() {
for (let m of migrations) {
await m();
const migrationVersion = await Config.findOrCreateDefault('migration-version');
let cache = {}; // Cache to prevent loading multiple times the same data from the db
const firstMigrationIndex = parseInt(migrationVersion.value, 10) + 1;
for (let m = firstMigrationIndex; m < migrations.length; m++) {
await migrations[m](cache);
migrationVersion.value = m.toString();
await migrationVersion.save();
}
}
......@@ -2,6 +2,7 @@ const DefaultSettings = new Map;
DefaultSettings.set('weboob-installed', 'false');
DefaultSettings.set('weboob-version', '?');
DefaultSettings.set('migration-version', '0');
DefaultSettings.set('standalone-mode', 'false');
DefaultSettings.set('url-prefix', '/');
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment