Commit 775a72de authored by Emmanuel Raviart's avatar Emmanuel Raviart
Browse files

Add full text search & autocompletion for "dossiers".

parent 9a73cc82
......@@ -28,6 +28,8 @@ createuser -D -P -R -S assemblee
Enter password for new role: assemblee
Enter it again: assemblee
createdb -E utf-8 -O assemblee assemblee
psql assemblee
CREATE EXTENSION IF NOT EXISTS pg_trgm;
```
### Configuration
......
......@@ -56,7 +56,7 @@ async function configureDatabase() {
`,
)
await db.none(
"CREATE INDEX IF NOT EXISTS acteurs_depute_actif_idx ON acteurs(depute_actif)",
"CREATE INDEX IF NOT EXISTS acteurs_depute_actif_idx ON acteurs (depute_actif)",
)
for (const command of [
"COMMENT ON TABLE acteurs IS 'persons (députés, sénateurs, …)'",
......@@ -122,10 +122,10 @@ async function configureDatabase() {
`,
)
await db.none(
"CREATE INDEX IF NOT EXISTS dossiers_date_dernier_acte_idx ON dossiers(date_dernier_acte)",
"CREATE INDEX IF NOT EXISTS dossiers_date_dernier_acte_idx ON dossiers (date_dernier_acte)",
)
await db.none(
"CREATE INDEX IF NOT EXISTS dossiers_date_prochain_acte_idx ON dossiers(date_prochain_acte)",
"CREATE INDEX IF NOT EXISTS dossiers_date_prochain_acte_idx ON dossiers (date_prochain_acte)",
)
for (const command of [
"COMMENT ON TABLE dossiers IS 'set of documents and operations produced during the" +
......@@ -137,6 +137,56 @@ async function configureDatabase() {
]) {
await db.none(command)
}
await db.none(
`
CREATE TABLE IF NOT EXISTS dossiers_autocomplete (
uid text NOT NULL REFERENCES dossiers(uid) ON DELETE CASCADE,
autocomplete text NOT NULL,
PRIMARY KEY (uid, autocomplete)
)
`,
)
await db.none(
`
CREATE INDEX IF NOT EXISTS dossiers_autocomplete_trigrams_idx
ON dossiers_autocomplete
USING GIST (autocomplete gist_trgm_ops)
`,
)
for (const command of [
"COMMENT ON TABLE dossiers_autocomplete IS 'autocompletions texts for dossiers'",
"COMMENT ON COLUMN dossiers_autocomplete.uid IS 'unique ID of dossier" +
" (same as dossiers)'",
"COMMENT ON COLUMN dossiers_autocomplete.autocomplete IS 'autocompletion text of" +
" dossier'",
]) {
await db.none(command)
}
await db.none(
`
CREATE TABLE IF NOT EXISTS dossiers_text_search (
uid text NOT NULL REFERENCES dossiers(uid) ON DELETE CASCADE,
text_search tsvector NOT NULL,
PRIMARY KEY (uid, text_search)
)
`,
)
await db.none(
`
CREATE INDEX IF NOT EXISTS dossiers_text_search_idx
ON dossiers_text_search
USING GIN (text_search)
`,
)
for (const command of [
"COMMENT ON TABLE dossiers_text_search IS 'full text search for dossiers'",
"COMMENT ON COLUMN dossiers_text_search.uid IS 'unique ID of dossier" +
" (same as dossiers)'",
"COMMENT ON COLUMN dossiers_text_search.text_search IS 'full text search vector" +
" of dossier'",
]) {
await db.none(command)
}
// Table: organes
await db.none(
......@@ -169,16 +219,16 @@ async function configureDatabase() {
`,
)
await db.none(
"CREATE INDEX IF NOT EXISTS reunions_jour_debut_idx ON reunions(jour_debut)",
"CREATE INDEX IF NOT EXISTS reunions_jour_debut_idx ON reunions (jour_debut)",
)
await db.none(
"CREATE INDEX IF NOT EXISTS reunions_jour_fin_idx ON reunions(jour_fin)",
"CREATE INDEX IF NOT EXISTS reunions_jour_fin_idx ON reunions (jour_fin)",
)
await db.none(
"CREATE INDEX IF NOT EXISTS reunions_lieu_code_idx ON reunions(lieu_code)",
"CREATE INDEX IF NOT EXISTS reunions_lieu_code_idx ON reunions (lieu_code)",
)
await db.none(
"CREATE INDEX IF NOT EXISTS reunions_xsi_type_idx ON reunions(xsi_type)",
"CREATE INDEX IF NOT EXISTS reunions_xsi_type_idx ON reunions (xsi_type)",
)
for (const command of [
"COMMENT ON TABLE reunions IS 'public meetings'",
......
......@@ -2,13 +2,17 @@ import { NextFunction, Request, Response } from "express"
import { db, extractDataFromEntry } from "../database"
import { wrapAsyncMiddleware } from "../middlewares"
import { allFollows, DataGetter, DossierParlementaire, getDossier } from "../model"
import {
allFollows,
DataGetter,
DossierParlementaire,
getDossier,
} from "../model"
import { validateDateIso8601String } from "../validators/core"
validateDateIso8601String,
validateInteger,
validateMaybeTrimmedString,
validateMissing,
validateOption,
validateSetValue,
validateStringToNumber,
validateTest,
} from "../validators/core"
import { validateFollow } from "../validators/query"
interface DossierRequest extends Request {
......@@ -91,21 +95,37 @@ export const listDossiers = wrapAsyncMiddleware(async function listDossiers(
),
)
}
const { date, follow } = query
const { date, follow, limit, offset, q: term } = query
const joinClauses: string[] = []
const whereClauses: string[] = []
if (date !== null) {
whereClauses.push("(date_dernier_acte >= $<date> OR date_prochain_acte NOTNULL)")
}
if (term !== null) {
joinClauses.push("INNER JOIN dossiers_text_search ON dossiers.uid = dossiers_text_search.uid")
whereClauses.push("text_search @@ plainto_tsquery('french', $<term>)")
}
const joinsClause = joinClauses.length > 0 ? joinClauses.join(" ") : ""
const whereClause = whereClauses.length > 0 ? "WHERE " + whereClauses.join(" AND ") : ""
const dossiers = (await db.any(
`
SELECT *
FROM dossiers
WHERE
date_dernier_acte >= $<date>
OR date_prochain_acte NOTNULL
${joinsClause}
${whereClause}
ORDER BY
date_prochain_acte DESC NULLS LAST,
date_dernier_acte DESC NULLS LAST
OFFSET $<offset>
LIMIT $<limit>
`,
{
date,
limit,
offset,
term,
},
)).map(extractDataFromEntry)
......@@ -207,7 +227,9 @@ function validateListDossiersQuery(data: any) {
{
const key = "date"
remainingKeys.delete(key)
const [value, error] = validateDateIso8601String(data[key])
const [value, error] = validateOption(validateMissing, validateDateIso8601String)(
data[key],
)
data[key] = value
if (error !== null) {
errors[key] = error
......@@ -224,6 +246,50 @@ function validateListDossiersQuery(data: any) {
}
}
{
const key = "limit"
remainingKeys.delete(key)
const [value, error] = validateOption(
[validateMissing, validateSetValue(10)],
[
validateStringToNumber,
validateInteger,
validateTest(value => value >= 0, "Value must be greater than or equal to 0"),
],
)(data[key])
data[key] = value
if (error !== null) {
errors[key] = error
}
}
{
const key = "offset"
remainingKeys.delete(key)
const [value, error] = validateOption(
[validateMissing, validateSetValue(0)],
[
validateStringToNumber,
validateInteger,
validateTest(value => value >= 0, "Value must be greater than or equal to 0"),
],
)(data[key])
data[key] = value
if (error !== null) {
errors[key] = error
}
}
{
const key = "q"
remainingKeys.delete(key)
const [value, error] = validateMaybeTrimmedString(data[key])
data[key] = value
if (error !== null) {
errors[key] = error
}
}
for (let key of remainingKeys) {
errors[key] = "Unexpected entry"
}
......
......@@ -649,6 +649,22 @@ async function main() {
`,
)).map(({ uid }) => uid),
)
const existingDossiersAutocompletions = new Set(
(await db.any(
`
SELECT uid, autocomplete
FROM dossiers_autocomplete
`,
)).map(({ uid, autocomplete }) => `${uid}_${autocomplete}`),
)
const existingDossiersTextsSearches = new Set(
(await db.any(
`
SELECT uid, text_search
FROM dossiers_text_search
`,
)).map(({ uid, text_search }) => `${uid}_${text_search}`),
)
for (const dossier of Object.values(dossierParlementaireByUid)) {
const [dateDernierActe, dateProchainActe] = datesProchesFromActes(
......@@ -683,9 +699,81 @@ async function main() {
},
)
existingDossiersUids.delete(dossier.uid)
const autocomplete = dossier.titreDossier.titre
await db.none(
`
INSERT INTO dossiers_autocomplete (
uid,
autocomplete
)
VALUES (
$<uid>,
$<autocomplete>
)
ON CONFLICT (uid, autocomplete)
DO NOTHING
`,
{
autocomplete,
uid: dossier.uid,
},
)
existingDossiersAutocompletions.delete(`${dossier.uid}_${autocomplete}`)
const textSearch = dossier.titreDossier.titre
await db.none(
`
INSERT INTO dossiers_text_search (
uid,
text_search
)
VALUES (
$<uid>,
setweight(to_tsvector('french', $<textSearch>), 'A')
)
ON CONFLICT (uid, text_search)
DO NOTHING
`,
{
textSearch,
uid: dossier.uid,
},
)
existingDossiersTextsSearches.delete(`${dossier.uid}_${textSearch}`)
}
// Delete obsolete dossiers.
for (const key of existingDossiersAutocompletions) {
const [uid, autocomplete] = key.split("_")
await db.none(
`
DELETE FROM dossiers_autocomplete
WHERE
uid = $<uid>
AND autocomplete = $<autocomplete>
`,
{
autocomplete,
uid,
},
)
}
for (const key of existingDossiersTextsSearches) {
const [uid, textSearch] = key.split("_")
await db.none(
`
DELETE FROM dossiers_text_search
WHERE
uid = $<uid>
AND text_search = $<textSearch>
`,
{
textSearch,
uid,
},
)
}
for (const obsoleteDossierUid of existingDossiersUids) {
await db.none(
`
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment