webui.py 14.9 KB
Newer Older
koniu's avatar
import  
koniu committed
1
2
3
4
#{{{ imports
import os
import bottle
import time
5
import sys
koniu's avatar
import  
koniu committed
6
7
8
9
import datetime
import glob
import hashlib
import csv
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
10
import io
koniu's avatar
import  
koniu committed
11
12
import string
import shlex
13
14
from urllib.parse import urlencode, quote as urlquote
from recoll import recoll, rclextract, rclconfig
15

Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
16
17
def msg(s):
    print("%s" % s, file=sys.stderr)
18
19
20
21
22
23
24
25

# use ujson if avalible (faster than built in json)
try:
    import ujson as json
except ImportError:
    import json
    msg("ujson module not found, using (slower) built-in json module instead")

26
27
28

g_fscharset=sys.getfilesystemencoding()

koniu's avatar
import  
koniu committed
29
30
31
32
33
34
35
#}}}
#{{{ settings
# settings defaults
DEFAULTS = {
    'context': 30,
    'stem': 1,
    'timefmt': '%c',
36
    'dirdepth': 2,
koniu's avatar
koniu committed
37
    'maxchars': 500,
38
    'maxresults': 0,
koniu's avatar
koniu committed
39
    'perpage': 25,
40
    'csvfields': 'filename title author size time mtype url',
41
    'title_link': 'download',
koniu's avatar
import  
koniu committed
42
43
44
45
}

# sort fields/labels
SORTS = [
Laurent Defert's avatar
Laurent Defert committed
46
    ("relevancyrating", "Relevancy"),
koniu's avatar
import  
koniu committed
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
    ("mtime", "Date",),
    ("url", "Path"),
    ("filename", "Filename"),
    ("fbytes", "Size"),
    ("author", "Author"),
]

# doc fields
FIELDS = [
    # exposed by python api
    'ipath',
    'filename',
    'title',
    'author',
    'fbytes',
    'dbytes',
    'size',
    'fmtime',
    'dmtime',
    'mtime',
    'mtype',
    'origcharset',
    'sig',
    'relevancyrating',
    'url',
    'abstract',
    'keywords',
    # calculated
    'time',
    'snippet',
    'label',
]
#}}}
#{{{  functions
#{{{  helpers
def select(ls, invalid=[None]):
    for value in ls:
        if value not in invalid:
            return value

def timestr(secs, fmt):
88
89
    if secs == '' or secs is None:
        secs = '0'
koniu's avatar
import  
koniu committed
90
91
92
93
94
95
96
97
98
99
100
101
    t = time.gmtime(int(secs))
    return time.strftime(fmt, t)

def normalise_filename(fn):
    valid_chars = "_-%s%s" % (string.ascii_letters, string.digits)
    out = ""
    for i in range(0,len(fn)):
        if fn[i] in valid_chars:
            out += fn[i]
        else:
            out += "_"
    return out
102
103
104
105
106

def get_topdirs(confdir):
    rclconf = rclconfig.RclConfig(confdir)
    return rclconf.getConfParam('topdirs')

107
108
109
110
111
112
113
# Environment fetch for the cases where we don't care if unset or null
def safe_envget(varnm):
    try:
        return os.environ[varnm]
    except Exception as ex:
        return None

114
115
# get database directory from recoll.conf, defaults to confdir/xapiandb. Note
# that this is available as getDbDir() in newer rclconfig versions
116
def get_dbdir(confdir):
117
    confdir = os.path.expanduser(confdir)
118
119
    rclconf = rclconfig.RclConfig(confdir)
    dbdir = rclconf.getConfParam('dbdir')
120
121
122
123
124
125
126
    if not dbdir:
        dbdir = 'xapiandb'
    if not os.path.isabs(dbdir):
        cachedir = rclconf.getConfParam('cachedir')
        if not cachedir:
            cachedir = confdir
        dbdir = os.path.join(cachedir, dbdir)
127
    # recoll API expects bytes, not strings
128
    return os.path.normpath(dbdir).encode(g_fscharset)
koniu's avatar
import  
koniu committed
129
#}}}
130
131
132
#{{{ get_config
def get_config():
    config = {}
133
    envdir = safe_envget('RECOLL_CONFDIR')
koniu's avatar
import  
koniu committed
134
    # get useful things from recoll.conf
135
    rclconf = rclconfig.RclConfig(envdir)
koniu's avatar
koniu committed
136
    config['confdir'] = rclconf.getConfDir()
137
    config['dirs'] = dict.fromkeys([os.path.expanduser(d) for d in
138
139
                      shlex.split(rclconf.getConfParam('topdirs'))],
                                   config['confdir'])
140
    # add topdirs from extra config dirs
141
    extraconfdirs = safe_envget('RECOLL_EXTRACONFDIRS')
142
143
144
145
146
    if extraconfdirs:
        config['extraconfdirs'] = shlex.split(extraconfdirs)
        for e in config['extraconfdirs']:
            config['dirs'].update(dict.fromkeys([os.path.expanduser(d) for d in
                shlex.split(get_topdirs(e))],e))
147
        config['extradbs'] = list(map(get_dbdir, config['extraconfdirs']))
148
149
    else:
        config['extraconfdirs'] = None
150
        config['extradbs'] = None
151
    config['stemlang'] = rclconf.getConfParam('indexstemminglanguages')
koniu's avatar
import  
koniu committed
152
153
    # get config from cookies or defaults
    for k, v in DEFAULTS.items():
154
155
        value = select([bottle.request.get_cookie(k), v])
        config[k] = type(v)(value)
156
157
    # Fix csvfields: get rid of invalid ones to avoid needing tests in
    # the dump function
158
159
160
161
    cf = config['csvfields'].split()
    ncf = [f for f in cf if f in FIELDS]
    config['csvfields'] = ' '.join(ncf)
    config['fields'] = ' '.join(FIELDS)
162
163
164
    # get mountpoints
    config['mounts'] = {}
    for d in config['dirs']:
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
165
        name = 'mount_%s' % urlquote(d,'')
166
167
        config['mounts'][d] = select([bottle.request.get_cookie(name),
                                      'file://%s' % d], [None, ''])
168
169
170

    # Parameters set by the admin in the recoll configuration
    # file. These override anything else, so read them last
171
172
173
174
    val = rclconf.getConfParam('webui_nojsoncsv')
    val = 0 if val is None else int(val)
    config['rclc_nojsoncsv'] = val

175
176
177
178
179
    val = rclconf.getConfParam('webui_maxperpage')
    val = 0 if val is None else int(val)
    if val:
        if config['perpage'] == 0 or config['perpage'] > val:
            config['perpage'] = val
180
181
182
183
184

    val = rclconf.getConfParam('webui_nosettings')
    val = 0 if val is None else int(val)
    config['rclc_nosettings'] = val

koniu's avatar
import  
koniu committed
185
186
187
188
189
190
    return config
#}}}
#{{{ get_dirs
def get_dirs(tops, depth):
    v = []
    for top in tops:
191
192
193
194
        # We do the conversion to bytes here, because Python versions
        # before 3.7 won't do the right thing if the locale is C,
        # which would be the case with a default apache install
        top = top.encode('utf-8', 'surrogateescape')
koniu's avatar
import  
koniu committed
195
        dirs = [top]
koniu's avatar
koniu committed
196
        for d in range(1, depth+1):
197
            dirs = dirs + glob.glob(top + b'/*' * d)
koniu's avatar
import  
koniu committed
198
        dirs = filter(lambda f: os.path.isdir(f), dirs)
199
200
        top_path = top.rsplit(b'/', 1)[0]
        dirs = [w.replace(top_path+b'/', b'', 1) for w in dirs]
koniu's avatar
import  
koniu committed
201
        v = v + dirs
202
203
    for i in range(len(v)):
        v[i] = v[i].decode('utf-8', 'surrogateescape')
koniu's avatar
import  
koniu committed
204
205
206
207
208
    return ['<all>'] + v
#}}}
#{{{ get_query
def get_query():
    query = {
209
210
211
212
213
214
215
216
217
        'query': select([bottle.request.query.query, '']),
        'before': select([bottle.request.query.before, '']),
        'after': select([bottle.request.query.after, '']),
        'dir': select([bottle.request.query.dir, '', '<all>'], [None, '']),
        'sort': select([bottle.request.query.sort, SORTS[0][0]]),
        'ascending': int(select([bottle.request.query.ascending, 0], [None, ''])),
        'page': int(select([bottle.request.query.page, 0], [None, ''])),
        'highlight': int(select([bottle.request.query.highlight, 1], [None, ''])),
        'snippets': int(select([bottle.request.query.snippets, 1], [None, ''])),
koniu's avatar
import  
koniu committed
218
    }
219
    #msg("query['query'] : %s" % query['query'])
koniu's avatar
import  
koniu committed
220
221
222
223
    return query
#}}}
#{{{ query_to_recoll_string
def query_to_recoll_string(q):
224
    qs = q['query']
koniu's avatar
import  
koniu committed
225
226
    if len(q['after']) > 0 or len(q['before']) > 0:
        qs += " date:%s/%s" % (q['after'], q['before'])
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
227
228
229
    qdir = q['dir']
    if qdir != '<all>':
        qs += " dir:\"%s\" " % qdir
koniu's avatar
import  
koniu committed
230
231
    return qs
#}}}
232
233
#{{{ recoll_initsearch
def recoll_initsearch(q):
koniu's avatar
import  
koniu committed
234
    config = get_config()
235
236
237
238
239
240
241
242
243
244
245
    confdir = config['confdir']
    dbs = []
    """ The reason for this somewhat elaborate scheme is to keep the
    set size as small as possible by searching only those databases
    with matching topdirs """
    if q['dir'] == '<all>':
        if config['extraconfdirs']:
            dbs.extend(map(get_dbdir,config['extraconfdirs']))
    else:
        confdirs = []
        for d,conf in config['dirs'].items():
246
247
            tdbasename = os.path.basename(d)
            if os.path.commonprefix([tdbasename, q['dir']]) == tdbasename:
248
249
250
                confdirs.append(conf)
        if len(confdirs) == 0:
            # should not happen, using non-existing q['dir']?
251
            bottle.abort(400, 'no matching database for search directory ' + q['dir'])
252
253
254
255
256
257
        elif len(confdirs) == 1:
            # only one config (most common situation)
            confdir = confdirs[0]
        else:
            # more than one config with matching topdir, use 'm all
            confdir = confdirs[0]
258
            dbs.extend(map(get_dbdir, confdirs[1:]))
259

260
    if config['extradbs']:
261
262
263
264
        dbs.extend(config['extradbs'])

    if dbs:
        db = recoll.connect(confdir,dbs)
265
    else:
266
267
        db = recoll.connect(confdir)

koniu's avatar
koniu committed
268
    db.setAbstractParams(config['maxchars'], config['context'])
koniu's avatar
import  
koniu committed
269
    query = db.query()
koniu's avatar
koniu committed
270
    query.sortby(q['sort'], q['ascending'])
koniu's avatar
import  
koniu committed
271
    try:
koniu's avatar
koniu committed
272
        qs = query_to_recoll_string(q)
273
        query.execute(qs, config['stem'], config['stemlang'])
274
275
    except Exception as ex:
        msg("Query execute failed: %s" % ex)
276
277
278
        pass
    return query
#}}}
279
280
281
282
283
284
285
#{{{ HlMeths
class HlMeths:
    def startMatch(self, idx):
        return '<span class="search-result-highlight">'
    def endMatch(self):
        return '</span>'
#}}}
286
#{{{ recoll_search
287
def recoll_search(q):
288
289
290
291
292
293
    config = get_config()
    tstart = datetime.datetime.now()
    results = []
    query = recoll_initsearch(q)
    nres = query.rowcount

koniu's avatar
koniu committed
294
295
296
297
    if config['maxresults'] == 0:
        config['maxresults'] = nres
    if nres > config['maxresults']:
        nres = config['maxresults']
koniu's avatar
koniu committed
298
    if config['perpage'] == 0 or q['page'] == 0:
koniu's avatar
koniu committed
299
        config['perpage'] = nres
koniu's avatar
koniu committed
300
        q['page'] = 1
koniu's avatar
koniu committed
301
    offset = (q['page'] - 1) * config['perpage']
302

303
304
305
306
307
308
    if query.rowcount > 0:
        if type(query.next) == int:
            query.next = offset
        else:
            query.scroll(offset, mode='absolute')

309
    highlighter = HlMeths()
310
311
312
    for i in range(config['perpage']):
        try:
            doc = query.fetchone()
313
314
315
316
317
            # Later Recoll versions return None at EOL instead of
            # exception This change restores conformance to PEP 249
            # Python Database API Specification
            if not doc:
                break
318
319
        except:
            break
koniu's avatar
import  
koniu committed
320
321
        d = {}
        for f in FIELDS:
322
323
            v = getattr(doc, f)
            if v is not None:
324
                d[f] = v
325
            else:
326
                d[f] = ''
koniu's avatar
import  
koniu committed
327
        d['label'] = select([d['title'], d['filename'], '?'], [None, ''])
328
329
        d['sha'] = hashlib.sha1((d['url']+d['ipath']).encode('utf-8')).hexdigest()
        d['time'] = timestr(d['mtime'], config['timefmt'])
330
331
        if 'snippets' in q and q['snippets']:
            if 'highlight' in q and q['highlight']:
332
                d['snippet'] = query.makedocabstract(doc, highlighter)
333
            else:
334
                d['snippet'] = query.makedocabstract(doc)
335
336
337
338
339
            if not d['snippet']:
                try:
                    d['snippet'] = doc['abstract']
                except:
                    pass
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
340
341
        #for n,v in d.items():
        #    print("type(%s) is %s" % (n,type(v)))
koniu's avatar
import  
koniu committed
342
343
        results.append(d)
    tend = datetime.datetime.now()
koniu's avatar
koniu committed
344
    return results, nres, tend - tstart
koniu's avatar
import  
koniu committed
345
346
347
348
349
350
351
352
353
354
355
356
357
358
#}}}
#}}}
#{{{ routes
#{{{ static
@bottle.route('/static/:path#.+#')
def server_static(path):
    return bottle.static_file(path, root='./static')
#}}}
#{{{ main
@bottle.route('/')
@bottle.view('main')
def main():
    config = get_config()
    return { 'dirs': get_dirs(config['dirs'], config['dirdepth']),
359
            'query': get_query(), 'sorts': SORTS, 'config': config}
koniu's avatar
import  
koniu committed
360
361
362
363
364
365
366
367
#}}}
#{{{ results
@bottle.route('/results')
@bottle.view('results')
def results():
    config = get_config()
    query = get_query()
    qs = query_to_recoll_string(query)
koniu's avatar
koniu committed
368
369
370
371
372
    res, nres, timer = recoll_search(query)
    if config['maxresults'] == 0:
        config['maxresults'] = nres
    if config['perpage'] == 0:
        config['perpage'] = nres
koniu's avatar
import  
koniu committed
373
    return { 'res': res, 'time': timer, 'query': query, 'dirs':
374
             get_dirs(config['dirs'], config['dirdepth']),
375
             'qs': qs, 'sorts': SORTS, 'config': config,
376
377
             'query_string': bottle.request.query_string, 'nres': nres,
             'config': config}
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
#}}}
#{{{ preview
@bottle.route('/preview/<resnum:int>')
def preview(resnum):
    query = get_query()
    qs = query_to_recoll_string(query)
    rclq = recoll_initsearch(query)
    if resnum > rclq.rowcount - 1:
        return 'Bad result index %d' % resnum
    rclq.scroll(resnum)
    doc = rclq.fetchone()
    xt = rclextract.Extractor(doc)
    tdoc = xt.textextract(doc.ipath)
    if tdoc.mimetype == 'text/html':
        bottle.response.content_type = 'text/html; charset=utf-8'
    else:
        bottle.response.content_type = 'text/plain; charset=utf-8'
    return tdoc.text
#}}}
397
398
#{{{ download
@bottle.route('/download/<resnum:int>')
399
400
401
402
403
404
405
406
407
408
def edit(resnum):
    query = get_query()
    qs = query_to_recoll_string(query)
    rclq = recoll_initsearch(query)
    if resnum > rclq.rowcount - 1:
        return 'Bad result index %d' % resnum
    rclq.scroll(resnum)
    doc = rclq.fetchone()
    bottle.response.content_type = doc.mimetype
    pathismine = False
409
410
411
412
413

    xt = rclextract.Extractor(doc)
    path = xt.idoctofile(doc.ipath, doc.mimetype)
    pathismine = True

414
415
416
417
    if (not doc.ipath) and "filename" in doc.keys():
        filename = doc.filename
    else:
        filename = os.path.basename(path)
418
    bottle.response.headers['Content-Disposition'] = \
419
        'attachment; filename="%s"' % filename
420
    bottle.response.headers['Content-Length'] = os.stat(path).st_size
421
    f = open(path, 'rb')
422
423
424
    if pathismine:
        os.unlink(path)
    return f
koniu's avatar
import  
koniu committed
425
426
427
428
429
430
431
#}}}
#{{{ json
@bottle.route('/json')
def get_json():
    query = get_query()
    qs = query_to_recoll_string(query)
    bottle.response.headers['Content-Type'] = 'application/json'
432
433
    bottle.response.headers['Content-Disposition'] = \
      'attachment; filename=recoll-%s.json' % normalise_filename(qs)
koniu's avatar
koniu committed
434
    res, nres, timer = recoll_search(query)
435
436
437
438
439
440
441
    ures = []
    for d in res:
        ud={}
        for f,v in d.items():
            ud[f] = v
        ures.append(ud)
    res = ures
koniu's avatar
import  
koniu committed
442
443
444
445
446
    return json.dumps({ 'query': query, 'results': res })
#}}}
#{{{ csv
@bottle.route('/csv')
def get_csv():
447
    config = get_config()
koniu's avatar
import  
koniu committed
448
    query = get_query()
koniu's avatar
koniu committed
449
    query['page'] = 0
450
    query['snippets'] = 0
koniu's avatar
import  
koniu committed
451
452
    qs = query_to_recoll_string(query)
    bottle.response.headers['Content-Type'] = 'text/csv'
453
454
    bottle.response.headers['Content-Disposition'] = \
      'attachment; filename=recoll-%s.csv' % normalise_filename(qs)
455
    res, nres, timer = recoll_search(query)
456
    si = io.StringIO()
koniu's avatar
import  
koniu committed
457
    cw = csv.writer(si)
458
459
    fields = config['csvfields'].split()
    cw.writerow(fields)
koniu's avatar
import  
koniu committed
460
461
    for doc in res:
        row = []
462
        for f in fields:
463
            if f in doc:
464
                row.append(doc[f])
465
466
            else:
                row.append('')
koniu's avatar
import  
koniu committed
467
468
469
470
471
472
473
474
475
476
477
        cw.writerow(row)
    return si.getvalue().strip("\r\n")
#}}}
#{{{ settings/set
@bottle.route('/settings')
@bottle.view('settings')
def settings():
    return get_config()

@bottle.route('/set')
def set():
478
    config = get_config()
koniu's avatar
import  
koniu committed
479
    for k, v in DEFAULTS.items():
480
        bottle.response.set_cookie(k, str(bottle.request.query.get(k)), max_age=3153600000, expires=315360000)
481
    for d in config['dirs']:
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
482
        cookie_name = 'mount_%s' % urlquote(d, '')
483
        bottle.response.set_cookie(cookie_name, str(bottle.request.query.get('mount_%s' % d)), max_age=3153600000, expires=315360000)
484
    bottle.redirect('./')
koniu's avatar
import  
koniu committed
485
#}}}
486
487
488
489
490
491
492
493
#{{{ osd
@bottle.route('/osd.xml')
@bottle.view('osd')
def main():
    #config = get_config()
    url = bottle.request.urlparts
    url = '%s://%s' % (url.scheme, url.netloc)
    return {'url': url}
koniu's avatar
import  
koniu committed
494
495
#}}}
# vim: fdm=marker:tw=80:ts=4:sw=4:sts=4:et