webui.py 16.8 KB
Newer Older
koniu's avatar
import  
koniu committed
1
2
3
4
#{{{ imports
import os
import bottle
import time
5
import sys
koniu's avatar
import  
koniu committed
6
7
8
9
import datetime
import glob
import hashlib
import csv
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
10
import io
koniu's avatar
import  
koniu committed
11
12
import string
import shlex
13
14
from urllib.parse import urlencode, quote as urlquote
from recoll import recoll, rclextract, rclconfig
15

Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
16
17
def msg(s):
    print("%s" % s, file=sys.stderr)
18

19
# use ujson if avalaible (faster than built in json)
20
21
22
23
try:
    import ujson as json
except ImportError:
    import json
24
    #msg("ujson module not found, using (slower) built-in json module instead")
25

26
27
28

g_fscharset=sys.getfilesystemencoding()

koniu's avatar
import  
koniu committed
29
30
31
32
33
34
35
#}}}
#{{{ settings
# settings defaults
DEFAULTS = {
    'context': 30,
    'stem': 1,
    'timefmt': '%c',
36
    'dirdepth': 2,
koniu's avatar
koniu committed
37
    'maxchars': 500,
38
    'maxresults': 0,
koniu's avatar
koniu committed
39
    'perpage': 25,
40
    'csvfields': 'filename title author size time mtype url',
41
    'title_link': 'download',
koniu's avatar
import  
koniu committed
42
43
44
45
}

# sort fields/labels
SORTS = [
Laurent Defert's avatar
Laurent Defert committed
46
    ("relevancyrating", "Relevancy"),
koniu's avatar
import  
koniu committed
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
    ("mtime", "Date",),
    ("url", "Path"),
    ("filename", "Filename"),
    ("fbytes", "Size"),
    ("author", "Author"),
]

# doc fields
FIELDS = [
    # exposed by python api
    'ipath',
    'filename',
    'title',
    'author',
    'fbytes',
    'dbytes',
    'size',
    'fmtime',
    'dmtime',
    'mtime',
    'mtype',
    'origcharset',
    'sig',
    'relevancyrating',
    'url',
    'abstract',
    'keywords',
    # calculated
    'time',
    'snippet',
    'label',
]
#}}}
#{{{  functions
#{{{  helpers
def select(ls, invalid=[None]):
    for value in ls:
        if value not in invalid:
            return value

def timestr(secs, fmt):
88
89
    if secs == '' or secs is None:
        secs = '0'
koniu's avatar
import  
koniu committed
90
91
92
93
94
95
96
97
98
99
100
101
    t = time.gmtime(int(secs))
    return time.strftime(fmt, t)

def normalise_filename(fn):
    valid_chars = "_-%s%s" % (string.ascii_letters, string.digits)
    out = ""
    for i in range(0,len(fn)):
        if fn[i] in valid_chars:
            out += fn[i]
        else:
            out += "_"
    return out
102
103
104
105
106

def get_topdirs(confdir):
    rclconf = rclconfig.RclConfig(confdir)
    return rclconf.getConfParam('topdirs')

107
108
109
110
111
112
113
# Environment fetch for the cases where we don't care if unset or null
def safe_envget(varnm):
    try:
        return os.environ[varnm]
    except Exception as ex:
        return None

114
115
# get database directory from recoll.conf, defaults to confdir/xapiandb. Note
# that this is available as getDbDir() in newer rclconfig versions
116
def get_dbdir(confdir):
117
    confdir = os.path.expanduser(confdir)
118
119
    rclconf = rclconfig.RclConfig(confdir)
    dbdir = rclconf.getConfParam('dbdir')
120
121
122
123
124
125
126
    if not dbdir:
        dbdir = 'xapiandb'
    if not os.path.isabs(dbdir):
        cachedir = rclconf.getConfParam('cachedir')
        if not cachedir:
            cachedir = confdir
        dbdir = os.path.join(cachedir, dbdir)
127
    # recoll API expects bytes, not strings
128
    return os.path.normpath(dbdir).encode(g_fscharset)
koniu's avatar
import  
koniu committed
129
#}}}
130
131
#{{{ get_config
def get_config():
132
133
134
135
136
    # Arrange for apache wsgi SetEnv values to be reflected in the os environment.
    # This allows people to use either method
    for k in ("RECOLL_CONFDIR", "RECOLL_EXTRACONFDIRS"):
        if  k in bottle.request.environ:
            os.environ[k] = bottle.request.environ[k]
137
    config = {}
138
    envdir = safe_envget('RECOLL_CONFDIR')
koniu's avatar
import  
koniu committed
139
    # get useful things from recoll.conf
140
    rclconf = rclconfig.RclConfig(envdir)
koniu's avatar
koniu committed
141
    config['confdir'] = rclconf.getConfDir()
142
    config['dirs'] = dict.fromkeys([os.path.expanduser(d) for d in
143
                                    shlex.split(rclconf.getConfParam('topdirs'))],
144
                                   config['confdir'])
145
    # add topdirs from extra config dirs
146
    extraconfdirs = safe_envget('RECOLL_EXTRACONFDIRS')
147
148
149
150
151
    if extraconfdirs:
        config['extraconfdirs'] = shlex.split(extraconfdirs)
        for e in config['extraconfdirs']:
            config['dirs'].update(dict.fromkeys([os.path.expanduser(d) for d in
                shlex.split(get_topdirs(e))],e))
152
        config['extradbs'] = list(map(get_dbdir, config['extraconfdirs']))
153
154
    else:
        config['extraconfdirs'] = None
155
        config['extradbs'] = None
156
    config['stemlang'] = rclconf.getConfParam('indexstemminglanguages')
157
158
159
160
161
162
163
164

    # Possibly adjust user config defaults with data from recoll.conf. Some defaults which are
    # generally suitable like dirdepth=2 can be unworkable on big data sets (causing init errors so
    # that they can't even be adjusted from the UI). The 2nd parameter asks for an int conversion
    fetches = [("context", 1), ("stem", 1),("timefmt", 0),("dirdepth", 1),("maxchars", 1),
               ("maxresults", 1), ("perpage", 1), ("csvfields", 0), ("title_link", 0)]
    for k, isint in fetches:
        value = rclconf.getConfParam("webui_" + k)
165
        if value is not None:
166
            DEFAULTS[k] = int(value) if isint else value
koniu's avatar
import  
koniu committed
167
168
    # get config from cookies or defaults
    for k, v in DEFAULTS.items():
169
170
        value = select([bottle.request.get_cookie(k), v])
        config[k] = type(v)(value)
171
172
    # Fix csvfields: get rid of invalid ones to avoid needing tests in
    # the dump function
173
174
175
176
    cf = config['csvfields'].split()
    ncf = [f for f in cf if f in FIELDS]
    config['csvfields'] = ' '.join(ncf)
    config['fields'] = ' '.join(FIELDS)
177
178
179
    # get mountpoints
    config['mounts'] = {}
    for d in config['dirs']:
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
180
        name = 'mount_%s' % urlquote(d,'')
181
        config['mounts'][d] = select([bottle.request.get_cookie(name), 'file://%s' % d], [None, ''])
182
183
184

    # Parameters set by the admin in the recoll configuration
    # file. These override anything else, so read them last
185
186
187
188
    val = rclconf.getConfParam('webui_nojsoncsv')
    val = 0 if val is None else int(val)
    config['rclc_nojsoncsv'] = val

189
190
191
192
193
    val = rclconf.getConfParam('webui_maxperpage')
    val = 0 if val is None else int(val)
    if val:
        if config['perpage'] == 0 or config['perpage'] > val:
            config['perpage'] = val
194
195
196
197
198

    val = rclconf.getConfParam('webui_nosettings')
    val = 0 if val is None else int(val)
    config['rclc_nosettings'] = val

199
200
201
202
203
204
    val = str(rclconf.getConfParam('webui_defaultsort'))
    config['defsortidx'] = 0
    for i in range(len(SORTS)):
        if SORTS[i][0] == val or SORTS[i][1] == val:
            config['defsortidx'] = i
            break
koniu's avatar
import  
koniu committed
205
206
207
208
209
210
    return config
#}}}
#{{{ get_dirs
def get_dirs(tops, depth):
    v = []
    for top in tops:
211
212
213
214
        # We do the conversion to bytes here, because Python versions
        # before 3.7 won't do the right thing if the locale is C,
        # which would be the case with a default apache install
        top = top.encode('utf-8', 'surrogateescape')
koniu's avatar
import  
koniu committed
215
        dirs = [top]
koniu's avatar
koniu committed
216
        for d in range(1, depth+1):
217
            dirs = dirs + glob.glob(top + b'/*' * d)
koniu's avatar
import  
koniu committed
218
        dirs = filter(lambda f: os.path.isdir(f), dirs)
219
220
        top_path = top.rsplit(b'/', 1)[0]
        dirs = [w.replace(top_path+b'/', b'', 1) for w in dirs]
koniu's avatar
import  
koniu committed
221
        v = v + dirs
222
223
    for i in range(len(v)):
        v[i] = v[i].decode('utf-8', 'surrogateescape')
koniu's avatar
import  
koniu committed
224
225
226
    return ['<all>'] + v
#}}}
#{{{ get_query
227
228
def get_query(config=None):
    defsortidx = config['defsortidx'] if config and 'defsortidx' in config else 0
koniu's avatar
import  
koniu committed
229
    query = {
230
231
232
233
        'query': select([bottle.request.query.query, '']),
        'before': select([bottle.request.query.before, '']),
        'after': select([bottle.request.query.after, '']),
        'dir': select([bottle.request.query.dir, '', '<all>'], [None, '']),
234
        'sort': select([bottle.request.query.sort, SORTS[defsortidx][0]], [None, '']),
235
236
237
238
        'ascending': int(select([bottle.request.query.ascending, 0], [None, ''])),
        'page': int(select([bottle.request.query.page, 0], [None, ''])),
        'highlight': int(select([bottle.request.query.highlight, 1], [None, ''])),
        'snippets': int(select([bottle.request.query.snippets, 1], [None, ''])),
koniu's avatar
import  
koniu committed
239
    }
240
    #msg("query['query'] : %s" % query['query'])
koniu's avatar
import  
koniu committed
241
242
243
244
    return query
#}}}
#{{{ query_to_recoll_string
def query_to_recoll_string(q):
245
    qs = q['query']
koniu's avatar
import  
koniu committed
246
247
    if len(q['after']) > 0 or len(q['before']) > 0:
        qs += " date:%s/%s" % (q['after'], q['before'])
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
248
249
250
    qdir = q['dir']
    if qdir != '<all>':
        qs += " dir:\"%s\" " % qdir
koniu's avatar
import  
koniu committed
251
252
    return qs
#}}}
253
254
#{{{ recoll_initsearch
def recoll_initsearch(q):
koniu's avatar
import  
koniu committed
255
    config = get_config()
256
257
258
259
260
261
262
263
264
265
266
    confdir = config['confdir']
    dbs = []
    """ The reason for this somewhat elaborate scheme is to keep the
    set size as small as possible by searching only those databases
    with matching topdirs """
    if q['dir'] == '<all>':
        if config['extraconfdirs']:
            dbs.extend(map(get_dbdir,config['extraconfdirs']))
    else:
        confdirs = []
        for d,conf in config['dirs'].items():
267
268
            tdbasename = os.path.basename(d)
            if os.path.commonprefix([tdbasename, q['dir']]) == tdbasename:
269
270
271
                confdirs.append(conf)
        if len(confdirs) == 0:
            # should not happen, using non-existing q['dir']?
272
            bottle.abort(400, 'no matching database for search directory ' + q['dir'])
273
274
275
276
277
278
        elif len(confdirs) == 1:
            # only one config (most common situation)
            confdir = confdirs[0]
        else:
            # more than one config with matching topdir, use 'm all
            confdir = confdirs[0]
279
            dbs.extend(map(get_dbdir, confdirs[1:]))
280

281
    if config['extradbs']:
282
283
284
285
        dbs.extend(config['extradbs'])

    if dbs:
        db = recoll.connect(confdir,dbs)
286
    else:
287
288
        db = recoll.connect(confdir)

koniu's avatar
koniu committed
289
    db.setAbstractParams(config['maxchars'], config['context'])
koniu's avatar
import  
koniu committed
290
    query = db.query()
koniu's avatar
koniu committed
291
    query.sortby(q['sort'], q['ascending'])
koniu's avatar
import  
koniu committed
292
    try:
koniu's avatar
koniu committed
293
        qs = query_to_recoll_string(q)
294
        query.execute(qs, config['stem'], config['stemlang'])
295
296
    except Exception as ex:
        msg("Query execute failed: %s" % ex)
297
298
299
        pass
    return query
#}}}
300
301
302
303
304
305
306
#{{{ HlMeths
class HlMeths:
    def startMatch(self, idx):
        return '<span class="search-result-highlight">'
    def endMatch(self):
        return '</span>'
#}}}
307
#{{{ recoll_search
308
def recoll_search(q):
309
310
311
312
313
314
    config = get_config()
    tstart = datetime.datetime.now()
    results = []
    query = recoll_initsearch(q)
    nres = query.rowcount

koniu's avatar
koniu committed
315
316
317
318
    if config['maxresults'] == 0:
        config['maxresults'] = nres
    if nres > config['maxresults']:
        nres = config['maxresults']
koniu's avatar
koniu committed
319
    if config['perpage'] == 0 or q['page'] == 0:
koniu's avatar
koniu committed
320
        config['perpage'] = nres
koniu's avatar
koniu committed
321
        q['page'] = 1
koniu's avatar
koniu committed
322
    offset = (q['page'] - 1) * config['perpage']
323

324
325
326
327
328
329
    if query.rowcount > 0:
        if type(query.next) == int:
            query.next = offset
        else:
            query.scroll(offset, mode='absolute')

330
331
332
333
    if 'highlight' in q and q['highlight']:
        highlighter = HlMeths()
    else:
        highlighter = None
334
335
336
    for i in range(config['perpage']):
        try:
            doc = query.fetchone()
337
338
339
340
341
            # Later Recoll versions return None at EOL instead of
            # exception This change restores conformance to PEP 249
            # Python Database API Specification
            if not doc:
                break
342
343
        except:
            break
koniu's avatar
import  
koniu committed
344
345
        d = {}
        for f in FIELDS:
346
347
            v = getattr(doc, f)
            if v is not None:
348
                d[f] = v
349
            else:
350
                d[f] = ''
koniu's avatar
import  
koniu committed
351
        d['label'] = select([d['title'], d['filename'], '?'], [None, ''])
352
353
        d['sha'] = hashlib.sha1((d['url']+d['ipath']).encode('utf-8')).hexdigest()
        d['time'] = timestr(d['mtime'], config['timefmt'])
354
        if 'snippets' in q and q['snippets']:
355
            d['snippet'] = query.makedocabstract(doc, methods=highlighter)
356
357
358
359
360
            if not d['snippet']:
                try:
                    d['snippet'] = doc['abstract']
                except:
                    pass
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
361
362
        #for n,v in d.items():
        #    print("type(%s) is %s" % (n,type(v)))
koniu's avatar
import  
koniu committed
363
364
        results.append(d)
    tend = datetime.datetime.now()
koniu's avatar
koniu committed
365
    return results, nres, tend - tstart
koniu's avatar
import  
koniu committed
366
367
368
369
370
371
372
373
374
375
376
377
378
379
#}}}
#}}}
#{{{ routes
#{{{ static
@bottle.route('/static/:path#.+#')
def server_static(path):
    return bottle.static_file(path, root='./static')
#}}}
#{{{ main
@bottle.route('/')
@bottle.view('main')
def main():
    config = get_config()
    return { 'dirs': get_dirs(config['dirs'], config['dirdepth']),
380
            'query': get_query(config), 'sorts': SORTS, 'config': config}
koniu's avatar
import  
koniu committed
381
382
383
384
385
386
#}}}
#{{{ results
@bottle.route('/results')
@bottle.view('results')
def results():
    config = get_config()
387
    query = get_query(config)
koniu's avatar
import  
koniu committed
388
    qs = query_to_recoll_string(query)
koniu's avatar
koniu committed
389
390
391
392
393
    res, nres, timer = recoll_search(query)
    if config['maxresults'] == 0:
        config['maxresults'] = nres
    if config['perpage'] == 0:
        config['perpage'] = nres
koniu's avatar
import  
koniu committed
394
    return { 'res': res, 'time': timer, 'query': query, 'dirs':
395
             get_dirs(config['dirs'], config['dirdepth']),
396
             'qs': qs, 'sorts': SORTS, 'config': config,
397
398
             'query_string': bottle.request.query_string, 'nres': nres,
             'config': config}
399
400
401
402
#}}}
#{{{ preview
@bottle.route('/preview/<resnum:int>')
def preview(resnum):
403
404
    config = get_config()
    query = get_query(config)
405
406
407
408
409
410
411
412
413
    qs = query_to_recoll_string(query)
    rclq = recoll_initsearch(query)
    if resnum > rclq.rowcount - 1:
        return 'Bad result index %d' % resnum
    rclq.scroll(resnum)
    doc = rclq.fetchone()
    xt = rclextract.Extractor(doc)
    tdoc = xt.textextract(doc.ipath)
    if tdoc.mimetype == 'text/html':
414
        ishtml = 1
415
416
        bottle.response.content_type = 'text/html; charset=utf-8'
    else:
417
        ishtml = 0
418
        bottle.response.content_type = 'text/plain; charset=utf-8'
419
420
421
422
    if 'highlight' in query and query['highlight']:
        hl = HlMeths()
        txt = rclq.highlight(tdoc.text, ishtml=ishtml, methods=hl)
        pos = txt.find('<head>')
423
        ssref = '<link rel="stylesheet" type="text/css" href="../static/style.css">'
424
        if pos >= 0:
425
426
            txt = txt[0:pos+6] + ssref + txt[pos+6:]
        else:
427
428
            txt = '<html><head>' + ssref + \
                '<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"></head><body>'+ \
429
                txt
430
431
        bottle.response.content_type = 'text/html; charset=utf-8'
        return txt
432
433
    return tdoc.text
#}}}
434
435
#{{{ download
@bottle.route('/download/<resnum:int>')
436
def edit(resnum):
437
438
    config = get_config()
    query = get_query(config)
439
440
441
442
443
444
445
    qs = query_to_recoll_string(query)
    rclq = recoll_initsearch(query)
    if resnum > rclq.rowcount - 1:
        return 'Bad result index %d' % resnum
    rclq.scroll(resnum)
    doc = rclq.fetchone()
    bottle.response.content_type = doc.mimetype
446
447
    xt = rclextract.Extractor(doc)
    path = xt.idoctofile(doc.ipath, doc.mimetype)
448
    if "filename" in doc.keys():
449
450
451
        filename = doc.filename
    else:
        filename = os.path.basename(path)
452
    bottle.response.headers['Content-Disposition'] = f'attachment; filename="{filename}"'
453
    bottle.response.headers['Content-Length'] = os.stat(path).st_size
454
    f = open(path, 'rb')
455
    try:
456
        os.unlink(path)
457
458
    except:
        pass
459
    return f
koniu's avatar
import  
koniu committed
460
461
462
463
#}}}
#{{{ json
@bottle.route('/json')
def get_json():
464
465
    config = get_config()
    query = get_query(config)
koniu's avatar
import  
koniu committed
466
467
    qs = query_to_recoll_string(query)
    bottle.response.headers['Content-Type'] = 'application/json'
468
469
    bottle.response.headers['Content-Disposition'] = \
      'attachment; filename=recoll-%s.json' % normalise_filename(qs)
koniu's avatar
koniu committed
470
    res, nres, timer = recoll_search(query)
471
472
473
474
475
476
477
    ures = []
    for d in res:
        ud={}
        for f,v in d.items():
            ud[f] = v
        ures.append(ud)
    res = ures
koniu's avatar
import  
koniu committed
478
479
480
481
482
    return json.dumps({ 'query': query, 'results': res })
#}}}
#{{{ csv
@bottle.route('/csv')
def get_csv():
483
    config = get_config()
484
    query = get_query(config)
koniu's avatar
koniu committed
485
    query['page'] = 0
486
    query['snippets'] = 0
koniu's avatar
import  
koniu committed
487
488
    qs = query_to_recoll_string(query)
    bottle.response.headers['Content-Type'] = 'text/csv'
489
490
    bottle.response.headers['Content-Disposition'] = \
      'attachment; filename=recoll-%s.csv' % normalise_filename(qs)
491
    res, nres, timer = recoll_search(query)
492
    si = io.StringIO()
koniu's avatar
import  
koniu committed
493
    cw = csv.writer(si)
494
495
    fields = config['csvfields'].split()
    cw.writerow(fields)
koniu's avatar
import  
koniu committed
496
497
    for doc in res:
        row = []
498
        for f in fields:
499
            if f in doc:
500
                row.append(doc[f])
501
502
            else:
                row.append('')
koniu's avatar
import  
koniu committed
503
504
505
506
507
508
509
510
511
512
513
        cw.writerow(row)
    return si.getvalue().strip("\r\n")
#}}}
#{{{ settings/set
@bottle.route('/settings')
@bottle.view('settings')
def settings():
    return get_config()

@bottle.route('/set')
def set():
514
    config = get_config()
koniu's avatar
import  
koniu committed
515
    for k, v in DEFAULTS.items():
516
        bottle.response.set_cookie(k, str(bottle.request.query.get(k)), max_age=3153600000, expires=315360000)
517
    for d in config['dirs']:
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
518
        cookie_name = 'mount_%s' % urlquote(d, '')
519
        bottle.response.set_cookie(cookie_name, str(bottle.request.query.get('mount_%s' % d)), max_age=3153600000, expires=315360000)
520
    bottle.redirect('./')
koniu's avatar
import  
koniu committed
521
#}}}
522
523
524
525
526
527
528
529
#{{{ osd
@bottle.route('/osd.xml')
@bottle.view('osd')
def main():
    #config = get_config()
    url = bottle.request.urlparts
    url = '%s://%s' % (url.scheme, url.netloc)
    return {'url': url}
koniu's avatar
import  
koniu committed
530
531
#}}}
# vim: fdm=marker:tw=80:ts=4:sw=4:sts=4:et