Commit ec68abe1 authored by Jean-Francois Dockes's avatar Jean-Francois Dockes
Browse files

Python3 port:

- Updated bottle module to 0.12.13.
- bottle: include header ... -> include('header', ...)
- Imported up to date version of rclconfig.py (+ conftree.py)
- Adapt urllib imports
- Needed explicit integer division in places (//)
- Recoll result records contain utf-8 bytes: can't mix with 'xx' need b'xx'
- has_key -> in
- StringIO -> io.StringIO
parent 6e896daf
This diff is collapsed.
#!/usr/bin/env python
# Copyright (C) 2016 J.F.Dockes
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the
# Free Software Foundation, Inc.,
# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
from __future__ import print_function
import locale
import re
import os
import sys
import base64
import platform
def _debug(s):
print("%s"%s, file=sys.stderr)
class ConfSimple(object):
"""A ConfSimple class reads a recoll configuration file, which is
a typical ini file (see the Recoll manual). It's a dictionary of
dictionaries which lets you retrieve named values from the top
level or a subsection"""
def __init__(self, confname, tildexp = False, readonly = True):
self.submaps = {}
self.dotildexpand = tildexp
self.readonly = readonly
self.confname = confname
try:
f = open(confname, 'rb')
except Exception as exc:
#_debug("Open Exception: %s" % exc)
# File does not exist -> empty config, not an error.
self.submaps = {}
self.submaps[b''] = {}
return
self._parseinput(f)
def _parseinput(self, f):
appending = False
line = b''
submapkey = b''
for cline in f:
cline = cline.rstrip(b'\r\n')
if appending:
line = line + cline
else:
line = cline
line = line.strip()
if line == b'' or line[0] == b'#'[0]:
continue
if line[len(line)-1] == b'\\'[0]:
line = line[0:len(line)-1]
appending = True
continue
appending = False
#_debug(line)
if line[0] == b'['[0]:
line = line.strip(b'[]')
if self.dotildexpand:
submapkey = os.path.expanduser(line)
if type(submapkey) == type(u''):
submapkey = submapkey.encode('utf-8')
else:
submapkey = line
#_debug("Submapkey: [%s]" % submapkey)
continue
nm, sep, value = line.partition(b'=')
if sep == b'':
# No equal sign in line -> considered comment
continue
nm = nm.strip()
value = value.strip()
#_debug("sk [%s] nm: [%s] value: [%s]" % (submapkey, nm, value))
if not submapkey in self.submaps:
self.submaps[submapkey] = {}
self.submaps[submapkey][nm] = value
def getbin(self, nm, sk = b''):
'''Returns None if not found, empty string if found empty'''
if type(nm) != type(b'') or type(sk) != type(b''):
raise TypeError("getbin: parameters must be binary not unicode")
#_debug("ConfSimple::getbin nm [%s] sk [%s]" % (nm, sk))
if not sk in self.submaps:
return None
if not nm in self.submaps[sk]:
return None
return self.submaps[sk][nm]
def get(self, nm, sk = b''):
dodecode = False
if type(nm) == type(u''):
dodecode = True
nm = nm.encode('utf-8')
if type(sk) == type(u''):
sk = sk.encode('utf-8')
#v = ConfSimple.getbin(self, nm, sk)
v = self.getbin(nm, sk)
if v and dodecode:
v = v.decode('utf-8')
return v
def getNamesbin(self, sk = b''):
if not sk in self.submaps:
return None
return list(self.submaps[sk].keys())
def getNames(self, sk = ''):
if not sk in self.submaps:
return None
dodecode = False
if type(sk) == type(u''):
dodecode = True
sk = sk.encode('utf-8')
names = self.getNamesbin(sk)
if names and dodecode:
names = [nm.decode('utf-8') for nm in names]
return names
def _rewrite(self):
if self.readonly:
raise Exception("ConfSimple is readonly")
tname = self.confname + "-"
f = open(tname, 'wb')
# First output null subkey submap
if b'' in self.submaps:
for nm,value in self.submaps[b''].items():
f.write(nm + b'=' + value + b'\n')
for sk,mp in self.submaps.items():
if sk == b'':
continue
f.write(b'[' + sk + b']\n')
for nm,value in mp.items():
f.write(nm + b'=' + value + b'\n')
f.close()
os.rename(tname, self.confname)
def setbin(self, nm, value, sk = b''):
if self.readonly:
raise Exception("ConfSimple is readonly")
if sk not in self.submaps:
self.submaps[sk] = {}
self.submaps[sk][nm] = value
self._rewrite()
return True
def set(self, nm, value, sk = b''):
if self.readonly:
raise Exception("ConfSimple is readonly")
if type(nm) == type(u''):
nm = nm.encode('utf-8')
if type(value) == type(u''):
value = value.encode('utf-8')
if type(sk) == type(u''):
sk = sk.encode('utf-8')
return self.setbin(nm, value, sk)
class ConfTree(ConfSimple):
"""A ConfTree adds path-hierarchical interpretation of the section keys,
which should be '/'-separated values. When a value is requested for a
given path, it will also be searched in the sections corresponding to
the ancestors. E.g. get(name, '/a/b') will also look in sections '/a' and
'/' or '' (the last 2 are equivalent)"""
def getbin(self, nm, sk = b''):
if type(nm) != type(b'') or type(sk) != type(b''):
raise TypeError("getbin: parameters must be binary not unicode")
#_debug("ConfTree::getbin: nm [%s] sk [%s]" % (nm, sk))
if sk == b'' or sk[0] != b'/'[0]:
return ConfSimple.getbin(self, nm, sk)
if sk[len(sk)-1] == b'/'[0]:
sk = sk[:len(sk)-1]
# Try all sk ancestors as submaps (/a/b/c-> /a/b/c, /a/b, /a, b'')
while sk:
if sk in self.submaps:
return ConfSimple.getbin(self, nm, sk)
if sk + b'/' in self.submaps:
return ConfSimple.getbin(self, nm, sk+b'/')
i = sk.rfind(b'/')
if i == -1:
break
sk = sk[:i]
return ConfSimple.getbin(self, nm)
class ConfStack(object):
""" A ConfStack manages the superposition of a list of Configuration
objects. Values are looked for in each object from the list until found.
This typically provides for defaults overriden by sparse values in the
topmost file."""
def __init__(self, nm, dirs, tp = 'simple'):
fnames = []
for dir in dirs:
fnm = os.path.join(dir, nm)
fnames.append(fnm)
self._construct(tp, fnames)
def _construct(self, tp, fnames):
self.confs = []
for fname in fnames:
if tp.lower() == 'simple':
conf = ConfSimple(fname)
else:
conf = ConfTree(fname)
self.confs.append(conf)
# Accepts / returns binary strings (non-unicode)
def getbin(self, nm, sk = b''):
if type(nm) != type(b'') or type(sk) != type(b''):
raise TypeError("getbin: parameters must be binary not unicode")
for conf in self.confs:
value = conf.getbin(nm, sk)
if value is not None:
return value
return None
def get(self, nm, sk = b''):
dodecode = False
if type(nm) == type(u''):
dodecode = True
nm = nm.encode('utf-8')
if type(sk) == type(u''):
sk = sk.encode('utf-8')
#v = ConfSimple.getbin(self, nm, sk)
v = self.getbin(nm, sk)
if v and dodecode:
v = v.decode('utf-8')
return v
#!/usr/bin/env python
#!/usr/bin/env python3
from __future__ import print_function
import locale
import re
import os
import sys
import base64
import platform
class ConfSimple:
"""A ConfSimple class reads a recoll configuration file, which is a typical
ini file (see the Recoll manual). It's a dictionary of dictionaries which
lets you retrieve named values from the top level or a subsection"""
def __init__(self, confname, tildexp = False):
f = open(confname, 'r')
self.dotildexpand = tildexp
self.submaps = {}
self.parseinput(f)
def parseinput(self, f):
appending = False
line = ''
submapkey = ''
for cline in f:
cline = cline.rstrip("\r\n")
if appending:
line = line + cline
else:
line = cline
line = line.strip()
if line == '' or line[0] == '#':
continue
if line[len(line)-1] == '\\':
line = line[0:len(line)-1]
appending = True
continue
appending = False
#print line
if line[0] == '[':
line = line.strip('[]')
if self.dotildexpand:
submapkey = os.path.expanduser(line)
else:
submapkey = line
#print "Submapkey:", submapkey
continue
nm, sep, value = line.partition('=')
if sep == '':
continue
nm = nm.strip()
value = value.strip()
#print "Name:", nm, "Value:", value
if not self.submaps.has_key(submapkey):
self.submaps[submapkey] = {}
self.submaps[submapkey][nm] = value
def get(self, nm, sk = ''):
'''Returns None if not found, empty string if found empty'''
if not self.submaps.has_key(sk):
return None
if not self.submaps[sk].has_key(nm):
return None
return self.submaps[sk][nm]
def getNames(self, sk = ''):
if not self.submaps.has_key(sk):
return None
return self.submaps[sk].keys()
class ConfTree(ConfSimple):
"""A ConfTree adds path-hierarchical interpretation of the section keys,
which should be '/'-separated values. When a value is requested for a
given path, it will also be searched in the sections corresponding to
the ancestors. E.g. get(name, '/a/b') will also look in sections '/a' and
'/' or '' (the last 2 are equivalent"""
def get(self, nm, sk = ''):
if sk == '' or sk[0] != '/':
return ConfSimple.get(self, nm, sk)
if sk[len(sk)-1] != '/':
sk = sk + '/'
# Try all sk ancestors as submaps (/a/b/c-> /a/b/c, /a/b, /a, '')
while sk.find('/') != -1:
val = ConfSimple.get(self, nm, sk)
if val is not None:
return val
i = sk.rfind('/')
if i == -1:
break
sk = sk[:i]
return ConfSimple.get(self, nm)
class ConfStack:
""" A ConfStack manages the superposition of a list of Configuration
objects. Values are looked for in each object from the list until found.
This typically provides for defaults overriden by sparse values in the
topmost file."""
def __init__(self, nm, dirs, tp = 'simple'):
fnames = []
for dir in dirs:
fnm = os.path.join(dir, nm)
fnames.append(fnm)
self._construct(tp, fnames)
def _construct(self, tp, fnames):
self.confs = []
for fname in fnames:
if tp.lower() == 'simple':
conf = ConfSimple(fname)
else:
conf = ConfTree(fname)
self.confs.append(conf)
def get(self, nm, sk = ''):
for conf in self.confs:
value = conf.get(nm, sk)
if value is not None:
return value
return None
import conftree
class RclDynConf:
def __init__(self, fname):
self.data = ConfSimple(fname)
......@@ -138,33 +23,53 @@ class RclDynConf:
class RclConfig:
def __init__(self, argcnf = None):
self.config = None
platsys = platform.system()
# Find configuration directory
if argcnf is not None:
self.confdir = os.path.abspath(argcnf)
elif os.environ.has_key("RECOLL_CONFDIR"):
elif "RECOLL_CONFDIR" in os.environ:
self.confdir = os.environ["RECOLL_CONFDIR"]
else:
self.confdir = os.path.expanduser("~/.recoll")
#print "Confdir: [%s]" % self.confdir
if platsys == "Windows":
if "LOCALAPPDATA" in os.environ:
dir = os.environ["LOCALAPPDATA"]
else:
dir = os.path.expanduser("~")
self.confdir = os.path.join(dir, "Recoll")
else:
self.confdir = os.path.expanduser("~/.recoll")
#print("Confdir: [%s]" % self.confdir, file=sys.stderr)
# Also find datadir. This is trickier because this is set by
# "configure" in the C code. We can only do our best. Have to
# choose a preference order. Use RECOLL_DATADIR if the order is wrong
self.datadir = None
if os.environ.has_key("RECOLL_DATADIR"):
if "RECOLL_DATADIR" in os.environ:
self.datadir = os.environ["RECOLL_DATADIR"]
else:
dirs = ("/opt/local", "/usr", "/usr/local")
for dir in dirs:
dd = os.path.join(dir, "share/recoll")
if os.path.exists(dd):
self.datadir = dd
if platsys == "Windows":
self.datadir = os.path.join(os.path.dirname(sys.argv[0]), "..")
else:
dirs = ("/opt/local", "/usr", "/usr/local")
for dir in dirs:
dd = os.path.join(dir, "share/recoll")
if os.path.exists(dd):
self.datadir = dd
if self.datadir is None:
self.datadir = "/usr/share/recoll"
#print "Datadir: [%s]" % self.datadir
self.cdirs = [self.confdir,]
#print("Datadir: [%s]" % self.datadir, file=sys.stderr)
self.cdirs = []
# Additional config directory, values override user ones
if "RECOLL_CONFTOP" in os.environ:
self.cdirs.append(os.environ["RECOLL_CONFTOP"])
self.cdirs.append(self.confdir)
# Additional config directory, overrides system's, overridden by user's
if "RECOLL_CONFMID" in os.environ:
self.cdirs.append(os.environ["RECOLL_CONFMID"])
self.cdirs.append(os.path.join(self.datadir, "examples"))
#print self.cdirs
self.config = ConfStack("recoll.conf", self.cdirs, "tree")
#print("Config dirs: %s" % self.cdirs, file=sys.stderr)
self.keydir = ''
def getConfDir(self):
......@@ -174,6 +79,8 @@ class RclConfig:
self.keydir = dir
def getConfParam(self, nm):
if not self.config:
self.config = conftree.ConfStack("recoll.conf", self.cdirs, "tree")
return self.config.get(nm, self.keydir)
class RclExtraDbs:
......@@ -185,3 +92,8 @@ class RclExtraDbs:
dync = RclDynConf(dyncfile)
return dync.getStringList("actExtDbs")
if __name__ == '__main__':
config = RclConfig()
print(config.getConfParam("topdirs"))
extradbs = RclExtraDbs(config)
print(extradbs.getActDbs())
%import sys
%py3k = sys.version_info >= (3, 0, 0)
%if py3k:
%from urllib.parse import urlencode
%else:
%from urllib.parse import urlencode
%end
%import math
%q = dict(query)
%def page_href(page):
%q['page'] = page
%return './results?%s' % urllib.urlencode(q)
%return './results?%s' % urlencode(q)
%end
%if nres > 0:
%import math, urllib
%npages = int(math.ceil(nres/float(config['perpage'])))
%if npages > 1:
<div id="pages">
<a title="First" class="page" href="{{page_href(1)}}">&#171;</a>
<a title="Previous" class="page" href="{{page_href(max(1,query['page']-1))}}">&#8249;</a> &nbsp;
%offset = ((query['page'])/10)*10
%offset = ((query['page'])//10)*10
%for p in range(max(1,offset), min(offset+10,npages+1)):
%if p == query['page']:
%cls = "page current"
......
......@@ -2,12 +2,12 @@
<div class="search-result">
%number = (query['page'] - 1)*config['perpage'] + i + 1
<div class="search-result-number"><a href="#r{{d['sha']}}">#{{number}}</a></div>
%url = d['url'].replace('file://', '')
%url = d['url'].replace(b'file://', b'')
%for dr, prefix in config['mounts'].items():
%url = url.replace(dr, prefix)
%url = url.replace(dr.encode('utf-8'), prefix.encode('utf-8'))
%end
<div class="search-result-title" id="r{{d['sha']}}" title="{{d['abstract']}}">
%if config.has_key('title_link') and config['title_link'] != 'download':
%if 'title_link' in config and config['title_link'] != 'download':
%if config['title_link'] == 'open':
<a href="{{url}}">{{d['label']}}</a>
%elif config['title_link'] == 'preview':
......@@ -20,13 +20,13 @@
%if len(d['ipath']) > 0:
<div class="search-result-ipath">[{{d['ipath']}}]</div>
%end
%if d.has_key('author') and len(d['author']) > 0:
%if 'author' in d and len(d['author']) > 0:
<div class="search-result-author">{{d['author']}}</div>
%end
<div class="search-result-url">
%urllabel = os.path.dirname(d['url'].replace('file://', ''))
%urllabel = os.path.dirname(d['url'].replace(b'file://', b''))
%for r in config['dirs']:
%urllabel = urllabel.replace(r.rsplit('/',1)[0] + '/' , '')
%urllabel = urllabel.replace(r.encode('utf-8').rsplit(b'/',1)[0] + b'/' , b'')
%end
<a href="{{os.path.dirname(url)}}">{{urllabel}}</a>
</div>
......
%include header title=": " + query['query']+" ("+str(nres)+")"
%include search query=query, dirs=dirs, sorts=sorts, config=config
%include('header', title=": " + query['query']+" ("+str(nres)+")")
%include('search', query=query, dirs=dirs, sorts=sorts, config=config)
<div id="status">
<div id="found">
Found <b>{{nres}}</b> matching: <b><i>{{qs}}</i></b>
......@@ -13,13 +13,13 @@
%end
<br style="clear: both">
</div>
%include pages query=query, config=config, nres=nres
%include('pages', query=query, config=config, nres=nres)
<div id="results">
%for i in range(0, len(res)):
%include result d=res[i], i=i, query=query, config=config, query_string=query_string, hasrclextract=hasrclextract
%include('result', d=res[i], i=i, query=query, config=config, query_string=query_string, hasrclextract=hasrclextract)
%end
</div>
%include pages query=query, config=config, nres=nres
%include footer
%include('pages', query=query, config=config, nres=nres)
%include('footer')
<!-- vim: fdm=marker:tw=80:ts=4:sw=4:sts=4:et:ai
-->
#!/usr/bin/env python
from __future__ import print_function
#{{{ imports
import os
import bottle
......@@ -9,20 +9,28 @@ import glob
import hashlib
import json
import csv
import StringIO
import ConfigParser
import io
import string
import shlex
def msg(s):
print("%s" % s, file=sys.stderr)
py3k = sys.version_info >= (3, 0, 0)
if py3k:
from urllib.parse import urlencode, quote as urlquote
else: # 2.x
from urllib import urlencode, quote as urlquote
import urllib
# import recoll and rclextract
try:
from recoll import recoll
from recoll import rclextract
hasrclextract = True
except:
except Exception as err:
msg("Import recoll because: %s" % err)
import recoll
hasrclextract = False
# import rclconfig system-wide or local copy
# Import rclconfig system-wide or local copy
try:
from recoll import rclconfig
except:
......@@ -122,7 +130,7 @@ def get_config():
# get mountpoints
config['mounts'] = {}
for d in config['dirs']:
name = 'mount_%s' % urllib.quote(d,'')
name = 'mount_%s' % urlquote(d,'')
config['mounts'][d] = select([bottle.request.get_cookie(name), 'file://%s' % d], [None, ''])
# Parameters set by the admin in the recoll configuration
......@@ -171,11 +179,17 @@ def get_query():
#}}}
#{{{ query_to_recoll_string
def query_to_recoll_string(q):
qs = q['query'].decode('utf-8')
if type(q['query']) == type(u''):
qs = q['query']
else:
qs = q['query'].decode('utf-8')
if len(q['after']) > 0 or len(q['before']) > 0:
qs += " date:%s/%s" % (q['after'], q['before'])
if q['dir'] != '<all>':
qs += " dir:\"%s\" " % q['dir'].decode('utf-8')
qdir = q['dir']
if type(qdir) != type(u''):
qdir = qdir.decode('utf-8')