Commit 4f8d0a06 authored by Jean-Francois Dockes's avatar Jean-Francois Dockes
Browse files

uprcl: cleanup and small simplifications

parent 182c0f44
......@@ -17,10 +17,13 @@ import conftree
from uprclutils import uplog
class MinimConfig(object):
def __init__(self, fn):
self.conf = conftree.ConfSimple(fn)
uplog("Minim config read: contentDir: %s" %
self.conf.get("minimserver.contentDir"))
def __init__(self, fn = ''):
if fn:
self.conf = conftree.ConfSimple(fn)
uplog("Minim config read: contentDir: %s" %
self.conf.get("minimserver.contentDir"))
else:
self.conf = conftree.ConfSimple('/dev/null')
self.quotes = "\"'"
self.escape = ''
self.whitespace = ', '
......@@ -108,5 +111,21 @@ class MinimConfig(object):
def getsimplevalue(self, nm):
return self.conf.get(nm).strip()
s = self.conf.get(nm)
if s:
return s.strip()
else:
return s
def getboolvalue(self, nm, dflt):
val = self.getsimplevalue(nm)
if val is None or val == '':
return dflt
if val == '0' or val.lower()[0] == 'f' or val.lower()[0] == 'n':
return False
else:
return True
......@@ -97,6 +97,9 @@ class Folders(object):
self._maxrclcnt = 0
self._fetchalldocs(confdir)
self._rcl2folders(confdir)
self._enabletags = uprclinit.g_minimconfig.getboolvalue("showExtras",
True)
def rcldocs(self):
return self._rcldocs
......@@ -311,42 +314,30 @@ class Folders(object):
rcldb = recoll.connect(confdir=confdir)
rclq = rcldb.query()
rclq.execute("mime:*", stemming=0)
#rclq.execute('album:a*', stemming=0)
#rclq.execute('album:a* OR album:b* OR album:c*', stemming=0)
uplog("Estimated alldocs query results: %d" % (rclq.rowcount))
tagaliases = None
if uprclinit.g_minimconfig:
tagaliases = uprclinit.g_minimconfig.gettagaliases()
totcnt = 0
self._rcldocs = []
while True:
# There are issues at the end of list with fetchmany (sets
# an exception). Works in python2 for some reason, but
# breaks p3. Until recoll is fixed, catch exception
# here. Also does not work if we try to fetch by bigger
# slices (we get an exception and a truncated list)
try:
docs = rclq.fetchmany()
for doc in docs:
if tagaliases:
for orig,target,rep in tagaliases:
val = doc.get(orig)
#uplog("Rep %s doc[%s]=[%s] doc[%s]=[%s]"%
# (rep, orig, val, target, doc.get(target)))
if val and (rep or not doc.get(target)):
setattr(doc,target,val)
self._rcldocs.append(doc)
totcnt += 1
except:
docs = []
if (self._maxrclcnt > 0 and totcnt >= self._maxrclcnt) or \
len(docs) != rclq.arraysize:
for doc in rclq:
if tagaliases:
for orig, target, rep in tagaliases:
val = doc.get(orig)
#uplog("Rep %s doc[%s]=[%s] doc[%s]=[%s]"%
# (rep, orig, val, target, doc.get(target)))
if val and (rep or not doc.get(target)):
setattr(doc, target, val)
self._rcldocs.append(doc)
if self._maxrclcnt > 0 and len(self._rcldocs) >= self._maxrclcnt:
break
time.sleep(0)
end = timer()
uplog("Retrieved %d docs in %.2f Seconds" % (totcnt,end - start))
uplog("Retrieved %d docs in %.2f Seconds" %
(len(self._rcldocs), end - start))
##############
......@@ -437,7 +428,7 @@ class Folders(object):
entries = []
# Add "Browse subtree by tags" entry
if pid != self._idprefix:
if pid != self._idprefix and self._enabletags:
id = pid + '$tagview.0'
entries.append(rcldirentry(id, pid, ">> Tag View"))
......
......@@ -31,7 +31,7 @@ import uprclinit
@bottle.view('main')
def main():
what = bottle.request.forms.get('what')
uplog("Main: what value is %s" % what)
#uplog("bottle:main: what value is %s" % what)
status = uprclinit.updaterunning()
if not status:
......
......@@ -42,9 +42,7 @@ def _maybeinitconfdir(confdir, topdirs):
src = os.path.join(datadir, "rclconfig-" + fn)
shutil.copyfile(src, dst)
exclpats = ""
if uprclinit.g_minimconfig:
exclpats = uprclinit.g_minimconfig.getexcludepatterns()
exclpats = uprclinit.g_minimconfig.getexcludepatterns()
userconfig = uprclinit.g_upconfig.get("uprclconfrecolluser")
if not userconfig:
......@@ -59,6 +57,8 @@ def _maybeinitconfdir(confdir, topdirs):
f.write(b"topdirs = %s\n"% topdirs.encode(locale.getpreferredencoding()))
f.write(b"idxabsmlen = 0\n")
f.write(b"loglevel = 2\n")
#f.write(b"idxlogfilename = /tmp/loguprcl.txt\n")
#f.write(b"pylogfilename = /tmp/logpyuprcl.txt\n")
f.write(b"noaspell = 1\n")
f.write(b"nomd5types = rclaudio rclimg\n")
f.write(b"testmodifusemtime = 1\n")
......
......@@ -81,7 +81,7 @@ def _update_index(rebuild=False):
folders = Folders(g_rclconfdir, g_httphp, g_pathprefix)
untagged = Untagged(folders.rcldocs(), g_httphp, g_pathprefix)
playlists = Playlists(folders, g_httphp, g_pathprefix)
playlists = Playlists(folders.rcldocs(), g_httphp, g_pathprefix)
tagged = Tagged(folders.rcldocs(), g_httphp, g_pathprefix)
newtrees = {}
newtrees['folders'] = folders
......@@ -121,8 +121,7 @@ def _uprcl_init_worker():
global g_minimconfig
minimcfn = g_upconfig.get("uprclminimconfig")
if minimcfn:
g_minimconfig = minimconfig.MinimConfig(minimcfn)
g_minimconfig = minimconfig.MinimConfig(minimcfn)
global g_httphp
g_httphp = g_upconfig.get("uprclhostport")
......@@ -140,10 +139,9 @@ def _uprcl_init_worker():
global g_rcltopdirs
g_rcltopdirs = g_upconfig.get("uprclmediadirs")
if not g_rcltopdirs:
if g_minimconfig:
g_rcltopdirs = g_minimconfig.getcontentdirs()
if g_rcltopdirs:
g_rcltopdirs = conftree.stringsToString(g_rcltopdirs)
g_rcltopdirs = g_minimconfig.getcontentdirs()
if g_rcltopdirs:
g_rcltopdirs = conftree.stringsToString(g_rcltopdirs)
if not g_rcltopdirs:
raise Exception("uprclmediadirs not in config")
......
......@@ -27,26 +27,25 @@ PY3 = sys.version > '3'
from upmplgutils import uplog
from uprclutils import rcldoctoentry, rcldirentry, cmpentries
import uprclutils
import uprclinit
from recoll import recoll
class Playlists(object):
def __init__(self, folders, httphp, pathprefix):
def __init__(self, rcldocs, httphp, pathprefix):
self._idprefix = '0$uprcl$playlists'
self._httphp = httphp
self._pprefix = pathprefix
self._folders = folders
self.recoll2playlists(self._folders.rcldocs())
self.recoll2playlists(rcldocs)
# Create the untagged entries static vector by filtering the global
# doc vector, storing the indexes of the playlists.
# We keep a reference to the doc vector.
def recoll2playlists(self, docs):
self.rcldocs = docs
def recoll2playlists(self, rcldocs):
# The -1 entry is because we use index 0 for our root.
self.utidx = [-1]
for docidx in range(len(docs)):
doc = docs[docidx]
for docidx in range(len(rcldocs)):
doc = rcldocs[docidx]
if doc.mtype == 'audio/x-mpegurl':
self.utidx.append(docidx)
......@@ -56,9 +55,6 @@ class Playlists(object):
if not pid.startswith(self._idprefix):
raise Exception("playlists:browse: bad pid %s" % pid)
if len(self.rcldocs) == 0:
raise Exception("playlists:browse: no docs")
idx = pid[len(self._idprefix):]
if not idx:
# Browsing the root.
......@@ -85,11 +81,12 @@ class Playlists(object):
def browse(self, pid, flag):
idx = self._objidtoidx(pid)
rcldocs = uprclinit.g_trees['folders'].rcldocs()
entries = []
if idx == 0:
# Browsing root
for i in range(len(self.utidx))[1:]:
doc = self.rcldocs[self.utidx[i]]
doc = rcldocs[self.utidx[i]]
id = self._idprefix + '$p' + str(i)
title = doc.title if doc.title else doc.filename
e = rcldirentry(id, pid, title,
......@@ -98,7 +95,7 @@ class Playlists(object):
entries.append(e)
else:
pldoc = self.rcldocs[self.utidx[idx]]
pldoc = rcldocs[self.utidx[idx]]
plpath = uprclutils.docpath(pldoc)
#uplog("playlists: plpath %s" % plpath)
try:
......@@ -121,7 +118,7 @@ class Playlists(object):
if docidx < 0:
uplog("playlists: can't stat %s"%doc.getbinurl())
continue
doc = self._folders.rcldocs()[docidx]
doc = rcldocs[docidx]
id = pid + '$e' + str(len(entries))
e = rcldoctoentry(id, pid, self._httphp, self._pprefix, doc)
......
......@@ -37,13 +37,12 @@ from uprcltagscreate import recolltosql, _clid, g_tagtotable, \
# The browseable object which defines the tree of tracks organized by tags.
class Tagged(object):
def __init__(self, docs, httphp, pathprefix):
def __init__(self, rcldocs, httphp, pathprefix):
self._httphp = httphp
self._pprefix = pathprefix
self._conn = None
self._rcldocs = docs
self._init_sqconn()
recolltosql(self._conn, docs)
recolltosql(self._conn, rcldocs)
def _init_sqconn(self):
......@@ -66,22 +65,21 @@ class Tagged(object):
def rootentries(self, pid, path=''):
uplog("rootentries: pid %s path %s" % (pid, path))
entries = []
args = ()
nalbs = self._albcntforfolder(path)
entries.append(rcldirentry(pid + 'albums', pid, nalbs + ' albums'))
if path:
pthcl = 'path LIKE ?'
where = ' WHERE tracks.path LIKE ? '
args = (path + '%',)
else:
where = ' '
args = ()
c = self._conn.cursor()
stmt = "SELECT COUNT(*) from tracks"
if path:
stmt += " WHERE " + pthcl
uplog("Executing: %s" % stmt)
c.execute(stmt, args)
c.execute(stmt+where, args)
nitems = str(c.fetchone()[0])
entries.append(rcldirentry(pid + 'items', pid, nitems + ' items'))
for tt in sorted(g_tagdisplaytag.keys()):
subqs = self._subtreetags(where, args)
for tt in subqs:
entries.append(rcldirentry(pid + '=' + tt , pid,
g_tagdisplaytag[tt]))
return entries
......@@ -95,12 +93,12 @@ class Tagged(object):
tb = g_tagtotable[tt]
stmt = '''SELECT COUNT(DISTINCT %s) FROM tracks %s''' % \
(_clid(tb), where)
uplog("subtreetags: stmt:: [%s]" % stmt)
#uplog("subtreetags: stmt: [%s]" % stmt)
c.execute(stmt, values)
cnt = c.fetchone()[0]
if len(stmt) > 80:
stmt = stmt[:80] + "..."
uplog("subtreetags: %d values for %s (%s)"%(cnt,tb,stmt))
uplog("subtreetags: %d values for %s (%s,%s)"%(cnt,tb,stmt,values))
if cnt > 1:
tags.append(tt)
return tags
......@@ -109,11 +107,12 @@ class Tagged(object):
# Build a list of track directory entries for an SQL statement
# which selects docidxs (SELECT docidx,... FROM tracks WHERE...)
def _trackentriesforstmt(self, stmt, values, pid):
rcldocs = uprclinit.g_trees['folders'].rcldocs()
c = self._conn.cursor()
c.execute(stmt, values)
entries = [rcldoctoentry(pid + '$i' + str(r[0]),
pid, self._httphp, self._pprefix,
self._rcldocs[r[0]]) for r in c]
rcldocs[r[0]]) for r in c]
if PY3:
return sorted(entries, key=cmpentries)
else:
......@@ -346,9 +345,10 @@ class Tagged(object):
el[0]['id'] = id
entries.append(el[0])
if displaytracks:
rcldocs = uprclinit.g_trees['folders'].rcldocs()
entries += [rcldoctoentry(pid + '$i' + str(docid),
pid, self._httphp, self._pprefix,
self._rcldocs[docid]) for docid in docids]
rcldocs[docid]) for docid in docids]
if PY3:
return sorted(entries, key=cmpentries)
else:
......@@ -447,11 +447,12 @@ class Tagged(object):
id = pid + '$=' + tt
entries.append(rcldirentry(id, pid, g_tagdisplaytag[tt]))
elif displaytracks:
rcldocs = uprclinit.g_trees['folders'].rcldocs()
for docidx in docids:
id = pid + '$*i' + str(docidx)
entries.append(rcldoctoentry(id, pid, self._httphp,
self._pprefix,
self._rcldocs[docidx]))
rcldocs[docidx]))
if PY3:
entries = sorted(entries, key=cmpentries)
else:
......@@ -497,7 +498,7 @@ class Tagged(object):
uplog("Tags:browsFolder: qpath %s"%qpath)
if qpath[0] == 'items':
args = (folder + '%',) if folder else ()
folderwhere = ' WHERE path LIKE ? ' if folder else ' '
folderwhere = ' WHERE tracks.path LIKE ? ' if folder else ' '
stmt = 'SELECT docidx FROM tracks' + folderwhere
entries = self._trackentriesforstmt(stmt, args, pid)
elif qpath[0] == 'albums':
......
......@@ -142,11 +142,8 @@ def _prepareTags():
global g_tagtotable
global g_indextags
indextagsp = []
itemtags = []
if uprclinit.g_minimconfig:
indextagsp = uprclinit.g_minimconfig.getindextags()
itemtags = uprclinit.g_minimconfig.getitemtags()
indextagsp = uprclinit.g_minimconfig.getindextags()
itemtags = uprclinit.g_minimconfig.getitemtags()
if not indextagsp:
indextagsp = [('Artist',''), ('Date',''), ('Genre',''), ('Composer','')]
......@@ -441,7 +438,7 @@ def _createmergedalbums(conn):
# Create the db and fill it up with the values we need, taken out of
# the recoll records list
def recolltosql(conn, docs):
def recolltosql(conn, rcldocs):
start = timer()
_createsqdb(conn)
......@@ -450,8 +447,8 @@ def recolltosql(conn, docs):
maxcnt = 0
totcnt = 0
c = conn.cursor()
for docidx in range(len(docs)):
doc = docs[docidx]
for docidx in range(len(rcldocs)):
doc = rcldocs[docidx]
totcnt += 1
if totcnt % 1000 == 0:
......
......@@ -33,24 +33,24 @@ import sys
from upmplgutils import uplog
from uprclutils import rcldoctoentry, rcldirentry
import uprclinit
class Untagged(object):
def __init__(self, docs, httphp, pathprefix):
def __init__(self, rcldocs, httphp, pathprefix):
self._idprefix = '0$uprcl$untagged'
self._httphp = httphp
self._pprefix = pathprefix
self.recoll2untagged(docs)
self.recoll2untagged(rcldocs)
# Create the untagged entries static vector by filtering the global
# doc vector, storing the indexes of all tracks without a title
# field. We keep a reference to the doc vector.
def recoll2untagged(self, docs):
self.rcldocs = docs
def recoll2untagged(self, rcldocs):
# The -1 entry is because we use index 0 for our root.
self.utidx = [-1]
for docidx in range(len(docs)):
doc = docs[docidx]
for docidx in range(len(rcldocs)):
doc = rcldocs[docidx]
if doc.mtype == 'inode/directory' or doc.mtype == 'audio/x-mpegurl':
continue
if not doc.title:
......@@ -61,9 +61,6 @@ class Untagged(object):
if not pid.startswith(self._idprefix):
raise Exception("untagged:browse: bad pid %s" % pid)
if len(self.rcldocs) == 0:
raise Exception("untagged:browse: no docs")
idx = pid[len(self._idprefix):]
if not idx:
# Browsing the root.
......@@ -90,17 +87,18 @@ class Untagged(object):
idx = self._objidtoidx(pid)
entries = []
rcldocs = uprclinit.g_trees['folders'].rcldocs()
if idx == 0:
# Browsing root
for i in range(len(self.utidx))[1:]:
doc = self.rcldocs[self.utidx[i]]
doc = rcldocs[self.utidx[i]]
id = self._idprefix + '$u' + str(i)
e = rcldoctoentry(id, pid, self._httphp, self._pprefix, doc)
if e:
entries.append(e)
else:
# Non root: only items in there. flag needs to be 'meta'
doc = self.rcldocs[idx]
doc = rcldocs[idx]
id = self._idprefix + '$u' + str(idx)
e = rcldoctoentry(id, pid, self._httphp, self._pprefix, doc)
if e:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment