Commit 31292246 authored by Jean-Francois Dockes's avatar Jean-Francois Dockes
Browse files

uprcl: fix init taking more time with each repetition because of adding trees...

uprcl: fix init taking more time with each repetition because of adding trees each time. Replace some global variables refs with method calls
parent f8bc38a4
...@@ -26,7 +26,7 @@ import uprclsearch ...@@ -26,7 +26,7 @@ import uprclsearch
import uprclindex import uprclindex
from upmplgutils import uplog, setidprefix from upmplgutils import uplog, setidprefix
from uprclutils import uplog, g_myprefix, rcldirentry, waitentry from uprclutils import rcldirentry, waitentry
import uprclinit import uprclinit
setidprefix("uprcl") setidprefix("uprcl")
...@@ -75,8 +75,8 @@ def _rootentries(): ...@@ -75,8 +75,8 @@ def _rootentries():
# let the different modules return their stuff, and we take note # let the different modules return their stuff, and we take note
# of the objid prefixes for later dispatching # of the objid prefixes for later dispatching
entries = [] entries = []
for treename in uprclinit.g_trees_order: for treename in uprclinit.getTreesOrder():
nents = uprclinit.g_trees[treename].rootentries(g_myprefix) nents = uprclinit.getTree(treename).rootentries(uprclinit.getObjPrefix())
for e in nents: for e in nents:
rootmap[e['id']] = treename rootmap[e['id']] = treename
entries += nents entries += nents
...@@ -88,14 +88,14 @@ def _browsedispatch(objid, bflg, offset, count): ...@@ -88,14 +88,14 @@ def _browsedispatch(objid, bflg, offset, count):
for id,treename in rootmap.items(): for id,treename in rootmap.items():
#uplog("Testing %s against %s" % (objid, id)) #uplog("Testing %s against %s" % (objid, id))
if objid.startswith(id): if objid.startswith(id):
return uprclinit.g_trees[treename].browse(objid, bflg, return uprclinit.getTree(treename).browse(objid, bflg,
offset, count) offset, count)
raise Exception("Browse: dispatch: bad objid not in rootmap: [%s]" % objid) raise Exception("Browse: dispatch: bad objid not in rootmap: [%s]" % objid)
@dispatcher.record('browse') @dispatcher.record('browse')
def browse(a): def browse(a):
msgproc.log("browse: %s. g_httphp [%s]" % (a, uprclinit.g_httphp)) msgproc.log("browse: %s. httphp [%s]" % (a, uprclinit.getHttphp()))
if 'objid' not in a: if 'objid' not in a:
raise Exception("No objid in args") raise Exception("No objid in args")
...@@ -109,10 +109,10 @@ def browse(a): ...@@ -109,10 +109,10 @@ def browse(a):
if 'count' in a: if 'count' in a:
count = int(a['count']) count = int(a['count'])
if not objid.startswith(g_myprefix): if not objid.startswith(uprclinit.getObjPrefix()):
raise Exception("bad objid <%s>" % objid) raise Exception("bad objid <%s>" % objid)
idpath = objid.replace(g_myprefix, '', 1) idpath = objid.replace(uprclinit.getObjPrefix(), '', 1)
msgproc.log("browse: idpath: <%s>" % idpath) msgproc.log("browse: idpath: <%s>" % idpath)
entries = [] entries = []
...@@ -122,8 +122,7 @@ def browse(a): ...@@ -122,8 +122,7 @@ def browse(a):
else: else:
try: try:
if not uprclinit.ready(): if not uprclinit.ready():
entries = [waitentry(objid + 'notready', objid, entries = [waitentry(objid + 'notready', objid, uprclinit.getHttphp()),]
uprclinit.g_httphp),]
elif not idpath: elif not idpath:
entries = _rootentries() entries = _rootentries()
else: else:
...@@ -157,12 +156,11 @@ def search(a): ...@@ -157,12 +156,11 @@ def search(a):
try: try:
if not uprclinit.ready(): if not uprclinit.ready():
entries = [waitentry(objid + 'notready', objid, entries = [waitentry(objid + 'notready', objid, uprclinit.getHttphp()),]
uprclinit.g_httphp),]
else: else:
entries = uprclsearch.search( entries = uprclsearch.search(
uprclinit.g_trees['folders'], uprclinit.g_rclconfdir, objid, uprclinit.getTree('folders'), uprclinit.getRclConfdir(), objid,
upnps, g_myprefix, uprclinit.g_httphp, uprclinit.g_pathprefix) upnps, uprclinit.getObjPrefix(), uprclinit.getHttphp(), uprclinit.getPathPrefix())
finally: finally:
uprclinit.g_dblock.release_read() uprclinit.g_dblock.release_read()
......
...@@ -476,8 +476,8 @@ class Folders(object): ...@@ -476,8 +476,8 @@ class Folders(object):
if pthremain: if pthremain:
if not pthremain.find('$tagview.0') == 0: if not pthremain.find('$tagview.0') == 0:
raise Exception("uprclfolders: pid [%s]. bad pthremain"%pid) raise Exception("uprclfolders: pid [%s]. bad pthremain"%pid)
return uprclinit.g_trees['tags'].browseFolder(pid, flag, pthremain, return uprclinit.getTree('tags').browseFolder(pid, flag, pthremain,
self.dirpath(pid)) self.dirpath(pid))
# If there is only one entry in root, skip it. This means that 0 # If there is only one entry in root, skip it. This means that 0
# and 1 point to the same dir, but this does not seem to be an # and 1 point to the same dir, but this does not seem to be an
......
...@@ -53,7 +53,7 @@ def main(): ...@@ -53,7 +53,7 @@ def main():
reloadsecs = '' reloadsecs = ''
return {'title':status, 'status':status, 'reloadsecs':reloadsecs, return {'title':status, 'status':status, 'reloadsecs':reloadsecs,
'friendlyname':uprclinit.g_friendlyname} 'friendlyname':uprclinit.getFriendlyname()}
@bottle.route('/static/<filepath:path>') @bottle.route('/static/<filepath:path>')
......
...@@ -35,19 +35,38 @@ import minimconfig ...@@ -35,19 +35,38 @@ import minimconfig
from upmplgutils import uplog, findmyip, getcachedir from upmplgutils import uplog, findmyip, getcachedir
from conftree import stringToStrings from conftree import stringToStrings
# Once initialization (not on imports) _g_pathprefix = ""
try: _g_httphp = ""
_s = g_httphp g_dblock = ReadWriteLock()
except: _g_rclconfdir = ""
# The recoll documents _g_friendlyname = "UpMpd-mediaserver"
g_pathprefix = "" _g_trees = {}
g_httphp = "" _g_trees_order = ['folders', 'playlists', 'tags', 'untagged']
g_dblock = ReadWriteLock() g_minimconfig = None
g_rclconfdir = "" # Prefix for object Ids. This must be consistent with what
g_friendlyname = "UpMpd-mediaserver" # contentdirectory.cxx does
g_trees = {} _g_myprefix = '0$uprcl$'
g_trees_order = ['folders', 'playlists', 'tags', 'untagged']
g_minimconfig = None def getObjPrefix():
return _g_myprefix
def getPathPrefix():
return _g_pathprefix
def getHttphp():
return _g_httphp
def getRclConfdir():
return _g_rclconfdir
def getFriendlyname():
return _g_friendlyname
def getTree(treename):
return _g_trees[treename]
def getTreesOrder():
return _g_trees_order
def _reset_index(): def _reset_index():
_update_index(True) _update_index(True)
...@@ -56,14 +75,14 @@ def _reset_index(): ...@@ -56,14 +75,14 @@ def _reset_index():
# runs in the separate uprcl_init_worker thread, and signals # runs in the separate uprcl_init_worker thread, and signals
# startup/completion by setting/unsetting the g_initrunning flag # startup/completion by setting/unsetting the g_initrunning flag
def _update_index(rebuild=False): def _update_index(rebuild=False):
uplog("Creating/updating index in %s for %s" % (g_rclconfdir, g_rcltopdirs)) uplog("Creating/updating index in %s for %s" % (_g_rclconfdir, g_rcltopdirs))
# We take the writer lock, making sure that no browse/search # We take the writer lock, making sure that no browse/search
# thread are active, then set the busy flag and release the # thread are active, then set the busy flag and release the
# lock. This allows future browse operations to signal the # lock. This allows future browse operations to signal the
# condition to the user instead of blocking (if we kept the write # condition to the user instead of blocking (if we kept the write
# lock). # lock).
global g_initrunning, g_trees global g_initrunning, _g_trees
g_dblock.acquire_write() g_dblock.acquire_write()
g_initrunning = "Rebuilding" if rebuild else "Updating" g_initrunning = "Rebuilding" if rebuild else "Updating"
g_dblock.release_write() g_dblock.release_write()
...@@ -71,23 +90,23 @@ def _update_index(rebuild=False): ...@@ -71,23 +90,23 @@ def _update_index(rebuild=False):
try: try:
start = timer() start = timer()
uprclindex.runindexer(g_rclconfdir, g_rcltopdirs, rebuild=rebuild) uprclindex.runindexer(_g_rclconfdir, g_rcltopdirs, rebuild=rebuild)
# Wait for indexer # Wait for indexer
while not uprclindex.indexerdone(): while not uprclindex.indexerdone():
time.sleep(.5) time.sleep(.5)
fin = timer() fin = timer()
uplog("Indexing took %.2f Seconds" % (fin - start)) uplog("Indexing took %.2f Seconds" % (fin - start))
folders = Folders(g_rclconfdir, g_httphp, g_pathprefix) folders = Folders(_g_rclconfdir, _g_httphp, _g_pathprefix)
untagged = Untagged(folders.rcldocs(), g_httphp, g_pathprefix) untagged = Untagged(folders.rcldocs(), _g_httphp, _g_pathprefix)
playlists = Playlists(folders.rcldocs(), g_httphp, g_pathprefix) playlists = Playlists(folders.rcldocs(), _g_httphp, _g_pathprefix)
tagged = Tagged(folders.rcldocs(), g_httphp, g_pathprefix) tagged = Tagged(folders.rcldocs(), _g_httphp, _g_pathprefix)
newtrees = {} newtrees = {}
newtrees['folders'] = folders newtrees['folders'] = folders
newtrees['untagged'] = untagged newtrees['untagged'] = untagged
newtrees['playlists'] = playlists newtrees['playlists'] = playlists
newtrees['tags'] = tagged newtrees['tags'] = tagged
g_trees = newtrees _g_trees = newtrees
finally: finally:
g_dblock.acquire_write() g_dblock.acquire_write()
g_initrunning = False g_initrunning = False
...@@ -102,18 +121,18 @@ def _uprcl_init_worker(): ...@@ -102,18 +121,18 @@ def _uprcl_init_worker():
####### #######
# Acquire configuration data. # Acquire configuration data.
global g_pathprefix global _g_pathprefix
# pathprefix would typically be something like "/uprcl". It's used # pathprefix would typically be something like "/uprcl". It's used
# for dispatching URLs to the right plugin for processing. We # for dispatching URLs to the right plugin for processing. We
# strip it whenever we need a real file path # strip it whenever we need a real file path
if "UPMPD_PATHPREFIX" not in os.environ: if "UPMPD_PATHPREFIX" not in os.environ:
raise Exception("No UPMPD_PATHPREFIX in environment") raise Exception("No UPMPD_PATHPREFIX in environment")
g_pathprefix = os.environ["UPMPD_PATHPREFIX"] _g_pathprefix = os.environ["UPMPD_PATHPREFIX"]
if "UPMPD_CONFIG" not in os.environ: if "UPMPD_CONFIG" not in os.environ:
raise Exception("No UPMPD_CONFIG in environment") raise Exception("No UPMPD_CONFIG in environment")
global g_friendlyname global _g_friendlyname
if "UPMPD_FNAME" in os.environ: if "UPMPD_FNAME" in os.environ:
g_friendlyname = os.environ["UPMPD_FNAME"] _g_friendlyname = os.environ["UPMPD_FNAME"]
global g_upconfig global g_upconfig
g_upconfig = conftree.ConfSimple(os.environ["UPMPD_CONFIG"]) g_upconfig = conftree.ConfSimple(os.environ["UPMPD_CONFIG"])
...@@ -122,20 +141,20 @@ def _uprcl_init_worker(): ...@@ -122,20 +141,20 @@ def _uprcl_init_worker():
minimcfn = g_upconfig.get("uprclminimconfig") minimcfn = g_upconfig.get("uprclminimconfig")
g_minimconfig = minimconfig.MinimConfig(minimcfn) g_minimconfig = minimconfig.MinimConfig(minimcfn)
global g_httphp global _g_httphp
g_httphp = g_upconfig.get("uprclhostport") _g_httphp = g_upconfig.get("uprclhostport")
if g_httphp is None: if _g_httphp is None:
ip = findmyip() ip = findmyip()
g_httphp = ip + ":" + "9090" _g_httphp = ip + ":" + "9090"
uplog("uprclhostport not in config, using %s" % g_httphp) uplog("uprclhostport not in config, using %s" % _g_httphp)
global g_rclconfdir global _g_rclconfdir
g_rclconfdir = g_upconfig.get("uprclconfdir") _g_rclconfdir = g_upconfig.get("uprclconfdir")
if g_rclconfdir: if _g_rclconfdir:
os.makedirs(g_rclconfdir) os.makedirs(_g_rclconfdir)
else: else:
g_rclconfdir = getcachedir(g_upconfig, "uprcl") _g_rclconfdir = getcachedir(g_upconfig, "uprcl")
uplog("uprcl: cachedir: %s" % g_rclconfdir) uplog("uprcl: cachedir: %s" % _g_rclconfdir)
global g_rcltopdirs global g_rcltopdirs
g_rcltopdirs = g_upconfig.get("uprclmediadirs") g_rcltopdirs = g_upconfig.get("uprclmediadirs")
...@@ -161,7 +180,7 @@ def _uprcl_init_worker(): ...@@ -161,7 +180,7 @@ def _uprcl_init_worker():
l = ptt.split(':') l = ptt.split(':')
pathmap[l[0]] = l[1] pathmap[l[0]] = l[1]
host,port = g_httphp.split(':') host,port = _g_httphp.split(':')
# Start the bottle app. Its' both the control/config interface and # Start the bottle app. Its' both the control/config interface and
# the file streamer # the file streamer
...@@ -169,7 +188,7 @@ def _uprcl_init_worker(): ...@@ -169,7 +188,7 @@ def _uprcl_init_worker():
kwargs = {'host':host , kwargs = {'host':host ,
'port':int(port), 'port':int(port),
'pthstr':pthstr, 'pthstr':pthstr,
'pathprefix':g_pathprefix}) 'pathprefix':_g_pathprefix})
httpthread.daemon = True httpthread.daemon = True
httpthread.start() httpthread.start()
......
...@@ -80,7 +80,7 @@ class Playlists(object): ...@@ -80,7 +80,7 @@ class Playlists(object):
def browse(self, pid, flag, offset, count): def browse(self, pid, flag, offset, count):
idx = self._objidtoidx(pid) idx = self._objidtoidx(pid)
folders = uprclinit.g_trees['folders'] folders = uprclinit.getTree('folders')
rcldocs = folders.rcldocs() rcldocs = folders.rcldocs()
entries = [] entries = []
if idx == 0: if idx == 0:
......
...@@ -29,10 +29,10 @@ import time ...@@ -29,10 +29,10 @@ import time
import tempfile import tempfile
from upmplgutils import uplog from upmplgutils import uplog
from uprclutils import g_myprefix, rcldirentry, rcldoctoentry, cmpentries from uprclutils import rcldirentry, rcldoctoentry, cmpentries
import uprclinit import uprclinit
from uprcltagscreate import recolltosql, _clid, g_tagtotable, \ import uprcltagscreate
g_tagdisplaytag, g_indextags from uprcltagscreate import _clid, recolltosql
# The browseable object which defines the tree of tracks organized by tags. # The browseable object which defines the tree of tracks organized by tags.
class Tagged(object): class Tagged(object):
...@@ -85,9 +85,9 @@ class Tagged(object): ...@@ -85,9 +85,9 @@ class Tagged(object):
nitems = str(c.fetchone()[0]) nitems = str(c.fetchone()[0])
entries.append(rcldirentry(pid + 'items', pid, nitems + ' items')) entries.append(rcldirentry(pid + 'items', pid, nitems + ' items'))
subqs = self._subtreetags(where, args) subqs = self._subtreetags(where, args)
tagdisplaytag = uprcltagscreate.getTagDisplayTag()
for tt in subqs: for tt in subqs:
entries.append(rcldirentry(pid + '=' + tt , pid, entries.append(rcldirentry(pid + '=' + tt , pid, tagdisplaytag[tt]))
g_tagdisplaytag[tt]))
return entries return entries
...@@ -95,8 +95,10 @@ class Tagged(object): ...@@ -95,8 +95,10 @@ class Tagged(object):
def _subtreetags(self, where, values): def _subtreetags(self, where, values):
c = self._conn.cursor() c = self._conn.cursor()
tags = [] tags = []
for tt in g_indextags: indextags = uprcltagscreate.getIndexTags()
tb = g_tagtotable[tt] tagtotable = uprcltagscreate.getTagToTable()
for tt in indextags:
tb = tagtotable[tt]
stmt = '''SELECT COUNT(DISTINCT %s) FROM tracks %s''' % \ stmt = '''SELECT COUNT(DISTINCT %s) FROM tracks %s''' % \
(_clid(tb), where) (_clid(tb), where)
#uplog("subtreetags: stmt: [%s]" % stmt) #uplog("subtreetags: stmt: [%s]" % stmt)
...@@ -149,7 +151,7 @@ class Tagged(object): ...@@ -149,7 +151,7 @@ class Tagged(object):
stmt += " LIMIT %d " % count stmt += " LIMIT %d " % count
if offset != 0: if offset != 0:
stmt += " OFFSET %d " % offset stmt += " OFFSET %d " % offset
rcldocs = uprclinit.g_trees['folders'].rcldocs() rcldocs = uprclinit.getTree('folders').rcldocs()
c = self._conn.cursor() c = self._conn.cursor()
c.execute(stmt, values) c.execute(stmt, values)
entries = [rcldoctoentry(pid + '$i' + str(r[0]), entries = [rcldoctoentry(pid + '$i' + str(r[0]),
...@@ -388,7 +390,7 @@ class Tagged(object): ...@@ -388,7 +390,7 @@ class Tagged(object):
el[0]['id'] = id el[0]['id'] = id
entries.append(el[0]) entries.append(el[0])
if displaytracks: if displaytracks:
rcldocs = uprclinit.g_trees['folders'].rcldocs() rcldocs = uprclinit.getTree('folders').rcldocs()
entries += [rcldoctoentry(pid + '$i' + str(docid), entries += [rcldoctoentry(pid + '$i' + str(docid),
pid, self._httphp, self._pprefix, pid, self._httphp, self._pprefix,
rcldocs[docid]) for docid in docids] rcldocs[docid]) for docid in docids]
...@@ -401,6 +403,8 @@ class Tagged(object): ...@@ -401,6 +403,8 @@ class Tagged(object):
def _tagsbrowse(self, pid, qpath, flag, path=''): def _tagsbrowse(self, pid, qpath, flag, path=''):
uplog("tagsbrowse. pid %s qpath %s" % (pid, qpath)) uplog("tagsbrowse. pid %s qpath %s" % (pid, qpath))
tagdisplaytag = uprcltagscreate.getTagDisplayTag()
# Walk the qpath, which was generated from the objid and # Walk the qpath, which was generated from the objid and
# defines what tracks are selected and what we want to # defines what tracks are selected and what we want to
# display. E.g =Artist$21$=Date would display all distinct # display. E.g =Artist$21$=Date would display all distinct
...@@ -414,6 +418,7 @@ class Tagged(object): ...@@ -414,6 +418,7 @@ class Tagged(object):
else: else:
selwhere = '' selwhere = ''
values = [] values = []
tagtotable = uprcltagscreate.getTagToTable()
i = 0 i = 0
while i < qlen: while i < qlen:
elt = qpath[i] elt = qpath[i]
...@@ -430,7 +435,7 @@ class Tagged(object): ...@@ -430,7 +435,7 @@ class Tagged(object):
# in different ways depending if this is the last element or # in different ways depending if this is the last element or
# not. # not.
if elt.startswith('='): if elt.startswith('='):
col = g_tagtotable[elt[1:]] col = tagtotable[elt[1:]]
selwhere = selwhere + ' AND ' if selwhere else ' WHERE ' selwhere = selwhere + ' AND ' if selwhere else ' WHERE '
if i == qlen - 1: if i == qlen - 1:
...@@ -485,9 +490,9 @@ class Tagged(object): ...@@ -485,9 +490,9 @@ class Tagged(object):
entries.append(rcldirentry(id, pid, label % len(docids))) entries.append(rcldirentry(id, pid, label % len(docids)))
for tt in subqs: for tt in subqs:
id = pid + '$=' + tt id = pid + '$=' + tt
entries.append(rcldirentry(id, pid, g_tagdisplaytag[tt])) entries.append(rcldirentry(id, pid, tagdisplaytag[tt]))
elif displaytracks: elif displaytracks:
rcldocs = uprclinit.g_trees['folders'].rcldocs() rcldocs = uprclinit.getTree('folders').rcldocs()
for docidx in docids: for docidx in docids:
id = pid + '$*i' + str(docidx) id = pid + '$*i' + str(docidx)
entries.append(rcldoctoentry(id, pid, self._httphp, entries.append(rcldoctoentry(id, pid, self._httphp,
...@@ -563,7 +568,7 @@ class Tagged(object): ...@@ -563,7 +568,7 @@ class Tagged(object):
# Top level browse routine. Handle the special cases and call the # Top level browse routine. Handle the special cases and call the
# appropriate worker routine. # appropriate worker routine.
def browse(self, pid, flag, offset, count): def browse(self, pid, flag, offset, count):
idpath = pid.replace(g_myprefix, '', 1) idpath = pid.replace(uprclinit.getObjPrefix(), '', 1)
uplog('tags:browse: idpath <%s>' % idpath) uplog('tags:browse: idpath <%s>' % idpath)
qpath = idpath.split('$') qpath = idpath.split('$')
return self._dobrowse(pid, flag, qpath, offset=offset, count=count) return self._dobrowse(pid, flag, qpath, offset=offset, count=count)
......
...@@ -66,6 +66,13 @@ g_indextags = [] ...@@ -66,6 +66,13 @@ g_indextags = []
g_tagdisplaytag = {} g_tagdisplaytag = {}
g_tagtotable = {} g_tagtotable = {}
def getIndexTags():
return g_indextags
def getTagDisplayTag():
return g_tagdisplaytag
def getTagToTable():
return g_tagtotable
def _clid(table): def _clid(table):
return table + '_id' return table + '_id'
...@@ -158,6 +165,10 @@ def _prepareTags(): ...@@ -158,6 +165,10 @@ def _prepareTags():
global g_tagtotable global g_tagtotable
global g_indextags global g_indextags
g_indextags = []
g_tagdisplaytag = {}
g_tagtotable = {}
indextagsp = uprclinit.g_minimconfig.getindextags() indextagsp = uprclinit.g_minimconfig.getindextags()
itemtags = uprclinit.g_minimconfig.getitemtags() itemtags = uprclinit.g_minimconfig.getitemtags()
if not indextagsp: if not indextagsp:
...@@ -301,8 +312,7 @@ def _maybecreatealbum(conn, doc, trackartid): ...@@ -301,8 +312,7 @@ def _maybecreatealbum(conn, doc, trackartid):
c.execute('''UPDATE albums SET albtartist = ? c.execute('''UPDATE albums SET albtartist = ?
WHERE album_id = ?''', (trackartid, album_id)) WHERE album_id = ?''', (trackartid, album_id))
else: else:
arturi = uprclutils.docarturi(doc, uprclinit.g_httphp, arturi = uprclutils.docarturi(doc, uprclinit.getHttphp(), uprclinit.getPathPrefix())
uprclinit.g_pathprefix)
c.execute('''INSERT INTO c.execute('''INSERT INTO
albums(albtitle, albfolder, artist_id, albdate, albarturi, albums(albtitle, albfolder, artist_id, albdate, albarturi,
......
...@@ -88,7 +88,7 @@ class Untagged(object): ...@@ -88,7 +88,7 @@ class Untagged(object):
idx = self._objidtoidx(pid) idx = self._objidtoidx(pid)
entries = [] entries = []
rcldocs = uprclinit.g_trees['folders'].rcldocs() rcldocs = uprclinit.getTree('folders').rcldocs()
if idx == 0: if idx == 0:
# Browsing root # Browsing root
for i in range(len(self.utidx))[1:]: for i in range(len(self.utidx))[1:]:
......