Commit 9424678b authored by Jean-Francois Dockes's avatar Jean-Francois Dockes
Browse files

uprcl ok 2/3

parent 2444556e
......@@ -30,6 +30,7 @@ from __future__ import print_function, unicode_literals
import posixpath
import re
import sys
PY3 = sys.version > '3'
import conftree
default_mime = "audio/mpeg"
......@@ -186,7 +187,10 @@ def getserviceuserpass(upconfig, servicename):
return username, password
def uplog(s):
if not type(s) == type(u''):
s = s.decode('utf-8', errors='replace')
print(("%s: %s" % (_idprefix, s)).encode('utf-8',errors='replace'),
file=sys.stderr)
if not type(s) == type(b''):
s = ("%s: %s" % (_idprefix, s)).encode('utf-8')
if PY3:
sys.stderr.buffer.write(s + b'\n')
else:
sys.stderr.write(s + b'\n')
sys.stderr.flush()
#!/usr/bin/env python2
#!/usr/bin/env python3
#
# Copyright (C) 2017 J.F.Dockes
#
......@@ -69,7 +69,7 @@ def _rootentries():
# let the different modules return their stuff, and we take note
# of the objid prefixes for later dispatching
entries = []
for treename in uprclinit.g_trees.keys():
for treename in uprclinit.g_trees_order:
nents = uprclinit.g_trees[treename].rootentries(g_myprefix)
for e in nents:
rootmap[e['id']] = treename
......
......@@ -48,8 +48,13 @@
import os
import shlex
import urllib
import sys
PY3 = sys.version > '3'
if PY3:
from urllib.parse import quote as urlquote
else:
from urllib import quote as urlquote
import time
from timeit import default_timer as timer
......@@ -107,6 +112,14 @@ class Folders(object):
# Walk the doc list and update the directory tree according to the
# url: create intermediary directories if needed, create leaf
# entry.
#
# Binary path issue: at the moment the python rclconfig can't
# handle binary (the underlying conftree.py can, we'd need a
# binary stringToStrings). So the topdirs entries have to be
# strings, and so we decode the binurl too. This probably
# could be changed we wanted to support binary, (non utf-8)
# paths. For now, for python3 all dir/file names in the tree
# are str
for docidx in range(len(self._rcldocs)):
doc = self._rcldocs[docidx]
......@@ -121,22 +134,19 @@ class Folders(object):
# it as a doc attribute
doc.albumarturi = arturi
url = doc.getbinurl()
url = doc.getbinurl().decode('utf-8', errors='replace')
url = url[7:]
try:
decoded = url.decode('utf-8')
except:
decoded = urllib.quote(url).decode('utf-8')
# Determine the root entry (topdirs element). Special because
# its path is not a simple name.
fathidx = -1
for rtpath,idx in self._dirvec[0].iteritems():
for rtpath,idx in self._dirvec[0].items():
#uplog("type(url) %s type(rtpath) %s rtpath %s url %s" %
# (type(url),type(rtpath),rtpath, url))
if url.startswith(rtpath):
fathidx = idx[0]
break
if fathidx == -1:
uplog("No parent in topdirs: %s" % decoded)
uplog("No parent in topdirs: %s" % url)
continue
# Compute rest of path
......@@ -148,8 +158,8 @@ class Folders(object):
# directory in the path. This only affects the visible tree,
# not the 'real' URLs of course.
if doc.contentgroup:
a = os.path.dirname(url1).decode('utf-8', errors='replace')
b = os.path.basename(url1).decode('utf-8', errors='replace')
a = os.path.dirname(url1)
b = os.path.basename(url1)
url1 = os.path.join(a, doc.contentgroup, b)
# Split path, then walk the vector, possibly creating
......@@ -164,8 +174,8 @@ class Folders(object):
# the doc idx (previous entries were created for
# intermediate elements without a Doc).
if idx == len(path) -1:
self._dirvec[fathidx][elt] = (self._dirvec[fathidx][elt][0], docidx)
#uplog("updating docidx for %s" % decoded)
self._dirvec[fathidx][elt] = \
(self._dirvec[fathidx][elt][0], docidx)
# Update fathidx for next iteration
fathidx = self._dirvec[fathidx][elt][0]
else:
......@@ -179,7 +189,6 @@ class Folders(object):
# Last element. If directory, needs a self._dirvec entry
if doc.mtype == 'inode/directory':
fathidx = self._createdir(fathidx, docidx, elt)
#uplog("Setting docidx for %s" % decoded)
else:
self._dirvec[fathidx][elt] = (-1, docidx)
......@@ -206,10 +215,18 @@ class Folders(object):
totcnt = 0
self._rcldocs = []
while True:
docs = rclq.fetchmany()
for doc in docs:
self._rcldocs.append(doc)
totcnt += 1
# There are issues at the end of list with fetchmany (sets
# an exception). Works in python2 for some reason, but
# breaks p3. Until recoll is fixed, catch exception
# here. Also does not work if we try to fetch by bigger
# slices (we get an exception and a truncated list)
try:
docs = rclq.fetchmany()
for doc in docs:
self._rcldocs.append(doc)
totcnt += 1
except:
docs = []
if (self._maxrclcnt > 0 and totcnt >= self._maxrclcnt) or \
len(docs) != rclq.arraysize:
break
......@@ -252,7 +269,7 @@ class Folders(object):
# Look all non-directory docs inside directory, and return the cover
# art we find.
def _arturifordir(self, diridx):
for nm,ids in self._dirvec[diridx].iteritems():
for nm,ids in self._dirvec[diridx].items():
if ids[1] >= 0:
doc = self._rcldocs[ids[1]]
if doc.mtype != 'inode/directory' and doc.albumarturi:
......@@ -276,7 +293,7 @@ class Folders(object):
# The basename call is just for diridx==0 (topdirs). Remove it if
# this proves a performance issue
for nm,ids in self._dirvec[diridx].iteritems():
for nm,ids in self._dirvec[diridx].items():
if nm == "..":
continue
thisdiridx = ids[0]
......@@ -309,7 +326,10 @@ class Folders(object):
if e:
entries.append(e)
return sorted(entries, cmp=cmpentries)
if PY3:
return sorted(entries, key=cmpentries)
else:
return sorted(entries, cmp=cmpentries)
# Return path for objid, which has to be a container.This is good old
# pwd... It is called from the search module for generating a 'dir:'
......@@ -328,7 +348,7 @@ class Folders(object):
lpath = []
while True:
fathidx = self._dirvec[diridx][".."][0]
for nm, ids in self._dirvec[fathidx].iteritems():
for nm, ids in self._dirvec[fathidx].items():
if ids[0] == diridx:
lpath.append(nm)
break
......
......@@ -44,6 +44,8 @@ except:
g_rclconfdir = ""
g_friendlyname = "UpMpd-mediaserver"
g_trees = {}
# order in python3 maps is unstable
g_trees_order = ['folders', 'untagged', 'tags']
# Create or update Recoll index, then read and process the data. This
# runs in the separate uprcl_init_worker thread, and signals
......
......@@ -223,8 +223,10 @@ def search(foldersobj, rclconfdir, objid, upnps, idprefix, httphp, pathprefix):
break
uplog("Search retrieved %d docs" % (len(entries),))
return sorted(entries, cmp=cmpentries)
if PY3:
return sorted(entries, key=cmpentries)
else:
return sorted(entries, cmp=cmpentries)
if __name__ == '__main__':
......
......@@ -15,6 +15,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys
PY3 = sys.version > '3'
import os
import sqlite3
import time
......@@ -284,7 +285,10 @@ class Tagged(object):
entries = [rcldoctoentry(pid + '$i' + str(r[0]),
pid, self._httphp, self._pprefix,
self._rcldocs[r[0]]) for r in c]
return sorted(entries, cmp=cmpentries)
if PY3:
return sorted(entries, key=cmpentries)
else:
return sorted(entries, cmp=cmpentries)
# Return a list of trackids as selected by the current
......@@ -463,7 +467,10 @@ class Tagged(object):
entries.append(rcldoctoentry(id, pid, self._httphp,
self._pprefix,
self._rcldocs[docidx]))
entries = sorted(entries, cmp=cmpentries)
if PY3:
entries = sorted(entries, key=cmpentries)
else:
entries = sorted(entries, cmp=cmpentries)
else:
for tt in subqs:
id = pid + '$=' + tt
......
......@@ -29,7 +29,6 @@
import os
import shlex
import urllib
import sys
from upmplgutils import uplog
......
......@@ -16,15 +16,18 @@
from __future__ import print_function
import sys
import urllib
PY3 = sys.version > '3'
if PY3:
from urllib.parse import quote as urlquote
import functools
else:
from urllib import quote as urlquote
import os
import glob
import subprocess
import mutagen
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
import io
from upmplgutils import uplog
......@@ -104,7 +107,7 @@ def rcldoctoentry(id, pid, httphp, pathprefix, doc):
# TBD
li['upnp:class'] = 'object.item.audioItem.musicTrack'
for oname,dname in upnp2rclfields.iteritems():
for oname,dname in upnp2rclfields.items():
val = getattr(doc, dname)
if val:
li[oname] = val
......@@ -135,8 +138,8 @@ def rcldoctoentry(id, pid, httphp, pathprefix, doc):
path = path[7:]
if 'tt' not in li:
li['tt'] = os.path.basename(path.decode('UTF-8', errors = 'replace'))
path = os.path.join(pathprefix, path)
li['uri'] = "http://%s%s" % (httphp, urllib.quote(path))
path = os.path.join(pathprefix.encode('ascii'), path)
li['uri'] = "http://%s%s" % (httphp, urlquote(path))
#uplog("rcldoctoentry: uri: %s" % li['uri'])
# The album art uri is precooked with httphp and prefix
......@@ -172,14 +175,14 @@ def docfolder(doc):
def embdimgurl(doc, httphp, pathprefix):
if doc.embdimg == 'jpg':
ext = '.jpg'
ext = b'.jpg'
elif doc.embdimg == '.png':
ext = 'png'
ext = b'png'
else:
return None
path = doc.getbinurl()
path = path[7:]
path = urllib.quote(os.path.join(pathprefix, path+ext))
path = urlquote(os.path.join(pathprefix.encode('ascii'), path+ext))
path += "?embed=1"
return "http://%s%s" % (httphp, path)
......@@ -187,7 +190,7 @@ def printable(s):
return s.decode('utf-8', errors='replace') if s else ""
def _httpurl(path, httphp, pathprefix):
return "http://%s%s" % (httphp, urllib.quote(path))
return "http://%s%s" % (httphp, urlquote(path))
# Find cover art for doc.
#
......@@ -196,7 +199,7 @@ def _httpurl(path, httphp, pathprefix):
# We are usually called repeatedly for the same directory, so we cache
# one result.
_foldercache = {}
_artexts = ('.jpg', '.png')
_artexts = (b'.jpg', b'.png')
_artnames = ('folder', 'cover')
def docarturi(doc, httphp, pathprefix):
global _foldercache, _artnames
......@@ -211,8 +214,8 @@ def docarturi(doc, httphp, pathprefix):
path,ext = os.path.splitext(docpath(doc))
for ext in _artexts:
if os.path.exists(path + ext):
return _httpurl(os.path.join(pathprefix, path+ext), httphp,
pathprefix)
return _httpurl(os.path.join(pathprefix.encode('ascii'),
path+ext), httphp, pathprefix)
# If doc is a directory, this returns itself, else the father dir.
folder = docfolder(doc)
......@@ -234,7 +237,7 @@ def docarturi(doc, httphp, pathprefix):
artnm = fsimple
if artnm:
_foldercache[folder] = _httpurl(
urllib.quote(os.path.join(pathprefix, folder, artnm)),
urlquote(os.path.join(pathprefix, folder, artnm)),
httphp, pathprefix)
break
except:
......@@ -257,7 +260,7 @@ def _logentry(nm, e1):
tn = _keyvalornull(e1, 'upnp:originalTrackNumber')
uplog("%s tp %s alb %s dir %s tno %s" % (nm, tp,al,dr,tn))
def cmpentries(e1, e2):
def _cmpentries_func(e1, e2):
#uplog("cmpentries");_logentry("e1", e1);_logentry("e2", e2)
tp1 = e1['tp']
tp2 = e2['tp']
......@@ -298,8 +301,16 @@ def cmpentries(e1, e2):
k = 'upnp:originalTrackNumber'
a1 = e1[k] if k in e1 else "0"
a2 = e2[k] if k in e2 else "0"
return int(a1) - int(a2)
try:
return int(a1) - int(a2)
except:
uplog("upnp:originalTrackNumber %s %s"% (a1, a2))
return 0
if PY3:
cmpentries=functools.cmp_to_key(_cmpentries_func)
else:
cmpentries=_cmpentries_func
def rcldirentry(id, pid, title, arturi=None, artist=None, upnpclass=None,
searchable='1', date=None):
......@@ -414,27 +425,27 @@ def embedded_open(path):
f = None
size = 0
if 'audio/mp3' in mutf.mime:
for tagname in mutf.iterkeys():
for tagname in mutf.keys():
if tagname.startswith('APIC:'):
#self.em.rclog("mp3 img: %s" % mutf[tagname].mime)
mtype = mutf[tagname].mime
s = mutf[tagname].data
size = len(s)
f = StringIO(s)
f = io.BytesIO(s)
elif 'audio/x-flac' in mutf.mime:
if mutf.pictures:
mtype = mutf.pictures[0].mime
size = len(mutf.pictures[0].data)
f = StringIO(mutf.pictures[0].data)
f = io.BytesIO(mutf.pictures[0].data)
elif 'audio/mp4' in mutf.mime:
if 'covr' in mutf.iterkeys():
if 'covr' in mutf.keys():
format = mutf['covr'][0].imageformat
if format == mutagen.mp4.AtomDataType.JPEG:
mtype = 'image/jpeg'
else:
mtype = 'image/png'
size = len(mutf['covr'][0])
f = StringIO(mutf['covr'][0])
f = io.BytesIO(mutf['covr'][0])
if f is None:
raise Exception("can't open embedded image")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment