Commit b1ab1bc1 authored by Jean-Francois Dockes's avatar Jean-Francois Dockes
Browse files

upnp to recoll search translation sort of working

parent c2a515fa
......@@ -66,7 +66,7 @@ class PlgWithSlave::Internal {
public:
Internal(PlgWithSlave *_plg, const string& exe, const string& hst,
int prt, const string& pp)
: plg(_plg), exepath(exe), upnphost(hst), upnpport(prt), pathprefix(pp),
: plg(_plg), exepath(exe), upnphost(hst), upnpport(prt), pathprefix(pp),
laststream(this) {
}
......@@ -118,7 +118,7 @@ static int answer_to_connection(void *cls, struct MHD_Connection *connection,
// and then dispatch the request.
PlgWithSlave::Internal *plgi = (PlgWithSlave::Internal*)cls;
PlgWithSlave *realplg =
dynamic_cast<PlgWithSlave*>(plgi->plg->m_services->getpluginforpath(url));
dynamic_cast<PlgWithSlave*>(plgi->plg->m_services->getpluginforpath(url));
if (nullptr == realplg) {
LOGERR("answer_to_connection: no plugin for path [" << url << endl);
return MHD_NO;
......@@ -245,22 +245,22 @@ string PlgWithSlave::get_media_url(const string& path)
{
LOGDEB0("PlgWithSlave::get_media_url: " << path << endl);
if (!m->maybeStartCmd()) {
return string();
return string();
}
time_t now = time(0);
if (m->laststream.path.compare(path) ||
(now - m->laststream.opentime > 10)) {
unordered_map<string, string> res;
if (!m->cmd.callproc("trackuri", {{"path", path}}, res)) {
LOGERR("PlgWithSlave::get_media_url: slave failure\n");
return string();
}
auto it = res.find("media_url");
if (it == res.end()) {
LOGERR("PlgWithSlave::get_media_url: no media url in result\n");
return string();
}
unordered_map<string, string> res;
if (!m->cmd.callproc("trackuri", {{"path", path}}, res)) {
LOGERR("PlgWithSlave::get_media_url: slave failure\n");
return string();
}
auto it = res.find("media_url");
if (it == res.end()) {
LOGERR("PlgWithSlave::get_media_url: no media url in result\n");
return string();
}
m->laststream.clear();
m->laststream.path = path;
m->laststream.media_url = it->second;
......@@ -287,7 +287,7 @@ PlgWithSlave::~PlgWithSlave()
}
static int resultToEntries(const string& encoded, int stidx, int cnt,
vector<UpSong>& entries)
vector<UpSong>& entries)
{
Json::Value decoded;
istringstream input(encoded);
......@@ -298,25 +298,31 @@ static int resultToEntries(const string& encoded, int stidx, int cnt,
for (unsigned int i = stidx; i < decoded.size(); i++) {
#define JSONTOUPS(fld, nm) {song.fld = decoded[i].get(#nm, "").asString();}
if (dolimit && --cnt < 0) {
break;
}
UpSong song;
// tp is container ("ct") or item ("it")
if (dolimit && --cnt < 0) {
break;
}
UpSong song;
JSONTOUPS(id, id);
JSONTOUPS(parentid, pid);
JSONTOUPS(title, tt);
JSONTOUPS(artUri, upnp:albumArtURI);
JSONTOUPS(artist, upnp:artist);
JSONTOUPS(upnpClass, upnp:class);
// tp is container ("ct") or item ("it")
string stp = decoded[i].get("tp", "").asString();
if (!stp.compare("ct")) {
song.iscontainer = true;
if (!stp.compare("ct")) {
song.iscontainer = true;
string ss = decoded[i].get("searchable", "").asString();
if (!ss.empty()) {
song.searchable = stringToBool(ss);
}
} else if (!stp.compare("it")) {
song.iscontainer = false;
JSONTOUPS(uri, uri);
JSONTOUPS(artist, dc:creator);
JSONTOUPS(genre, upnp:genre);
} else if (!stp.compare("it")) {
song.iscontainer = false;
JSONTOUPS(uri, uri);
JSONTOUPS(artist, dc:creator);
JSONTOUPS(genre, upnp:genre);
JSONTOUPS(album, upnp:album);
JSONTOUPS(tracknum, upnp:originalTrackNumber);
JSONTOUPS(tracknum, upnp:originalTrackNumber);
JSONTOUPS(mime, res:mime);
string srate = decoded[i].get("res:samplefreq", "").asString();
if (!srate.empty()) {
......@@ -326,18 +332,13 @@ static int resultToEntries(const string& encoded, int stidx, int cnt,
if (!sdur.empty()) {
song.duration_secs = atoi(sdur.c_str());
}
} else {
LOGERR("PlgWithSlave::result: bad type in entry: " << stp << endl);
continue;
}
JSONTOUPS(id, id);
JSONTOUPS(parentid, pid);
JSONTOUPS(title, tt);
JSONTOUPS(artUri, upnp:albumArtURI);
JSONTOUPS(artist, upnp:artist);
JSONTOUPS(upnpClass, upnp:class);
} else {
LOGERR("PlgWithSlave::result: bad type in entry: " << stp <<
"(title: " << song.title << ")\n");
continue;
}
LOGDEB1("PlgWitSlave::result: pushing: " << song.dump() << endl);
entries.push_back(song);
entries.push_back(song);
}
// We return the total match size, the count of actually returned
// entries can be obtained from the vector
......@@ -463,7 +464,7 @@ int PlgWithSlave::browse(const string& objid, int stidx, int cnt,
LOGDEB1("PlgWithSlave::browse\n");
entries.clear();
if (!m->maybeStartCmd()) {
return errorEntries(objid, entries);
return errorEntries(objid, entries);
}
string sbflg;
switch (flg) {
......@@ -489,13 +490,13 @@ int PlgWithSlave::browse(const string& objid, int stidx, int cnt,
unordered_map<string, string> res;
if (!m->cmd.callproc("browse", {{"objid", objid}, {"flag", sbflg}}, res)) {
LOGERR("PlgWithSlave::browse: slave failure\n");
return errorEntries(objid, entries);
LOGERR("PlgWithSlave::browse: slave failure\n");
return errorEntries(objid, entries);
}
auto it = res.find("entries");
if (it == res.end()) {
LOGERR("PlgWithSlave::browse: no entries returned\n");
LOGERR("PlgWithSlave::browse: no entries returned\n");
return errorEntries(objid, entries);
}
......@@ -523,9 +524,14 @@ int PlgWithSlave::search(const string& ctid, int stidx, int cnt,
LOGDEB("PlgWithSlave::search: [" << searchstr << "]\n");
entries.clear();
if (!m->maybeStartCmd()) {
return errorEntries(ctid, entries);
return errorEntries(ctid, entries);
}
// Computing a pre-cooked query. For simple-minded plugins.
// Note that none of the qobuz/gmusic/tidal plugins actually use
// the slavefield part (defining in what field the term should
// match).
//
// Ok, so the upnp query language is quite powerful, but us, not
// so much. We get rid of parenthesis and then try to find the
// first searchExp on a field we can handle, pretend the operator
......@@ -543,9 +549,9 @@ int PlgWithSlave::search(const string& ctid, int stidx, int cnt,
// The sequence can now be either [field, op, value], or
// [field, op, value, and/or, field, op, value,...]
if ((vs.size() + 1) % 4 != 0) {
LOGERR("PlgWithSlave::search: bad search string: [" << searchstr <<
LOGERR("PlgWithSlave::search: bad search string: [" << searchstr <<
"]\n");
return errorEntries(ctid, entries);
return errorEntries(ctid, entries);
}
string slavefield;
string value;
......@@ -571,7 +577,7 @@ int PlgWithSlave::search(const string& ctid, int stidx, int cnt,
}
classfilter = what;
} else if (!upnpproperty.compare("upnp:artist") ||
!upnpproperty.compare("dc:author")) {
!upnpproperty.compare("dc:author")) {
slavefield = "artist";
value = vs[i+2];
break;
......@@ -585,14 +591,10 @@ int PlgWithSlave::search(const string& ctid, int stidx, int cnt,
break;
}
}
if (slavefield.empty()) {
LOGERR("PlgWithSlave: unsupported search: [" << searchstr << "]\n");
return errorEntries(ctid, entries);
}
// In cache ?
ContentCacheEntry *cep;
string cachekey(m_name + ":" + objkind + ":" + slavefield + ":" + value);
string cachekey(m_name + ":" + ctid + ":" + searchstr);
if ((cep = o_scache.get(cachekey)) != nullptr) {
int total = cep->toResult(classfilter, stidx, cnt, entries);
delete cep;
......@@ -602,19 +604,19 @@ int PlgWithSlave::search(const string& ctid, int stidx, int cnt,
// Run query
unordered_map<string, string> res;
if (!m->cmd.callproc("search", {
{"objid", ctid},
{"objkind", objkind},
{"objid", ctid},
{"objkind", objkind},
{"origsearch", searchstr},
{"field", slavefield},
{"value", value} }, res)) {
LOGERR("PlgWithSlave::search: slave failure\n");
return errorEntries(ctid, entries);
{"value", value} }, res)) {
LOGERR("PlgWithSlave::search: slave failure\n");
return errorEntries(ctid, entries);
}
auto it = res.find("entries");
if (it == res.end()) {
LOGERR("PlgWithSlave::search: no entries returned\n");
return errorEntries(ctid, entries);
LOGERR("PlgWithSlave::search: no entries returned\n");
return errorEntries(ctid, entries);
}
// Convert the whole set and store in cache
ContentCacheEntry e;
......
......@@ -23,6 +23,7 @@ import posixpath
import re
import conftree
import cmdtalkplugin
import urllib
import uprclfolders
import uprclsearch
......@@ -76,7 +77,7 @@ def trackuri(a):
msgproc.log("trackuri: [%s]" % a)
if 'path' not in a:
raise Exception("trackuri: no 'path' in args")
path = a['path']
path = urllib.quote(a['path'])
media_url = rclpathtoreal(path, pathprefix, uprclhost, pathmap)
msgproc.log("trackuri: returning: %s" % media_url)
return {'media_url' : media_url}
......@@ -131,7 +132,8 @@ def search(a):
upnps = a['origsearch']
entries = uprclsearch.search(rclconfdir, objid, upnps, g_myprefix, httphp, pathprefix)
entries = uprclsearch.search(rclconfdir, objid, upnps, g_myprefix,
httphp, pathprefix)
encoded = json.dumps(entries)
return {"entries" : encoded}
......
......@@ -144,40 +144,6 @@ def inittree(confdir):
g_dirvec = _rcl2folders(g_alldocs, confdir)
def _cmpentries(e1, e2):
tp1 = e1['tp']
tp2 = e2['tp']
isct1 = tp1 == 'ct'
isct2 = tp2 == 'ct'
# Containers come before items, and are sorted in alphabetic order
if isct1 and not isct2:
return 1
elif not isct1 and isct2:
return -1
elif isct1 and isct2:
tt1 = e1['tt']
tt2 = e2['tt']
if tt1 < tt2:
return -1
elif tt1 > tt2:
return 1
else:
return 0
# 2 tracks. Sort by album then track number
k = 'upnp:album'
a1 = e1[k] if k in e1 else ""
a2 = e2[k] if k in e2 else ""
if a1 < a2:
return -1
elif a1 > a2:
return 1
k = 'upnp:originalTrackNumber'
a1 = e1[k] if k in e1 else "0"
a2 = e2[k] if k in e2 else "0"
return int(a1) - int(a2)
def _objidtodiridx(pid):
if not pid.startswith(g_myprefix):
raise Exception("folders.browse: bad pid %s" % pid)
......@@ -230,13 +196,21 @@ def browse(pid, flag, httphp, pathprefix):
if e:
entries.append(e)
return sorted(entries, cmp=_cmpentries)
return sorted(entries, cmp=cmpentries)
# return path for objid, which has to be a container. This is good old pwd
def dirpath(objid):
diridx = _objidtodiridx(objid)
# We may get called from search, on the top dir (above [folders]). Return
# empty in this case
try:
diridx = _objidtodiridx(objid)
except:
return ""
if diridx == 0:
return "/"
lpath = []
while True:
fathidx = g_dirvec[diridx][".."][0]
......
......@@ -21,29 +21,70 @@ def _readword(s, i):
w += s[j]
return j,w
# Called with '"' already read:
def _readstring(s, i):
str = '"'
# Called with '"' already read.
# Upnp search term strings are double quoted, but we should not take
# them as recoll phrases. We separate parts which are internally
# quoted, and become phrases, and lists of words which we interpret as
# an and search (comma-separated). Internal quotes come backslash-escaped
def _parsestring(s, i=0):
uplog("parseString: input: <%s>" % s[i:])
# First change '''"hello \"one phrase\"''' world" into
# '''hello "one phrase" world'''
# Note that we can't handle quoted dquotes inside string
str = ''
escape = False
instring = False
for j in range(i, len(s)):
#print("s[j] [%s] out now [%s]" % (s[j],out))
if s[j] == '\\':
if not escape:
escape = True
str += '\\'
continue
if instring:
if escape:
if s[j] == '"':
str += '"'
instring = False
else:
str += '\\' + s[j]
escape = False
else:
if s[j] == '\\':
escape = True
else:
str += s[j]
if s[j] == '"':
str += '"'
if not escape:
return j+1, str
else:
str += s[j]
escape = False
return len(s), str
if escape:
str += s[j]
escape = False
if s[j] == '"':
instring = True
else:
if s[j] == '\\':
escape = True
elif s[j] == '"':
j += 2
break
else:
str += s[j]
tokens = stringToStrings(str)
return j, tokens
def _appendterms(out, v, field, oper):
uplog("_appendterms: v %s field <%s> oper <%s>" % (v,field,oper))
swords = ""
phrases = []
for w in v:
if len(w.split()) == 1:
if swords:
swords += ","
swords += w
else:
phrases.append(w)
out.append(swords)
for ph in phrases:
out.append(field)
out.append(oper)
out.append('"' + ph + '"')
def upnpsearchtorecoll(s):
uplog("upnpsearchtorecoll:in: <%s>" % s)
......@@ -52,6 +93,8 @@ def upnpsearchtorecoll(s):
out = []
hadDerived = False
i = 0
field = ""
oper = ""
while True:
i,c = _getchar(s, i)
if not c:
......@@ -67,13 +110,18 @@ def upnpsearchtorecoll(s):
out = ["mime:*"]
break
if c == '(' or c == ')' or c == '>' or c == '<' or c == '=':
if c == '(' or c == ')':
out.append(c)
elif c == '>' or c == '<' or c == '=':
oper += c
else:
if c == '"':
i,w = _readstring(s, i)
if not w.endswith('"'):
raise Exception("Unterminated string in [%s]" % out)
i,v = _parsestring(s, i)
uplog("_parsestring ret: %s" % v)
_appendterms(out, v, field, oper)
oper = ""
field = ""
continue
else:
i -= 1
i,w = _readword(s, i)
......@@ -81,20 +129,25 @@ def upnpsearchtorecoll(s):
#print("Got word [%s]" % w)
if w == 'contains':
out.append(':')
oper = ':'
elif w == 'doesNotContain':
if len(out) < 1:
raise Exception("doesNotContain can't be the first word")
out.insert(-1, "-")
out.append(':')
oper = ':'
elif w == 'derivedFrom':
hadDerived = True
out.append(':')
oper = ':'
elif w == 'true':
out.append('*')
oper = ""
elif w == 'false':
out.append('xxxjanzocsduochterrrrm')
elif w == 'exists':
out.append(':')
oper = ':'
elif w == 'and':
# Recoll has implied AND, but see next
pass
......@@ -105,13 +158,9 @@ def upnpsearchtorecoll(s):
# use parentheses
out.append('OR')
else:
if hadDerived:
hadDerived = False
if len(w) >= 1 and w[-1] == '"':
w = w[:-1] + '*' + '"'
else:
w += '*'
out.append(w)
field = upnp2rclfields[w]
out.append(field)
oper = ""
ostr = ""
for tok in out:
......@@ -124,9 +173,10 @@ def search(rclconfdir, objid, upnps, idprefix, httphp, pathprefix):
rcls = upnpsearchtorecoll(upnps)
filterdir = uprclfolders.dirpath(objid)
rcls += " dir:\"" + filterdir + "\""
if filterdir and filterdir != "/":
rcls += " dir:\"" + filterdir + "\""
uplog("Search: recoll search: %s" % rcls)
uplog("Search: recoll search: <%s>" % rcls)
rcldb = recoll.connect(confdir=rclconfdir)
try:
......@@ -142,19 +192,19 @@ def search(rclconfdir, objid, upnps, idprefix, httphp, pathprefix):
entries = []
maxcnt = 0
totcnt = 0
while True:
docs = rclq.fetchmany()
for doc in docs:
id = idprefix + '$' + 'seeyoulater'
e = rcldoctoentry(id, objid, httphp, pathprefix, doc)
entries.append(e)
totcnt += 1
if (maxcnt > 0 and totcnt >= maxcnt) or len(docs) != rclq.arraysize:
if e:
entries.append(e)
if (maxcnt > 0 and len(entries) >= maxcnt) or \
len(docs) != rclq.arraysize:
break
uplog("Search retrieved %d docs" % (totcnt,))
uplog("Search retrieved %d docs" % (len(entries),))
return entries
return sorted(entries, cmp=cmpentries)
......
......@@ -3,6 +3,7 @@ from __future__ import print_function
import sys
import posixpath
import urllib
import os
audiomtypes = frozenset([
'audio/mpeg',
......@@ -11,9 +12,20 @@ audiomtypes = frozenset([
'audio/aac',
'audio/mp4',
'audio/x-aiff',
'audio/x-wav'
'audio/x-wav',
'inode/directory'
])
upnp2rclfields = {'upnp:album': 'album',
'releasedate' : 'date',
'upnp:originalTrackNumber' : 'tracknumber',
'upnp:artist' : 'artist',
'upnp:genre' : 'genre',
'res:mime' : 'mtype',
'duration' : 'duration',
'res:samplefreq' : 'sample_rate'
}
def rcldoctoentry(id, pid, httphp, pathprefix, doc):
"""
Transform a Doc objects into the format expected by the parent
......@@ -38,8 +50,8 @@ def rcldoctoentry(id, pid, httphp, pathprefix, doc):
http://host:port/pathprefix/track?version=1&trackId=<trackid>
"""
uplog("rcldoctoentry: pid %s id %s httphp %s pathprefix %s" %
(pid, id, httphp, pathprefix))
uplog("rcldoctoentry: pid %s id %s mtype %s" %
(pid, id, doc.mtype))
li = {}
if doc.mtype not in audiomtypes:
......@@ -47,7 +59,7 @@ def rcldoctoentry(id, pid, httphp, pathprefix, doc):
li['pid'] = pid
li['id'] = id
li['tp'] = 'it'
li['tp'] = 'ct' if doc.mtype == 'inode/directory' else 'it'
# Why no dc.title??
li['tt'] = doc.title
......@@ -72,11 +84,7 @@ def rcldoctoentry(id, pid, httphp, pathprefix, doc):
#lyricist=
#lyrics=
for oname,dname in [('upnp:album', 'album'), ('releasedate','date'),
('upnp:originalTrackNumber', 'tracknumber'),
('upnp:artist', 'artist'), ('upnp:genre', 'genre'),
('res:mime', 'mtype'), ('duration', 'duration'),
('res:samplefreq', 'sample_rate')]:
for oname,dname in upnp2rclfields.iteritems():
val = getattr(doc, dname)
if val:
li[oname] = val
......@@ -98,6 +106,49 @@ def rcldoctoentry(id, pid, httphp, pathprefix, doc):
uplog("rcldoctoentry: uri: %s" % li['uri'])
return li
def cmpentries(e1, e2):
tp1 = e1['tp']
tp2 = e2['tp']
isct1 = tp1 == 'ct'
isct2 = tp2 == 'ct'
# Containers come before items, and are sorted in alphabetic order
if isct1 and not isct2:
return 1
elif not isct1 and isct2:
return -1