Commit e5435600 authored by Jean-Francois Dockes's avatar Jean-Francois Dockes
Browse files

Switch to doc subscript not attr access to prepare switch to querystore. Works...

Switch to doc subscript not attr access to prepare switch to querystore. Works with recoll querystore branch, maybe also with regular. Not fully ready for querystore because of doc attributes assignments
parent 2a7538d7
......@@ -32,6 +32,10 @@
# Container: $d<diridx> where <diridx> indexes into _dirvec
# Item: $i<docidx> where <docidx> indexes into _rcldocs
#
# Note: this is very different from what Minim does. Minim uses actual
# objid paths as objids. E.g. 0$folders$f1589$f1593$f1604$*i11609
# Must make pwd and any walk up the tree much easier.
#
# Each _dirvec entry is a Python dict, mapping the directory entries'
# names to a pair (diridx,docidx), where:
#
......@@ -48,6 +52,10 @@
# think that it is ever used. The Recoll doc for a directory has
# nothing very interesting in it.
#
# Note: We could probably use a single value, with a convention
# saying, e.g., that > 0 is for docs and < -1 for folders. Check if
# this saves a significant amount of memory.
#
# Each directory has a special ".." entry with a diridx pointing to
# the parent directory. This allows building a path from a container
# id (aka pwd).
......@@ -69,12 +77,7 @@
import os
import shlex
import sys
PY3 = sys.version > '3'
if PY3:
from urllib.parse import quote as urlquote
else:
from urllib import quote as urlquote
from urllib.parse import quote as urlquote
import time
from timeit import default_timer as timer
......@@ -133,7 +136,7 @@ class Folders(object):
pldoc = self._rcldocs[pldocidx]
arturi = uprclutils.docarturi(pldoc, self._httphp,self._pprefix)
if arturi:
pldoc.albumarturi = arturi
pldoc["albumarturi"] = arturi
plpath = uprclutils.docpath(pldoc)
try:
m3u = uprclutils.M3u(plpath)
......@@ -144,16 +147,16 @@ class Folders(object):
if m3u.urlRE.match(url):
# Actual URL (usually http). Create bogus doc
doc = recoll.Doc()
doc.setbinurl(bytearray(url))
doc["url"] = url
elt = os.path.split(url)[1]
doc.title = elt.decode('utf-8', errors='ignore')
doc.mtype = "audio/mpeg"
doc["title"] = elt.decode('utf-8', errors='ignore')
doc["mtype"] = "audio/mpeg"
self._rcldocs.append(doc)
docidx = len(self._rcldocs) -1
self._dirvec[diridx][elt] = (-1, docidx)
else:
doc = recoll.Doc()
doc.setbinurl(bytearray(b'file://' + url))
doc["url"] = b'file://' + url
fathidx, docidx = self._stat(doc)
if docidx >= 0 and docidx < len(self._rcldocs):
elt = os.path.split(url)[1]
......@@ -191,10 +194,10 @@ class Folders(object):
# If there is a contentgroup field, just add it as a virtual
# directory in the path. This only affects the visible tree,
# not the 'real' URLs of course.
if doc.contentgroup:
if doc["contentgroup"]:
a = os.path.dirname(url1)
b = os.path.basename(url1)
url1 = os.path.join(a, doc.contentgroup, b)
url1 = os.path.join(a, doc["contentgroup"], b)
# Split path. The caller will walk the list (possibly creating
# directory entries as needed, or doing something else).
......@@ -245,20 +248,20 @@ class Folders(object):
# Only include selected mtypes: tracks, playlists,
# directories etc.
if doc.mtype not in audiomtypes:
if doc["mtype"] not in audiomtypes:
continue
# For linking item search results to the main
# array. Deactivated for now as it does not seem to be
# needed.
#self._xid2idx[doc.xdocid] = docidx
#self._xid2idx[doc["xdocid"]] = docidx
# Possibly enrich the doc entry with a cover art uri.
arturi = docarturi(doc, self._httphp, self._pprefix)
if arturi:
# The uri is quoted, so it's ascii and we can just store
# it as a doc attribute
doc.albumarturi = arturi
doc["albumarturi"] = arturi
fathidx, path = self._pathbeyondtopdirs(doc)
if not fathidx:
......@@ -286,9 +289,9 @@ class Folders(object):
fathidx = self._createdir(fathidx, -1, elt)
else:
# Last element. If directory, needs a self._dirvec entry
if doc.mtype == 'inode/directory':
if doc["mtype"] == 'inode/directory':
fathidx = self._createdir(fathidx, docidx, elt)
elif doc.mtype == 'audio/x-mpegurl':
elif doc["mtype"] == 'audio/x-mpegurl':
fathidx = self._createpldir(fathidx, docidx,doc,elt)
else:
self._dirvec[fathidx][elt] = (-1, docidx)
......@@ -325,10 +328,10 @@ class Folders(object):
for doc in rclq:
if tagaliases:
for orig, target, rep in tagaliases:
val = doc.get(orig)
val = doc[orig]
#uplog("Rep %s doc[%s]=[%s] doc[%s]=[%s]"%
# (rep, orig, val, target, doc.get(target)))
if val and (rep or not doc.get(target)):
# (rep, orig, val, target, doc[target]))
if val and (rep or not doc[target]):
setattr(doc, target, val)
self._rcldocs.append(doc)
......@@ -397,8 +400,8 @@ class Folders(object):
for nm,ids in self._dirvec[diridx].items():
if ids[1] >= 0:
doc = self._rcldocs[ids[1]]
if doc.mtype != 'inode/directory' and doc.albumarturi:
return doc.albumarturi
if doc["mtype"] != 'inode/directory' and doc["albumarturi"]:
return doc["albumarturi"]
# Folder hierarchy browse method.
......@@ -444,8 +447,8 @@ class Folders(object):
if len(self._dirvec[thisdiridx]) == 1:
continue
id = self._idprefix + '$d' + str(thisdiridx)
if doc and doc.albumarturi:
arturi = doc.albumarturi
if doc and doc["albumarturi"]:
arturi = doc["albumarturi"]
else:
arturi = self._arturifordir(thisdiridx)
entries.append(rcldirentry(id, pid, os.path.basename(nm),
......@@ -519,9 +522,9 @@ class Folders(object):
# Compute object id for doc out of recoll search. Not used at the moment.
# and _xid2idx is not built
def _objidforxdocid(self, doc):
if doc.xdocid not in self._xid2idx:
if doc["xdocid"] not in self._xid2idx:
return None
return self._idprefix + '$i' + str(self._xid2idx[doc.xdocid])
return self._idprefix + '$i' + str(self._xid2idx[doc["xdocid"]])
def _stat(self, doc):
......@@ -546,7 +549,7 @@ class Folders(object):
def objidfordoc(self, doc):
if doc.mtype == 'inode/directory':
if doc["mtype"] == 'inode/directory':
id = self._objidforpath(doc)
else:
id = self._objidforxdocid(doc)
......
......@@ -22,7 +22,6 @@
# inside the global document vector.
import os, sys
PY3 = sys.version > '3'
from upmplgutils import uplog
from uprclutils import rcldoctoentry, rcldirentry, cmpentries
......@@ -46,7 +45,7 @@ class Playlists(object):
for docidx in range(len(rcldocs)):
doc = rcldocs[docidx]
if doc.mtype == 'audio/x-mpegurl':
if doc["mtype"] == 'audio/x-mpegurl':
self.utidx.append(docidx)
# Compute index into our entries vector by 'parsing' the objid.
......@@ -89,7 +88,7 @@ class Playlists(object):
for i in range(len(self.utidx))[1:]:
doc = rcldocs[self.utidx[i]]
id = self._idprefix + '$p' + str(i)
title = doc.title if doc.title else doc.filename
title = doc["title"] if doc["title"] else doc["filename"]
e = rcldirentry(id, pid, title,
upnpclass='object.container.playlistContainer')
if e:
......@@ -109,15 +108,15 @@ class Playlists(object):
doc = recoll.Doc()
if m3u.urlRE.match(url):
# Actual URL (usually http). Create bogus doc
doc.setbinurl(bytearray(url))
doc["url"] = url
elt = os.path.split(url)[1]
doc.title = elt.decode('utf-8', errors='ignore')
doc.mtype = "audio/mpeg"
doc["title"] = elt.decode('utf-8', errors='ignore')
doc["mtype"] = "audio/mpeg"
else:
doc.setbinurl(bytearray(b'file://' + url))
doc["url"] = b'file://' + url
fathidx, docidx = folders._stat(doc)
if docidx < 0:
uplog("playlists: can't stat %s"%doc.getbinurl())
uplog("playlists: can't stat %s"%doc["url"])
continue
doc = rcldocs[docidx]
......
......@@ -15,7 +15,6 @@
from __future__ import print_function
import sys
PY3 = sys.version > '3'
import re
from recoll import recoll
......@@ -255,7 +254,7 @@ def search(foldersobj, rclconfdir, objid, upnps, idprefix, httphp, pathprefix):
if arturi:
# The uri is quoted, so it's ascii and we can just store
# it as a doc attribute
doc.albumarturi = arturi
doc["albumarturi"] = arturi
id = foldersobj.objidfordoc(doc)
e = uprclutils.rcldoctoentry(id, objid, httphp, pathprefix, doc)
if e:
......
......@@ -23,7 +23,6 @@
# Object Ids inside the subsections: complicated
import sys
PY3 = sys.version > '3'
import os
import sqlite3
import time
......
......@@ -36,7 +36,7 @@ import uprclinit
# The keys must stay same as what Minim uses as we filter them with
# indexTags from minim config
#
# Note: artist is actually doc.albumartist if set, else doc.artist and
# Note: artist is actually doc["albumartist"] if set, else doc["artist"] and
# is processed separately
#
# Only a smaller set of entries is normally used, as filtered by indexTags
......@@ -198,10 +198,10 @@ def _auxtableinsert(conn, tb, value):
# should not arrive here, but let's play it safe.
def _tracknofordoc(doc):
try:
return int(doc.tracknumber.split('/')[0])
return int(doc["tracknumber"].split('/')[0])
except:
#uplog("_tracknofordoc: doc.tracknumber %s title %s url %s" %
# (doc.tracknumber, doc.title, doc.url))
#uplog("_tracknofordoc: doc["tracknumber"] %s title %s url %s" %
# (doc["tracknumber"], doc["title"], doc["url"]))
return 1
......@@ -220,19 +220,19 @@ def _maybecreatealbum(conn, doc, trackartid):
album = getattr(doc, 'album', None)
if not album:
album = uprclutils.basename(folder)
#uplog("Using %s for alb MIME %s title %s" % (album,doc.mtype,doc.url))
#uplog("Using %s for alb MIME %s title %s" % (album,doc["mtype"],doc["url"]))
if doc.albumartist:
albartist_id = _auxtableinsert(conn, 'artist', doc.albumartist)
if doc["albumartist"]:
albartist_id = _auxtableinsert(conn, 'artist', doc["albumartist"])
else:
albartist_id = None
# See if there is a discnumber, either explicit or from album
# title
discnumber = None
if doc.discnumber:
if doc["discnumber"]:
try:
discnumber = int(doc.discnumber)
discnumber = int(doc["discnumber"])
#uplog("discnumber %d folder %s" % (discnumber, folder))
except:
pass
......@@ -288,8 +288,8 @@ def _maybecreatealbum(conn, doc, trackartid):
c.execute('''INSERT INTO
albums(albtitle, albfolder, artist_id, albdate, albarturi,
albtdisc, albartok, albtartist) VALUES (?,?,?,?,?,?,?,?)''',
(album, folder, albartist_id, doc.date,
doc.albumarturi, discnumber, 1, trackartid))
(album, folder, albartist_id, doc["date"],
doc["albumarturi"], discnumber, 1, trackartid))
album_id = c.lastrowid
#uplog("Created album %d %s disc %s artist %s folder %s" %
# (album_id, album, discnumber, albartist_id, folder))
......@@ -455,30 +455,30 @@ def recolltosql(conn, rcldocs):
time.sleep(0)
# No need to include non-audio or non-tagged types
if doc.mtype not in audiomtypes or doc.mtype == 'inode/directory' \
or doc.mtype == 'audio/x-mpegurl':
if doc["mtype"] not in audiomtypes or doc["mtype"] == 'inode/directory' \
or doc["mtype"] == 'audio/x-mpegurl':
continue
# Do the artist apart from the other attrs, as we need the
# value for album creation.
if doc.albumartist:
trackartid = _auxtableinsert(conn, 'artist', doc.albumartist)
elif doc.artist:
trackartid = _auxtableinsert(conn, 'artist', doc.artist)
if doc["albumartist"]:
trackartid = _auxtableinsert(conn, 'artist', doc["albumartist"])
elif doc["artist"]:
trackartid = _auxtableinsert(conn, 'artist', doc["artist"])
else:
trackartid = None
album_id, albartist_id = _maybecreatealbum(conn, doc, trackartid)
trackno = _tracknofordoc(doc)
if doc.url.find('file://') == 0:
path = doc.url[7:]
if doc["url"].find('file://') == 0:
path = doc["url"][7:]
else:
path = ''
# Set base values for column names, values list, placeholders
columns = ['docidx','album_id','trackno','title','path','artist_id']
values = [docidx, album_id, trackno, doc.title, path, trackartid]
values = [docidx, album_id, trackno, doc["title"], path, trackartid]
placehold = ['?', '?', '?', '?', '?', '?']
# Append data for each auxiliary table if the doc has a value
# for the corresponding field (else let SQL set a dflt/null value)
......@@ -497,7 +497,7 @@ def recolltosql(conn, rcldocs):
stmt='INSERT INTO tracks(' + ','.join(columns) + \
') VALUES(' + ','.join(placehold) + ')'
c.execute(stmt, values)
#uplog(doc.title)
#uplog(doc["title"])
## End Big doc loop
......
......@@ -51,9 +51,10 @@ class Untagged(object):
for docidx in range(len(rcldocs)):
doc = rcldocs[docidx]
if doc.mtype == 'inode/directory' or doc.mtype == 'audio/x-mpegurl':
if doc["mtype"] == 'inode/directory' or \
doc["mtype"] == 'audio/x-mpegurl':
continue
if not doc.title:
if not doc["title"]:
self.utidx.append(docidx)
# Compute index into our entries vector by 'parsing' the objid.
......
......@@ -16,13 +16,8 @@
from __future__ import print_function
import sys
PY3 = sys.version > '3'
if PY3:
from urllib.parse import quote as urlquote
import functools
else:
from urllib import quote as urlquote
from urllib.parse import quote as urlquote, unquote_to_bytes as urlunquotetobytes
import functools
import glob
import io
import locale
......@@ -31,6 +26,7 @@ import os
import re
import traceback
from recoll import recoll
from upmplgutils import uplog
# This must be consistent with what contentdirectory.cxx does
......@@ -100,15 +96,15 @@ def rcldoctoentry(id, pid, httphp, pathprefix, doc):
A dict representing an UPnP item, with the
keys as expected in the plgwithslave.cxx resultToEntries() function.
"""
#uplog("rcldoctoentry: pid %s id %s mtype %s" % (pid, id, doc.mtype))
#uplog("rcldoctoentry: pid %s id %s mtype %s" % (pid, id, doc["mtype"]))
li = {}
if doc.mtype not in audiomtypes:
if doc["mtype"] not in audiomtypes:
return li
li['pid'] = pid
li['id'] = id
if doc.mtype == 'inode/directory':
if doc["mtype"] == 'inode/directory':
li['tp'] = 'ct'
li['upnp:class'] = 'object.container'
else:
......@@ -121,8 +117,8 @@ def rcldoctoentry(id, pid, httphp, pathprefix, doc):
if val:
li[oname] = val
if 'upnp:artist' not in li and doc.albumartist:
li['upnp:artist'] = doc.albumartist
if 'upnp:artist' not in li and doc["albumartist"]:
li['upnp:artist'] = doc["albumartist"]
# TBD Date format ?
#comment=
......@@ -207,25 +203,28 @@ def basename(path):
# Compute fs path for URL. All Recoll URLs are like file://xx
def docpath(doc):
return doc.getbinurl()[7:]
p = urlunquotetobytes(doc["url"])
if len(p) > 7:
return p[7:]
else:
return b''
def docfolder(doc):
path = docpath(doc)
if doc.mtype != 'inode/directory':
if doc["mtype"] != 'inode/directory':
path = os.path.dirname(path)
if path[-1] != b'/'[0]:
path += b'/'
return path
def embdimgurl(doc, httphp, pathprefix):
if doc.embdimg == 'jpg':
if doc["embdimg"] == 'jpg':
ext = b'.jpg'
elif doc.embdimg == '.png':
elif doc["embdimg"] == '.png':
ext = b'png'
else:
return None
path = doc.getbinurl()
path = path[7:]
path = docpath(doc)
path = os.path.join(pathprefix.encode('utf-8'), path+ext)
query = "?embed=1"
return _httpurl(httphp, path, query)
......@@ -248,7 +247,7 @@ def tag2fn(s):
replace(b'|', b'')
##########################
# Find cover art for doc.
# Find cover art for doc
#
# We return a special uri if the file has embedded image data, else an
# uri for for the directory cover art (if any).
......@@ -283,15 +282,16 @@ def docarturi(doc, httphp, pathprefix):
return _httpurl(httphp, os.path.join(bpp, artpath))
# Else try to use an embedded img
if doc.embdimg:
if doc["embdimg"]:
arturi = embdimgurl(doc, httphp, pathprefix)
if arturi:
#uplog("docarturi: embedded: %s"%printable(arturi))
return arturi
# won't work for the virtual group directory itself: it has no doc
if doc.contentgroup:
base = os.path.join(os.path.dirname(objpath), tag2fn(doc.contentgroup))
if doc["contentgroup"]:
base = os.path.join(os.path.dirname(objpath),
tag2fn(doc["contentgroup"]))
for artpath in _artnamegen(base):
#uplog("docarturi: testing %s" % artpath)
if os.path.exists(artpath):
......@@ -330,7 +330,7 @@ def docarturi(doc, httphp, pathprefix):
arturi = _foldercache[folder]
if arturi:
#uplog("folder %s arturi %s"% (printable(folder), arturi))
if doc.mtype == 'inode/directory':
if doc["mtype"] == 'inode/directory':
#uplog("docarturi: external: %s->%s" %
# (printable(folder), printable(arturi)))
pass
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment