uprcltags.py 25.6 KB
Newer Older
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
1
#
2
# Copyright (C) 2017-2020 J.F.Dockes
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
3
4
5
6
7
8
9
10
11
12
13
14
15
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
16

17
18
19
20
21
22
23
24
# Manage the tags sections of the tree.
#
# Object Id prefixes:
#     0$uprcl$=[tagname]
#     0$uprcl$albums
#     0$uprcl$items
# Object Ids inside the subsections: complicated

25
import sys
26
import os
27
import sqlite3
28
29
import time
import tempfile
30

31
from upmplgutils import uplog
32
from uprclutils import rcldirentry, rcldoctoentry, cmpentries
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
33
import uprclinit
34
35
import uprcltagscreate
from uprcltagscreate import _clid, recolltosql
Jean-Francois Dockes's avatar
ckpt    
Jean-Francois Dockes committed
36

37
# The browseable object which defines the tree of tracks organized by tags.
38
class Tagged(object):
39
40
41
42
43
    # We maintain a cache for the total count of query statements
    # which are expensive to compute when we are returning a partial
    # slice (which we currently only do when displaying the top-level
    # items list, so the cache can actually never have more than 1 element...)
    _stmt_cnt_cachesize = 20
44
    def __init__(self, rcldocs, httphp, pathprefix):
45
46
47
48
        self._httphp = httphp
        self._pprefix = pathprefix
        self._conn = None
        self._init_sqconn()
49
50
        self._stmt_cnt_cache = {}
        self._stmt_cnt_cachequeue = []
51
        recolltosql(self._conn, rcldocs)
52
        
53

54
55
56
57
58
59
60
61
62
63
64
65
66
67
    def _init_sqconn(self):
        # We use a separate thread for building the db to ensure
        # responsiveness during this phase.  :memory: handles normally
        # can't be shared between threads, and different :memory: handles
        # access different dbs. The following would work, but it needs
        # python 3.4
        #self._conn = sqlite3.connect('file:uprcl_db?mode=memory&cache=shared')
        # As we can guarantee that 2 threads will never access the db at
        # the same time (the init thread just goes away when it's done),
        # we just disable the same_thread checking on :memory:
        if self._conn is None:
            self._conn = sqlite3.connect(':memory:', check_same_thread=False)


68
69
70
    # Create our top-level directories, with fixed entries, and stuff
    # from the tags tables. This may be called (indirectly) from the folders
    # hierarchy, with a path restriction
71
72
73
74
75
76
    def rootentries(self, pid, path=''):
        uplog("rootentries: pid %s path %s" % (pid, path))
        entries = []
        nalbs = self._albcntforfolder(path)
        entries.append(rcldirentry(pid + 'albums', pid, nalbs + ' albums'))
        if path:
77
            where = ' WHERE tracks.path LIKE ? '
78
            args = (path + '%',)
79
80
81
        else:
            where = ' '
            args = ()
82
        c = self._conn.cursor()
83
        stmt = "SELECT COUNT(*) from tracks"
84
        c.execute(stmt+where, args)
85
        nitems = str(c.fetchone()[0])
86
        entries.append(rcldirentry(pid + 'items', pid, nitems + ' items'))
87
        subqs = self._subtreetags(where, args)
88
        tagdisplaytag = uprcltagscreate.getTagDisplayTag()
89
        for tt in subqs:
90
            entries.append(rcldirentry(pid + '=' + tt , pid, tagdisplaytag[tt]))
91
92
93
        return entries


94
95
    # List all tags which still have multiple values inside this selection level
    def _subtreetags(self, where, values):
96
97
        c = self._conn.cursor()
        tags = []
98
99
100
101
        indextags = uprcltagscreate.getIndexTags()
        tagtotable = uprcltagscreate.getTagToTable()
        for tt in indextags:
            tb = tagtotable[tt]
102
103
            stmt = '''SELECT COUNT(DISTINCT %s) FROM tracks %s''' % \
                   (_clid(tb), where)
104
            #uplog("subtreetags: stmt: [%s]" % stmt)
105
            c.execute(stmt, values)
106
            cnt = c.fetchone()[0]
107
108
            if len(stmt) > 80:
                stmt = stmt[:80] + "..."
109
            uplog("subtreetags: %d values for %s (%s,%s)"%(cnt,tb,stmt,values))
110
111
            if cnt > 1:
                tags.append(tt)
112
113
        return tags

114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
    def _stmt_total(self, stmt, values):
        try:
            total = self._stmt_cnt_cache[stmt]
        except:
            c = self._conn.cursor()
            c.execute(stmt, values)
            ids = [r[0] for r in c]
            total = len(ids)
            self._stmt_cnt_cache[stmt] = total
            self._stmt_cnt_cachequeue.append(stmt)
            if len(self._stmt_cnt_cachequeue) > self._stmt_cnt_cachesize:
                del(self._stmt_cnt_cache[self._stmt_cnt_cachequeue[0]])
                self._stmt_cnt_cachequeue = self._stmt_cnt_cachequeue[1:]
        return total
        
129
130
    # Build a list of track directory entries for an SQL statement
    # which selects docidxs (SELECT docidx,... FROM tracks WHERE...)
131
132
133
134
135
136
137
138
139
    #
    # NOTE: setting offset or cnt currently precludes sorting, which
    # is unavoidable if we don't cache the whole result (which would
    # use a lot of memory for big collections, around 100MB for 10k
    # songs, significant on a small SBC). At the moment, we only do
    # this for the full top-level items list, which is not really
    # useful anyway.
    def _trackentriesforstmt(self, stmt, values, pid, offset=0, count=0):
        uplog("trackentries: offset %d count %d" % (offset, count))
140
        total = 0
141
142
143
144
145
146
147
148
149
150
151
152
153
        if offset != 0 or count != 0:
            total = self._stmt_total(stmt, values)
        if total < 1000:
            # Then return the full set: this is acceptable in terms of
            # performances, and will allow the result to be sorted.
            # plgwithslave knows how to deal with a full set returned
            # from a partial request.
            count = 0
            offset = 0
        if count != 0:
            stmt += " LIMIT %d " % count
        if offset != 0:
            stmt += " OFFSET %d " % offset
154
        rcldocs = uprclinit.getTree('folders').rcldocs()
155
156
        c = self._conn.cursor()
        c.execute(stmt, values)
157
158
        entries = [rcldoctoentry(pid + '$i' + str(r[0]),
                                 pid, self._httphp, self._pprefix,
159
                                 rcldocs[r[0]]) for r in c]
160
161
162
163
164
165
        uplog("trackentries: stmt returns %d entries" % len(entries))
        if offset != 0 or count != 0:
            return (offset, total, entries)
        else:
            return sorted(entries, key=cmpentries)

166

167
168
169
170
171
172
173
174
175
176
177
    # Return a list of trackids as selected by the current
    # path <selwhere> is like: WHERE col1_id = ? AND col2_id = ? [...], and
    # <values> holds the corresponding values
    def _docidsforsel(self, selwhere, values):
        c = self._conn.cursor()
        stmt = 'SELECT docidx FROM tracks ' + selwhere + ' ORDER BY trackno'
        uplog("docidsforsel: executing <%s> values %s" % (stmt, values))
        c.execute(stmt, values)
        return [r[0] for r in c.fetchall()]


178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
    # Expand multiple possibly merged albums to real ones. The tracks
    # always refer to the raw albid, so this is necessary to compute a
    # track list. Multiple albums as input, no sorting.
    def _albids2rawalbids(self, albids):
        c = self._conn.cursor()
        rawalbids = []
        for albid in albids:
            c.execute('''SELECT album_id FROM albums WHERE albalb = ?''',
                      (albid,))
            rows = c.fetchall()
            if len(rows):
                for r in rows:
                    rawalbids.append(r[0])
            else:
                rawalbids.append(albid)
        return rawalbids
    

    # Expand single possibly merged album into list of ids for component discs
    def _albid2rawalbidssorted(self, albid):
        c = self._conn.cursor()
        c.execute('''SELECT album_id FROM albums
          WHERE albalb = ? ORDER BY albtdisc''', (albid,))
        rows = c.fetchall()
        if len(rows) <= 1:
            return (albid,)
        else:
            return [r[0] for r in rows]


    # Translate albids so that the ones which are part of a merged
    # album become the merged id. The returned list is same size or
    # smaller because there maybe duplicate merged ids. Used to show
    # album lists
    def _rawalbids2albids(self, rawalbids):
        albids = set()
        c = self._conn.cursor()
        for rawalbid in rawalbids:
            c.execute('''SELECT album_id, albalb FROM albums
                WHERE album_id = ?''', (rawalbid,))
            alb = c.fetchone()
            if alb[1]:
                albids.add(alb[1])
            else:
                albids.add(alb[0])
        #
        return [id for id in albids]


    # Count albums under file system path. We use albalb because
    # merged albums may come from multiple folders, and have no
    # albfolder. So this returns merged albums for which at least one
    # disk is under this folder path
231
232
233
234
235
236
237
238
239
240
241
242
243
244
    def _albcntforfolder(self, path):
        c = self._conn.cursor()
        if path:
            stmt = '''SELECT COUNT(DISTINCT albalb) FROM ALBUMS
            WHERE albfolder LIKE ?'''
            args = (path + "%",)
        else:
            stmt = "SELECT COUNT(*) FROM albums WHERE albtdisc is NULL"
            args = ()
        uplog("_albcntforfolder: stmt %s args %s" % (stmt, args))
        c.execute(stmt, args)
        return str(c.fetchone()[0])


245
246
    # Track list for possibly merged album: get tracks from all
    # components, then renumber trackno
247
    def _trackentriesforalbum(self, albid, pid):
248
        albids = self._albid2rawalbidssorted(albid)
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
        uplog("_trackentriesforalbid: %d -> %s" % (albid, albids))
        # I don't see a way to use a select..in statement and get the
        # order right
        tracks = []
        for albid in albids:
            stmt = '''SELECT docidx FROM tracks
            WHERE album_id = ? ORDER BY trackno'''
            tracks += self._trackentriesforstmt(stmt, (albid,), pid)

        tno = None
        for track in tracks:
            tn = 1
            if 'upnp:originalTrackNumber' in track:
                tn = int(track['upnp:originalTrackNumber'])
            if tno:
                if tn <= tno:
                    tn = tno + 1
                tno = tn
            else:
                tno = tn
            track['upnp:originalTrackNumber'] = str(tno)
        return tracks
            
272

273
274
275
276
277
278
279
280
281
282
283
284
    # Return all albums ids to which any of the selected tracks belong
    def _subtreealbums(self, selwhere, values):
        stmt = 'SELECT DISTINCT album_id FROM tracks ' + selwhere 
        c = self._conn.cursor()
        uplog('subtreealbums: executing %s' % stmt)
        c.execute(stmt, values)
        rawalbids = [r[0] for r in c]
        albids = self._rawalbids2albids(rawalbids)
        uplog('subtreealbums: returning %s' % albids)
        return albids
    

285
    def _direntriesforalbums(self, pid, where, path=''):
286
        uplog("_direntriesforalbums. where: %s" % where)
287
        c = self._conn.cursor()
288
289
290
291
292
293
294
295
        args = (path + '%',) if path else ()
        if path:
            if not where:
                where = '''WHERE albfolder LIKE ?'''
            else:
                where += ''' AND albfolder LIKE ?'''
            substmt = '''SELECT DISTINCT albalb FROM ALBUMS %s'''%where
            where = '''WHERE album_id IN (%s)''' % substmt
296
        else:
297
298
299
300
            if not where:
                where = '''WHERE albtdisc IS NULL'''
            else:
                where += ''' AND albtdisc IS NULL'''
301
302

        stmt = '''SELECT album_id, albtitle, albarturi, albdate, artist.value
303
304
        FROM albums LEFT JOIN artist ON artist.artist_id = albums.artist_id
        %s ORDER BY albtitle''' % where
305
306

        uplog('direntriesforalbums: %s' % stmt)
307
        c.execute(stmt, args)
308
        entries = []
309
310
311
        for r in c:
            id = pid + '$' + str(r[0])
            entries.append(
312
                rcldirentry(id, pid, r[1], arturi=r[2], date=r[3],artist=r[4],
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
                            upnpclass='object.container.album.musicAlbum'))
        return entries


    # This is called when an 'albums' element is encountered in the
    # selection path. i is the index of the albums element. The tree under
    # albums has a well defined structure: ql=len(qpath), we have an
    # albums list if i is the last element (i == ql-1), an album track
    # list for i == ql-2 (we then have an albid at ql-1), and a 'Complete
    # album' query if i == ql-3 (...$albums$xxx$showca)
    def _tagsbrowsealbums(self, pid, qpath, i, selwhere, values):
        uplog("_tagsbrowsealbums: pid %s qpath %s i %s selwhere %s values %s" %
              (pid, qpath, i, selwhere, values))
        c = self._conn.cursor()
        entries = []
        if i == len(qpath)-1:
329
330
            # List of albums to which belong any track from selection
            albidsl = self._subtreealbums(selwhere, values)
331
332
333
334
            albids = ','.join([str(a) for a in albidsl])
            where = ' WHERE album_id in (' + albids + ') '
            entries = self._direntriesforalbums(pid, where)
        elif i == len(qpath)-2:
335
            # Album track list. Maybe a merged album->multiple phys albids
336
            albid = int(qpath[-1])
337
338
339
340
341
            rawalbids = self._albids2rawalbids((albid,))
            uplog("_tagsbrowsealbums: albid %s rawalbids %s"%(albid,rawalbids))
            stmt = '''SELECT COUNT(docidx) FROM tracks
                WHERE album_id IN (%s)''' % ','.join('?'*len(rawalbids))
            c.execute(stmt, rawalbids)
342
343
            r = c.fetchone()
            ntracks = int(r[0])
344
            docidsl = self._docidsforsel(selwhere, values)
345
346
347
348
            stmt = '''SELECT docidx FROM tracks 
                WHERE album_id IN (%s) AND docidx IN (%s)''' % \
            (','.join('?'*len(rawalbids)), ','.join('?'*len(docidsl)))
            entries = self._trackentriesforstmt(stmt, rawalbids+docidsl, pid)
349
350
351
352
            if ntracks != len(entries):
                id = pid + '$' + 'showca'
                entries = [rcldirentry(id, pid, '>> Complete Album')] + entries
        elif i == len(qpath)-3:
353
            # 'Complete album' entry
354
355
356
357
358
359
360
            # Note that minim has an additional level here, probably to
            # present groups or multiple groups ? The trackids ids are
            # like: 
            #    0$=Composer$17738$albums$2$showca.0$hcalbum$*i13458
            # I don't know what the .0 is for.
            # The 'hcalbum' level usually has 2 entries '>> Hide Content' 
            # and the album title. TBD
361
            albid = int(qpath[-2])
362
            entries = self._trackentriesforalbum(albid, pid)
363
        
364
365
366
367
        return entries


    # This is called when an 'items' element is encountered in the
368
    # selection path. 
369
370
    def _tagsbrowseitems(self, pid, qpath, i, selwhere, values):
        stmt = 'SELECT docidx FROM tracks ' + selwhere
371
372
373
        c = self._conn.cursor()
        rows = c.execute(stmt, values)
        docids = [r[0] for r in rows]
374
        albids = self._subtreealbums(selwhere, values)
375
376
377
378
379
380
381
382
383
384
385
386
387
388
        entries = []
        displaytracks = True
        if len(albids) == 1:
            # Only display '>> Complete album' if not all tracks
            # already there. If all tracks are there, we display
            # the album entry (with the same id value: show album)
            albid = albids[0]
            tlist = self._trackentriesforalbum(albid, pid)
            # Replace $items with $albums for the album entry
            id = pid.replace('$items', '$albums') + '$' + str(albid) + '$showca'
            if len(tlist) != len(docids):
                entries.append(rcldirentry(id, pid, '>> Complete Album'))
            else:
                displaytracks = False
389
                el = self._direntriesforalbums(pid, "WHERE album_id = %s"%albid)
390
391
392
                el[0]['id'] = id
                entries.append(el[0])
        if displaytracks:
393
            rcldocs = uprclinit.getTree('folders').rcldocs()
394
395
            entries += [rcldoctoentry(pid + '$i' + str(docid),
                                      pid, self._httphp, self._pprefix,
396
                                      rcldocs[docid]) for docid in docids]
397
        return sorted(entries, key=cmpentries)
398
399
400
401
402


    # Main browsing routine. Given an objid, translate it into a select
    # statement, plus further processing, and return the corresponding
    # records
403
    def _tagsbrowse(self, pid, qpath, flag, path=''):
404
        uplog("tagsbrowse. pid %s qpath %s" % (pid, qpath))
405

406
407
        tagdisplaytag = uprcltagscreate.getTagDisplayTag()

408
409
410
411
412
        # Walk the qpath, which was generated from the objid and
        # defines what tracks are selected and what we want to
        # display. E.g =Artist$21$=Date would display all distinct
        # dates for tracks by Artist #21. =Artist$21$=Date$48 the data
        # for date 48 (the numbers are indexes into the aux tables)
413
414
        qlen = len(qpath)
        selwhat = ''
415
416
417
418
419
420
        if path:
            selwhere = ' WHERE tracks.path LIKE ? '
            values = [path + '%',]
        else:
            selwhere = ''
            values = []
421
        tagtotable = uprcltagscreate.getTagToTable()
422
423
424
425
426
427
428
429
430
431
432
        i = 0
        while i < qlen:
            elt = qpath[i]

            # Detect the special values: albums items etc. here. Their
            # presence changes how we process the rest (showing tracks and
            # albums and not dealing with other tags any more)
            if elt == 'albums':
                return self._tagsbrowsealbums(pid, qpath, i, selwhere, values)
            elif elt == 'items':
                return self._tagsbrowseitems(pid, qpath, i, selwhere, values)
433
            
434
435
436
437
            # '=colname'. Set the current column name, which will be used
            # in different ways depending if this is the last element or
            # not.
            if elt.startswith('='):
438
                col = tagtotable[elt[1:]] 
439

440
441
            selwhere = selwhere + ' AND ' if selwhere else ' WHERE '
            if i == qlen - 1:
442
443
444
445
446
447
448
449
450
                # We can only get here if the qpath ends with '=colname'
                # (otherwise the else branch below fetches the 2 last
                # elements and breaks the loop). We want to fetch all
                # unique values for the column inside the current selection.

                # e.g. artist.artist_id, artist.value
                selwhat = '%s.%s, %s.value' % (col, _clid(col), col)
                # e.g. tracks.artist_id = artist.artist_id
                selwhere += 'tracks.%s = %s.%s' % (_clid(col), col, _clid(col))
451
452
453
454
            else:
                # Look at the value specified for the =xx column. The
                # selwhat value is only used as a flag
                selwhat = 'tracks.docidx'
455
                selwhere += 'tracks.%s =  ?' % _clid(col)
456
457
                i += 1
                values.append(int(qpath[i]))
458
            i += 1
459

460
461
        entries = []
        if selwhat == 'tracks.docidx':
462
            # We are displaying content for a given value of a given tag
463
            docids = self._docidsforsel(selwhere, values)
464
465
            albids = self._subtreealbums(selwhere, values)
            subqs = self._subtreetags(selwhere, values)
466
            displaytracks = True
467
468
            if len(albids) > 1:
                id = pid + '$albums'
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
                label = '%d albums'
                entries.append(rcldirentry(id, pid, label % len(albids)))
            elif len(albids) == 1 and not subqs:
                # Only display '>> Complete album' if not all tracks
                # already there. If all tracks are there, we display
                # the album entry (with the same id value: show album)
                albid = albids[0]
                tlist = self._trackentriesforalbum(albid, pid)
                id = pid + '$albums$' + str(albid) + '$showca'
                if len(tlist) != len(docids):
                    entries.append(rcldirentry(id, pid, '>> Complete Album'))
                else:
                    displaytracks = False
                    el = self._direntriesforalbums(pid,
                                                   "WHERE album_id = %s"%albid)
                    el[0]['id'] = id
                    entries.append(el[0])

            if subqs:
488
                id = pid + '$items'
489
490
491
492
                label = '%d items'
                entries.append(rcldirentry(id, pid, label % len(docids)))
                for tt in subqs:
                    id = pid + '$=' + tt
493
                    entries.append(rcldirentry(id, pid, tagdisplaytag[tt]))
494
            elif displaytracks:
495
                rcldocs = uprclinit.getTree('folders').rcldocs()
496
497
498
499
                for docidx in docids:
                    id = pid + '$*i' + str(docidx)
                    entries.append(rcldoctoentry(id, pid, self._httphp,
                                                 self._pprefix,
500
                                                 rcldocs[docidx]))
501
                    entries = sorted(entries, key=cmpentries)
502
        else:
503
            # SELECT col.col_id, col.value FROM tracks, col
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
            # WHERE tracks.col_id = col.col_id
            # GROUP BY tracks.col_id
            # ORDER BY col.value
            stmt = "SELECT " + selwhat + " FROM tracks, " + col + \
                   selwhere + \
                   " GROUP BY tracks." + _clid(col) + \
                   " ORDER BY value"
            uplog("tagsbrowse: executing <%s> values %s" % (stmt,values))
            c = self._conn.cursor()
            c.execute(stmt, values)
            for r in c:
                id = pid + '$' + str(r[0])
                entries.append(rcldirentry(id, pid, r[1]))
        return entries


    # Browse the top-level tree named like 'xxx albums'. There are just 2
    # levels: the whole albums list, then for each entry the specified
    # albums track list
523
    def _albumsbrowse(self, pid, qpath, flag, path=''):
524
525
526
        c = self._conn.cursor()
        entries = []
        if len(qpath) == 1:
527
            entries = self._direntriesforalbums(pid, '', path)
528
529
530
531
532
533
534
535
536
537
        elif len(qpath) == 2:
            e1 = qpath[1]
            album_id = int(e1)
            entries = self._trackentriesforalbum(album_id, pid)
        else:
            raise Exception("Bad path in album tree (too deep): <%s>" % qpath)

        return entries


538
    # Implement the common part of browse() and browseFolder()
539
    def _dobrowse(self, pid, flag, qpath, folder='', offset=0, count=0):
540
541
542
        uplog("Tags:browsFolder: qpath %s"%qpath)
        if qpath[0] == 'items':
            args = (folder + '%',) if folder else ()
543
            folderwhere = ' WHERE tracks.path LIKE ? ' if folder else ' '
544
            stmt = 'SELECT docidx FROM tracks' + folderwhere
545
            entries = self._trackentriesforstmt(stmt, args, pid, offset, count)
546
547
548
549
550
551
552
553
554
        elif qpath[0] == 'albums':
            entries = self._albumsbrowse(pid, qpath, flag, folder)
        elif qpath[0].startswith('='):
            entries = self._tagsbrowse(pid, qpath, flag, folder)
        else:
            raise Exception('Bad path in tags tree (start): <%s>' % qpath)
        return entries
        

555
556
    # Call from the folders tree when Tag View is selected.
    def browseFolder(self, pid, flag, pthremain, folder):
557
        uplog("Tags:browseFolder: pid %s pth %s fld %s"%(pid,pthremain,folder))
558
        l = pthremain.split('$')
559
560
        # 1st elt in list is empty because pthremain begins with $. so
        # len(l)==2 is the root of tags from this folder
561
562
563
564
        if len(l) == 2:
            return self.rootentries(pid + '$', folder)
        else:
            qpath = l[2:]
565
            return self._dobrowse(pid, flag, qpath, folder)
566
567

        
568
569
    # Top level browse routine. Handle the special cases and call the
    # appropriate worker routine.
570
    def browse(self, pid, flag, offset, count):
571
        idpath = pid.replace(uprclinit.getObjPrefix(), '', 1)
572
573
        uplog('tags:browse: idpath <%s>' % idpath)
        qpath = idpath.split('$')
574
        return self._dobrowse(pid, flag, qpath, offset=offset, count=count)
Jean-Francois Dockes's avatar
ckpt    
Jean-Francois Dockes committed
575
576
577
578
579
580
581
582
583







############ Misc test/trial code, not used by uprcl ########################

584
def misctries():
585
    c = self._conn.cursor()
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
    c.execute('''SELECT COUNT(*) FROM tracks''')
    uplog("Count(*) %d" % (c.fetchone()[0],))
    
    #for row in c.execute('''SELECT album
    #                        FROM tracks where artist LIKE "%Gould%"
    #                        GROUP BY album'''):
    #    uplog("%s" % (row,))

    # For some strange reason it appears that GROUP BY is faster than SELECT
    # DISTINCT
    stmt = '''SELECT album FROM tracks GROUP BY album ORDER BY album'''
    start = timer()
    for row in c.execute(stmt):
        #uplog("%s" % (row[0].encode('UTF-8')))
        pass
    end = timer()
    uplog("Select took %.2f Seconds" % (end - start))
    for row in c.execute('''SELECT COUNT(DISTINCT album) from tracks'''):
        uplog("Album count %d" % row[0])


if __name__ == '__main__':
    confdir = "/home/dockes/.recoll-mp3"
    from recoll import recoll

    def fetchalldocs(confdir):
        allthedocs = []
        rcldb = recoll.connect(confdir=confdir)
        rclq = rcldb.query()
        rclq.execute("mime:*", stemming=0)
        uplog("Estimated alldocs query results: %d" % (rclq.rowcount))
        maxcnt = 1000
        totcnt = 0
        while True:
            docs = rclq.fetchmany()
            for doc in docs:
                allthedocs.append(doc)
                totcnt += 1
            if (maxcnt > 0 and totcnt >= maxcnt) or \
                   len(docs) != rclq.arraysize:
                break
        uplog("Retrieved %d docs" % (totcnt,))
        return allthedocs
    
    start = timer()
    docs = fetchalldocs(confdir)
    end = timer()
    uplog("Recoll extract took %.2f Seconds" % (end - start))
    start = timer()
    recolltosql(docs)
    end = timer()
    uplog("SQL db create took %.2f Seconds" % (end - start))