uprclsearch.py 8.87 KB
Newer Older
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright (C) 2017 J.F.Dockes
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
Jean-Francois Dockes's avatar
none    
Jean-Francois Dockes committed
15
16
from __future__ import print_function

17
18
import sys

Jean-Francois Dockes's avatar
uprcl    
Jean-Francois Dockes committed
19
import re
20
21
from recoll import recoll

22
from upmplgutils import uplog
23
from conftree import stringToStrings
24
import uprclutils
25
import uprclinit
26

Jean-Francois Dockes's avatar
none    
Jean-Francois Dockes committed
27
28
29
30
31
32
33
34
35
36
37
38
39
40
def _getchar(s, i):
    if i < len(s):
        return i+1,s[i]
    else:
        return i,None

def _readword(s, i):
    w = ''
    for j in range(i, len(s)):
        if s[j].isspace():
            return j,w
        w += s[j]
    return j,w

41
42
43
44
# Called with '"' already read.
# Upnp search term strings are double quoted, but we should not take
# them as recoll phrases. We separate parts which are internally
# quoted, and become phrases, and lists of words which we interpret as
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
45
# an AND search (comma-separated). Internal quotes come backslash-escaped
46
def _parsestring(s, i=0):
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
47
    #uplog("parseString: input: <%s>" % s[i:])
48
49
50
51
    # First change '''"hello \"one phrase\"''' world" into
    #  '''hello "one phrase" world'''
    # Note that we can't handle quoted dquotes inside string
    str = ''
Jean-Francois Dockes's avatar
none    
Jean-Francois Dockes committed
52
    escape = False
53
    instring = False
Jean-Francois Dockes's avatar
none    
Jean-Francois Dockes committed
54
    for j in range(i, len(s)):
55
56
57
58
59
60
61
62
63
64
65
66
67
        if instring:
            if escape:
                if s[j] == '"':
                    str += '"'
                    instring = False
                else:
                    str += '\\' + s[j]
                escape = False
            else:
                if s[j] == '\\':
                    escape = True
                else:
                    str += s[j]
Jean-Francois Dockes's avatar
none    
Jean-Francois Dockes committed
68
69

        else:
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
            if escape:
                str += s[j]
                escape = False
                if s[j] == '"':
                    instring = True
            else:
                if s[j] == '\\':
                    escape = True
                elif s[j] == '"':
                    j += 2
                    break
                else:
                    str += s[j]
                
    tokens = stringToStrings(str)
    return j, tokens

Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
87
88
89
90
91
92
93
94
95
96
97
98
99
100

def _searchClauses(out, field, oper, words, phrases):
    if words:
        out.append(field)
        out.append(oper)
        out.append(words)
    for ph in phrases:
        out.append(field)
        out.append(oper)
        out.append('"' + ph + '"')
    return out


def _separatePhrasesAndWords(v):
101
102
103
104
105
106
107
108
109
    swords = ""
    phrases = []
    for w in v:
        if len(w.split()) == 1:
            if swords:
                swords += ","
            swords += w
        else:
            phrases.append(w)
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
    return (swords, phrases)

# the v list contains terms and phrases. Fields maybe several space
# separated field specs, which we should OR (ex: for search title or
# filename).
def _makeSearchExp(out, v, field, oper, neg):
    uplog("_makeSearchExp: v <%s> field <%s> oper <%s> neg <%s>" %
          (v, field, oper, neg))

    if oper == 'I':
        return

    swords,phrases = _separatePhrasesAndWords(v)

    if neg:
        out.append(" -")

    # Special-case 'title' because we want to also match directory names
    # ((title:keyword) OR (filename:keyword AND mime:inode/directory))
    if field == 'title':
        fields = (field, 'filename')
    else:
        fields = (field,)
        
    if len(fields) > 1:
        out.append(" (")

    for i in range(len(fields)):
        field = fields[i]
        out.append(" (")
        _searchClauses(out, field, oper, swords, phrases)
141
142
143
        # We'd like to do the following to avoid matching reg file names but
        # recoll takes all mime: clause as global filters, so can't work
        # if i == 1: out.append(" AND mime:inode/directory")
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
144
145
146
147
148
149
150
151
        out.append(")")
        if len(fields) == 2 and i == 0:
            out.append(" OR ")

    if len(fields) > 1:
        out.append(") ")
        

152
153
def _upnpsearchtorecoll(s):
    uplog("_upnpsearchtorecoll:in: <%s>" % s)
Jean-Francois Dockes's avatar
uprcl    
Jean-Francois Dockes committed
154
155

    s = re.sub('[\t\n\r\f ]+', ' ', s)
Jean-Francois Dockes's avatar
none    
Jean-Francois Dockes committed
156
157

    out = []
158
159
    field = ""
    oper = ""
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
160
161
    neg = False
    i = 0
Jean-Francois Dockes's avatar
none    
Jean-Francois Dockes committed
162
163
164
165
    while True:
        i,c = _getchar(s, i)
        if not c:
            break
166
        #uplog("_upnpsearchtorecoll: nextchar: <%s>" % c)
Jean-Francois Dockes's avatar
none    
Jean-Francois Dockes committed
167
168
169
170
171
172
173
174
175
176
177

        if c.isspace():
            continue

        if c == "*":
            if (len(out) > 1 or (len(out) == 1 and not out[-1].isspace())) or \
                   (len(s[i:]) and not s[i:].isspace()):
                raise Exception("If * is used it must be the only input")
            out = ["mime:*"]
            break

178
        if c == '(' or c == ')': 
Jean-Francois Dockes's avatar
none    
Jean-Francois Dockes committed
179
            out.append(c)
180
181
        elif c == '>' or c == '<' or c == '=':
            oper += c
Jean-Francois Dockes's avatar
none    
Jean-Francois Dockes committed
182
183
        else:
            if c == '"':
184
                i,v = _parsestring(s, i)
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
185
                _makeSearchExp(out, v, field, oper, neg)
186
                field = ""
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
187
188
                oper = ""
                neg = False
189
                continue
Jean-Francois Dockes's avatar
none    
Jean-Francois Dockes committed
190
191
192
            else:
                i -= 1
                i,w = _readword(s, i)
Jean-Francois Dockes's avatar
ckpt    
Jean-Francois Dockes committed
193
                #uplog("_readword returned <%s>" % w)
Jean-Francois Dockes's avatar
none    
Jean-Francois Dockes committed
194
195

            if w == 'contains':
196
                oper = ':'
Jean-Francois Dockes's avatar
none    
Jean-Francois Dockes committed
197
            elif w == 'doesNotContain':
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
198
                neg = True
199
                oper = ':'
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
200
201
202
203
204
205
206
207
208
            elif w == 'derivedFrom' or w == 'exists':
                # upnp:class derivedfrom "object.container.album"
                # exists??
                # can't use this, will be ignored
                oper = 'I'
            elif w == 'true' or w == 'false':
                # Don't know what to do with this. Just ignore it,
                # by not calling makeSearchExp.
                pass
Jean-Francois Dockes's avatar
none    
Jean-Francois Dockes committed
209
210
211
212
213
214
215
            elif w == 'and':
                # Recoll has implied AND, but see next
                pass
            elif w == 'or':
                # Does not work because OR/AND priorities are reversed
                # between recoll and upnp. This would be very
                # difficult to correct, let's hope that the callers
216
                # use parentheses
Jean-Francois Dockes's avatar
none    
Jean-Francois Dockes committed
217
218
                out.append('OR')
            else:
Jean-Francois Dockes's avatar
ckpt    
Jean-Francois Dockes committed
219
                try:
220
                    field = uprclutils.upnp2rclfields[w]
Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
221
222
223
                except Exception as ex:
                    #uplog("Field translation error: %s"%ex)
                    field = (w,)
Jean-Francois Dockes's avatar
none    
Jean-Francois Dockes committed
224

Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
225
    return " ".join(out)
Jean-Francois Dockes's avatar
none    
Jean-Francois Dockes committed
226

227
228
229
# inobjid is for the container this search is run from.
def search(foldersobj, rclconfdir, inobjid, upnps, idprefix, httphp, pathprefix):
    tags = uprclinit.getTree('tags')
230
    rcls = _upnpsearchtorecoll(upnps)
231

232
    filterdir = foldersobj.dirpath(inobjid)
233
234
    if filterdir and filterdir != "/":
        rcls += " dir:\"" + filterdir + "\""
235
    
236
    uplog("Search: recoll search: <%s>" % rcls)
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254

    rcldb = recoll.connect(confdir=rclconfdir)
    try:
        rclq = rcldb.query()
        rclq.execute(rcls)
    except Exception as e:
        uplog("Search: recoll query raised: %s" % e)
        return []
    
    uplog("Estimated query results: %d" % (rclq.rowcount))
    if rclq.rowcount == 0:
        return []
    
    entries = []
    maxcnt = 0
    while True:
        docs = rclq.fetchmany()
        for doc in docs:
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
            # The doc is either an actual recollindex product from the
            # FS or a synthetic one from uprcltags creating album
            # entries. Different processing for either
            e = None
            if doc["rcludi"].find("albid") == 0:
                albid = doc["rcludi"][5:]
                uplog("rcludi %s albid %s" % (doc["rcludi"], albid))
                e = tags.direntryforalbid(albid)
            else:
                # Objidfordoc uses the path from the url to walk the
                # _dirvec and determine the right entry if doc is a
                # container. If doc is an item, the returned id is bogus
                # (0$uprcl$folders$seeyoulater), because it's not useful
                # at the moment. Of course, this breaks the recommendation
                # for the objids to be consistent and unchanging
                id = foldersobj.objidfordoc(doc)
                uplog("Search: id [%s] for doc udi [%s]\n" % (id, doc["rcludi"]))
                e = uprclutils.rcldoctoentry(id, inobjid, httphp, pathprefix, doc)
273
274
275
276
            if e:
                entries.append(e)
        if (maxcnt > 0 and len(entries) >= maxcnt) or \
               len(docs) != rclq.arraysize:
277
            break
278
    uplog("Search retrieved %d docs" % (len(entries),))
279

280
281
    entries.sort(key=uprclutils.cmpentries)
    return entries
282

Jean-Francois Dockes's avatar
none    
Jean-Francois Dockes committed
283
284
285
286
287
if __name__ == '__main__':
    s = '(upnp:artist derivedFrom  "abc\\"def\\g") or (dc:title:xxx) '
    print("INPUT: %s" % s)
    o = upnpsearchtorecoll(s)
    print("OUTPUT: %s" % o)