rclaspell.cpp 16.4 KB
Newer Older
dockes's avatar
dockes committed
1
#ifndef TEST_RCLASPELL
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
/* Copyright (C) 2006 J.F.Dockes
 *   This program is free software; you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation; either version 2 of the License, or
 *   (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with this program; if not, write to the
 *   Free Software Foundation, Inc.,
 *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 */
dockes's avatar
dockes committed
18
19
20
21
22
23
#ifdef HAVE_CONFIG_H
#include "autoconfig.h"
#endif

#ifdef RCL_USE_ASPELL

dockes's avatar
dockes committed
24
25
#include <unistd.h>
#include <dlfcn.h>
dockes's avatar
dockes committed
26
#include <stdlib.h>
27

dockes's avatar
dockes committed
28
#include ASPELL_INCLUDE
dockes's avatar
dockes committed
29
30
31
32

#include "pathut.h"
#include "execmd.h"
#include "rclaspell.h"
33
#include "debuglog.h"
34
#include "unacpp.h"
35
36
#include "ptmutex.h"

Jean-Francois Dockes's avatar
Jean-Francois Dockes committed
37
38
using namespace std;

dockes's avatar
dockes committed
39
40
41
42
43
44
45
46
47
48
49
50
51
// Just a place where we keep the Aspell library entry points together
class AspellApi {
public:
    struct AspellConfig *(*new_aspell_config)();
    int (*aspell_config_replace)(struct AspellConfig *, const char * key, 
				 const char * value);
    struct AspellCanHaveError *(*new_aspell_speller)(struct AspellConfig *);
    void (*delete_aspell_config)(struct AspellConfig *);
    void (*delete_aspell_can_have_error)(struct AspellCanHaveError *);
    struct AspellSpeller * (*to_aspell_speller)(struct AspellCanHaveError *);
    struct AspellConfig * (*aspell_speller_config)(struct AspellSpeller *);
    const struct AspellWordList * (*aspell_speller_suggest)
	(struct AspellSpeller *, const char *, int);
52
    int (*aspell_speller_check)(struct AspellSpeller *, const char *, int);
dockes's avatar
dockes committed
53
54
55
56
57
58
59
60
61
62
63
64
65
    struct AspellStringEnumeration * (*aspell_word_list_elements)
	(const struct AspellWordList * ths);
    const char * (*aspell_string_enumeration_next)
	(struct AspellStringEnumeration * ths);
    void (*delete_aspell_string_enumeration)(struct AspellStringEnumeration *);
    const struct AspellError *(*aspell_error)
	(const struct AspellCanHaveError *);
    const char *(*aspell_error_message)(const struct AspellCanHaveError *);
    const char *(*aspell_speller_error_message)(const struct AspellSpeller *);
    void (*delete_aspell_speller)(struct AspellSpeller *);

};
static AspellApi aapi;
66
static PTMutexInit o_aapi_mutex;
dockes's avatar
dockes committed
67
68
69
70
71
72

#define NMTOPTR(NM, TP)							\
    if ((aapi.NM = TP dlsym(m_data->m_handle, #NM)) == 0) {		\
	badnames += #NM + string(" ");					\
    }

dockes's avatar
dockes committed
73
74
75
76
77
78
static const char *aspell_lib_suffixes[] = {
  ".so",
  ".so.15",
  ".so.16"
};
static const unsigned int nlibsuffs  = sizeof(aspell_lib_suffixes) / sizeof(char *);
dockes's avatar
dockes committed
79

80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
// Stuff that we don't wish to see in the .h (possible sysdeps, etc.)
class AspellData {
public:
    AspellData() 
        : m_handle(0), m_speller(0) 
    {}
    ~AspellData() {
        LOGDEB2(("~AspellData\n"));
	if (m_handle) {
	    dlclose(m_handle);
            m_handle = 0;
        }
        if (m_speller) {
            // Dumps core if I do this?? 
            //aapi.delete_aspell_speller(m_speller);
            m_speller = 0;
            LOGDEB2(("~AspellData: speller done\n"));
        }
    }

    void  *m_handle;
    string m_exec;
    AspellSpeller *m_speller;    
};

Aspell::Aspell(RclConfig *cnf)
    : m_config(cnf), m_data(0) 
{
}

Aspell::~Aspell()
{
    deleteZ(m_data);
}

dockes's avatar
dockes committed
115
bool Aspell::init(string &reason)
dockes's avatar
dockes committed
116
{
117
    PTMutexLocker locker(o_aapi_mutex);
118
119
    deleteZ(m_data);

dockes's avatar
dockes committed
120
121
122
    // Language: we get this from the configuration, else from the NLS
    // environment. The aspell language names used for selecting language 
    // definition files (used to create dictionaries) are like en, fr
dockes's avatar
dockes committed
123
    if (!m_config->getConfParam("aspellLanguage", m_lang) || m_lang.empty()) {
dockes's avatar
dockes committed
124
125
	string lang = "en";
	const char *cp;
dockes's avatar
dockes committed
126
	if ((cp = getenv("LC_ALL")))
dockes's avatar
dockes committed
127
	    lang = cp;
dockes's avatar
dockes committed
128
	else if ((cp = getenv("LANG")))
dockes's avatar
dockes committed
129
130
131
132
	    lang = cp;
	if (!lang.compare("C"))
	    lang = "en";
	m_lang = lang.substr(0, lang.find_first_of("_"));
dockes's avatar
dockes committed
133
134
    }

dockes's avatar
dockes committed
135
    m_data = new AspellData;
136
137
138
139
140
141
142
143
144
145

    const char *aspell_prog_from_env = getenv("ASPELL_PROG");
    if (aspell_prog_from_env && access(aspell_prog_from_env, X_OK) == 0) {
	m_data->m_exec = aspell_prog_from_env;
#ifdef ASPELL_PROG
    } else if (access(ASPELL_PROG, X_OK) == 0) {
	m_data->m_exec = ASPELL_PROG;
#endif // ASPELL_PROG
    } else {
	ExecCmd::which("aspell", m_data->m_exec);
dockes's avatar
dockes committed
146
    }
147

dockes's avatar
dockes committed
148
149
    if (m_data->m_exec.empty()) {
	reason = "aspell program not found or not executable";
150
        deleteZ(m_data);
dockes's avatar
dockes committed
151
152
	return false;
    }
dockes's avatar
dockes committed
153

154
    reason = "Could not open shared library ";
155
156
157
158
159
160
161
162
163
    string libbase("libaspell");
    string lib;
    for (unsigned int i = 0; i < nlibsuffs; i++) {
        lib = libbase + aspell_lib_suffixes[i];
        reason += string("[") + lib + "] ";
        if ((m_data->m_handle = dlopen(lib.c_str(), RTLD_LAZY)) != 0) {
            reason.erase();
            goto found;
        }
dockes's avatar
dockes committed
164
    }
165
166
    
 found:
dockes's avatar
dockes committed
167
    if (m_data->m_handle == 0) {
168
169
170
        reason += string(" : ") + dlerror();
        deleteZ(m_data);
        return false;
dockes's avatar
dockes committed
171
172
    }

dockes's avatar
dockes committed
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
    string badnames;
    NMTOPTR(new_aspell_config, (struct AspellConfig *(*)()));
    NMTOPTR(aspell_config_replace, (int (*)(struct AspellConfig *, 
					    const char *, const char *)));
    NMTOPTR(new_aspell_speller, 
	    (struct AspellCanHaveError *(*)(struct AspellConfig *)));
    NMTOPTR(delete_aspell_config, 
	    (void (*)(struct AspellConfig *)));
    NMTOPTR(delete_aspell_can_have_error, 
	    (void (*)(struct AspellCanHaveError *)));
    NMTOPTR(to_aspell_speller, 
	    (struct AspellSpeller *(*)(struct AspellCanHaveError *)));
    NMTOPTR(aspell_speller_config, 
	    (struct AspellConfig *(*)(struct AspellSpeller *)));
    NMTOPTR(aspell_speller_suggest, 
	    (const struct AspellWordList *(*)(struct AspellSpeller *, 
					       const char *, int)));
190
191
    NMTOPTR(aspell_speller_check, 
	    (int (*)(struct AspellSpeller *, const char *, int)));
dockes's avatar
dockes committed
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
    NMTOPTR(aspell_word_list_elements, 
	    (struct AspellStringEnumeration *(*)
	     (const struct AspellWordList *)));
    NMTOPTR(aspell_string_enumeration_next, 
	    (const char * (*)(struct AspellStringEnumeration *)));
    NMTOPTR(delete_aspell_string_enumeration, 
	    (void (*)(struct AspellStringEnumeration *)));
    NMTOPTR(aspell_error, 
	    (const struct AspellError*(*)(const struct AspellCanHaveError *)));
    NMTOPTR(aspell_error_message,
	    (const char *(*)(const struct AspellCanHaveError *)));
    NMTOPTR(aspell_speller_error_message, 
	    (const char *(*)(const struct AspellSpeller *)));
    NMTOPTR(delete_aspell_speller, (void (*)(struct AspellSpeller *)));

    if (!badnames.empty()) {
	reason = string("Aspell::init: symbols not found:") + badnames;
209
        deleteZ(m_data);
dockes's avatar
dockes committed
210
211
	return false;
    }
dockes's avatar
dockes committed
212

dockes's avatar
dockes committed
213
214
215
    return true;
}

216
bool Aspell::ok() const
dockes's avatar
dockes committed
217
218
219
220
221
222
{
    return m_data != 0 && m_data->m_handle != 0;
}

string Aspell::dicPath()
{
dockes's avatar
dockes committed
223
    return path_cat(m_config->getConfDir(), 
dockes's avatar
dockes committed
224
225
226
		    string("aspdict.") + m_lang + string(".rws"));
}

227

228
229
230
231
232
// The data source for the create dictionary aspell command. We walk
// the term list, filtering out things that are probably not words.
// Note that the manual for the current version (0.60) of aspell
// states that utf-8 is not well supported, so that we should maybe
// also filter all 8bit chars.
233
234
235
236
237
238
239
240
241
242
class AspExecPv : public ExecCmdProvide {
public:
    string *m_input; // pointer to string used as input buffer to command
    Rcl::TermIter *m_tit;
    Rcl::Db &m_db;
    AspExecPv(string *i, Rcl::TermIter *tit, Rcl::Db &db) 
	: m_input(i), m_tit(tit), m_db(db)
    {}
    void newData() {
	while (m_db.termWalkNext(m_tit, *m_input)) {
243
244
245
	    LOGDEB2(("Aspell::buildDict: term: [%s]\n", m_input->c_str()));
	    if (!Rcl::Db::isSpellingCandidate(*m_input)) {
		LOGDEB2(("Aspell::buildDict: SKIP\n"));
246
		continue;
247
	    }
248
249
250
251
252
253
	    if (!o_index_stripchars) {
		string lower;
		if (!unacmaybefold(*m_input, lower, "UTF-8", UNACOP_FOLD))
		    continue;
		m_input->swap(lower);
	    }
254
255
	    // Got a non-empty sort-of appropriate term, let's send it to
	    // aspell
256
	    LOGDEB2(("Apell::buildDict: SEND\n"));
257
258
259
260
261
262
263
264
265
	    m_input->append("\n");
	    return;
	}
	// End of data. Tell so. Exec will close cmd.
	m_input->erase();
    }
};


dockes's avatar
dockes committed
266
267
bool Aspell::buildDict(Rcl::Db &db, string &reason)
{
dockes's avatar
dockes committed
268
269
270
    if (!ok())
	return false;

271
272
273
    string addCreateParam;
    m_config->getConfParam("aspellAddCreateParam", addCreateParam);

dockes's avatar
dockes committed
274
275
    // We create the dictionary by executing the aspell command:
    // aspell --lang=[lang] create master [dictApath]
276
    string cmdstring(m_data->m_exec);
dockes's avatar
dockes committed
277
    ExecCmd aspell;
278
    vector<string> args;
dockes's avatar
dockes committed
279
    args.push_back(string("--lang=")+ m_lang);
280
    cmdstring += string(" ") + string("--lang=") + m_lang;
281
    args.push_back("--encoding=utf-8");
282
    cmdstring += string(" ") + "--encoding=utf-8";
283
284
285
286
    if (!addCreateParam.empty()) {
	args.push_back(addCreateParam);
	cmdstring += string(" ") + addCreateParam;
    }
dockes's avatar
dockes committed
287
    args.push_back("create");
288
    cmdstring += string(" ") + "create";
dockes's avatar
dockes committed
289
    args.push_back("master");
290
    cmdstring += string(" ") + "master";
dockes's avatar
dockes committed
291
    args.push_back(dicPath());
292
    cmdstring += string(" ") + dicPath();
293
294
295
296
297
298
299
300

    // Have to disable stderr, as numerous messages about bad strings are
    // printed. We'd like to keep errors about missing databases though, so
    // make it configurable for diags
    bool keepStderr = false;
    m_config->getConfParam("aspellKeepStderr", &keepStderr);
    if (!keepStderr)
	aspell.setStderr("/dev/null");
dockes's avatar
dockes committed
301
302
303
304
305
306

    Rcl::TermIter *tit = db.termWalkOpen();
    if (tit == 0) {
	reason = "termWalkOpen failed\n";
	return false;
    }
307
308
309
    string termbuf;
    AspExecPv pv(&termbuf, tit, db);
    aspell.setProvide(&pv);
310
    
311
    if (aspell.doexec(m_data->m_exec, args, &termbuf)) {
312
313
314
315
316
	ExecCmd cmd;
	args.clear();
	args.push_back("dicts");
	string dicts;
	bool hasdict = false;
317
	if (cmd.doexec(m_data->m_exec, args, 0, &dicts)) {
318
319
320
321
322
323
324
325
	    vector<string> vdicts;
	    stringToTokens(dicts, vdicts, "\n\r\t ");
	    if (find(vdicts.begin(), vdicts.end(), m_lang) != vdicts.end()) {
		hasdict = true;
	    }
	}
	if (hasdict)
	    reason = string(
326
327
		"\naspell dictionary creation command [") +
                cmdstring + string("] failed. Reason unknown.\n"
328
329
330
331
		"Try to set aspellKeepStderr = 1 in recoll.conf, and execute \n"
		"the indexing command in a terminal to see the aspell "
		"diagnostic output.\n");
	else
332
333
334
335
336
	    reason = string("aspell dictionary creation command failed:\n") +
                cmdstring + "\n"
                "One possible reason might be missing language "
                "data files for lang = " + m_lang +
                ". Maybe try to execute the command by hand for a better diag.";
dockes's avatar
dockes committed
337
338
	return false;
    }
339
    db.termWalkClose(tit);
dockes's avatar
dockes committed
340
    return true;
dockes's avatar
dockes committed
341
342
343
}


344
bool Aspell::make_speller(string& reason)
dockes's avatar
dockes committed
345
{
dockes's avatar
dockes committed
346
347
    if (!ok())
	return false;
348
349
    if (m_data->m_speller != 0)
        return true;
dockes's avatar
dockes committed
350

dockes's avatar
dockes committed
351
352
    AspellCanHaveError *ret;

353
    AspellConfig *config = aapi.new_aspell_config();
dockes's avatar
dockes committed
354
355
356
    aapi.aspell_config_replace(config, "lang", m_lang.c_str());
    aapi.aspell_config_replace(config, "encoding", "utf-8");
    aapi.aspell_config_replace(config, "master", dicPath().c_str());
dockes's avatar
dockes committed
357
358
    aapi.aspell_config_replace(config, "sug-mode", "fast");
    //    aapi.aspell_config_replace(config, "sug-edit-dist", "2");
dockes's avatar
dockes committed
359
360
361
362
363
364
365
366
    ret = aapi.new_aspell_speller(config);
    aapi.delete_aspell_config(config);

    if (aapi.aspell_error(ret) != 0) {
	reason = aapi.aspell_error_message(ret);
	aapi.delete_aspell_can_have_error(ret);
	return false;
    }
367
368
369
370
    m_data->m_speller = aapi.to_aspell_speller(ret);
    return true;
}

371
bool Aspell::check(const string &iterm, string& reason)
372
{
373
374
    LOGDEB2(("Aspell::check [%s]\n", iterm.c_str()));
    string mterm(iterm);
375
376
377

    if (!ok() || !make_speller(reason))
	return false;
378
    if (iterm.empty())
379
380
        return true; //??

381
382
383
384
385
386
387
388
389
    if (!o_index_stripchars) {
	string lower;
	if (!unacmaybefold(mterm, lower, "UTF-8", UNACOP_FOLD)) {
	    LOGERR(("Aspell::check : cant lowercase input\n"));
	    return false;
	}
	mterm.swap(lower);
    }

390
    int ret = aapi.aspell_speller_check(m_data->m_speller, 
391
                                        mterm.c_str(), mterm.length());
392
393
394
395
396
397
398
399
400
401
402
403
    reason.clear();
    switch (ret) {
    case 0: return false;
    case 1: return true;
    default:
    case -1:
        reason.append("Aspell error: ");
        reason.append(aapi.aspell_speller_error_message(m_data->m_speller));
        return false;
    }
}

404
bool Aspell::suggest(Rcl::Db &db, const string &_term, 
405
406
407
408
                     list<string>& suggestions, string& reason)
{
    if (!ok() || !make_speller(reason))
	return false;
409
410
    string mterm(_term);
    if (mterm.empty())
411
412
        return true; //??

413
414
415
416
417
418
419
420
421
    if (!o_index_stripchars) {
	string lower;
	if (!unacmaybefold(mterm, lower, "UTF-8", UNACOP_FOLD)) {
	    LOGERR(("Aspell::check : cant lowercase input\n"));
	    return false;
	}
	mterm.swap(lower);
    }

422
423
    AspellCanHaveError *ret;

dockes's avatar
dockes committed
424
    const AspellWordList *wl = 
425
	aapi.aspell_speller_suggest(m_data->m_speller, 
426
                                    mterm.c_str(), mterm.length());
dockes's avatar
dockes committed
427
    if (wl == 0) {
428
	reason = aapi.aspell_speller_error_message(m_data->m_speller);
dockes's avatar
dockes committed
429
430
431
432
433
	return false;
    }
    AspellStringEnumeration *els = aapi.aspell_word_list_elements(wl);
    const char *word;
    while ((word = aapi.aspell_string_enumeration_next(els)) != 0) {
434
435
436
437
438
439
440
	// Check that the word exists in the index (we don't want
	// aspell computed stuff, only exact terms from the
	// dictionary).  We used to also check that it stems
	// differently from the base word but this is complicated
	// (stemming on/off + language), so we now leave this to the
	// caller.
	if (db.termExists(word))
dockes's avatar
dockes committed
441
	    suggestions.push_back(word);
dockes's avatar
dockes committed
442
443
444
445
446
    }
    aapi.delete_aspell_string_enumeration(els);
    return true;
}

dockes's avatar
dockes committed
447
#endif // RCL_USE_ASPELL
dockes's avatar
dockes committed
448
449
450

#else // TEST_RCLASPELL test driver ->

dockes's avatar
dockes committed
451
452
453
454
455
456
#ifdef HAVE_CONFIG_H
#include "autoconfig.h"
#endif

#ifdef RCL_USE_ASPELL

dockes's avatar
dockes committed
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>

#include <iostream>
using namespace std;

#include "rclinit.h"
#include "rclconfig.h"
#include "rcldb.h"
#include "rclaspell.h"

static char *thisprog;
RclConfig *rclconfig;

static char usage [] =
dockes's avatar
dockes committed
475
476
" -b : build dictionary\n"
" -s <term>: suggestions for term\n"
477
478
" -c <term>: check term\n"
"\n"
dockes's avatar
dockes committed
479
480
481
482
483
484
485
486
487
488
489
490
;
static void
Usage(void)
{
    fprintf(stderr, "%s: usage:\n%s", thisprog, usage);
    exit(1);
}

static int     op_flags;
#define OPT_MOINS 0x1
#define OPT_s	  0x2 
#define OPT_b	  0x4 
491
#define OPT_c     0x8
dockes's avatar
dockes committed
492
493
494

int main(int argc, char **argv)
{
dockes's avatar
dockes committed
495
    string word;
dockes's avatar
dockes committed
496
    
dockes's avatar
dockes committed
497
498
499
500
501
502
503
504
505
506
507
    thisprog = argv[0];
    argc--; argv++;

    while (argc > 0 && **argv == '-') {
	(*argv)++;
	if (!(**argv))
	    /* Cas du "adb - core" */
	    Usage();
	while (**argv)
	    switch (*(*argv)++) {
	    case 'b':	op_flags |= OPT_b; break;
508
509
510
511
	    case 'c':	op_flags |= OPT_c; if (argc < 2)  Usage();
		word = *(++argv);
		argc--; 
		goto b1;
dockes's avatar
dockes committed
512
513
514
515
516
517
518
519
	    case 's':	op_flags |= OPT_s; if (argc < 2)  Usage();
		word = *(++argv);
		argc--; 
		goto b1;
	    default: Usage();	break;
	    }
    b1: argc--; argv++;
    }
dockes's avatar
dockes committed
520

dockes's avatar
dockes committed
521
522
    if (argc != 0 || op_flags == 0)
	Usage();
dockes's avatar
dockes committed
523

dockes's avatar
dockes committed
524
525
526
527
528
    string reason;
    rclconfig = recollinit(0, 0, reason);
    if (!rclconfig || !rclconfig->ok()) {
	fprintf(stderr, "Configuration problem: %s\n", reason.c_str());
	exit(1);
dockes's avatar
dockes committed
529
530
    }

dockes's avatar
dockes committed
531
532
533
534
535
    string dbdir = rclconfig->getDbDir();
    if (dbdir.empty()) {
	fprintf(stderr, "No db directory in configuration");
	exit(1);
    }
dockes's avatar
dockes committed
536

537
538
539
    Rcl::Db rcldb(rclconfig);

    if (!rcldb.open(Rcl::Db::DbRO, 0)) {
dockes's avatar
dockes committed
540
541
542
	fprintf(stderr, "Could not open database in %s\n", dbdir.c_str());
	exit(1);
    }
dockes's avatar
dockes committed
543

dockes's avatar
dockes committed
544
    Aspell aspell(rclconfig);
dockes's avatar
dockes committed
545

dockes's avatar
dockes committed
546
    if (!aspell.init(reason)) {
dockes's avatar
dockes committed
547
548
549
550
551
552
553
554
	cerr << "Init failed: " << reason << endl;
	exit(1);
    }
    if (op_flags & OPT_b) {
	if (!aspell.buildDict(rcldb, reason)) {
	    cerr << "buildDict failed: " << reason << endl;
	    exit(1);
	}
555
556
557
558
559
560
561
562
563
564
565
566
    } else if (op_flags & OPT_c) {
	bool ret = aspell.check(word, reason);
	if (!ret && reason.size()) {
	    cerr << "Aspell error: " << reason << endl;
	    return 1;
	}
	cout << word;
	if (ret) {
	    cout << " is in dictionary" << endl;
	} else {
	    cout << " not in dictionary" << endl;
	}
dockes's avatar
dockes committed
567
568
569
570
571
572
573
574
575
576
577
578
579
    } else {
	list<string> suggs;
	if (!aspell.suggest(rcldb, word, suggs, reason)) {
	    cerr << "suggest failed: " << reason << endl;
	    exit(1);
	}
	cout << "Suggestions for " << word << ":" << endl;
	for (list<string>::iterator it = suggs.begin(); 
	     it != suggs.end(); it++) {
	    cout << *it << endl;
	}
    }
    exit(0);
dockes's avatar
dockes committed
580
}
dockes's avatar
dockes committed
581
582
583
584
#else
int main(int argc, char **argv)
{return 1;}
#endif // RCL_USE_ASPELL
dockes's avatar
dockes committed
585
586

#endif // TEST_RCLASPELL test driver