Commit bcb0a62d authored by Sacha's avatar Sacha Committed by SXibolet@2PITAU
Browse files

nohup finally running after some bugs.

parent ab9aa668
......@@ -230,9 +230,7 @@ def keyword_exists(kw):
poss = col.find({
'keyword': kw
})
print(poss.count())
print('here')
return poss.count()
return poss.count() != 0
def article_is_new(url, col=None):
......@@ -243,9 +241,7 @@ def article_is_new(url, col=None):
col = get_articles_collection()
poss = col.find({
'url': {
'$exists': url
}
'url': url
})
return ~poss
return poss.count() == 0
......@@ -51,12 +51,15 @@ if __name__ == '__main__':
print(HELP_STR)
sys.exit(0)
print('running ... this could take a while')
n_tasks = len(terms)
n_done = 0
n_docs = 0
n_dup_articles = 0
n_dup_keywords = 0
for term in terms:
if mturk.keyword_exists(term):
n_dup_keywords += 1
continue
try:
......@@ -66,14 +69,14 @@ if __name__ == '__main__':
continue
col = mturk.get_articles_collection()
social['articles'] = filter(lambda x: mturk.article_is_new(x['url']),
social['articles'])
social['result']['articles'] = filter(lambda x: mturk.article_is_new(x['url']),
social['result']['articles'])
n_docs += mturk.new_doc(social)
n_done += 1
update_progress(n_done, n_tasks)
print('\t... done')
summary = 'summary: %s doc(s) ready for training from %s terms' % (n_docs, n_tasks)
summary = 'summary: %s doc(s) ready for training from %s terms (%s skipped)' % (n_docs, n_tasks, n_dup_keywords)
sms_admin('mturk_loader finished. %s' % summary)
print('\t ...' % summary)
......@@ -6,6 +6,8 @@ import db
import redis
import json
import datetime
from functools import partial
from operator import is_not
sr = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT)
......@@ -18,8 +20,9 @@ def perform(keyword, training=False):
if training:
res = {
'articles': map(lambda x: x.to_dict(),
articles),
'articles': filter(partial(is_not, None),
map(lambda x: x.to_dict(),
articles)),
'kw_tweets': map(lambda x: x.to_dict(),
twitter_search(keyword, training=training))
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment