Commit bd59e9c2 authored by Sacha's avatar Sacha Committed by SXibolet@2PITAU
Browse files

various tweaks

parent 1eaf6666
......@@ -55,7 +55,7 @@ def handle_500(error):
from flask import jsonify
from error import UsageError
client = TwilioRestClient(TWILIO_SID, TWILIO_AUTH_TOKEN)
client.messages.create(body="problem on Linode: %s" % repr(error), to=ADMIN_PHONE, from_="+19089982913")
client.messages.create(body='problem on Linode: %s' % repr(error), to=ADMIN_PHONE, from_='+19089982913')
raise UsageError('our-fault', status_code=500)
......
......@@ -22,10 +22,13 @@ from operator import is_not
MAX_ATTEMPTS = 6
MAX_COMMENTARY = 500
TAG_RE = re.compile(r'<[^>]+>')
ARTICLE_SEARCH_BASE = 'http://api.nytimes.com/svc/search/v2/articlesearch.json?'
COMMENT_BASE = 'http://api.nytimes.com/svc/community/v3/user-content/url.json?'
class SocialContent(object):
"""Subclasses are wrappers for social APIs
"""
def __init__(self, clean, dirty, training=False):
self.clean = clean
self.dirty = dirty
......@@ -43,7 +46,7 @@ class SocialContent(object):
class Tweet(SocialContent):
"""A tweet
holds basic attributes and finds sentiment
holds basic attributes and finds sentiment.
"""
def __init__(self, j, training=False):
......@@ -102,7 +105,11 @@ class Article(object):
self.xlarge = 'https://www.nytimes.com/%s' % j['multimedia'][1]['url'] if len(j['multimedia']) > 1 else None
self.published = j['pub_date'][:10]
self.full = self._full_text(training)
self.comments = article_comments(self.url, trainging=training)
if training:
self.title_tweets = twitter_search(self.title, training=training)
self.comments = article_comments(self.url, trainging=training)
# notice that this doesn't include tweets
self.n_comments = len(self.comments)
def to_dict(self):
return self.__dict__ if (self.full is not None and len(self.full)) != 0 else None
......@@ -120,7 +127,9 @@ class Article(object):
opener.addheaders = [('User-Agent', 'Mozilla/5.0')]
response = opener.open(urllib2.Request(self.url))
soup = BeautifulSoup(response.read(), 'html.parser')
body = soup.findAll('p', {'class' : ['story-body-text', 'story-content']})
body = soup.findAll('p', {
'class' : ['story-body-text', 'story-content']
})
# we'll split into paragraphs for easier reading if training
res = ('|*^*|' if training else ' ').join(p.text for p in body)
jar.clear()
......@@ -145,10 +154,12 @@ def article_search(keyword, training=False):
'facet_field': 'source'
})
response = urllib2.urlopen('http://api.nytimes.com/svc/search/v2/articlesearch.json?%s' % params)
response = urllib2.urlopen('%s%s' % (ARTICLE_SEARCH_BASE, params))
# an Article will be None if it doesn't have body text (thus the partial)
# return an array of Article objects that have a body text
return filter(partial(is_not, None), map(lambda x: Article(x, training=training), json.loads(response.read())['response']['docs']))
return filter(partial(is_not, None),
map(lambda x: Article(x, training=training),
json.loads(response.read())['response']['docs']))
def article_comments(url, offset=0, training=False):
......@@ -164,7 +175,7 @@ def article_comments(url, offset=0, training=False):
'offset': i * 25
})
response = urllib2.urlopen('http://api.nytimes.com/svc/community/v3/user-content/url.json?%s' % params)
response = urllib2.urlopen('%s%s' % (COMMENT_BASE, params))
try:
comment_batch = json.loads(response.read())['results']['comments']
except ValueError:
......@@ -198,7 +209,6 @@ def twitter_search(keyword, training=False):
})
response = twitter.search(**_kwargs)
tweets += map(lambda x: Tweet(x, training=training), response['statuses'])
try:
next_res = response['search_metadata']['next_results']
......
......@@ -159,6 +159,7 @@ def toggle_being_read(col, url, dest):
def new_doc(doc):
"""given an API response, make an entry for each article,
preserving the timestamp of the entire response and keyword.
Returns number of articles found.
"""
col = get_collection()
n_docs = 0
......
......@@ -7,7 +7,7 @@ from error import UsageError
import sys
# source of terms ===> http://arxiv.org/pdf/1409.8152v1.pdf
# source of terms --> http://arxiv.org/pdf/1409.8152v1.pdf
TT_DIR = 'training_terms'
......@@ -19,8 +19,10 @@ def get_file(name):
def update_progress(n_done, n_tasks):
pprogress = int(100 * (float(n_done) / n_tasks))
sys.stdout.write('\r working ... [ %s ] %s%%'
% ('#' * (pprogress / 5), pprogress))
fifthp = pprogress / 5
fourfifthp = (n_tasks / 5) - fifthp
sys.stdout.write('\r working ... [ %s%s ] %s%%'
% ('#' * (fifthp), '_' * fourfifthp, pprogress))
sys.stdout.flush()
......@@ -35,7 +37,7 @@ if __name__ == '__main__':
print('using small subset of training terms')
terms = list(set(terms))[:1]
elif sys.argv[1] in {'--help', 'help'}:
print('''options:\n\t``test`` <== uses small subset of training terms \n\t\t\tfor testing mturk\n\t```` <== (no options) loads all training keywords\n\t``help`` or ``--help`` <== displays this message\n''')
print('''options:\n\t``test`` <-- uses small subset of training terms \n\t\t\tfor testing mturk\n\t```` <-- (no options) loads all training keywords\n\t``help`` or ``--help`` <-- displays this message\n''')
sys.exit(0)
n_tasks = len(terms)
......@@ -54,4 +56,4 @@ if __name__ == '__main__':
update_progress(n_done, n_tasks)
print('\t... done')
print('\t ... summary: %s doc(s) ready for training' % n_docs)
print('\t ... summary: %s doc(s) ready for training from %s terms' % (n_docs, n_tasks))
......@@ -36,7 +36,6 @@ class BM25:
self.DocAvgLen = 0
self.fn_docs = fn_docs
self.DocLen = []
#self.raw_data = []
self.build_dictionary()
self.tf_idf_generator()
......@@ -53,13 +52,11 @@ class BM25:
def tf_idf_generator(self, base=math.e):
docTotalLen = 0.0
# print (self.dictionary.token2id)
for i in xrange(len(self.fn_docs)):
text = self.fn_docs[i].clean
doc = preprocess(text)
docTotalLen += len(doc)
self.DocLen.append(len(doc))
# print self.dictionary.doc2bow(doc)
bow = dict([(term, freq * 1.0 / len(doc)) for term, freq in self.dictionary.doc2bow(doc)])
for term, tf in bow.items():
if term not in self.DF:
......@@ -85,7 +82,7 @@ class BM25:
below = ((doc[term]) + k1 * (1 - b + b * doc_terms_len / self.DocAvgLen))
tmp_score.append(self.DocIDF[term] * upper / below)
score = sum(tmp_score)
if score!=0:
if score != 0:
scores.append((score, idx))
# descending order
return sorted(scores, reverse=True)
......@@ -95,15 +92,12 @@ class BM25:
tfidf = []
for doc in self.DocTF:
doc_tfidf = [(term, tf * self.DocIDF[term]) for term, tf in doc.items()]
# print doc, tf, self.DocIDF[term]
doc_tfidf.sort()
tfidf.append(doc_tfidf)
# print "tfidf:", tfidf
return tfidf
def items(self):
# return a list [(term_idx, term_desc),]
it = self.dictionary.items()
it.sort()
return it
......@@ -129,7 +123,7 @@ def preprocess(file_content):
def score_entropy(li):
"""Get entropy of list ``li``.
"""Get entropy of list ``li``
"""
ret = dict((x, (li.count(x) / float(len(li)))) for x in set(li)).values()
#ret = map(lambda x: li.count(x) / float(len(li)), li)
......
......@@ -42,11 +42,13 @@
{% else %}
&hellip; nothing yet. May we interest you in <a href="../">a try</a>?
{% endif %}
{% endif %}
</ul>
<h3>This account</h3>
<p>This account is owned by <b>{{ user['Name'] }}</b> who goes by <b>{{ user['Id'] }}</b> and is ostensibly associated with <b>{{ user['School'] }}</b>.</p>
{% if user['Id'] != 'bourbaki@illinois.edu' %}
<p>You're welcome to close your account. We'll delete anything associated with you. This doesn't include contributions to trending queries, since those are anonymous.</p>
<span>
<form method=POST action="{{ url_for('account') }}">
......@@ -66,10 +68,9 @@
{% endfor %}
</p>
{% endif %}
</p>
{% else %}
<!-- bourbaki@illinois.edu cannot be closed -->
{% endif %}
<h3>Usage notes</h3>
<p>We encrypt passwords with a one-way hash and cannot even see your password ourselves. As of fall 2015, we force SSL as you browse the site. This means your connection, namely the data we exchange, is private by default. We'd like to thank StartSSL for providing this service for free.</p>
......
PUT KEYWORDS (NEWLINE DELINEATED) IN TEXT FILES IN THIS DIRECTORY WITH EXTENSION ".txt"
IF THIS IS DONE, THE LOADER WILL FIND THEM AND SCORE THE KEYWORDS
ONLY USE THE ".txt" EXTENSION FOR KEYWORD FILES
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment