Commit 1cfcaaa3 authored by Sacha's avatar Sacha Committed by SXibolet@2PITAU
Browse files

various improvements (see log)

parent 51c14356
......@@ -26,13 +26,15 @@ from digest import digest
application = Flask(__name__)
application.register_blueprint(api, url_prefix='/api')
application.register_blueprint(mturk, url_prefix='/training')
application.secret_key = SECRET_KEY
application.config['RECAPTCHA_PUBLIC_KEY'] = CAPTCHA_PUBLIC
application.config['RECAPTCHA_PRIVATE_KEY'] = CAPTCHA_PRIVATE
application.config['testing'] = DEBUG
application.config['version'] = 'v0.3'
application.config['testing'] = DEBUG
def get_added_styles():
......@@ -69,8 +71,9 @@ def require_login(view):
@application.route('/bourbaki')
def bourbaki():
session['username'] = 'bourbaki@illinois.edu'
session['user'] = db.dump_user(session['username'])
username = 'bourbaki@illinois.edu'
session['username'] = username
session['user'] = db.dump_user(username)
return redirect('/')
......
......@@ -14,7 +14,7 @@ from twython import Twython
import re, json
import time, datetime
import urllib, urllib2, cookielib
from sentiment import textblob, is_positive, is_negative
from sentiment import textblob, is_positive, is_negative, sentistrength
from functools import partial
from operator import is_not
......@@ -26,18 +26,18 @@ TAG_RE = re.compile(r'<[^>]+>')
class SocialContent(object):
def __init__(self, clean, dirty):
def __init__(self, clean, dirty, sentis=False):
self.clean = clean
self.dirty = dirty
self.sentiment = self._sentiment()
self.sentiment = self._sentiment(sentis)
self.is_negative = is_negative(self.sentiment)
self.is_positive = is_positive(self.sentiment)
def to_dict(self):
return self.__dict__
def _sentiment(self):
return textblob(self.clean)
def _sentiment(self, sentis):
return sentistrength(self.clean) if sentis else textblob(self.clean)
class Tweet(SocialContent):
......@@ -45,9 +45,9 @@ class Tweet(SocialContent):
holds basic attributes and finds sentiment
"""
def __init__(self, j):
def __init__(self, j, sentis=False):
dirty = j['text']
SocialContent.__init__(self, self._clean(dirty), dirty)
SocialContent.__init__(self, self._clean(dirty), dirty, sentis=sentis)
self.ts = j['created_at']
self.retweets = j['retweet_count']
......@@ -71,9 +71,9 @@ class Comment(SocialContent):
holds basic attributes and finds sentiment
"""
def __init__(self, j):
def __init__(self, j, sentis=False):
dirty = j['commentBody']
SocialContent.__init__(self, self._clean(dirty), dirty)
SocialContent.__init__(self, self._clean(dirty), dirty, sentis)
self.userLocation = j['userLocation']
self.n_replies = j['replyCount']
......@@ -90,7 +90,7 @@ class Article(object):
holds basic attibutes and gets full-text
"""
def __init__(self, j):
def __init__(self, j, sentis=False):
self.lead = j['lead_paragraph']
self.abstract = j['abstract']
self.title = j['headline']['main']
......@@ -101,7 +101,7 @@ class Article(object):
self.xlarge = 'https://www.nytimes.com/%s' % j['multimedia'][1]['url'] if len(j['multimedia']) > 1 else None
self.published = j['pub_date'][:10]
self.full = self._full_text()
self.comments = article_comments(self.url)
self.comments = article_comments(self.url, sentis=sentis)
def to_dict(self):
return self.__dict__ if (self.full is not None and len(self.full)) != 0 else None
......@@ -129,7 +129,7 @@ def nyt_query_date(s):
return s.strftime('%Y%m%d')
def article_search(keyword):
def article_search(keyword, sentis=False):
"""Get articles based on keyword."""
# tweets are < 10 days old; articles should match
today = datetime.date.today()
......@@ -146,10 +146,10 @@ def article_search(keyword):
response = urllib2.urlopen('http://api.nytimes.com/svc/search/v2/articlesearch.json?%s' % params)
# an Article will be None if it doesn't have body text (thus the partial)
# return an array of Article objects that have a body text
return filter(partial(is_not, None), map(lambda x: Article(x), json.loads(response.read())['response']['docs']))
return filter(partial(is_not, None), map(lambda x: Article(x, sentis=sentis), json.loads(response.read())['response']['docs']))
def article_comments(url, offset=0):
def article_comments(url, offset=0, sentis=False):
comments = []
for i in xrange(MAX_ATTEMPTS):
......@@ -169,12 +169,12 @@ def article_comments(url, offset=0):
# no json could be decoded
break
comments += map(lambda x: Comment(x).to_dict(), comment_batch)
comments += map(lambda x: Comment(x, sentis=sentis).to_dict(), comment_batch)
return comments
def twitter_search(keyword):
def twitter_search(keyword, sentis=False):
twitter = get_auth()
tweets = []
......@@ -183,7 +183,7 @@ def twitter_search(keyword):
break
response = twitter.search(q=keyword, count=100, lang='en') if i == 0 else twitter.search(q=keyword, include_entities=True, max_id=next_max)
tweets += map(lambda x: Tweet(x), response['statuses'])
tweets += map(lambda x: Tweet(x, sentis=sentis), response['statuses'])
try:
next_res = response['search_metadata']['next_results']
next_max = next_res.split('max_id=')[1].split('&')[0]
......
......@@ -57,6 +57,10 @@ def index():
return render_template('mturk_welcome.html', **locals())
article = get_next_doc()
if article is None:
return render_template('mturk_no_tasks.html',
css=css)
n_sentences = len(article['full'])
# minimum reading time in milliseconds (at 750 wpm, fast skimming pace)
minimum_time = int(100 * (float(60 * len(tokenizer.tokenize(article['full']))) / 75))
......@@ -78,6 +82,7 @@ def index():
@mturk.route('/mark_available')
@require_human
def mark_available():
print('\n\n\nhere\n\n\n')
toggle_being_read(get_collection(), session['reading_url'], False)
return success()
......@@ -100,10 +105,16 @@ def update_doc():
url = session['reading_url']
increment_reads(col, url)
toggle_being_read(col, url, False)
print('\n\n\ndone\n\n\n')
return success()
@mturk.route('/submitted')
@require_human
def submitted():
css = digest('mturk/highlight.css')
return render_template('mturk_submitted.html', **locals())
def get_collection():
client = MongoClient('localhost', MONGO_PORT)
db = client.controversy
......@@ -160,6 +171,10 @@ def get_next_doc():
}).sort([
('being_read', 1)
])
if poss is None:
return None
to_be_read = poss[:][0]
toggle_being_read(col, to_be_read['url'], True)
return to_be_read
......@@ -43,7 +43,7 @@ if __name__ == '__main__':
for term in terms:
update_progress(n_done, n_tasks)
try:
scored_keyword = querier.perform(term)
scored_keyword = querier.perform(term, sentis=True)
except UsageError:
# no articles
continue
......
......@@ -9,15 +9,15 @@ import datetime
sr = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT)
def perform(keyword):
def perform(keyword, sentis=False):
"""Provide ``keyword`` for content retrieval, scoring.
"""
articles = article_search(keyword)
articles = article_search(keyword, sentis=sentis)
if len(articles) == 0:
raise UsageError('no-articles', status_code=200)
return {
'result': controversy(articles, twitter_search(keyword)),
'result': controversy(articles, twitter_search(keyword, sentis=sentis)),
'ts': datetime.datetime.utcnow(),
'keyword': keyword,
'ok': 1
......
......@@ -87,7 +87,8 @@ class BM25:
score = sum(tmp_score)
if score!=0:
scores.append((score, idx))
return sorted(scores, reverse=True) # descending order
# descending order
return sorted(scores, reverse=True)
def tf_idf(self):
......@@ -136,11 +137,11 @@ def score_entropy(li):
def sentiments_of_i(i, C):
res = len(filter(lambda x: x.sentiment == i, C))
"""$p(X_{sent} = x_i) = number of tweets
with sentiment $x_i$ / total number of comments C.
Provides number of tweets in corpus C with sentiment i
"""
res = len(filter(lambda x: x.sentiment == i, C))
return res
......@@ -216,7 +217,7 @@ def controversy(articles, social_content):
})
# 10% of the sentence count
n = int(math.ceil(len(sentences) * .10))
# n (15%) largest scores (recall greater entropy ==> more controversial)
# ``n`` largest scores (recall greater entropy ==> more controversial)
nlargest = heapq.nlargest(n, map(lambda x: x['score'], sentences))
# only provide controversial sentences with "enough" related tweets
filtered = filter(lambda x: any(x['score'] >= i for i in nlargest) and len(x['tweets']) > 5, sentences)
......
@import url(bootstrap.min.css);*{font-family:'Open Sans', 'Roboto', 'Ubuntu', 'Source Sans Pro', 'Helvetica Nueue', 'Helvetica', sans-serif}h1{font-weight:700}blockquote{font-size:16px}blockquote.good{background-color:#dff0d8;border-left:5px solid #3c763d}blockquote.bad{background-color:#f2dede;border-left:5px solid #a94442}.col-md-3{background-color:#eee}#full_article{font-size:16px}#full_article span{cursor:pointer}#full_article span.controversial{color:#bada55;background-color:#800}#copyright{margin-bottom:14em}.top_bar{height:8px;background-color:#d9d9d9;margin-bottom:12px}#full_article{text-align:justify}#bot_bar{position:fixed;z-index:1;bottom:0}#bot_bar .col-md-12{padding:8px;background-color:#eee;margin-top:8px;text-align:right;height:61px}#bot_bar .col-md-12 .pull-left{padding:8px}@media (max-height: 400px){#bot_bar .col-md-12{height:40px}.btn-primary{height:30px;padding:0px 4px;font-size:14px}}@media (min-width: 1200px){#bot_bar{width:1170px}}@media (min-width: 992px){#bot_bar{width:970}}@media (min-width: 768px){#bot_bar{width:750}}
@import url(bootstrap.min.css);*{font-family:'Open Sans', Roboto, Ubuntu, 'Source Sans Pro', 'Helvetica Nueue', Helvetica, sans-serif}h1{font-weight:700}blockquote{font-size:16px}blockquote.good{background-color:#dff0d8;border-left:5px solid #3c763d}blockquote.bad{background-color:#f2dede;border-left:5px solid #a94442}.col-md-3{background-color:#eee}#full_article{font-size:16px}#full_article span{cursor:pointer}#full_article span.controversial{color:#bada55;background-color:#800}#copyright{margin-bottom:14em}.btn{font-weight:700}.top_bar{height:8px;background-color:#d9d9d9;margin-bottom:12px}#full_article{text-align:justify}#bot_bar{position:fixed;z-index:1;bottom:0}#bot_bar .col-md-12{padding:8px;background-color:#eee;margin-top:8px;text-align:right;height:61px}#bot_bar .col-md-12 .pull-left{padding:8px}@media (max-height: 400px){#bot_bar{position:relative}}@media (min-width: 1200px){#bot_bar{width:1170px}}@media (min-width: 992px){#bot_bar{width:970}}@media (min-width: 768px){#bot_bar{width:750}}
/*# sourceMappingURL=highlight.css.map */
......@@ -11,10 +11,10 @@ function timer() {
$("#time_left").addClass('alert-success');
setTimeout(function() {
updateCanSubmit();
}, 1500);
}, 1100);
setTimeout(function() {
markAvailable();
}, 30000);
}, 45000);
return;
}
......@@ -60,9 +60,7 @@ function toggleControversial(i, el) {
$(el).removeClass('controversial');
} else {
controversial.push(i);
$(el)
.addClass('controversial')
.prop('title', 'currently controversial');
$(el).addClass('controversial')
}
updateCanSubmit();
......@@ -103,13 +101,9 @@ function enable(el) {
.removeClass('disabled');
}
function submitComplete() {
}
function submitSuccess(data) {
$("#bot_bar img").hide();
console.log(data);
window.location.replace('submitted');
}
function submitFailure(data) {
......@@ -120,7 +114,7 @@ function submitFailure(data) {
$("#submit").click(function() {
// these checks will be done server-side too. They are duplicated here to reduce strain on the server.
if (!pastMinTime()) {
showError("your words per minute is too high");
showError("your reading speed is too high");
return;
}
......@@ -149,7 +143,7 @@ $("#submit").click(function() {
return;
}
if (n_highlights < (n_sentences * 0.1) && !CONFIRMED_HIGHLIGHTS) {
if (n_highlights < (n_sentences * 0.05) && !CONFIRMED_HIGHLIGHTS) {
showError("are you sure you've highlighted all controversial sentences? Submit again to confirm.");
CONFIRMED_HIGHLIGHTS = true;
return;
......
@import 'bootstrap.min.css';
* { font-family: 'Open Sans', 'Roboto', 'Ubuntu', 'Source Sans Pro', 'Helvetica Nueue', 'Helvetica', sans-serif; }
* { font-family: 'Open Sans', Roboto, Ubuntu, 'Source Sans Pro', 'Helvetica Nueue', Helvetica, sans-serif; }
h1 { font-weight: 700; }
blockquote {
font-size: 16px; }
......@@ -18,6 +18,7 @@ blockquote.bad {
background-color: #800; }
}
#copyright { margin-bottom: 14em; }
.btn { font-weight: 700; }
.top_bar {
height: 8px;
background-color: #d9d9d9;
......@@ -35,12 +36,7 @@ blockquote.bad {
z-index: 1;
bottom: 0; }
@media (max-height: 400px) {
#bot_bar .col-md-12 { height: 40px; }
.btn-primary {
height: 30px;
padding: 0px 4px;
font-size: 14px;
}
#bot_bar { position: relative; }
}
@media (min-width: 1200px) {
#bot_bar { width: 1170px; }
......
......@@ -13,7 +13,6 @@ Controversy &middot; hotspots of news articles</title>
<meta name="viewport" id="viewport" content="width=device-width, minimum-scale=1, maximum-scale=1" />
<link rel=icon href="{{ url_for('static', filename='img/fav.ico') }}" />
<link rel=stylesheet href="//2pitau.org/os.css" />
<link rel=stylesheet href="{{ css }}" />
{% if webkit is not none %}
{% if webkit is not none and webkit|length > 0%}
......
{% extends "mturk_base.html" %}
{% block content %}
<div class="row">
<div class="col-md-4">
<p class="lead">Tasks completed</p>
</div>
<div class="col-md-8">
<p>All articles in our training set have been annotated. <a href="https://github.com/gdyer/controversy">Learn more about this project</a>.</p>
</div>
</div>
{% endblock %}
{% extends "mturk_base.html" %}
{% block content %}
<div class="row">
<div class="col-md-4">
<p class="lead">You've completed this HIT</p>
</div>
<div class="col-md-8">
<p>Report feedback to us: <b><tt>gdyer</tt>@<tt>seznam</tt>.<tt>cz</tt></b> or <a href="https://github.com/gdyer/controversy">learn more about this project</a>. Thank you.</p>
</div>
</div>
{% endblock %}
......@@ -5,7 +5,7 @@
</div>
<div class="row">
<div class="col-md-12">
<p class="lead">Each HIT involves reading an article from The New York Times. While reading on the next page, click any sentences which you believe to be controversial, i.e. expressing an opinion or weighted (with qualifying words) fact that you believe could be debated. Use your knowledge of current events while determining whether a sentence is controversial. When done, click <b><q>Submit</q></b>.</p>
<p class="lead">Each HIT involves reading an article from The New York Times. While reading on the next page, click any sentences which you believe to be controversial, i.e. expressing an opinion or qualified fact (e.g. <q>terrible losses</q>) that you believe could be debated. Use your knowledge of current events while determining whether a sentence is controversial. When done, click <b><q>Submit</q></b>.</p>
<p class="lead">You must be fluent in English to participate.</p>
<p>This sentence is controversial because it describes some heavy action being enforced:</p>
<blockquote class="good">
......@@ -18,13 +18,13 @@
<p>Worried about unemployment and political unrest, the government pushed back when Avtovaz, the maker of Lada cars, <i>moved quickly to cut</i> workers as the economy slowed.</p>
<p>In 1994, a pilot <i>let</i> his 16-year-old son fly an Airbus that <i>promptly</i> crashed, killing <i>all</i> 75 aboard.</p>
</blockquote>
<p>The following non-controversial sentence just states numbers, albeit depressing:</p>
<p>The following non-controversial sentence just states numbers:</p>
<blockquote class="bad">
<p><strike>During a demonstration flight in 2012, the plane crashed into a mountain in Indonesia with 37 aviation executives and journalists and eight crew members aboard, killing everybody.</strike></p>
</blockquote>
</div>
<div class="col-md-8">
<p>To continue, complete the reCAPTCHA. Once done, click <b><q>Next</q></b>.</p>
<p>Complete the reCAPTCHA. Once done, click <b><q>Next</q></b>.</p>
<form method=POST>
<div class="form-group">
{{ form.hidden_tag() }}
......@@ -35,7 +35,7 @@
<br>
</div>
<div class="col-md-4">
<p>Sometimes it can happen that technical difficulties cause experimental scripts to freeze so that you will not be able to submit a HIT. We are trying our best to avoid these problems. Should they nevertheless occur, we urge you to email us: <b><tt>gdyer</tt>@<tt>seznam</tt>.<tt>cz</tt></b></p>
<p>Problems preventing submission of an HIT are unlikely. Should they nevertheless occur, email us your browser info: <b><tt>gdyer</tt>@<tt>seznam</tt>.<tt>cz</tt></b></p>
<br>
</div>
</div>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment