Commit ddea9545 authored by Sacha's avatar Sacha Committed by SXibolet@2PITAU
Browse files

paragraphs for training, bug fix

parent 85e3f141
......@@ -7,7 +7,6 @@
Runs server.
:version: 0.3
:copyright: (c) 2015 I Lourentzou, G Dyer, A Sharma, C Zhai. Some rights reserved.
:license: CC BY-NC-SA 4.0, see LICENSE for more details.
"""
......@@ -33,7 +32,7 @@ application.secret_key = SECRET_KEY
application.config['RECAPTCHA_PUBLIC_KEY'] = CAPTCHA_PUBLIC
application.config['RECAPTCHA_PRIVATE_KEY'] = CAPTCHA_PRIVATE
application.config['version'] = 'v0.3'
application.config['version'] = '0.3'
application.config['testing'] = DEBUG
......
......@@ -120,8 +120,7 @@ class Article(object):
response = opener.open(urllib2.Request(self.url))
soup = BeautifulSoup(response.read(), 'html.parser')
body = soup.findAll('p', {'class' : ['story-body-text', 'story-content']})
res = ' '.join(p.text for p in body)
#res = ''.join(map(lambda x: '<p>%s</p>' % x.text, body)) if sentis else ' '.join(p.text for p in body)
res = ('|*^*|' if sentis else ' ').join(p.text for p in body)
jar.clear()
return res
......
......@@ -10,6 +10,7 @@ import time
import forms
from error import UsageError
import nltk.data
import itertools
from nltk.tokenize import RegexpTokenizer
......@@ -54,7 +55,10 @@ def index():
if form.validate_on_submit():
session['human'] = 'yes'
else:
return render_template('mturk_welcome.html', **locals())
return render_template('mturk_welcome.html',
form=form,
css=css,
js=js)
article = get_next_doc()
if article is None:
......@@ -64,11 +68,16 @@ def index():
n_sentences = len(article['full'])
# minimum reading time in milliseconds (at 750 wpm, fast skimming pace)
minimum_time = int(100 * (float(60 * len(tokenizer.tokenize(article['full']))) / 75))
article['full'] = sent_detector.tokenize(article['full'].strip())
# split by paragraph and then sentences
paras = map(lambda x: sent_detector.tokenize(x.strip()), article['full'].split('|*^*|'))
article['full'] = list(itertools.chain(*paras))
# not the cleanest solution for paragraphs, but it'll be fine
sentence_lookup = lambda x: article['full'].index(x)
session['minimum_time'] = minimum_time
session['started_reading'] = time.time()
session['n_sentences'] = len(article['full'])
session['n_sentences'] = sum(map(lambda x: len(x), article['full']))
session['reading_url'] = article['url']
return render_template('mturk_highlight.html',
......@@ -76,6 +85,8 @@ def index():
js=js,
minimum_time=minimum_time,
n_sentences=n_sentences,
paras=paras,
sentence_lookup=sentence_lookup,
article=article)
......@@ -102,7 +113,7 @@ def update_doc():
col = get_collection()
url = session['reading_url']
increment_reads(col, url)
increment_reads(col, url, inds)
toggle_being_read(col, url, False)
return success()
......@@ -121,7 +132,7 @@ def get_collection():
return db.training
def increment_reads(col, url):
def increment_reads(col, url, highlights):
return col.update_one({
'url': url
}, {
......@@ -129,6 +140,8 @@ def increment_reads(col, url):
'ts': datetime.datetime.utcnow()
}, '$inc': {
'n_reads': 1
}, '$push': {
'highlights': highlights
}
}).modified_count
......@@ -155,7 +168,8 @@ def new_doc(doc):
'ts': doc['ts'],
'keyword': doc['keyword'],
'n_reads': 0,
'being_read': False
'being_read': False,
'highlights': []
}
a.update(article)
col.insert_one(a).inserted_id
......@@ -166,7 +180,7 @@ def get_next_doc():
col = get_collection()
poss = col.find({
'n_reads' : {
'$lt': 5
'$lt': 3
}
}).sort([
('being_read', 1)
......
......@@ -7,7 +7,7 @@ from error import UsageError
import sys
# http://arxiv.org/pdf/1409.8152v1.pdf <== source of terms
# source of terms ===> http://arxiv.org/pdf/1409.8152v1.pdf
TT_DIR = 'training_terms'
......
......@@ -17,7 +17,7 @@ def perform(keyword, sentis=False):
raise UsageError('no-articles', status_code=200)
return {
'result': controversy(articles, twitter_search(keyword, sentis=sentis), _filter=~sentis),
'result': controversy(articles, twitter_search(keyword, sentis=sentis), _filter=not sentis),
'ts': datetime.datetime.utcnow(),
'keyword': keyword,
'ok': 1
......
......@@ -171,7 +171,7 @@ def controversy(articles, social_content, _filter=True):
ranked_articles[article_index] = dict_art
sentiment_score, linguistic_score = 0, 0 # (entropy) score of the entire article
sentences = [] # scores & metadata for each sentence
query = tokenizer.tokenize(articles[article_index].full)
query = tokenizer.tokenize(articles[article_index].full.replace('|*^*|', ''))
# for every sentence in the article
for sentence in query:
......@@ -216,7 +216,7 @@ def controversy(articles, social_content, _filter=True):
'sentiment_score': sentence_sentiment_score
})
filtered = []
if not _filter:
if _filter:
# 10% of the sentence count
n = int(math.ceil(len(sentences) * .10))
# ``n`` largest scores (recall greater entropy ==> more controversial)
......
@import url(bootstrap.min.css);*{font-family:'Open Sans', Roboto, Ubuntu, 'Source Sans Pro', 'Helvetica Nueue', Helvetica, sans-serif}h1{font-weight:700}blockquote{font-size:16px}blockquote.good{background-color:#dff0d8;border-left:5px solid #3c763d}blockquote.bad{background-color:#f2dede;border-left:5px solid #a94442}.col-md-3{color:#ddd;background-color:rgba(245,245,245,0.3)}.col-md-3:hover{color:#000}#full_article{font-size:16px}#full_article span{cursor:pointer}#full_article span.controversial{color:#bada55;background-color:#800}#copyright{margin-bottom:14em}.btn{font-weight:700}.top_bar{height:8px;background-color:#d9d9d9;margin-bottom:12px}#full_article{text-align:justify}#bot_bar{position:fixed;z-index:1;bottom:0}#bot_bar .col-md-12{padding:8px;background-color:#eee;margin-top:8px;text-align:right;height:61px}#bot_bar .col-md-12 .pull-left{padding:8px}@media (max-height: 400px){#bot_bar{position:relative}}@media (min-width: 1200px){#bot_bar{width:1170px}}@media (min-width: 992px){#bot_bar{width:970}}@media (min-width: 768px){#bot_bar{width:750}}
@import url(bootstrap.min.css);*{font-family:'Open Sans', Roboto, Ubuntu, 'Source Sans Pro', 'Helvetica Nueue', Helvetica, sans-serif}h1{font-weight:700}blockquote{font-size:16px}blockquote.good{background-color:#dff0d8;border-left:5px solid #3c763d}blockquote.bad{background-color:#f2dede;border-left:5px solid #a94442}.col-md-3{color:#ddd;background-color:rgba(245,245,245,0.3)}div#full_article{font-size:16px}div#full_article span{cursor:pointer}div#full_article span.controversial{color:#bada55;background-color:#800}#copyright{margin-bottom:14em}.btn{font-weight:700}.top_bar{height:8px;background-color:#d9d9d9;margin-bottom:12px}#full_article{text-align:justify}#bot_bar{position:fixed;z-index:1;bottom:0}#bot_bar .col-md-12{padding:8px;background-color:#eee;margin-top:8px;text-align:right;height:61px}#bot_bar .col-md-12 .pull-left{padding:8px}@media (max-height: 400px){#bot_bar{position:relative}}@media (min-width: 1200px){#bot_bar{width:1170px}}@media (min-width: 992px){#bot_bar{width:970}}@media (min-width: 768px){#bot_bar{width:750}}
/*# sourceMappingURL=highlight.css.map */
@import 'bootstrap.min.css';
* { font-family: 'Open Sans', Roboto, Ubuntu, 'Source Sans Pro', 'Helvetica Nueue', Helvetica, sans-serif; }
h1 { font-weight: 700; }
h1 {
font-weight: 700; }
blockquote {
font-size: 16px; }
blockquote.good {
......@@ -12,9 +13,7 @@ blockquote.bad {
.col-md-3 {
color: #ddd;
background-color: rgba(245, 245, 245, .3); }
.col-md-3:hover {
color: #000; }
#full_article {
div#full_article {
font-size: 16px;
span {
cursor: pointer; }
......@@ -22,8 +21,10 @@ blockquote.bad {
color: #bada55;
background-color: #800; }
}
#copyright { margin-bottom: 14em; }
.btn { font-weight: 700; }
#copyright {
margin-bottom: 14em; }
.btn {
font-weight: 700; }
.top_bar {
height: 8px;
background-color: #d9d9d9;
......
......@@ -79,7 +79,7 @@
<a class="github-button" href="https://github.com/gdyer/controversy" data-icon="octicon-star" data-style="mega" aria-label="Star gdyer/controversy on GitHub">Star</a>
<a class="github-button" href="https://github.com/gdyer/controversy" data-icon="octicon-eye" data-style="mega" aria-label="Watch gdyer/controversy on GitHub">Watch</a>
<a class="github-button" href="https://github.com/gdyer/controversy/issues" data-icon="octicon-issue-opened" data-style="mega" data-count-api="/repos/ntkme/github-buttons#open_issues_count" data-count-aria-label="# issues on GitHub" aria-label="Issue gdyer/controversy on GitHub">Known bugs</a>
<p>This is <b>Controversy</b>, version {{ version }}. Available <a target=_blank href="http://github.com/gdyer/controversy">on GitHub</a>.</p>
<p>This is <b>Controversy</b>, version <b>v{{ version }}</b>. Available <a target=_blank href="http://github.com/gdyer/controversy">on GitHub</a>.</p>
<div>
<p>Found a bug? <a target=_blank href="https://github.com/gdyer/controversy/issues/new">Please file a report</a>.</p>
<ul ng-controller="IssuesController" class=issues_list>
......
......@@ -25,7 +25,7 @@
<div id=login>
<h1>Controversy</h1>
<span id=details>
Please login or <a href="register">create a free account</a>. <b>New</b><a href="training">annotate articles</a> to help us build a dataset for future study!
Please login or <a href="register">create a free account</a>. <a href="training">Annotate articles</a> to help us build a dataset for future study!
<center>
<a class="github-button" href="https://github.com/gdyer/controversy" data-icon="octicon-star" data-style="mega" aria-label="Star gdyer/controversy on GitHub">Star</a>
<a class="github-button" href="https://github.com/gdyer/controversy/issues" data-icon="octicon-issue-opened" data-style="mega" data-count-api="/repos/gdyer/controversy#open_issues_count" data-count-aria-label="# issues on GitHub" aria-label="Issue gdyer/controversy on GitHub">Known bugs</a>
......
......@@ -13,11 +13,16 @@
<hr>
</div>
<div class="col-md-12">
<p id="full_article">
{% for sentence in article.full %}
<span onclick="toggleControversial({{ loop.index-1 }}, this)">{{ sentence }}</span>
{% endfor %}
</p>
<div id="full_article">
{% for paragraph in paras %}
{% set _loop = loop %}
<p>
{% for sentence in paragraph %}
<span onclick="toggleControversial({{ sentence_lookup(sentence) }}, this)">{{ sentence }}</span>
{% endfor %}
</p>
{% endfor %}
</div>
</div>
<div class="col-md-6">
<div class="alert" id="time_left" role="alert">
......
......@@ -5,8 +5,8 @@
</div>
<div class="row">
<div class="col-md-12">
<p class="lead">Each HIT involves reading an article from The New York Times. While reading on the next page, click any sentences which you believe to be controversial, i.e. expressing an opinion or qualified fact (e.g. <q>terrible losses</q>) that you believe could be debated. Use your knowledge of current events while determining whether a sentence is controversial. When done, click <b><q>Submit</q></b>.</p>
<p class="lead">You must be fluent in English to participate.</p>
<p class="lead">Each HIT involves reading an article from <em>The New York Times</em>. You must be fluent in English to particiate.</p>
<p class="lead">While reading on the next page, click any sentences which you believe to be controversial, i.e. expressing an opinion or qualified fact (e.g. <q>terrible losses</q>) that you believe could be debated. Use your knowledge of current events while determining whether a sentence is controversial. When done, click <b><q>Submit</q></b>. You have unlimited time to read, however there is a minimum time shown at the bottom that is determined by the length of the article. You may not submit before this time or with no or too many highlights. Examples are below.</p>
<p>This sentence is controversial because it describes some heavy action being enforced:</p>
<blockquote class="good">
<p>Across Russia, companies are <i>being forced</i> to prioritize patriotism over profits.</p>
......
{% raw %}
<header>
<div>
<a class=lg href="#results">&larr; all articles</a>
<a class=mini href="#results">&larr;</a>
&middot;
<div>
<a class=lg href="#results">&larr; all articles</a>
<a class=mini href="#results">&larr;</a>
&middot;
<span title="{{ article.title }}" ng-bind-html="article.title | limitTo: lim"></span>
{{ article.title.length > lim ? '&hellip;' : ''}}
<span id=right-header>
<a ng-show="previousExists" ng-click="change(-1)" title="more controversial article">
&lArr; previous
</a>
<span ng-show="previousExists && nextExists">
&middot;
</span>
<a ng-show="nextExists" ng-click="change(1)" title="less controversial article">
next &rArr;
</a>
</span>
</div>
{{ article.title.length > lim ? '&hellip;' : ''}}
<span id=right-header>
<a ng-show="previousExists" ng-click="change(-1)" title="more controversial article">
&lArr; previous
</a>
<span ng-show="previousExists && nextExists">
&middot;
</span>
<a ng-show="nextExists" ng-click="change(1)" title="less controversial article">
next &rArr;
</a>
</span>
</div>
</header>
<main>
<ul class=details>
<li ng-repeat="key in ['byline', 'source']">
<span ng-show="article[key]" title="{{ key }}">
{{ article[key] }}
</span>&nbsp;
</li>
<li>
<span title="publication date">
{{ article['published'] | date: "EEEE, d MMMM yyyy" }}
</span>
</li>
<li ng-repeat="key in ['byline', 'source']">
<span ng-show="article[key]" title="{{ key }}">
{{ article[key] }}
</span>&nbsp;
</li>
<li>
<span title="publication date">
{{ article['published'] | date: "EEEE, d MMMM yyyy" }}
</span>
</li>
</ul>
<article ng-bind-html="highlight(article.full, article.sentences)"></article>
<p>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment