Commit f684b249 authored by Sacha's avatar Sacha
Browse files

download of training dataset available and minor tweaks

parent be3bc8a2
# -*- coding: utf-8 -*-
from flask import Blueprint, jsonify, request, session, render_template
from flask import Blueprint, jsonify, request, session, render_template, Response
from functools import wraps
from digest import digest
from pymongo import MongoClient
......@@ -13,6 +13,7 @@ from error import UsageError
import nltk.data
import itertools
from nltk.tokenize import RegexpTokenizer
from bson import json_util
highlighting = Blueprint('/training', __name__)
......@@ -134,6 +135,20 @@ def dataset():
css = digest('highlighting/highlight.css')
return render_template('highlighting_dataset.html', **locals())
def justmime(r, status=200):
return Response(r, mimetype='application/json', status=status)
@highlighting.route('/dataset/all')
def dataset_all():
col = get_articles_collection()
return justmime(json.dumps(list(col.find()),
default=json_util.default))
@highlighting.route('/dataset/annotated')
def dataset_annotated():
col = get_articles_collection()
return justmime(json.dumps(list(col.find({'n_reads': { '$gt': 0 }})),
default=json_util.default))
def get_db():
......@@ -150,18 +165,18 @@ def get_tweets_collection():
def increment_reads(col, url, highlights, author):
return col.update_one({
'url': url
}, {
'$set': {
'ts': datetime.datetime.utcnow()
}, '$inc': {
'n_reads': 1
}, '$push': {
'highlights_author': author,
'highlights': highlights
}
}).modified_count
return col.update_one({
'url': url
}, {
'$set': {
'ts': datetime.datetime.utcnow()
}, '$inc': {
'n_reads': 1
}, '$push': {
'highlights_author': author,
'highlights': highlights
}
}).modified_count
def toggle_being_read(col, url, dest):
......
......@@ -32,8 +32,8 @@
</div>
</div>
<div class='col-md-6' id='copyright'>
<p class='text-muted'>If you supply your name or some moniker below, we will thank you in any publications that use the labor you've performed here.</p>
<p class='text-muted'>Study from the University of Illinois at Urbana&ndash;Champaign. Articles &copy; 2015, The New York Times Company <img width='17px' src='{{ url_for("static", filename="img/NYT.svg") }}' />. All rights reserved.</p>
<p class='text-muted'>If you supply your name or some moniker (perhaps one that&rsquo;s not <i>too</i> vulgar), we&rsquo;ll thank you in any publications that use the labour you&rsquo;ve performed here.</p>
<p class='text-muted'>Study from the University of Illinois at Urbana&ndash;Champaign. Articles &copy; The New York Times Company <img width='17px' src='{{ url_for("static", filename="img/NYT.svg") }}' />. All rights reserved.</p>
</div>
<div class='row' id='bot_bar'>
<div class='col-md-12'>
......
......@@ -2,15 +2,18 @@
{% block content %}
<div class='row'>
<div class='col-md-12'>
<p><b>Thank you for your interest</b>. We will not employ microtask workers to annotate our training articles for <a href='https://www.jacobinmag.com/2017/12/middle-east-digital-labor-microwork-gaza-refugees-amazon'>ethical</a> <a href='https://www.jacobinmag.com/2015/08/amazon-google-facebook-privacy-bezos'>reasons</a> and personal frugality. A small subset of the articles has been annotated by volunteers.</p>
<p>The data are compiled dynamically and will update with new annotations as we receive them.</p>
<p class=text-danger>As of December 2017, downloads aren't ready; we're working on building a more efficient scraper.</p>
<p><b>Dataset</b>. We will not employ microtask workers to annotate our training articles for <a href='https://www.jacobinmag.com/2017/12/middle-east-digital-labor-microwork-gaza-refugees-amazon'>ethical</a> <a href='https://www.jacobinmag.com/2015/08/amazon-google-facebook-privacy-bezos'>reasons</a> and personal frugality. A small subset of the articles has been annotated by volunteers.</p>
<p>The data are compiled dynamically and will update with new annotations as we receive them. <b>Downloads are now available</b>.</p>
<p style='line-height: 275%'>
<b>download &hellip;</b>
<button class='btn btn-success' type='submit'>annotated only (json)</button>
<button class='btn btn-danger' type='submit'>annotated &amp; unannotated (json, 0.953GB)</button>
<b>download &hellip;</b>
<a class='btn btn-success' type='submit' href='dataset/annotated'>annotated only (json)</a>
<a class='btn btn-danger' type='submit' href='dataset/all'>annotated &amp; unannotated (json)</a>
</p>
<p>Copyright (&copy;) 2017, with some rights reserved under CC BY-NC-SA 4.0. See <a href='https://creativecommons.org/licenses/by-nc-sa/4.0/'>here</a> for details.</p>
<p>
<b>Meaning of the file:</b>
you will find those sentences judged by readers to be controversial in the <code>highlights</code> array of each article. Each entry in the array is one person's reading of the article. Within each reading, there is another array that lists the indices of the sentences of the article that were judged controversial. Please use <a href='https://framagit.org/gdyer/CONTROVERSY/blob/master/controversy/highlighting.py#L74'>our tokenizer in</a> to separate the <code>full</code> article into sentences so that these indices are meaningful (we do some unusual things).
</p>
<p>Articles are &copy; The New York Times Company. Assume all rights reserved or, for content produce by our research team, CC BY-NC-SA 4.0 (see <a href='https://creativecommons.org/licenses/by-nc-sa/4.0/'>here</a> for details).</p>
</div>
</div>
{% endblock %}
......@@ -5,7 +5,7 @@
<p class="lead"><b>You've completed this article &hellip; thanks!</b></p>
</div>
<div class='col-md-8'>
<p>Report feedback to us at <b><tt>gdyer2</tt>@<tt>illinois</tt>.<tt>edu</tt></b> or <a href='https://framagit.org/gdyer/CONTROVERSY'>learn more about this project</a>.</p>
<p><a href='https://framagit.org/gdyer/CONTROVERSY'>Learn more about this project</a>.</p>
</div>
</div>
{% endblock %}
......@@ -16,7 +16,7 @@
</header>
{% raw %}
<main style='padding: 8em 1em'>
<main style='padding: 5em 1em'>
<ul id='articles' ng-show='json' ng-style='$root.results_style'>
<li ng-repeat='article in json'>
<div class='article' ng-class='{ "most-controversial" : $first, "least-controversial" : $last }'>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment