mturk.py 6.1 KB
Newer Older
Sacha's avatar
Sacha committed
1
2
# -*- coding: utf-8 -*-
from flask import Blueprint, jsonify, request, session, render_template
Sacha's avatar
Sacha committed
3
from functools import wraps
Sacha's avatar
Sacha committed
4
5
6
7
8
from digest import digest
from pymongo import MongoClient
from config import MONGO_PORT
import content
import datetime
Sacha's avatar
Sacha committed
9
import time
Sacha's avatar
Sacha committed
10
11
12
import forms
from error import UsageError
import nltk.data
Sacha's avatar
Sacha committed
13
import itertools
Sacha's avatar
Sacha committed
14
from nltk.tokenize import RegexpTokenizer
Sacha's avatar
Sacha committed
15
16
17
18


mturk = Blueprint('/training', __name__)
sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')
Sacha's avatar
Sacha committed
19
20
21
22
23
24
tokenizer = RegexpTokenizer(r'\w+')


def require_human(view):
    @wraps(view)
    def protected_view(*args, **kwargs):
Sacha's avatar
Sacha committed
25
        if is_human():
Sacha's avatar
Sacha committed
26
27
28
29
30
31
            return view(*args, **kwargs)
        else:
            raise UsageError('not human!')
    return protected_view


Sacha's avatar
Sacha committed
32
33
34
35
def is_human():
    return session.get('human') == 'yes'


Sacha's avatar
Sacha committed
36
37
38
39
def success():
    return jsonify({
        'ok': 1
    })
Sacha's avatar
Sacha committed
40
41
42
43
44
45
46
47
48
49
50
51


@mturk.errorhandler(UsageError)
def handle_error(error):
    response = jsonify(error.to_dict())
    response.status_code = error.status_code
    return response


@mturk.route('/', methods=['GET', 'POST'])
def index():
    css = digest('mturk/highlight.css')
Sacha's avatar
Sacha committed
52
    js = digest('mturk/highlight.js')
Sacha's avatar
Sacha committed
53
    if not is_human():
Sacha's avatar
Sacha committed
54
55
56
57
        form = forms.BeginHIT()
        if form.validate_on_submit():
            session['human'] = 'yes'
        else:
Sacha's avatar
Sacha committed
58
59
60
61
            return render_template('mturk_welcome.html',
                                   form=form,
                                   css=css,
                                   js=js)
Sacha's avatar
Sacha committed
62

Sacha's avatar
Sacha committed
63
    article = get_next_doc()
Sacha's avatar
Sacha committed
64
65
66
67
    if article is None:
        return render_template('mturk_no_tasks.html',
                               css=css)

Sacha's avatar
Sacha committed
68
    n_sentences = len(article['full'])
Sacha's avatar
Sacha committed
69
70
    # minimum reading time in milliseconds (at 750 wpm, fast skimming pace)
    minimum_time = int(100 * (float(60 * len(tokenizer.tokenize(article['full']))) / 75))
Sacha's avatar
Sacha committed
71
72
73
74
75
76

    # split by paragraph and then sentences
    paras = map(lambda x: sent_detector.tokenize(x.strip()), article['full'].split('|*^*|'))
    article['full'] = list(itertools.chain(*paras))
    # not the cleanest solution for paragraphs, but it'll be fine
    sentence_lookup = lambda x: article['full'].index(x)
Sacha's avatar
Sacha committed
77

Sacha's avatar
Sacha committed
78
79
    session['minimum_time'] = minimum_time
    session['started_reading'] = time.time()
Sacha's avatar
Sacha committed
80
    session['n_sentences'] = sum(map(lambda x: len(x), article['full']))
Sacha's avatar
Sacha committed
81
82
    session['reading_url'] = article['url']

Sacha's avatar
Sacha committed
83
84
    return render_template('mturk_highlight.html',
                           css=css,
Sacha's avatar
Sacha committed
85
86
87
                           js=js,
                           minimum_time=minimum_time,
                           n_sentences=n_sentences,
Sacha's avatar
Sacha committed
88
89
                           paras=paras,
                           sentence_lookup=sentence_lookup,
Sacha's avatar
Sacha committed
90
91
92
                           article=article)


Sacha's avatar
Sacha committed
93
94
95
@mturk.route('/mark_available')
@require_human
def mark_available():
96
97
98
    toggle_being_read(get_articles_collection(),
                      session['reading_url'],
                      False)
Sacha's avatar
Sacha committed
99
100
101
    return success()


Sacha's avatar
Sacha committed
102
@mturk.route('/submit', methods=['GET', 'POST'])
Sacha's avatar
Sacha committed
103
@require_human
Sacha's avatar
Sacha committed
104
def update_doc():
Sacha's avatar
Sacha committed
105
    if 'checked' not in request.args:
Sacha's avatar
Sacha committed
106
107
108
109
110
111
112
113
114
        raise UsageError('missing ``checked`` arg!')

    inds = request.args['checked'].split(',')
    n_inds = len(inds)
    if time.time() - session['started_reading'] < (float(session['minimum_time']) / 1000):
        raise UsageError('reading speed too fast')

    if n_inds > session['n_sentences'] or n_inds < 1:
        raise UsageError('no or too many highlights')
Sacha's avatar
Sacha committed
115

116
    col = get_articles_collection()
Sacha's avatar
Sacha committed
117
    url = session['reading_url']
Sacha's avatar
Sacha committed
118
    increment_reads(col, url, inds)
Sacha's avatar
Sacha committed
119
    toggle_being_read(col, url, False)
120

Sacha's avatar
Sacha committed
121
    return success()
Sacha's avatar
Sacha committed
122
123


Sacha's avatar
Sacha committed
124
125
126
127
128
129
130
@mturk.route('/submitted')
@require_human
def submitted():
    css = digest('mturk/highlight.css')
    return render_template('mturk_submitted.html', **locals())


131
def get_db():
Sacha's avatar
Sacha committed
132
    client = MongoClient('localhost', MONGO_PORT)
133
134
135
136
137
138
139
140
141
    return client.controversy


def get_articles_collection():
    return get_db().articles


def get_tweets_collection():
    return get_db().tweets
Sacha's avatar
Sacha committed
142
143


Sacha's avatar
Sacha committed
144
def increment_reads(col, url, highlights):
Sacha's avatar
Sacha committed
145
146
147
    return col.update_one({
        'url': url
    }, {
Sacha's avatar
Sacha committed
148
149
150
        '$set': {
            'ts': datetime.datetime.utcnow()
        }, '$inc': {
Sacha's avatar
Sacha committed
151
            'n_reads': 1
Sacha's avatar
Sacha committed
152
153
        }, '$push': {
            'highlights': highlights
Sacha's avatar
Sacha committed
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
        }
    }).modified_count


def toggle_being_read(col, url, dest):
    return col.update_one({
        'url': url
    }, {
        '$set': {
            'being_read': dest
        }
    })


def new_doc(doc):
    """given an API response, make an entry for each article,
    preserving the timestamp of the entire response and keyword.
171
    Cache tweets by keyword (not associated with article corpus for speed).
Sacha's avatar
Sacha committed
172
    Returns number of articles found.
Sacha's avatar
Sacha committed
173
    """
174
175
176
177
178
179
180
181
182
183
    if doc['ok'] != 1:
        return 0

    ar_col = get_articles_collection()
    tw_col = get_tweets_collection()
    res = doc['result']
    
    tw_col.insert_one({
        'tweets': res['kw_tweets'],
        'ts': doc['ts'],
Sacha's avatar
Sacha committed
184
        'keyword': doc['keyword']
185
186
    })

Sacha's avatar
Sacha committed
187
    n_docs = 0
188
189
190
    for article in res['articles']:
        if article is None:
            continue
Sacha's avatar
Sacha committed
191
192
193
194
195
        n_docs += 1
        a = {
            'ts': doc['ts'],
            'keyword': doc['keyword'],
            'n_reads': 0,
Sacha's avatar
Sacha committed
196
197
            'being_read': False,
            'highlights': []
Sacha's avatar
Sacha committed
198
199
        }
        a.update(article)
200
        ar_col.insert_one(a).inserted_id
Sacha's avatar
Sacha committed
201
202
203
204
    return n_docs


def get_next_doc():
Sacha's avatar
Sacha committed
205
    col = get_articles_collection()
Sacha's avatar
Sacha committed
206
207
    poss = col.find({
        'n_reads' : {
Sacha's avatar
Sacha committed
208
            '$lt': 3
Sacha's avatar
Sacha committed
209
210
211
212
        }
    }).sort([
        ('being_read', 1)
    ])
Sacha's avatar
Sacha committed
213
214
215
216

    if poss is None:
        return None

Sacha's avatar
Sacha committed
217
218
219
220
    try:
        to_be_read = poss[:][0]
    except IndexError:
        return None
Sacha's avatar
Sacha committed
221
222
    toggle_being_read(col, to_be_read['url'], True)
    return to_be_read
Sacha's avatar
Sacha committed
223
224
225


def keyword_exists(kw):
Sacha's avatar
Sacha committed
226
227
228
    """Checks if ``kw`` has been saved already
    Forced collection update
    """
Sacha's avatar
Sacha committed
229
230
    col = get_tweets_collection()
    poss = col.find({
Sacha's avatar
Sacha committed
231
232
        'keyword': kw
    })
Sacha's avatar
Sacha committed
233
    return poss.count() != 0
Sacha's avatar
Sacha committed
234
235
236
237
238
239
240
241
242
243


def article_is_new(url, col=None):
    """Checks if article @ ``url``
    has been saved already
    """
    if col is None:
        col = get_articles_collection()

    poss = col.find({
Sacha's avatar
Sacha committed
244
        'url': url
Sacha's avatar
Sacha committed
245
246
    })

Sacha's avatar
Sacha committed
247
    return poss.count() == 0