highlighting.py 6.94 KB
Newer Older
Sacha's avatar
Sacha committed
1
# -*- coding: utf-8 -*-
2
from flask import Blueprint, jsonify, request, session, render_template, Response
Sacha's avatar
Sacha committed
3
from functools import wraps
Sacha's avatar
Sacha committed
4
5
6
7
8
from digest import digest
from pymongo import MongoClient
from config import MONGO_PORT
import content
import datetime
Sacha's avatar
Sacha committed
9
import time
Sacha's avatar
Sacha committed
10
import forms
11
import json
Sacha's avatar
Sacha committed
12
13
from error import UsageError
import nltk.data
Sacha's avatar
Sacha committed
14
import itertools
Sacha's avatar
Sacha committed
15
from nltk.tokenize import RegexpTokenizer
16
from bson import json_util
Sacha's avatar
Sacha committed
17
18


19
highlighting = Blueprint('/training', __name__)
Sacha's avatar
Sacha committed
20
sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')
Sacha's avatar
Sacha committed
21
22
23
24
25
26
tokenizer = RegexpTokenizer(r'\w+')


def require_human(view):
    @wraps(view)
    def protected_view(*args, **kwargs):
Sacha's avatar
Sacha committed
27
        if is_human():
Sacha's avatar
Sacha committed
28
29
30
31
32
33
            return view(*args, **kwargs)
        else:
            raise UsageError('not human!')
    return protected_view


Sacha's avatar
Sacha committed
34
35
36
37
def is_human():
    return session.get('human') == 'yes'


Sacha's avatar
Sacha committed
38
39
40
41
def success():
    return jsonify({
        'ok': 1
    })
Sacha's avatar
Sacha committed
42
43


44
@highlighting.errorhandler(UsageError)
Sacha's avatar
Sacha committed
45
46
47
48
49
50
def handle_error(error):
    response = jsonify(error.to_dict())
    response.status_code = error.status_code
    return response


51
@highlighting.route('/', methods=['GET', 'POST'])
Sacha's avatar
Sacha committed
52
def index():
53
54
    css = digest('highlighting/highlight.css')
    js = digest('highlighting/highlight.js')
Sacha's avatar
Sacha committed
55
    if not is_human():
Sacha's avatar
Sacha committed
56
57
58
59
        form = forms.BeginHIT()
        if form.validate_on_submit():
            session['human'] = 'yes'
        else:
60
            return render_template('highlighting_welcome.html',
Sacha's avatar
Sacha committed
61
62
63
                                   form=form,
                                   css=css,
                                   js=js)
Sacha's avatar
Sacha committed
64

Sacha's avatar
Sacha committed
65
    article = get_next_doc()
Sacha's avatar
Sacha committed
66
    if article is None:
67
        return render_template('highlighting_none.html',
Sacha's avatar
Sacha committed
68
69
                               css=css)

Sacha's avatar
Sacha committed
70
    n_sentences = len(article['full'])
Sacha's avatar
Sacha committed
71
72
    # minimum reading time in milliseconds (at 750 wpm, fast skimming pace)
    minimum_time = int(100 * (float(60 * len(tokenizer.tokenize(article['full']))) / 75))
Sacha's avatar
Sacha committed
73
74
75
76
77
78

    # split by paragraph and then sentences
    paras = map(lambda x: sent_detector.tokenize(x.strip()), article['full'].split('|*^*|'))
    article['full'] = list(itertools.chain(*paras))
    # not the cleanest solution for paragraphs, but it'll be fine
    sentence_lookup = lambda x: article['full'].index(x)
Sacha's avatar
Sacha committed
79

Sacha's avatar
Sacha committed
80
81
    session['minimum_time'] = minimum_time
    session['started_reading'] = time.time()
Sacha's avatar
Sacha committed
82
    session['n_sentences'] = sum(map(lambda x: len(x), article['full']))
Sacha's avatar
Sacha committed
83
84
    session['reading_url'] = article['url']

85
    return render_template('highlight.html',
Sacha's avatar
Sacha committed
86
                           css=css,
Sacha's avatar
Sacha committed
87
88
89
                           js=js,
                           minimum_time=minimum_time,
                           n_sentences=n_sentences,
Sacha's avatar
Sacha committed
90
91
                           paras=paras,
                           sentence_lookup=sentence_lookup,
Sacha's avatar
Sacha committed
92
93
94
                           article=article)


95
@highlighting.route('/mark_available')
Sacha's avatar
Sacha committed
96
97
@require_human
def mark_available():
98
99
100
    toggle_being_read(get_articles_collection(),
                      session['reading_url'],
                      False)
Sacha's avatar
Sacha committed
101
102
103
    return success()


104
@highlighting.route('/submit', methods=['GET', 'POST'])
Sacha's avatar
Sacha committed
105
@require_human
Sacha's avatar
Sacha committed
106
def update_doc():
Sacha's avatar
Sacha committed
107
    if 'checked' not in request.args:
Sacha's avatar
Sacha committed
108
109
110
111
112
113
114
115
116
        raise UsageError('missing ``checked`` arg!')

    inds = request.args['checked'].split(',')
    n_inds = len(inds)
    if time.time() - session['started_reading'] < (float(session['minimum_time']) / 1000):
        raise UsageError('reading speed too fast')

    if n_inds > session['n_sentences'] or n_inds < 1:
        raise UsageError('no or too many highlights')
Sacha's avatar
Sacha committed
117

118
    col = get_articles_collection()
Sacha's avatar
Sacha committed
119
    url = session['reading_url']
120
    increment_reads(col, url, inds, request.args.get('author'))
Sacha's avatar
Sacha committed
121
    toggle_being_read(col, url, False)
122

Sacha's avatar
Sacha committed
123
    return success()
Sacha's avatar
Sacha committed
124
125


126
@highlighting.route('/submitted')
Sacha's avatar
Sacha committed
127
128
@require_human
def submitted():
129
130
    css = digest('highlighting/highlight.css')
    return render_template('highlighting_submitted.html', **locals())
Sacha's avatar
Sacha committed
131
132


133
@highlighting.route('/dataset')
134
def dataset():
135
136
    css = digest('highlighting/highlight.css')
    return render_template('highlighting_dataset.html', **locals())
137

138
139
140
141
142
143
144
145
146
147
148
149
150
151
def justmime(r, status=200):
    return Response(r, mimetype='application/json', status=status)

@highlighting.route('/dataset/all')
def dataset_all():
  col = get_articles_collection()
  return justmime(json.dumps(list(col.find()),
                             default=json_util.default))

@highlighting.route('/dataset/annotated')
def dataset_annotated():
  col = get_articles_collection()
  return justmime(json.dumps(list(col.find({'n_reads': { '$gt': 0 }})),
                             default=json_util.default))
152
153


154
def get_db():
Sacha's avatar
Sacha committed
155
    client = MongoClient('localhost', MONGO_PORT)
156
157
158
159
160
161
162
163
164
    return client.controversy


def get_articles_collection():
    return get_db().articles


def get_tweets_collection():
    return get_db().tweets
Sacha's avatar
Sacha committed
165
166


167
def increment_reads(col, url, highlights, author):
168
169
170
171
172
173
174
175
176
177
178
179
  return col.update_one({
    'url': url
  }, {
    '$set': {
      'ts': datetime.datetime.utcnow()
    }, '$inc': {
      'n_reads': 1
    }, '$push': {
      'highlights_author': author,
      'highlights': highlights
    }
  }).modified_count
Sacha's avatar
Sacha committed
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194


def toggle_being_read(col, url, dest):
    return col.update_one({
        'url': url
    }, {
        '$set': {
            'being_read': dest
        }
    })


def new_doc(doc):
    """given an API response, make an entry for each article,
    preserving the timestamp of the entire response and keyword.
195
    Cache tweets by keyword (not associated with article corpus for speed).
Sacha's avatar
Sacha committed
196
    Returns number of articles found.
Sacha's avatar
Sacha committed
197
    """
198
199
200
201
202
203
204
205
206
207
    if doc['ok'] != 1:
        return 0

    ar_col = get_articles_collection()
    tw_col = get_tweets_collection()
    res = doc['result']
    
    tw_col.insert_one({
        'tweets': res['kw_tweets'],
        'ts': doc['ts'],
Sacha's avatar
Sacha committed
208
        'keyword': doc['keyword']
209
210
    })

Sacha's avatar
Sacha committed
211
    n_docs = 0
212
213
214
    for article in res['articles']:
        if article is None:
            continue
Sacha's avatar
Sacha committed
215
216
217
218
219
        n_docs += 1
        a = {
            'ts': doc['ts'],
            'keyword': doc['keyword'],
            'n_reads': 0,
Sacha's avatar
Sacha committed
220
221
            'being_read': False,
            'highlights': []
Sacha's avatar
Sacha committed
222
223
        }
        a.update(article)
224
        ar_col.insert_one(a).inserted_id
Sacha's avatar
Sacha committed
225
226
227
228
    return n_docs


def get_next_doc():
Sacha's avatar
Sacha committed
229
    col = get_articles_collection()
Sacha's avatar
Sacha committed
230
231
    poss = col.find({
        'n_reads' : {
Sacha's avatar
Sacha committed
232
            '$lt': 3
Sacha's avatar
Sacha committed
233
234
        }
    }).sort([
Sacha's avatar
Sacha committed
235
236
        ('being_read', 1),
        ('full', -1)
Sacha's avatar
Sacha committed
237
    ])
Sacha's avatar
Sacha committed
238
239
240
241

    if poss is None:
        return None

Sacha's avatar
Sacha committed
242
243
244
245
    try:
        to_be_read = poss[:][0]
    except IndexError:
        return None
Sacha's avatar
Sacha committed
246
247
    toggle_being_read(col, to_be_read['url'], True)
    return to_be_read
Sacha's avatar
Sacha committed
248
249
250


def keyword_exists(kw):
Sacha's avatar
Sacha committed
251
    """Checks if ``kw`` has been saved already.
Sacha's avatar
Sacha committed
252
253
    Forced collection update
    """
Sacha's avatar
Sacha committed
254
255
    col = get_tweets_collection()
    poss = col.find({
Sacha's avatar
Sacha committed
256
257
        'keyword': kw
    })
Sacha's avatar
Sacha committed
258
    return poss.count() != 0
Sacha's avatar
Sacha committed
259
260
261


def article_is_new(url, col=None):
Sacha's avatar
Sacha committed
262
    """Checks if article with ``url`` has been saved already
Sacha's avatar
Sacha committed
263
264
265
266
267
    """
    if col is None:
        col = get_articles_collection()

    poss = col.find({
Sacha's avatar
Sacha committed
268
        'url': url
Sacha's avatar
Sacha committed
269
270
    })

Sacha's avatar
Sacha committed
271
    return poss.count() == 0