highlighting.py 6.44 KB
Newer Older
Sacha's avatar
Sacha committed
1
2
# -*- coding: utf-8 -*-
from flask import Blueprint, jsonify, request, session, render_template
Sacha's avatar
Sacha committed
3
from functools import wraps
Sacha's avatar
Sacha committed
4
5
6
7
8
from digest import digest
from pymongo import MongoClient
from config import MONGO_PORT
import content
import datetime
Sacha's avatar
Sacha committed
9
import time
Sacha's avatar
Sacha committed
10
import forms
11
import json
Sacha's avatar
Sacha committed
12
13
from error import UsageError
import nltk.data
Sacha's avatar
Sacha committed
14
import itertools
Sacha's avatar
Sacha committed
15
from nltk.tokenize import RegexpTokenizer
Sacha's avatar
Sacha committed
16
17


18
highlighting = Blueprint('/training', __name__)
Sacha's avatar
Sacha committed
19
sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')
Sacha's avatar
Sacha committed
20
21
22
23
24
25
tokenizer = RegexpTokenizer(r'\w+')


def require_human(view):
    @wraps(view)
    def protected_view(*args, **kwargs):
Sacha's avatar
Sacha committed
26
        if is_human():
Sacha's avatar
Sacha committed
27
28
29
30
31
32
            return view(*args, **kwargs)
        else:
            raise UsageError('not human!')
    return protected_view


Sacha's avatar
Sacha committed
33
34
35
36
def is_human():
    return session.get('human') == 'yes'


Sacha's avatar
Sacha committed
37
38
39
40
def success():
    return jsonify({
        'ok': 1
    })
Sacha's avatar
Sacha committed
41
42


43
@highlighting.errorhandler(UsageError)
Sacha's avatar
Sacha committed
44
45
46
47
48
49
def handle_error(error):
    response = jsonify(error.to_dict())
    response.status_code = error.status_code
    return response


50
@highlighting.route('/', methods=['GET', 'POST'])
Sacha's avatar
Sacha committed
51
def index():
52
53
    css = digest('highlighting/highlight.css')
    js = digest('highlighting/highlight.js')
Sacha's avatar
Sacha committed
54
    if not is_human():
Sacha's avatar
Sacha committed
55
56
57
58
        form = forms.BeginHIT()
        if form.validate_on_submit():
            session['human'] = 'yes'
        else:
59
            return render_template('highlighting_welcome.html',
Sacha's avatar
Sacha committed
60
61
62
                                   form=form,
                                   css=css,
                                   js=js)
Sacha's avatar
Sacha committed
63

Sacha's avatar
Sacha committed
64
    article = get_next_doc()
Sacha's avatar
Sacha committed
65
    if article is None:
66
        return render_template('highlighting_none.html',
Sacha's avatar
Sacha committed
67
68
                               css=css)

Sacha's avatar
Sacha committed
69
    n_sentences = len(article['full'])
Sacha's avatar
Sacha committed
70
71
    # minimum reading time in milliseconds (at 750 wpm, fast skimming pace)
    minimum_time = int(100 * (float(60 * len(tokenizer.tokenize(article['full']))) / 75))
Sacha's avatar
Sacha committed
72
73
74
75
76
77

    # split by paragraph and then sentences
    paras = map(lambda x: sent_detector.tokenize(x.strip()), article['full'].split('|*^*|'))
    article['full'] = list(itertools.chain(*paras))
    # not the cleanest solution for paragraphs, but it'll be fine
    sentence_lookup = lambda x: article['full'].index(x)
Sacha's avatar
Sacha committed
78

Sacha's avatar
Sacha committed
79
80
    session['minimum_time'] = minimum_time
    session['started_reading'] = time.time()
Sacha's avatar
Sacha committed
81
    session['n_sentences'] = sum(map(lambda x: len(x), article['full']))
Sacha's avatar
Sacha committed
82
83
    session['reading_url'] = article['url']

84
    return render_template('highlight.html',
Sacha's avatar
Sacha committed
85
                           css=css,
Sacha's avatar
Sacha committed
86
87
88
                           js=js,
                           minimum_time=minimum_time,
                           n_sentences=n_sentences,
Sacha's avatar
Sacha committed
89
90
                           paras=paras,
                           sentence_lookup=sentence_lookup,
Sacha's avatar
Sacha committed
91
92
93
                           article=article)


94
@highlighting.route('/mark_available')
Sacha's avatar
Sacha committed
95
96
@require_human
def mark_available():
97
98
99
    toggle_being_read(get_articles_collection(),
                      session['reading_url'],
                      False)
Sacha's avatar
Sacha committed
100
101
102
    return success()


103
@highlighting.route('/submit', methods=['GET', 'POST'])
Sacha's avatar
Sacha committed
104
@require_human
Sacha's avatar
Sacha committed
105
def update_doc():
Sacha's avatar
Sacha committed
106
    if 'checked' not in request.args:
Sacha's avatar
Sacha committed
107
108
109
110
111
112
113
114
115
        raise UsageError('missing ``checked`` arg!')

    inds = request.args['checked'].split(',')
    n_inds = len(inds)
    if time.time() - session['started_reading'] < (float(session['minimum_time']) / 1000):
        raise UsageError('reading speed too fast')

    if n_inds > session['n_sentences'] or n_inds < 1:
        raise UsageError('no or too many highlights')
Sacha's avatar
Sacha committed
116

117
    col = get_articles_collection()
Sacha's avatar
Sacha committed
118
    url = session['reading_url']
119
    increment_reads(col, url, inds, request.args.get('author'))
Sacha's avatar
Sacha committed
120
    toggle_being_read(col, url, False)
121

Sacha's avatar
Sacha committed
122
    return success()
Sacha's avatar
Sacha committed
123
124


125
@highlighting.route('/submitted')
Sacha's avatar
Sacha committed
126
127
@require_human
def submitted():
128
129
    css = digest('highlighting/highlight.css')
    return render_template('highlighting_submitted.html', **locals())
Sacha's avatar
Sacha committed
130
131


132
@highlighting.route('/dataset')
133
def dataset():
134
135
    css = digest('highlighting/highlight.css')
    return render_template('highlighting_dataset.html', **locals())
136
137
138



139
def get_db():
Sacha's avatar
Sacha committed
140
    client = MongoClient('localhost', MONGO_PORT)
141
142
143
144
145
146
147
148
149
    return client.controversy


def get_articles_collection():
    return get_db().articles


def get_tweets_collection():
    return get_db().tweets
Sacha's avatar
Sacha committed
150
151


152
def increment_reads(col, url, highlights, author):
Sacha's avatar
Sacha committed
153
154
155
    return col.update_one({
        'url': url
    }, {
Sacha's avatar
Sacha committed
156
157
158
        '$set': {
            'ts': datetime.datetime.utcnow()
        }, '$inc': {
Sacha's avatar
Sacha committed
159
            'n_reads': 1
Sacha's avatar
Sacha committed
160
        }, '$push': {
161
            'highlights_author': author,
Sacha's avatar
Sacha committed
162
            'highlights': highlights
Sacha's avatar
Sacha committed
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
        }
    }).modified_count


def toggle_being_read(col, url, dest):
    return col.update_one({
        'url': url
    }, {
        '$set': {
            'being_read': dest
        }
    })


def new_doc(doc):
    """given an API response, make an entry for each article,
    preserving the timestamp of the entire response and keyword.
180
    Cache tweets by keyword (not associated with article corpus for speed).
Sacha's avatar
Sacha committed
181
    Returns number of articles found.
Sacha's avatar
Sacha committed
182
    """
183
184
185
186
187
188
189
190
191
192
    if doc['ok'] != 1:
        return 0

    ar_col = get_articles_collection()
    tw_col = get_tweets_collection()
    res = doc['result']
    
    tw_col.insert_one({
        'tweets': res['kw_tweets'],
        'ts': doc['ts'],
Sacha's avatar
Sacha committed
193
        'keyword': doc['keyword']
194
195
    })

Sacha's avatar
Sacha committed
196
    n_docs = 0
197
198
199
    for article in res['articles']:
        if article is None:
            continue
Sacha's avatar
Sacha committed
200
201
202
203
204
        n_docs += 1
        a = {
            'ts': doc['ts'],
            'keyword': doc['keyword'],
            'n_reads': 0,
Sacha's avatar
Sacha committed
205
206
            'being_read': False,
            'highlights': []
Sacha's avatar
Sacha committed
207
208
        }
        a.update(article)
209
        ar_col.insert_one(a).inserted_id
Sacha's avatar
Sacha committed
210
211
212
213
    return n_docs


def get_next_doc():
Sacha's avatar
Sacha committed
214
    col = get_articles_collection()
Sacha's avatar
Sacha committed
215
216
    poss = col.find({
        'n_reads' : {
Sacha's avatar
Sacha committed
217
            '$lt': 3
Sacha's avatar
Sacha committed
218
219
        }
    }).sort([
Sacha's avatar
Sacha committed
220
221
        ('being_read', 1),
        ('full', -1)
Sacha's avatar
Sacha committed
222
    ])
Sacha's avatar
Sacha committed
223
224
225
226

    if poss is None:
        return None

Sacha's avatar
Sacha committed
227
228
229
230
    try:
        to_be_read = poss[:][0]
    except IndexError:
        return None
Sacha's avatar
Sacha committed
231
232
    toggle_being_read(col, to_be_read['url'], True)
    return to_be_read
Sacha's avatar
Sacha committed
233
234
235


def keyword_exists(kw):
Sacha's avatar
Sacha committed
236
    """Checks if ``kw`` has been saved already.
Sacha's avatar
Sacha committed
237
238
    Forced collection update
    """
Sacha's avatar
Sacha committed
239
240
    col = get_tweets_collection()
    poss = col.find({
Sacha's avatar
Sacha committed
241
242
        'keyword': kw
    })
Sacha's avatar
Sacha committed
243
    return poss.count() != 0
Sacha's avatar
Sacha committed
244
245
246


def article_is_new(url, col=None):
Sacha's avatar
Sacha committed
247
    """Checks if article with ``url`` has been saved already
Sacha's avatar
Sacha committed
248
249
250
251
252
    """
    if col is None:
        col = get_articles_collection()

    poss = col.find({
Sacha's avatar
Sacha committed
253
        'url': url
Sacha's avatar
Sacha committed
254
255
    })

Sacha's avatar
Sacha committed
256
    return poss.count() == 0