Commit f1c6bfe0 authored by Sacha's avatar Sacha
Browse files

problems with NYTimes website

parent baa0b4bb
......@@ -13,7 +13,6 @@ import stats
import datetime
import json
import db
import redis
QUERY_PARAM = 'q'
......
......@@ -144,17 +144,23 @@ class Article(object):
"""
if 'Paid Notice:' in self.title or 'video/multimedia' in self.url:
return None
jar = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(jar))
opener.addheaders = [('User-Agent', NYT_UA)]
response = opener.open(urllib2.Request(self.url))
soup = BeautifulSoup(response.read(), 'html.parser')
body = soup.findAll('p', {
'class' : ['css-18sbwfn', 'css-1ygdjhk', 'css-exrw3m']
paragraphs = soup.findAll('div', {
'class': ['StoryBodyCompanionColumn']
})
print(paragraphs)
if paragraphs == []:
paragraphs = soup.findAll('p')
# we'll split into paragraphs for easier reading if training
res = ('|*^*|' if training else ' ').join(p.text for p in body)
#res = ('|*^*|').join(p.text for p in body)
res = ('|*^*|' if training else ' ').join(p.text for p in paragraphs)
#res = ('|*^*|').join(p.text for p in paragraphs)
jar.clear()
return res
......@@ -171,19 +177,20 @@ def article_search(keyword, training=False):
params = urllib.urlencode({
'q': keyword,
'begin_date': nyt_query_date(last_week),
'end_date': nyt_query_date(today),
#'begin_date': nyt_query_date(last_week),
#'end_date': nyt_query_date(today),
'api-key': NYT_ARTICLE_SEARCH_KEY,
'facet_field': 'source'
#'facet': 'true',
#'facet_field': 'source'
})
#tor_query.renew_connection()
#response = tor_query.query('%s%s' % (ARTICLE_SEARCH_BASE, params))
response = urllib2.urlopen('%s%s' % (ARTICLE_SEARCH_BASE, params))
response = tor_query.query('%s%s' % (ARTICLE_SEARCH_BASE, params))
# an Article will be None if it doesn't have body text (thus the partial)
# return an array of Article objects that have a body text
parsed_response = json.loads(response.read())['response']['docs']
r = json.loads(response.read())
parsed_response = r['response']['docs']
return filter(partial(is_not, None),
map(lambda x: Article(x, training=training),
parsed_response))
......
......@@ -13,3 +13,4 @@ gensim
scipy
matplotlib
pymongo
newspaper
......@@ -88,7 +88,7 @@ function dismiss_error() {
<p>This is <span class='logo'>CONTROVERSY</span>, available <a target='_blank' href='http://framagit.org/gdyer/CONTROVERSY'>on Framagit</a>.</p>
<p>We&rsquo;ve setup an annotation system for people to select pieces of news articles they believe to be controversial: <a href='../training'>training</a>.</p>
<p>We&rsquo;ve set up an annotation system for people to select pieces of news articles they believe to be controversial: <a href='../training'>training</a>.</p>
<p>We encrypt passwords with a one-way hash. As of fall 2015, we force SSL as you browse the project.</p>
......
......@@ -16,8 +16,9 @@
</span>
</div>
</header>
<header id='sub' class='mobile'>
Uncached queries may take a minute to score
<header id='sub'>
As of 20 October 2020, <em>The New York Times</em> has changed their website to make it very hard to retrieve the full text. Thus, our system may not behave as expected or may be unable to find articles.
<!-- Uncached queries may take a minute to score -->
</header>
{% raw %}
<main style='padding: 8em 1em'>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment