Commit 41e29ac1 authored by Sacha's avatar Sacha Committed by SXibolet@2PITAU
Browse files

tweaks to NYT article search

parent 107daf36
......@@ -9,6 +9,7 @@ TWITTER_KEYS = ['twitter api'] # REQUIRED change. If you train, you'll need a fe
TWITTER_SECRETS = ['twitter api secret'] # REQUIRED change
NYT_ARTICLE_SEARCH_KEY = 'nyt article search key' # REQUIRED change
NYT_COMMUNITY_KEYS = ['nyt community api key'] # REQUIRED change
NYT_UA = 'Mozilla/5.0' # optional change
REDIS_HOST = 'localhost'
REDIS_PORT = '6378'
TWILIO_SID = 'twilio key' # optional change
......
......@@ -27,8 +27,8 @@ from twython.exceptions import TwythonAuthError
MAX_ATTEMPTS = 6
MAX_COMMENTARY = 500
TAG_RE = re.compile(r'<[^>]+>')
ARTICLE_SEARCH_BASE = 'http://api.nytimes.com/svc/search/v2/articlesearch.json?'
COMMENT_BASE = 'http://api.nytimes.com/svc/community/v3/user-content/url.json?'
ARTICLE_SEARCH_BASE = 'https://api.nytimes.com/svc/search/v2/articlesearch.json?'
COMMENT_BASE = 'https://api.nytimes.com/svc/community/v3/user-content/url.json?'
class SocialContent(object):
......@@ -142,7 +142,7 @@ class Article(object):
return None
jar = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(jar))
opener.addheaders = [('User-Agent', 'Mozilla/5.0')]
opener.addheaders = [('User-Agent', NYT_UA)]
response = opener.open(urllib2.Request(self.url))
soup = BeautifulSoup(response.read(), 'html.parser')
body = soup.findAll('p', {
......@@ -164,6 +164,10 @@ def article_search(keyword, training=False):
today = datetime.date.today()
last_week = today - datetime.timedelta(days=11)
opener = urllib2.build_opener()
opener.addheaders = [('User-Agent', NYT_UA)]
params = urllib.urlencode({
'q': keyword,
'begin_date': nyt_query_date(last_week),
......@@ -172,7 +176,8 @@ def article_search(keyword, training=False):
'facet_field': 'source'
})
response = urllib2.urlopen('%s%s' % (ARTICLE_SEARCH_BASE, params))
print '%s%s' % (ARTICLE_SEARCH_BASE, params)
response = opener.open('%s%s' % (ARTICLE_SEARCH_BASE, params))
# an Article will be None if it doesn't have body text (thus the partial)
# return an array of Article objects that have a body text
return filter(partial(is_not, None),
......
......@@ -13,7 +13,7 @@
{% endif %}
{% with messages = get_flashed_messages() %}
{% if messages %}
<div id=messages onclick="this.style.display='none';">
<div id=message onclick="this.style.display='none';">
{% for message in messages %}
{{ message|safe }}
{% if not loop.last %}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment