Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Sacha
CONTROVERSY
Commits
41e29ac1
Commit
41e29ac1
authored
Dec 28, 2016
by
Sacha
Committed by
SXibolet@2PITAU
Dec 28, 2016
Browse files
tweaks to NYT article search
parent
107daf36
Changes
3
Hide whitespace changes
Inline
Side-by-side
controversy/config.py.default
View file @
41e29ac1
...
...
@@ -9,6 +9,7 @@ TWITTER_KEYS = ['twitter api'] # REQUIRED change. If you train, you'll need a fe
TWITTER_SECRETS = ['twitter api secret'] # REQUIRED change
NYT_ARTICLE_SEARCH_KEY = 'nyt article search key' # REQUIRED change
NYT_COMMUNITY_KEYS = ['nyt community api key'] # REQUIRED change
NYT_UA = 'Mozilla/5.0' # optional change
REDIS_HOST = 'localhost'
REDIS_PORT = '6378'
TWILIO_SID = 'twilio key' # optional change
...
...
controversy/content.py
View file @
41e29ac1
...
...
@@ -27,8 +27,8 @@ from twython.exceptions import TwythonAuthError
MAX_ATTEMPTS
=
6
MAX_COMMENTARY
=
500
TAG_RE
=
re
.
compile
(
r
'<[^>]+>'
)
ARTICLE_SEARCH_BASE
=
'http://api.nytimes.com/svc/search/v2/articlesearch.json?'
COMMENT_BASE
=
'http://api.nytimes.com/svc/community/v3/user-content/url.json?'
ARTICLE_SEARCH_BASE
=
'http
s
://api.nytimes.com/svc/search/v2/articlesearch.json?'
COMMENT_BASE
=
'http
s
://api.nytimes.com/svc/community/v3/user-content/url.json?'
class
SocialContent
(
object
):
...
...
@@ -142,7 +142,7 @@ class Article(object):
return
None
jar
=
cookielib
.
CookieJar
()
opener
=
urllib2
.
build_opener
(
urllib2
.
HTTPCookieProcessor
(
jar
))
opener
.
addheaders
=
[(
'User-Agent'
,
'Mozilla/5.0'
)]
opener
.
addheaders
=
[(
'User-Agent'
,
NYT_UA
)]
response
=
opener
.
open
(
urllib2
.
Request
(
self
.
url
))
soup
=
BeautifulSoup
(
response
.
read
(),
'html.parser'
)
body
=
soup
.
findAll
(
'p'
,
{
...
...
@@ -164,6 +164,10 @@ def article_search(keyword, training=False):
today
=
datetime
.
date
.
today
()
last_week
=
today
-
datetime
.
timedelta
(
days
=
11
)
opener
=
urllib2
.
build_opener
()
opener
.
addheaders
=
[(
'User-Agent'
,
NYT_UA
)]
params
=
urllib
.
urlencode
({
'q'
:
keyword
,
'begin_date'
:
nyt_query_date
(
last_week
),
...
...
@@ -172,7 +176,8 @@ def article_search(keyword, training=False):
'facet_field'
:
'source'
})
response
=
urllib2
.
urlopen
(
'%s%s'
%
(
ARTICLE_SEARCH_BASE
,
params
))
print
'%s%s'
%
(
ARTICLE_SEARCH_BASE
,
params
)
response
=
opener
.
open
(
'%s%s'
%
(
ARTICLE_SEARCH_BASE
,
params
))
# an Article will be None if it doesn't have body text (thus the partial)
# return an array of Article objects that have a body text
return
filter
(
partial
(
is_not
,
None
),
...
...
controversy/templates/login.html
View file @
41e29ac1
...
...
@@ -13,7 +13,7 @@
{% endif %}
{% with messages = get_flashed_messages() %}
{% if messages %}
<div
id=
message
s
onclick=
"this.style.display='none';"
>
<div
id=
message
onclick=
"this.style.display='none';"
>
{% for message in messages %}
{{ message|safe }}
{% if not loop.last %}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment