Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Sacha
CONTROVERSY
Commits
79dc17ab
Commit
79dc17ab
authored
May 23, 2016
by
Sacha
Committed by
SXibolet@2PITAU
May 23, 2016
Browse files
see log
parent
927e54ef
Changes
3
Hide whitespace changes
Inline
Side-by-side
controversy/fetcher/README
0 → 100644
View file @
79dc17ab
0. fetch article URLs and tweets by keyword from mturk_loader.py
1. fetch each article result if not already saved (matching URL)
2. store article for each keyword with URL and keyword
2. store tweet for each keyword in tweets collection
controversy/fetcher/fetcher.c
View file @
79dc17ab
...
...
@@ -19,6 +19,7 @@ pthread_mutex_t n_fetches_lock = PTHREAD_MUTEX_INITIALIZER;
typedef
struct
page
{
char
*
string
;
size_t
len
;
char
*
keyword
;
}
page_t
;
size_t
w_callback
(
void
*
ptr
,
size_t
size
,
size_t
nmemb
,
page_t
*
pg
)
{
...
...
@@ -33,7 +34,7 @@ size_t w_callback(void *ptr, size_t size, size_t nmemb, page_t *pg) {
}
// write ``page`` to db
void
write_page
(
page_t
*
page
,
mongoc_client_pool_t
*
pool
)
{
void
write_page
(
page_t
*
page
,
mongoc_client_pool_t
*
pool
,
char
*
url
,
char
*
keyword
)
{
mongoc_client_t
*
client
=
mongoc_client_pool_pop
(
pool
);
mongoc_collection_t
*
collection
=
mongoc_client_get_collection
(
client
,
"controversy"
,
"training_html"
);
...
...
@@ -42,10 +43,11 @@ void write_page(page_t *page, mongoc_client_pool_t *pool) {
bson_oid_init
(
&
oid
,
NULL
);
BSON_APPEND_OID
(
doc
,
"_id"
,
&
oid
);
BSON_APPEND_UTF8
(
doc
,
"hello"
,
"world"
);
BSON_APPEND_UTF8
(
doc
,
"url"
,
url
);
BSON_APPEND_UTF8
(
doc
,
"url"
,
url
);
// TODO: write to article
s
collection
//
ts, keyword, article,
// TODO: write to article collection
with HTML, keyword, and url
//
TODO: write to tweets collection entire JSON response
bson_error_t
error
;
if
(
!
mongoc_collection_insert
(
collection
,
MONGOC_INSERT_NONE
,
doc
,
NULL
,
&
error
))
{
...
...
@@ -104,12 +106,12 @@ void *perform(void *arg) {
}
if
(
page
->
len
>
0
)
{
write_page
(
page
,
pool
);
write_page
(
page
,
pool
,
url
,
keyword
);
}
else
{
fprintf
(
stderr
,
"(!) ===> page %s is NULL
\n
"
,
url
);
}
free
(
page
->
string
);
free
(
page
->
string
);
// url
free
(
page
);
curl_easy_cleanup
(
curl
);
}
else
{
...
...
@@ -122,7 +124,7 @@ void *perform(void *arg) {
return
0
;
}
void
Fetcher_fetch
(
char
**
sources
)
{
void
Fetcher_fetch
(
char
**
sources
,
char
*
keyword
)
{
tasks
=
Queue_init
();
while
(
*
sources
!=
NULL
)
{
...
...
controversy/fetcher/fetcher.h
View file @
79dc17ab
...
...
@@ -3,7 +3,7 @@
#define FETCHER_H
#include
"queue.h"
//
HTML of URLs in ``sources``
to ``key``
void
Fetcher_fetch
(
char
**
sources
);
//
fetch HTML from ``sources[i]`` cooresponding
to ``key
word
``
void
Fetcher_fetch
(
char
**
sources
,
char
*
keyword
);
#endif
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment