Commit 927e54ef authored by Sacha's avatar Sacha Committed by SXibolet@2PITAU
Browse files

progress with fetcher

parent 5d7256bf
*.html linguist-vendored
*.scss linguist-vendored
*.css linguist-vendored
controversy/fetcher/mongo-c-driver linguist-vendored
controversy/fetcher/libbson linguist-vendored
......@@ -15,4 +15,3 @@ venv
controversy/sentistrength
notes.rst
nohup.out
mongo-c-driver
......@@ -57,7 +57,7 @@ def handle_500(error):
client = TwilioRestClient(TWILIO_SID, TWILIO_AUTH_TOKEN)
client.messages.create(body='problem on Linode: %s' % repr(error),
to=ADMIN_PHONE,
from_='+19089982913')
from_='+19085437718')
raise UsageError('our-fault', status_code=500)
......
rst2html.py -stg README.rst README.html
docusty --externals=documents/externals.txt --stydest=documents README.html README.html
CC=gcc
INC=-I.
INC=-isystem
LOPTS=-pthread
EXEC=fetcher_test queue_test
all: $(EXEC)
fetcher_test: fetcher_test.c fetcher.o queue.o
$(CC) $(INC) $(LOPTS) -std=c99 -ggdb $^ -o $@ -lcurl
$(CC) $(shell pkg-config --cflags --libs libmongoc-1.0) $(INC) $(LOPTS) -ggdb $^ -o $@ -lcurl
fetcher.o: fetcher.c fetcher.h queue.o
$(CC) $(INC) $(LOPTS) -std=c99 -ggdb -c $<
$(CC) $(shell pkg-config --cflags --libs libmongoc-1.0) $(INC) $(LOPTS) -lcurl -ggdb -c $<
queue_test: queue_test.c queue.o
$(CC) $(INC) $(LOPTS) -std=c99 -ggdb $^ -o $@
$(CC) $(INC) $(LOPTS) -ggdb $^ -o $@
queue.o: queue.c queue.h
$(CC) $(INC) $(LOPTS) -std=c99 -ggdb -c $<
$(CC) $(INC) $(LOPTS) -ggdb -c $<
clean:
rm *.o $(EXEC)
......@@ -3,6 +3,8 @@
#include <string.h>
#include <pthread.h>
#include <curl/curl.h>
#include <mongoc.h>
#include <bson.h>
#include "queue.h"
#include "fetcher.h"
......@@ -13,12 +15,13 @@ Queue *tasks;
int n_fetches, n_todo;
pthread_mutex_t n_fetches_lock = PTHREAD_MUTEX_INITIALIZER;
typedef struct Page {
typedef struct page {
char *string;
size_t len;
} Page;
} page_t;
size_t w_callback(void *ptr, size_t size, size_t nmemb, Page *pg) {
size_t w_callback(void *ptr, size_t size, size_t nmemb, page_t *pg) {
size_t pckt_size = size * nmemb,
nl = pg->len + pckt_size;
pg->string = realloc(pg->string, nl + 1);
......@@ -30,11 +33,32 @@ size_t w_callback(void *ptr, size_t size, size_t nmemb, Page *pg) {
}
// write ``page`` to db
void write_page(Page *page) {
// TODO
void write_page(page_t *page, mongoc_client_pool_t *pool) {
mongoc_client_t *client = mongoc_client_pool_pop(pool);
mongoc_collection_t *collection = mongoc_client_get_collection(client, "controversy", "training_html");
bson_t *doc = bson_new();
bson_oid_t oid;
bson_oid_init(&oid, NULL);
BSON_APPEND_OID(doc, "_id", &oid);
BSON_APPEND_UTF8(doc, "hello", "world");
// TODO: write to articles collection
// ts, keyword, article,
bson_error_t error;
if (!mongoc_collection_insert(collection, MONGOC_INSERT_NONE, doc, NULL, &error)) {
}
bson_destroy(doc);
mongoc_collection_destroy(collection);
mongoc_client_pool_push(pool, client);
}
void *perform(void *arg) {
mongoc_client_pool_t *pool = arg;
while (1) {
pthread_mutex_lock(&n_fetches_lock);
int end = (n_fetches == n_todo);
......@@ -60,12 +84,13 @@ void *perform(void *arg) {
curl = curl_easy_init();
if (curl) {
Page *page = malloc(sizeof(page));
page_t *page = malloc(sizeof(page));
page->len = 0;
// allocate enough space for NULL-char
page->string = malloc(1);
page->string[0] = '\0';
// TODO: get twitter search result too!
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_COOKIEFILE, "");
......@@ -75,11 +100,11 @@ void *perform(void *arg) {
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
exit(1);
exit(EXIT_FAILURE);
}
if (page->len > 0) {
write_page(page);
write_page(page, pool);
} else {
fprintf(stderr, "(!) ===> page %s is NULL\n", url);
}
......@@ -88,7 +113,7 @@ void *perform(void *arg) {
free(page);
curl_easy_cleanup(curl);
} else {
exit(1);
exit(EXIT_FAILURE);
}
curl_easy_cleanup(curl);
......@@ -110,12 +135,21 @@ void Fetcher_fetch(char **sources) {
pthread_t pids[n_threads];
memset(&pids, 0, sizeof(pthread_t) * n_threads);
for (int i = 0; i < n_threads; i++) {
pthread_create(&pids[i], NULL, perform, NULL);
mongoc_init();
mongoc_uri_t *uri = mongoc_uri_new("mongodb://localhost/27017/");
mongoc_client_pool_t *pool = mongoc_client_pool_new(uri);
int i, j;
for (i = 0; i < n_threads; i++) {
pthread_create(&pids[i], NULL, perform, pool);
}
for (int i = 0; i < n_threads; i++) {
mongoc_uri_destroy(uri);
for (i = 0; i < n_threads; i++) {
pthread_join(pids[i], NULL);
}
mongoc_cleanup();
curl_global_cleanup();
}
libbson @ 89edcc7a
Subproject commit 89edcc7ad7a65552139905ad4818716a2b1a8ae7
mongo-c-driver @ 7bd6891a
Subproject commit 7bd6891aa14a6f0e8811b0caf2de3ca0cbf843ef
......@@ -21,7 +21,7 @@ def sms_admin(m):
client = TwilioRestClient(TWILIO_SID, TWILIO_AUTH_TOKEN)
client.messages.create(body=m,
to=ADMIN_PHONE,
from_='+19089982913')
from_='+19085437718')
def get_file(name):
with open(name, 'r') as f:
......
......@@ -2,7 +2,6 @@ pymysql
twython
beautifulsoup4
requests
requests[security]
requests_oauthlib
oauthlib==0.7.2
redis
......
abuse
administration
afghanistan
aid
america
american
army
attack
attacks
authorities
authority
ban
banks
benefits
bill
bills
border
budget
campaign
candidate
candidates
catholic
china
chinese
church
concerns
congress
conservative
control
country
court
crime
criminal
crisis
cuts
......@@ -39,107 +25,55 @@ defense
deficit
democrats
disease
dollar
drug
drugs
economy
education
egypt
election
elections
enforcement
fighting
finance
fiscal
force
funding
gas
government
gun
health
immigration
inaccuracies
india
insurance
investigation
investigators
iran
israel
job
jobs
judge
justice
killing
korea
labor
land
law
lawmakers
laws
lawsuit
leadership
legislation
marriage
media
mexico
military
money
murder
nation
nations
news
obama
offensive
officials
oil
parties
peace
police
policies
policy
politics
poll
power
president
prices
primary
prison
progress
race
reform
republican
republicans
restrictions
rule
rules
ruling
russia
russian
school
security
senate
sex
shooting
society
spending
strategy
strike
support
syria
syrian
tax
taxes
threat
trial
unemployment
union
usa
victim
victims
violence
vote
voters
war
washington
weapons
world
\ No newline at end of file
60s
70s
addition
address
afternoon
agreed
amp
angeles
answer
april
attention
avenue
average
ball
base
bay
beach
beginning
bit
block
blue
bowl
box
boy
boys
brother
building
bus
call
calling
calls
camp
car
cars
central
cents
click
close
cloudy
club
coast
cup
dallas
date
daughter
davis
day
decade
decades
december
def
delivery
door
download
drive
eagles
end
entire
era
evening
face
faces
facility
fall
fans
father
feel
feeling
feet
fell
field
finish
floor
form
fort
francisco
friday
friend
friends
fun
girl
girls
ground
gt
guy
guys
half
hall
hand
hands
hawaii
heart
heat
heavy
hill
hits
hold
hopes
host
hotel
hour
hours
house
houston
hundreds
husband
ice
illinois
index
indiana
innings
island
january
johnson
jones
june
kansas
kind
lack
lake
leave
lee
letter
levels
light
line
lines
lot
lows
lt
main
make
makes
mark
mass
material
matter
medium
men
mid
middle
miles
mind
minneapolis
minutes
moment
monday
month
months
morning
mother
mountain
move
mph
museum
names
natural
net
night
north
note
notes
november
october
opening
park
part
parts
pass
period
person
philadelphia
pick
pitch
plant
play
player
playing
pm
point
post
practice
put
quarter
rain
read
reading
red
rest
restaurant
rise
rock
rose
round
sale
san
saturday
scene
search
season
seasons
seconds
selling
september
series
set
showers
showing
shows
sign
signs
smith
son
sox
special
spot
spring
square
stadium
stage
start
starting
starts
station
stay
step
stores
street
student
summer
sun
sunday
thing
things
thinking
thought
thousands
thunderstorms
thursday
time
title
top
total
transportation
type
unit
valley
vehicle
version
village
visit
wait
walk
wall
watch
water
ways
wednesday
week
weekend
weeks
williams
wind
winds
winner
winter
word
writer
yards
year
years
york
\ No newline at end of file
afternoon
agreed
avenue
average
ball
beach
camp
click
close
cloudy
club
coast
cup
drive