Commit a6f3d1d7 authored by Sacha's avatar Sacha Committed by SXibolet@2PITAU
Browse files

progress

parent 9f435052
......@@ -5,6 +5,9 @@
*.rdb
*.aux
*.log
*.o
*.out
*.dSYM
dump.rdb
.DS_Store
config.py
......
......@@ -27,8 +27,8 @@ from twython.exceptions import TwythonAuthError
MAX_ATTEMPTS = 6
MAX_COMMENTARY = 500
TAG_RE = re.compile(r'<[^>]+>')
ARTICLE_SEARCH_BASE = 'http://api-beta.nytimes.com/svc/search/v2/articlesearch.json?'
COMMENT_BASE = 'http://api-beta.nytimes.com/svc/community/v3/user-content/url.json?'
ARTICLE_SEARCH_BASE = 'http://api.nytimes.com/svc/search/v2/articlesearch.json?'
COMMENT_BASE = 'http://api.nytimes.com/svc/community/v3/user-content/url.json?'
class SocialContent(object):
......
language: c
before_install:
- sudo apt-get -qq update
- sudo apt-get install -y valgrind libcurl4-openssl-dev
script:
- cd controversy/dev
- make && ./queue_test && ./fetch_test
......@@ -6,16 +6,16 @@ EXEC=fetcher_test queue_test
all: $(EXEC)
fetcher_test: fetcher_test.c fetcher.o queue.o
$(CC) $(INC) $(LOPTS) -ggdb -lcurl $^ -o $@
$(CC) $(INC) $(LOPTS) -std=c99 -ggdb $^ -o $@ -lcurl
fetcher.o: fetcher.c fetcher.h queue.o
$(CC) $(INC) $(LOPTS) -ggdb -c $<
$(CC) $(INC) $(LOPTS) -std=c99 -ggdb -c $<
queue_test: queue_test.c queue.o
$(CC) $(INC) $(LOPTS) -ggdb $^ -o $@
$(CC) $(INC) $(LOPTS) -std=c99 -ggdb $^ -o $@
queue.o: queue.c queue.h
$(CC) $(INC) $(LOPTS) -ggdb -c $<
$(CC) $(INC) $(LOPTS) -std=c99 -ggdb -c $<
clean:
rm *.o $(EXEC)
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <pthread.h>
#include <curl/curl.h>
#include "queue.h"
#include "fetcher.h"
int perform() {
CURL *curl;
CURLcode res;
#define THREAD_MAX 8
#define MIN(x, y) (((x) < (y)) ? (x) : (y))
curl = curl_easy_init();
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://sxibolet.github.io");
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
int n_fetches, n_todo;
pthread_mutex_t n_fetches_lock = PTHREAD_MUTEX_INITIALIZER;
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
void *perform(void *arg) {
while (1) {
pthread_mutex_lock(&n_fetches_lock);
int end = (n_fetches == n_todo);
pthread_mutex_unlock(&n_fetches_lock);
if (end) {
tasks->cancel = 1;
break;
}
char *url = (char *)Queue_pull(tasks);
if (!url) {
tasks->cancel = 1;
break;
}
curl_easy_cleanup(curl);
pthread_mutex_lock(&n_fetches_lock);
n_fetches++;
pthread_mutex_unlock(&n_fetches_lock);
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
}
curl_easy_cleanup(curl);
}
}
return 0;
}
void Fetcher_init(char **sources) {
tasks = Queue_init();
while (sources) {
Queue_push(tasks, *sources);
sources++;
}
n_todo = tasks->size;
int n_threads = MIN(n_todo, THREAD_MAX);
pthread_t pids[n_threads];
memset(&pids, 0, sizeof(pthread_t) * n_threads);
for (int i = 0; i < n_threads; i++) {
pthread_create(&pids[i], NULL, perform, NULL);
}
for (int i = 0; i < n_threads; i++) {
pthread_join(pids[i], NULL);
}
}
#include <stdlib.h>
#include <stdio.h>
#include <curl/curl.h>
#include <pthread.h>
/* multithreaded article fetcher */
#ifndef FETCHER_H
#define FETCHER_H
#include "queue.h"
Queue *tasks;
void Fetcher_init(char **sources);
#endif
......@@ -7,6 +7,7 @@ Queue *Queue_init() {
q->size = 0;
q->head = NULL;
q->tail = NULL;
q->cancel = 0;
pthread_mutex_init(&q->lock, NULL);
pthread_cond_init(&q->not_empty, NULL);
return q;
......@@ -17,9 +18,12 @@ void Queue_free(Queue *q) {
pthread_cond_destroy(&q->not_empty);
QueueNode *i = q->head;
while (i) {
fprintf(stderr, "here!\n");
free(i);
i = i->next;
}
free(q);
}
void Queue_push(Queue *q, void *data) {
......@@ -48,15 +52,23 @@ void *Queue_pull(Queue *q) {
pthread_cond_wait(&q->not_empty, &q->lock);
}
if (q->cancel) {
pthread_mutex_unlock(&q->lock);
return NULL;
}
q->size--;
QueueNode *res = q->head;
QueueNode *old_head = q->head;
if (q->head == q->tail) {
q->head = q->tail = NULL;
} else {
q->head = q->head->next;
}
void *res = old_head->data;
free(old_head);
pthread_mutex_unlock(&q->lock);
return res->data;
return res;
}
/* a thread-safe queue implementation for
article/comment fetcher */
/* a thread-safe queue implementation for article/comment fetcher */
#ifndef QUEUE_H
#define QUEUE_H
#include <pthread.h>
#define true 1
#define false 0
#define bool int
typedef struct QueueNode {
struct QueueNode *next;
void *data;
......@@ -16,13 +11,15 @@ typedef struct QueueNode {
typedef struct {
int size;
int cancel;
QueueNode *head, *tail;
pthread_cond_t not_empty;
pthread_mutex_t lock;
bool cancel;
} Queue;
Queue *Queue_init();
// frees q and its nodes but not their data
void Queue_free(Queue *q);
// add element to queue; won't block
......
No preview for this file type
#define _GNU_SOURCE
#include <stdlib.h>
#include <stdio.h>
#include <assert.h>
#include <string.h>
#include "queue.h"
char *v = NULL;
void *push(void *arg) {
v = strdup("bye!");
Queue_push((Queue *)arg, v);
return NULL;
}
int main(void) {
Queue *q = Queue_init();
char *s = "hello";
Queue_push(q, s);
assert(strcmp(s, (char *)Queue_pull(q)) == 0);
char *t = "hello!";
char *u = "again...";
Queue_push(q, strdup(t));
Queue_push(q, strdup(t));
pthread_t p;
pthread_create(&p, NULL, push, (void *)q);
char *x = (char *)Queue_pull(q);
assert(strcmp(t, x) == 0);
free(x);
char *y = (char *)Queue_pull(q);
assert(strcmp(t, y) == 0);
free(y);
assert(strcmp(v, (char *)Queue_pull(q)) == 0);
pthread_join(p, NULL);
free(v);
Queue_push(q, u);
assert(strcmp(u, (char *)Queue_pull(q)) == 0);
Queue_free(q);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment