Commit 9f435052 authored by Sacha's avatar Sacha Committed by SXibolet@2PITAU
Browse files

begin work on faster processing

parent 815d593e
......@@ -27,8 +27,8 @@ from twython.exceptions import TwythonAuthError
MAX_ATTEMPTS = 6
MAX_COMMENTARY = 500
TAG_RE = re.compile(r'<[^>]+>')
ARTICLE_SEARCH_BASE = 'http://api.nytimes.com/svc/search/v2/articlesearch.json?'
COMMENT_BASE = 'http://api.nytimes.com/svc/community/v3/user-content/url.json?'
ARTICLE_SEARCH_BASE = 'http://api-beta.nytimes.com/svc/search/v2/articlesearch.json?'
COMMENT_BASE = 'http://api-beta.nytimes.com/svc/community/v3/user-content/url.json?'
class SocialContent(object):
......
CC=gcc
INC=-I.
LOPTS=-pthread
EXEC=fetcher_test queue_test
all: $(EXEC)
fetcher_test: fetcher_test.c fetcher.o queue.o
$(CC) $(INC) $(LOPTS) -ggdb -lcurl $^ -o $@
fetcher.o: fetcher.c fetcher.h queue.o
$(CC) $(INC) $(LOPTS) -ggdb -c $<
queue_test: queue_test.c queue.o
$(CC) $(INC) $(LOPTS) -ggdb $^ -o $@
queue.o: queue.c queue.h
$(CC) $(INC) $(LOPTS) -ggdb -c $<
clean:
rm *.o $(EXEC)
#include "fetcher.h"
int perform() {
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://sxibolet.github.io");
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
}
curl_easy_cleanup(curl);
}
return 0;
}
#include <stdlib.h>
#include <stdio.h>
#include <curl/curl.h>
#include <pthread.h>
#include "queue.h"
#include "fetcher.h"
int main(void) {
// TODO
return 0;
}
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleDevelopmentRegion</key>
<string>English</string>
<key>CFBundleIdentifier</key>
<string>com.apple.xcode.dsym.fetcher_test</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundlePackageType</key>
<string>dSYM</string>
<key>CFBundleSignature</key>
<string>????</string>
<key>CFBundleShortVersionString</key>
<string>1.0</string>
<key>CFBundleVersion</key>
<string>1</string>
</dict>
</plist>
#include <stdlib.h>
#include <stdio.h>
#include "queue.h"
Queue *Queue_init() {
Queue *q = malloc(sizeof(Queue));
q->size = 0;
q->head = NULL;
q->tail = NULL;
pthread_mutex_init(&q->lock, NULL);
pthread_cond_init(&q->not_empty, NULL);
return q;
}
void Queue_free(Queue *q) {
pthread_mutex_destroy(&q->lock);
pthread_cond_destroy(&q->not_empty);
QueueNode *i = q->head;
while (i) {
free(i);
i = i->next;
}
}
void Queue_push(Queue *q, void *data) {
pthread_mutex_lock(&q->lock);
q->size++;
QueueNode *node = malloc(sizeof(QueueNode));
node->data = data;
node->next = NULL;
if (!q->head) {
q->head = q->tail = node;
} else {
q->tail->next = node;
q->tail = node;
}
pthread_cond_signal(&q->not_empty);
pthread_mutex_unlock(&q->lock);
}
void *Queue_pull(Queue *q) {
pthread_mutex_lock(&q->lock);
while (!q->cancel && q->size == 0) {
pthread_cond_wait(&q->not_empty, &q->lock);
}
q->size--;
QueueNode *res = q->head;
if (q->head == q->tail) {
q->head = q->tail = NULL;
} else {
q->head = q->head->next;
}
pthread_mutex_unlock(&q->lock);
return res->data;
}
/* a thread-safe queue implementation for
article/comment fetcher */
#ifndef QUEUE_H
#define QUEUE_H
#include <pthread.h>
#define true 1
#define false 0
#define bool int
typedef struct QueueNode {
struct QueueNode *next;
void *data;
} QueueNode;
typedef struct {
int size;
QueueNode *head, *tail;
pthread_cond_t not_empty;
pthread_mutex_t lock;
bool cancel;
} Queue;
Queue *Queue_init();
void Queue_free(Queue *q);
// add element to queue; won't block
void Queue_push(Queue *q, void *data);
// removes node; will block until q->size > 0
void *Queue_pull(Queue *q);
#endif
#include <stdlib.h>
#include <stdio.h>
#include <assert.h>
#include <string.h>
#include "queue.h"
int main(void) {
Queue *q = Queue_init();
char *s = "hello";
Queue_push(q, s);
assert(strcmp(s, (char *)Queue_pull(q)) == 0);
Queue_free(q);
return 0;
}
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleDevelopmentRegion</key>
<string>English</string>
<key>CFBundleIdentifier</key>
<string>com.apple.xcode.dsym.queue_test</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundlePackageType</key>
<string>dSYM</string>
<key>CFBundleSignature</key>
<string>????</string>
<key>CFBundleShortVersionString</key>
<string>1.0</string>
<key>CFBundleVersion</key>
<string>1</string>
</dict>
</plist>
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment