Commit 5d7256bf authored by Sacha's avatar Sacha Committed by SXibolet@2PITAU
Browse files

more progress with new fetcher

parent 540800a1
......@@ -15,3 +15,4 @@ venv
controversy/sentistrength
notes.rst
nohup.out
mongo-c-driver
......@@ -27,7 +27,7 @@ We're currently improving upon our IEEE paper.
Running for development
-----------------------
If you'd like to reproduce our results, we encourage you to use the demo or look at our `API spec`_ for raw ``json`` results. Alternatively, you can clone this repository and run the server yourself. We wrote `some instructions to help you`_.
If you'd like to reproduce our results, we encourage you to use the demo online or look at our `API spec`_ for raw ``json`` results. Alternatively, you can clone this repository and either call the scoring function or run the demo server yourself. We wrote `some instructions to help you`_.
------
......
#include "fetcher.h"
int main(void) {
// TODO
return 0;
}
.. |...| unicode:: U+2026 .. ldots
This is not for the faint-hearted; we really do recommend you use the web client.
Running the server locally
~~~~~~~~~~~~~~~~~~~~~~~~~~
This is not for the faint-hearted.
Please use Python 2.7.x
......
......@@ -9,9 +9,31 @@
#define THREAD_MAX 8
#define MIN(x, y) (((x) < (y)) ? (x) : (y))
Queue *tasks;
int n_fetches, n_todo;
pthread_mutex_t n_fetches_lock = PTHREAD_MUTEX_INITIALIZER;
typedef struct Page {
char *string;
size_t len;
} Page;
size_t w_callback(void *ptr, size_t size, size_t nmemb, Page *pg) {
size_t pckt_size = size * nmemb,
nl = pg->len + pckt_size;
pg->string = realloc(pg->string, nl + 1);
memcpy(pg->string + pg->len, ptr, pckt_size);
pg->len = nl;
pg->string[nl] = '\0';
return pckt_size;
}
// write ``page`` to db
void write_page(Page *page) {
// TODO
}
void *perform(void *arg) {
while (1) {
pthread_mutex_lock(&n_fetches_lock);
......@@ -38,25 +60,48 @@ void *perform(void *arg) {
curl = curl_easy_init();
if (curl) {
Page *page = malloc(sizeof(page));
page->len = 0;
// allocate enough space for NULL-char
page->string = malloc(1);
page->string[0] = '\0';
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_COOKIEFILE, "");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, w_callback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, page);
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
exit(1);
}
if (page->len > 0) {
write_page(page);
} else {
fprintf(stderr, "(!) ===> page %s is NULL\n", url);
}
free(page->string);
free(page);
curl_easy_cleanup(curl);
} else {
exit(1);
}
curl_easy_cleanup(curl);
}
return 0;
}
void Fetcher_init(char **sources) {
void Fetcher_fetch(char **sources) {
tasks = Queue_init();
while (sources) {
Queue_push(tasks, *sources);
while (*sources != NULL) {
Queue_push(tasks, *sources++);
sources++;
}
......@@ -71,4 +116,6 @@ void Fetcher_init(char **sources) {
for (int i = 0; i < n_threads; i++) {
pthread_join(pids[i], NULL);
}
curl_global_cleanup();
}
......@@ -3,7 +3,7 @@
#define FETCHER_H
#include "queue.h"
Queue *tasks;
void Fetcher_init(char **sources);
// HTML of URLs in ``sources`` to ``key``
void Fetcher_fetch(char **sources);
#endif
#include "fetcher.h"
int main(void) {
// TODO
char *urls[] = { "http://www.nytimes.com/2016/05/22/arts/television/roots-remade-for-a-new-era.html?hp&action=click&pgtype=Homepage&clickSource=image&module=photo-spot-region&region=top-news&WT.nav=top-news&_r=0", "http://www.nytimes.com/2016/05/18/us/politics/bernie-sanders-oregon-results.html?action=click&contentCollection=Television&module=Trending&version=Full&region=Marginalia&pgtype=article" };
Fetcher_fetch(urls);
return 0;
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment