Commit 5d176340 authored by yPhil's avatar yPhil
Browse files

Let's give rss-parser another go

parent 8847fb46
......@@ -49,5 +49,51 @@
"nyc": "^15.1.0",
"rollup": "^2.47.0",
"should": "^13.2.3"
},
"eslintConfig": {
"parserOptions": {
"sourceType": "module"
},
"env": {
"browser": true,
"es6": true
},
"globals": {
"PTL": "readonly"
},
"extends": [
"eslint:recommended"
],
"rules": {
"semi-style": ["error", "last"],
"quotes": [
"error",
"double",
{
"avoidEscape": true
}
],
"comma-dangle": [
"error",
{
"arrays": "ignore",
"objects": "always-multiline",
"imports": "never",
"exports": "never",
"functions": "ignore"
}
],
"arrow-parens": 0,
"no-tabs": 0,
"indent": [
"error",
"tab",
{
"SwitchCase": 1
}
],
"no-console": 1,
"generator-star-spacing": 0
}
}
}
......@@ -469,7 +469,7 @@ PTL.dialog = {
},
change: function( event, ui ) {
$feedLimitInput.val(ui.value);
$dataStore.attr('data-limit', ui.value);
$dataStore.data('limit', ui.value);
}
});
......@@ -496,7 +496,7 @@ PTL.dialog = {
},
change: function( event, ui ) {
$feedNbItemsInput.val(ui.value);
$dataStore.attr('data-nbitems', ui.value);
$dataStore.data('nbitems', ui.value);
}
});
......
This diff is collapsed.
......@@ -6,108 +6,117 @@ const fetch = require('node-fetch'),
exports.getFeed = getFeed;
function maybeTranslate (res, charset) {
var iconvStream;
// Decode using iconv-lite if its not utf8 already.
if (!iconvStream && charset && !/utf-*8/i.test(charset)) {
try {
iconvStream = iconv.decodeStream(charset);
console.error('ICONV: Converting from charset %s to utf-8', charset);
iconvStream.on('error', () => {return;});
// If we're using iconvStream, stream will be the output of iconvStream
// otherwise it will remain the output of request
res = res.pipe(iconvStream);
} catch(err) {
res.emit('error', err);
}
var iconvStream;
// Decode using iconv-lite if its not utf8 already.
if (!iconvStream && charset && !/utf-*8/i.test(charset)) {
try {
iconvStream = iconv.decodeStream(charset);
console.error('ICONV: Converting from charset %s to utf-8', charset);
iconvStream.on('error', () => {return;});
// If we're using iconvStream, stream will be the output of iconvStream
// otherwise it will remain the output of request
res = res.pipe(iconvStream);
} catch(err) {
res.emit('error', err);
}
return res;
}
return res;
}
function getParams(str) {
var params = str.split(';').reduce(function (params, param) {
var parts = param.split('=').map(function (part) { return part.trim(); });
if (parts.length === 2) {
params[parts[0]] = parts[1];
}
return params;
}, {});
var params = str.split(';').reduce(function (params, param) {
var parts = param.split('=').map(function (part) { return part.trim(); });
if (parts.length === 2) {
params[parts[0]] = parts[1];
}
return params;
}, {});
return params;
}
function getFeed (feedUrl, lastItem, callback) {
// Get a response stream
console.error('feedUrl: %s (%s)', feedUrl);
try {
fetch(feedUrl, {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36',
'accept': 'text/html,application/xhtml+xml',
redirect: 'follow'
}).then(function (res) {
console.error('res: %s (%s)', res);
// if (res.status != 200) {
// return callback({errno:res.status, message:'Bad server response'});
// }
var feedparser = new FeedParser();
var feedItems = [];
var charset = getParams(res.headers.get('content-type') || '').charset;
var responseStream = res.body;
responseStream = maybeTranslate(responseStream, charset);
responseStream.pipe(feedparser);
var newLastItem;
var thereArenewItems = false;
var i = 0;
return new Promise((resolve, reject) => {
feedparser.on('error', function(error) {
console.error('## feedParserErr: %s (%s)', error.message, feedUrl);
reject('woopsie');
return callback({error:error, errno:res.status, message:error.message});
}).on('readable', function() {
try {
var item;
while ((item = this.read())) {
if (item !== null) {
i++;
if (typeof newLastItem === 'undefined') newLastItem = item.link;
if (item.link !== lastItem) {
console.error('item.link: %s (lastItem %s)', item.link, lastItem);
feedItems.push(item);
} else {
console.error('### i:[%s], item:[%s], last:[%s], new:[%s]', i, item.link, lastItem, newLastItem);
this.resume();
}
}
}
}
catch (err) {
console.error('## feedParserCatchErr: %s (%s)', err, feedUrl);
}
}).on ('end', function () {
var meta = this.meta;
resolve();
if (i > 1) thereArenewItems = true;
console.error('### Return i:%s, lastItem: [%s], newLastItem: %s', i, lastItem, newLastItem);
return callback(null, feedItems, meta.title || 'Untitled', meta.link || feedUrl, newLastItem, thereArenewItems);
});
});
}).catch((error) => {
console.error('Whooaps: %s (%s)', error.message, error.errno);
callback(error);
// Get a response stream
console.error('feedUrl: %s (%s)', feedUrl);
try {
fetch(feedUrl, {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36',
'accept': 'text/html,application/xhtml+xml',
redirect: 'follow'
}).then(function (res) {
console.error('res: %s (%s)', res);
// if (res.status != 200) {
// return callback({errno:res.status, message:'Bad server response'});
// }
var feedparser = new FeedParser();
var feedItems = [];
var charset = getParams(res.headers.get('content-type') || '').charset;
var responseStream = res.body;
responseStream = maybeTranslate(responseStream, charset);
responseStream.pipe(feedparser);
var newLastItem;
var thereArenewItems = false;
var i = 0;
return new Promise((resolve, reject) => {
feedparser.on('error', function(error) {
console.error('## feedParserErr: %s (%s)', error.message, feedUrl);
reject('woopsie');
return callback({error:error, errno:res.status, message:error.message});
}).on('readable', function() {
try {
var item;
while ((item = this.read())) {
if (item !== null) {
// console.error('item.link: %s (lastItem %s) item:[%s]', item.link, lastItem, JSON.stringify(item));
console.error('Seeing item.link: %s (lastItem %s) item:[%s]', item.link, lastItem);
feedItems.push(item);
// if (lastItem == 'none') {
// if (typeof newLastItem === 'undefined') newLastItem = item.link;
// console.error('Pushing item.link: %s (lastItem %s) item:[%s]', item.link, lastItem);
// i++;
// feedItems.push(item);
// } else if (item.link !== lastItem && item.link !== newLastItem) {
// i++;
// if (typeof newLastItem === 'undefined') newLastItem = item.link;
// console.error('Pushing item.link: %s (lastItem %s)', item.link, lastItem);
// feedItems.push(item);
// } else {
// console.error('###Else i:[%s], item:[%s], last:[%s], new:[%s]', i, item.link, lastItem, newLastItem);
// // this.resume();
// }
}
}
}
catch (err) {
console.error('## feedParserCatchErr: %s (%s)', err, feedUrl);
}
}).on ('end', function () {
var meta = this.meta;
resolve();
if (i > 1) thereArenewItems = true;
console.error('### Return i:%s, lastItem: [%s], newLastItem: %s', i, lastItem, newLastItem);
return callback(null, feedItems, meta.title || 'Untitled', meta.link || feedUrl, newLastItem, thereArenewItems);
});
});
} catch (err) {
console.error('Whoops: %s (%s)');
}
}).catch((error) => {
console.error('Whooaps: %s (%s)', error.message, error.errno);
callback(error);
});
} catch (err) {
console.error('Whoops: %s (%s)');
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment