...
 
Commits (3)
......@@ -14,26 +14,37 @@ module.exports = function(url, callback) {
linkTypes = ['application/atom+xml',
'application/rss+xml',
'application/xhtml+xml'],
usualSuspects = [url,
p.hostname + '/rss',
p.hostname + '/feed',
p.hostname + '/atom',
p.hostname + '/rss.xml',
p.hostname + '.rss',
p.hostname + './rss',
p.hostname + '/atom.xml',
p.hostname + '/feed.xml'];
usualSuspects = [url + '/rss',
url + '/feed',
url + '/atom',
url + '/rss.xml',
url + '.rss',
url + './rss',
url + '/atom.xml',
url + '/feed.xml'];
if (!p.protocol) return callback(null, searchPrefix + url.split(' '));
if (p.host.endsWith('youtube.com') && p.path.startsWith('/channel') && p.path.split('/').length === 3)
return callback(null, videoPrefix + p.path.split('/')[2]);
// console.log('whoa: (%s)');
for (let i = 0; i < usualSuspects.length; i++) {
does_it_render(usualSuspects[i], function(err, renders) {
// console.log('dir: (%s)', usualSuspects[i]);
if (renders) {
feed = usualSuspects[i];
}
});
}
does_it_render(feed, function(err, renders) {
if (err) return callback(err);
if (renders) {
console.log('renders: (%s)', renders);
return callback(null, feed);
}
request(feed, function(err, res, body) {
const $ = cheerio.load(body);
......@@ -50,21 +61,12 @@ module.exports = function(url, callback) {
feed = $('a[href$=".xml"]').attr('href');
}
if ($('a[href$=".xml"]').length === 0 && $('link').length === 0) {
for (let i = 0; i < usualSuspects.length; i++) {
does_it_render(usualSuspects[i], function(err, renders) {
if (renders) {
feed = usualSuspects[i];
}
});
}
}
if (!isUrl.test(feed)) feed = root + feed;
return callback(null, feed);
});
});
};
function does_it_render(url, callback) {
......
{
"name": "feedrat",
"version": "0.1.51",
"version": "0.1.53",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
......
{
"name": "feedrat",
"version": "0.1.51",
"version": "0.1.53",
"description": "Search, find and report a web page's RSS feed",
"main": "feedrat.js",
"license": "SEE LICENSE IN LICENSE",
......
......@@ -11,6 +11,14 @@ describe('feedrat', function() {
});
});
it('can find a feed from the usual suspects list', function(done) {
feedrat('https://www.reddit.com/r/news', function(err, url) {
if (err) return done(err);
url.should.be.oneOf('https://www.reddit.com/r/news.rss', 'https://www.reddit.com/r/news/.rss', 'https://www.reddit.com/r/news/feed.xml');
done();
});
});
// it('handles a website that has blocked us', function(done) {
// feedrat('http://www.leparisien.fr/', function(err, url) {
// if (err) return done(err);
......@@ -77,11 +85,4 @@ describe('feedrat', function() {
});
});
it('can find a bunch of feeds', function(done) {
feedrat('http://www.futura-sciences.com', function(err, url) {
if (err) return done(err);
url.should.be.oneOf('https://www.futura-sciences.com/rss/photos.xml', 'https://www.futura-sciences.com/rss/actualites.xml');
done();
});
});
});