Commit 0f3304c1 authored by yPhil's avatar yPhil
Browse files

Iconv out, request out, fetch in, MediaRSS OK (#64)

parent bef78c46
Pipeline #399255 failed with stage
in 1 minute and 31 seconds
#!/usr/bin/env node
const petrolette = require('../petrolette'),
path = require('path'),
http = require('http'),
https = require('https'),
fs = require('fs');
var httpServer = http.createServer(petrolette);
const httpServer = http.createServer(petrolette);
httpServer.listen(8000, () => {
console.debug('HTTP Server running');
......
This diff is collapsed.
......@@ -35,13 +35,12 @@
"feedparser": "^2.2.10",
"feedrat": "git+https://framagit.org/yphil/feedrat.git",
"helmet": "^4.4.1",
"iconv": "^3.0.0",
"iconv-lite": "^0.4.24",
"morgan": "^1.10.0",
"node-fetch": "^2.6.1",
"npm": "^7.5.2",
"pm2": "^4.5.4",
"request": "^2.88.2",
"sanitize": "^2.1.0",
"xml2json": "^0.12.0"
"sanitize": "^2.1.0"
},
"devDependencies": {
"mocha": "^8.3.0",
......
......@@ -20,15 +20,15 @@ app.set('view engine', 'html');
app.disable('x-powered-by');
app.use(compression());
app.use(helmet.originAgentCluster());
app.use(helmet.noSniff());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use('/favicons', express.static(path.join(__dirname, pjson.FAVICONS_CACHE_DIR)));
app.use('/static', express.static(path.join(__dirname, 'public')));
app.use('/bower', express.static(path.join(__dirname, 'bower_components')));
app.use('/', router);
app.use(helmet.originAgentCluster());
app.use(helmet.noSniff());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
module.exports = app;
const request = require('request'),
const fetch = require('node-fetch'),
zlib = require('zlib'),
iconv = require('iconv-lite'),
FeedParser = require('feedparser');
exports.getFeed = getFeed;
......@@ -14,17 +15,35 @@ function maybeDecompress (res, encoding) {
return decompress ? res.pipe(decompress) : res;
}
// function maybeTranslate (res, charset) {
// var iconv;
// // Use iconv if its not utf8 already.
// if (!iconv && charset && !/utf-*8/i.test(charset)) {
// try {
// iconv = new Iconv(charset, 'utf-8');
// console.error('Converting from charset %s to utf-8', charset);
// iconv.on('error', done);
// // If we're using iconv, stream will be the output of iconv
// // otherwise it will remain the output of request
// res = res.pipe(iconv);
// } catch(err) {
// res.emit('error', err);
// }
// }
// return res;
// }
function maybeTranslate (res, charset) {
var iconv;
// Use iconv if its not utf8 already.
if (!iconv && charset && !/utf-*8/i.test(charset)) {
var iconvStream;
// Decode using iconv-lite if its not utf8 already.
if (!iconvStream && charset && !/utf-*8/i.test(charset)) {
try {
iconv = new Iconv(charset, 'utf-8');
console.error('Converting from charset %s to utf-8', charset);
iconv.on('error', done);
// If we're using iconv, stream will be the output of iconv
iconvStream = iconv.decodeStream(charset);
console.log('Converting from charset %s to utf-8', charset);
iconvStream.on('error', done);
// If we're using iconvStream, stream will be the output of iconvStream
// otherwise it will remain the output of request
res = res.pipe(iconv);
res = res.pipe(iconvStream);
} catch(err) {
res.emit('error', err);
}
......@@ -32,6 +51,7 @@ function maybeTranslate (res, charset) {
return res;
}
function getParams(str) {
var params = str.split(';').reduce(function (params, param) {
var parts = param.split('=').map(function (part) { return part.trim(); });
......@@ -43,31 +63,72 @@ function getParams(str) {
return params;
}
function getFeed (urlfeed, callback) {
var req = request(urlfeed, {timeout: 10000, pool: false});
req.setMaxListeners(50);
req.setHeader('user-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36');
req.setHeader('accept', 'text/html,application/xhtml+xml');
var feedparser = new FeedParser ();
var feedItems = [];
req
.on ('error', function (err) {
callback(err);
})
.on ('response', function (res) {
if (res.statusCode != 200) return this.emit('error', new Error('Bad status code'));
var encoding = res.headers['content-encoding'] || 'identity',
charset = getParams(res.headers['content-type'] || '').charset;
res = maybeDecompress(res, encoding);
res = maybeTranslate(res, charset);
res.pipe (feedparser);
});
// function getFeed (urlfeed, callback) {
// var req = request(urlfeed, {timeout: 10000, pool: false});
// req.setMaxListeners(50);
// req.setHeader('user-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36');
// req.setHeader('accept', 'text/html,application/xhtml+xml');
// var feedparser = new FeedParser ();
// var feedItems = [];
// req
// .on ('error', function (err) {
// console.error('AAAh: %s (%s)');
// callback(err);
// })
// .on ('response', function (res) {
// if (res.statusCode != 200) return this.emit('error', new Error('Bad status code'));
// var encoding = res.headers['content-encoding'] || 'identity',
// charset = getParams(res.headers['content-type'] || '').charset;
// res = maybeDecompress(res, encoding);
// res = maybeTranslate(res, charset);
// res.pipe (feedparser);
// });
// feedparser
// .on ('readable', function () {
// try {
// var item = this.read ();
// if (item !== null) feedItems.push (item);
// }
// catch (err) {
// console.error('ERR (%s)', err.message);
// }
// })
// .on ('error', function (err) {
// var meta = this.meta;
// console.error('Error reading feed [%s] (%s)', err.message, urlfeed);
// callback (err.message);
// })
// .on ('end', function () {
// var meta = this.meta;
// callback ('Feed OK', feedItems, meta.title, meta.link);
// return;
// });
// }
feedparser
.on ('readable', function () {
function done(err) {
if (err) {
console.log(err, err.stack);
return;
}
}
function getFeed (feedUrl, callback) {
// Get a response stream
fetch(feedUrl, { 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36', 'accept': 'text/html,application/xhtml+xml' }).then(function (res) {
// Setup feedparser stream
var feedparser = new FeedParser();
var feedItems = [];
feedparser.on('error', done);
feedparser.on('end', done);
feedparser.on('readable', function() {
try {
var item = this.read ();
if (item !== null) feedItems.push (item);
......@@ -75,17 +136,21 @@ function getFeed (urlfeed, callback) {
catch (err) {
console.error('ERR (%s)', err.message);
}
})
.on ('error', function (err) {
var meta = this.meta;
console.error('Error reading feed [%s] (%s)', err.message, urlfeed);
callback (err.message);
})
.on ('end', function () {
}).on ('end', function () {
var meta = this.meta;
callback ('Feed OK', feedItems, meta.title, meta.link);
return;
});
// Handle our response and pipe it to feedparser
if (res.status != 200) throw new Error('Bad status code');
var charset = getParams(res.headers.get('content-type') || '').charset;
var responseStream = res.body;
responseStream = maybeTranslate(responseStream, charset);
// And boom goes the dynamite
responseStream.pipe(feedparser);
}).catch(done);
}
......@@ -2,16 +2,14 @@ const express = require('express'),
router = express.Router(),
favrat = require('favrat'),
feeder = require('./feeder'),
request = require('request'),
fetch = require('node-fetch'),
feedrat = require('feedrat'),
// feedrat = require(__dirname + '/../../feedrat/'),
fs = require('fs'),
path = require('path'),
crypto = require('crypto'),
pjson = require('../package.json'),
Iconv = require('iconv').Iconv,
sanitize = require('sanitize').middleware,
parser = require('xml2json'),
morgan = require('morgan');
require('events').EventEmitter.defaultMaxListeners = 15;
......@@ -36,31 +34,19 @@ router.use(sanitize);
router.get('/feed', function(req, res) {
var dnsreq = request(req.query.feedurl);
dnsreq
.on('error', function(error) {
// The only way so far to catch a DNS error
console.error('Err: %s (%s)', {error:error.code}, req.query.feedurl);
res.send({error:error.code});
})
.on('response', function(response) {
feeder.getFeed(req.query.feedurl, function (err, feedItems, feedTitle, feedLink) {
if (feedItems && !res.headersSent) {
res.send({
feedItems: feedItems,
feedLink: feedLink,
feedTitle: feedTitle
});
// return;
} else if (!res.headersSent) {
res.send({error:err});
}
feeder.getFeed(req.query.feedurl, function (err, feedItems, feedTitle, feedLink) {
if (feedItems && !res.headersSent) {
res.send({
feedItems: feedItems,
feedLink: feedLink,
feedTitle: feedTitle
});
});
} else if (!res.headersSent) {
res.send({error:err});
}
});
});
router.get('/favicon', function(req, res) {
......@@ -78,7 +64,7 @@ router.get('/favicon', function(req, res) {
if (fs.existsSync(filePath)) {
res.send('/favicons/' + fileName);
} else {
var p = new Promise(resolve => request(url)
var p = new Promise(resolve => fetch(url)
.pipe(fs.createWriteStream(filePath, {'Content-Type': 'image/x-icon'}))
.on('finish', resolve({fileName, url})));
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment