Commit 2cba9592 authored by yPhil's avatar yPhil
Browse files

New items OK

- But no unread count per se yet
parent bcc3f17e
Pipeline #413313 passed with stage
in 34 seconds
......@@ -15,12 +15,15 @@
"ejs": "^3.1.5",
"express": "^4.17.1",
"favrat": "git+https://framagit.org/yphil/favrat.git",
"feedparser": "^2.2.10",
"feedrat": "git+https://framagit.org/yphil/feedrat.git",
"helmet": "^4.4.1",
"iconv-lite": "^0.6.2",
"idb": "^6.0.0",
"morgan": "^1.10.0",
"node-fetch": "^2.6.1",
"npm": "^7.10.0",
"pm2": "^4.5.6",
"rss-parser": "^3.12.0",
"sanitize": "^2.1.0"
},
"devDependencies": {
......@@ -722,6 +725,11 @@
"node": ">= 0.6"
}
},
"node_modules/addressparser": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/addressparser/-/addressparser-1.0.1.tgz",
"integrity": "sha1-R6++GiqSYhkdtoOOT9HTm0CCF0Y="
},
"node_modules/agent-base": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
......@@ -856,6 +864,11 @@
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
"integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
},
"node_modules/array-indexofobject": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/array-indexofobject/-/array-indexofobject-0.0.1.tgz",
"integrity": "sha1-qqEo5iybPDWAlFaMIZ/2T+SJ1Co="
},
"node_modules/ast-types": {
"version": "0.13.4",
"resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz",
......@@ -1784,6 +1797,50 @@
"resolved": "https://registry.npmjs.org/fclone/-/fclone-1.0.11.tgz",
"integrity": "sha1-EOhdo4v+p/xZk0HClu4ddyZu5kA="
},
"node_modules/feedparser": {
"version": "2.2.10",
"resolved": "https://registry.npmjs.org/feedparser/-/feedparser-2.2.10.tgz",
"integrity": "sha512-WoAOooa61V8/xuKMi2pEtK86qQ3ZH/M72EEGdqlOTxxb3m6ve1NPvZcmPFs3wEDfcBbFLId2GqZ4YjsYi+h1xA==",
"dependencies": {
"addressparser": "^1.0.1",
"array-indexofobject": "~0.0.1",
"lodash.assign": "^4.2.0",
"lodash.get": "^4.4.2",
"lodash.has": "^4.5.2",
"lodash.uniq": "^4.5.0",
"mri": "^1.1.5",
"readable-stream": "^2.3.7",
"sax": "^1.2.4"
},
"bin": {
"feedparser": "bin/feedparser.js"
},
"engines": {
"node": ">= 10.18.1"
}
},
"node_modules/feedparser/node_modules/readable-stream": {
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"dependencies": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"node_modules/feedparser/node_modules/string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"dependencies": {
"safe-buffer": "~5.1.0"
}
},
"node_modules/feedrat": {
"resolved": "git+https://framagit.org/yphil/feedrat.git#1f5669b483930e13bdb2b3b87aca8ac6f82f4e78",
"dependencies": {
......@@ -2249,6 +2306,22 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/iconv-lite": {
"version": "0.6.2",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.2.tgz",
"integrity": "sha512-2y91h5OpQlolefMPmUlivelittSWy0rP+oYVpn6A7GwVHNE8AWzoYOBNmlwks3LobaJxgHCYZAnyNo2GgpNRNQ==",
"dependencies": {
"safer-buffer": ">= 2.1.2 < 3.0.0"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/idb": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/idb/-/idb-6.0.0.tgz",
"integrity": "sha512-+M367poGtpzAylX4pwcrZIa7cFQLfNkAOlMMLN2kw/2jGfJP6h+TB/unQNSVYwNtP8XqkLYrfuiVnxLQNP1tjA=="
},
"node_modules/imurmurhash": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
......@@ -2390,6 +2463,11 @@
"node": ">=0.10.0"
}
},
"node_modules/isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
},
"node_modules/isexe": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
......@@ -2663,12 +2741,32 @@
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
"integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
},
"node_modules/lodash.assign": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz",
"integrity": "sha1-DZnzzNem0mHRm9rrkkUAXShYCOc="
},
"node_modules/lodash.flattendeep": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz",
"integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=",
"dev": true
},
"node_modules/lodash.get": {
"version": "4.4.2",
"resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
"integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk="
},
"node_modules/lodash.has": {
"version": "4.5.2",
"resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz",
"integrity": "sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI="
},
"node_modules/lodash.uniq": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz",
"integrity": "sha1-0CJTc662Uq3BvILklFM5qEJ1R3M="
},
"node_modules/log-driver": {
"version": "1.2.7",
"resolved": "https://registry.npmjs.org/log-driver/-/log-driver-1.2.7.tgz",
......@@ -3006,6 +3104,14 @@
"node": ">= 0.8"
}
},
"node_modules/mri": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/mri/-/mri-1.1.6.tgz",
"integrity": "sha512-oi1b3MfbyGa7FJMP9GmLTttni5JoICpYBRlq+x5V16fZbLsnL9N3wFqqIm/nIG43FjUFkFh9Epzp/kzUGUnJxQ==",
"engines": {
"node": ">=4"
}
},
"node_modules/ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
......@@ -6700,6 +6806,11 @@
"node": ">= 0.8.0"
}
},
"node_modules/process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"node_modules/process-on-spawn": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/process-on-spawn/-/process-on-spawn-1.0.0.tgz",
......@@ -6970,15 +7081,6 @@
"fsevents": "~2.3.1"
}
},
"node_modules/rss-parser": {
"version": "3.12.0",
"resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.12.0.tgz",
"integrity": "sha512-aqD3E8iavcCdkhVxNDIdg1nkBI17jgqF+9OqPS1orwNaOgySdpvq6B+DoONLhzjzwV8mWg37sb60e4bmLK117A==",
"dependencies": {
"entities": "^2.0.3",
"xml2js": "^0.4.19"
}
},
"node_modules/run-series": {
"version": "1.1.9",
"resolved": "https://registry.npmjs.org/run-series/-/run-series-1.1.9.tgz",
......@@ -7451,6 +7553,11 @@
"node": ">= 0.8"
}
},
"node_modules/util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"node_modules/utils-merge": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
......@@ -7674,26 +7781,6 @@
}
}
},
"node_modules/xml2js": {
"version": "0.4.23",
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz",
"integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==",
"dependencies": {
"sax": ">=0.6.0",
"xmlbuilder": "~11.0.0"
},
"engines": {
"node": ">=4.0.0"
}
},
"node_modules/xmlbuilder": {
"version": "11.0.1",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
"integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==",
"engines": {
"node": ">=4.0"
}
},
"node_modules/xregexp": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/xregexp/-/xregexp-2.0.0.tgz",
......@@ -8413,6 +8500,11 @@
"negotiator": "0.6.2"
}
},
"addressparser": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/addressparser/-/addressparser-1.0.1.tgz",
"integrity": "sha1-R6++GiqSYhkdtoOOT9HTm0CCF0Y="
},
"agent-base": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
......@@ -8522,6 +8614,11 @@
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
"integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
},
"array-indexofobject": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/array-indexofobject/-/array-indexofobject-0.0.1.tgz",
"integrity": "sha1-qqEo5iybPDWAlFaMIZ/2T+SJ1Co="
},
"ast-types": {
"version": "0.13.4",
"resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz",
......@@ -9271,6 +9368,46 @@
"resolved": "https://registry.npmjs.org/fclone/-/fclone-1.0.11.tgz",
"integrity": "sha1-EOhdo4v+p/xZk0HClu4ddyZu5kA="
},
"feedparser": {
"version": "2.2.10",
"resolved": "https://registry.npmjs.org/feedparser/-/feedparser-2.2.10.tgz",
"integrity": "sha512-WoAOooa61V8/xuKMi2pEtK86qQ3ZH/M72EEGdqlOTxxb3m6ve1NPvZcmPFs3wEDfcBbFLId2GqZ4YjsYi+h1xA==",
"requires": {
"addressparser": "^1.0.1",
"array-indexofobject": "~0.0.1",
"lodash.assign": "^4.2.0",
"lodash.get": "^4.4.2",
"lodash.has": "^4.5.2",
"lodash.uniq": "^4.5.0",
"mri": "^1.1.5",
"readable-stream": "^2.3.7",
"sax": "^1.2.4"
},
"dependencies": {
"readable-stream": {
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"requires": {
"safe-buffer": "~5.1.0"
}
}
}
},
"feedrat": {
"version": "git+https://framagit.org/yphil/feedrat.git#1f5669b483930e13bdb2b3b87aca8ac6f82f4e78",
"from": "feedrat@git+https://framagit.org/yphil/feedrat.git",
......@@ -9620,6 +9757,19 @@
}
}
},
"iconv-lite": {
"version": "0.6.2",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.2.tgz",
"integrity": "sha512-2y91h5OpQlolefMPmUlivelittSWy0rP+oYVpn6A7GwVHNE8AWzoYOBNmlwks3LobaJxgHCYZAnyNo2GgpNRNQ==",
"requires": {
"safer-buffer": ">= 2.1.2 < 3.0.0"
}
},
"idb": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/idb/-/idb-6.0.0.tgz",
"integrity": "sha512-+M367poGtpzAylX4pwcrZIa7cFQLfNkAOlMMLN2kw/2jGfJP6h+TB/unQNSVYwNtP8XqkLYrfuiVnxLQNP1tjA=="
},
"imurmurhash": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
......@@ -9725,6 +9875,11 @@
"integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==",
"dev": true
},
"isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
},
"isexe": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
......@@ -9943,12 +10098,32 @@
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
"integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
},
"lodash.assign": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz",
"integrity": "sha1-DZnzzNem0mHRm9rrkkUAXShYCOc="
},
"lodash.flattendeep": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz",
"integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=",
"dev": true
},
"lodash.get": {
"version": "4.4.2",
"resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
"integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk="
},
"lodash.has": {
"version": "4.5.2",
"resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz",
"integrity": "sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI="
},
"lodash.uniq": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz",
"integrity": "sha1-0CJTc662Uq3BvILklFM5qEJ1R3M="
},
"log-driver": {
"version": "1.2.7",
"resolved": "https://registry.npmjs.org/log-driver/-/log-driver-1.2.7.tgz",
......@@ -10207,6 +10382,11 @@
}
}
},
"mri": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/mri/-/mri-1.1.6.tgz",
"integrity": "sha512-oi1b3MfbyGa7FJMP9GmLTttni5JoICpYBRlq+x5V16fZbLsnL9N3wFqqIm/nIG43FjUFkFh9Epzp/kzUGUnJxQ=="
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
......@@ -12727,6 +12907,11 @@
"resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",
"integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ="
},
"process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"process-on-spawn": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/process-on-spawn/-/process-on-spawn-1.0.0.tgz",
......@@ -12941,15 +13126,6 @@
"fsevents": "~2.3.1"
}
},
"rss-parser": {
"version": "3.12.0",
"resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.12.0.tgz",
"integrity": "sha512-aqD3E8iavcCdkhVxNDIdg1nkBI17jgqF+9OqPS1orwNaOgySdpvq6B+DoONLhzjzwV8mWg37sb60e4bmLK117A==",
"requires": {
"entities": "^2.0.3",
"xml2js": "^0.4.19"
}
},
"run-series": {
"version": "1.1.9",
"resolved": "https://registry.npmjs.org/run-series/-/run-series-1.1.9.tgz",
......@@ -13324,6 +13500,11 @@
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
"integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
},
"util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"utils-merge": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
......@@ -13492,20 +13673,6 @@
"integrity": "sha512-C34cIU4+DB2vMyAbmEKossWq2ZQDr6QEyuuCzWrM9zfw1sGc0mYiJ0UnG9zzNykt49C2Fi34hvr2vssFQRS6EA==",
"requires": {}
},
"xml2js": {
"version": "0.4.23",
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz",
"integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==",
"requires": {
"sax": ">=0.6.0",
"xmlbuilder": "~11.0.0"
}
},
"xmlbuilder": {
"version": "11.0.1",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
"integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="
},
"xregexp": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/xregexp/-/xregexp-2.0.0.tgz",
......
......@@ -33,12 +33,15 @@
"ejs": "^3.1.5",
"express": "^4.17.1",
"favrat": "git+https://framagit.org/yphil/favrat.git",
"feedparser": "^2.2.10",
"feedrat": "git+https://framagit.org/yphil/feedrat.git",
"helmet": "^4.4.1",
"iconv-lite": "^0.6.2",
"idb": "^6.0.0",
"morgan": "^1.10.0",
"node-fetch": "^2.6.1",
"npm": "^7.10.0",
"pm2": "^4.5.6",
"rss-parser": "^3.12.0",
"sanitize": "^2.1.0"
},
"devDependencies": {
......
This diff is collapsed.
let Parser = require('rss-parser');
let parser = new Parser({
// timeout: 5000,
headers: {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36',
'accept': 'text/html,application/xhtml+xml'
},
maxRedirects: 100,
requestOptions: {
rejectUnauthorized: false
},
defaultRSS: 2.0,
xml2js: {
emptyTag: 'media:community',
},
customFields: {
item: [
['media:community', 'media:content', {keepArray: true}],
]
}
});
const fetch = require('node-fetch'),
zlib = require('zlib'),
iconv = require('iconv-lite'),
FeedParser = require('feedparser');
exports.getFeed = getFeed;
function getFeed (feedUrl, lastItem, maxItems, callback) {
const CORS_PROXY = 'https://cors-anywhere.herokuapp.com/';
(async () => {
var feedItems = [];
var reached = false;
var newItems = 0;
parser.parseURL(feedUrl).then((feed) => {
var newLastItem;
console.log(feed.title);
feed.items.every(function(item, index) {
console.log(item.title);
feedItems.push(item);
function maybeTranslate (res, charset) {
var iconvStream;
// Decode using iconv-lite if its not utf8 already.
if (!iconvStream && charset && !/utf-*8/i.test(charset)) {
try {
iconvStream = iconv.decodeStream(charset);
console.error('ICONV: Converting from charset %s to utf-8', charset);
iconvStream.on('error', () => {return;});
// If we're using iconvStream, stream will be the output of iconvStream
// otherwise it will remain the output of request
res = res.pipe(iconvStream);
} catch(err) {
res.emit('error', err);
}
}
return res;
}
newLastItem = item.link;
function getParams(str) {
var params = str.split(';').reduce(function (params, param) {
var parts = param.split('=').map(function (part) { return part.trim(); });
if (parts.length === 2) {
params[parts[0]] = parts[1];
}
return params;
}, {});
return params;
}
console.error('item.link: %s (%s)',item.link);
function getFeed (feedUrl, lastItem, nbItems, callback) {
// Get a response stream
fetch(feedUrl, {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36',
'accept': 'text/html,application/xhtml+xml',
redirect: 'follow'
}).then(function (res) {
newItems = index;
if (res.status != 200) {
callback({error:'error', errno:res.status, message:'Bad server response'});
return reject();
}
if (item.link === lastItem) {
reached = true;
return false;
var feedparser = new FeedParser();
var feedItems = [];
var charset = getParams(res.headers.get('content-type') || '').charset;
var responseStream = res.body;
responseStream = maybeTranslate(responseStream, charset);
responseStream.pipe(feedparser);
var newLastItem;
var thereArenewItems = false;
var i = 0;
return new Promise((resolve, reject) => {
feedparser.on('error', function(error) {
console.error('## feedParserErr: %s (%s)', error.message, feedUrl);
reject();
return callback({error:error, errno:res.status, message:error.message});
}).on('readable', function() {
try {
var item = this.read();
if (item !== null){
i++;
console.error('item.link: %s (%s)', item.link);
if (typeof newLastItem == 'undefined') {
newLastItem = item.link;
}
if (newLastItem == lastItem) {
console.error('### Count reached i:%s, lastItem: [%s], newLastItem: %s', i, lastItem, newLastItem);
this.resume();
} else {
feedItems.push(item);
}
}
}
return true;
catch (err) {
console.error('## feedParserCatchErr: %s (%s)', err, feedUrl);
}
}).on ('end', function () {
var meta = this.meta;
resolve();
if (i > 1) thereArenewItems = true;
console.error('### Return i:%s, lastItem: [%s], newLastItem: %s', i, lastItem, newLastItem);
return callback(null, feedItems, meta.title || 'Untitled', meta.link || feedUrl, newLastItem, thereArenewItems);
});
console.error('### FEEDER: lastItem [%s] newLastItem [%s] newItems [%s] reached: [%s]', lastItem, newLastItem, newItems, reached);
callback (null, feedItems, feed.title || 'Untitled', feed.link || feedUrl, newLastItem);
}).catch((e) => {
console.error('Catched Error: %s (%s)', e);
callback (e);
});