Verified Commit 20bafcb6 authored by Chocobozzz's avatar Chocobozzz
Browse files

Move apicache in peertube

Allow us to upgrade to node 16
parent 13e13377
...@@ -74,7 +74,6 @@ ...@@ -74,7 +74,6 @@
}, },
"dependencies": { "dependencies": {
"@uploadx/core": "^4.4.0", "@uploadx/core": "^4.4.0",
"apicache": "1.6.2",
"async": "^3.0.1", "async": "^3.0.1",
"async-lru": "^1.1.1", "async-lru": "^1.1.1",
"bcrypt": "5.0.1", "bcrypt": "5.0.1",
......
...@@ -24,7 +24,7 @@ import { ...@@ -24,7 +24,7 @@ import {
videosCustomGetValidator, videosCustomGetValidator,
videosShareValidator videosShareValidator
} from '../../middlewares' } from '../../middlewares'
import { cacheRoute } from '../../middlewares/cache' import { cacheRoute } from '../../middlewares/cache/cache'
import { getAccountVideoRateValidatorFactory, videoCommentGetValidator } from '../../middlewares/validators' import { getAccountVideoRateValidatorFactory, videoCommentGetValidator } from '../../middlewares/validators'
import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy' import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy'
import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '../../middlewares/validators/videos/video-playlists' import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '../../middlewares/validators/videos/video-playlists'
...@@ -77,7 +77,7 @@ activityPubClientRouter.get('/accounts?/:name/dislikes/:videoId', ...@@ -77,7 +77,7 @@ activityPubClientRouter.get('/accounts?/:name/dislikes/:videoId',
activityPubClientRouter.get( activityPubClientRouter.get(
[ '/videos/watch/:id', '/w/:id' ], [ '/videos/watch/:id', '/w/:id' ],
executeIfActivityPub, executeIfActivityPub,
asyncMiddleware(cacheRoute()(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS)), cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS),
asyncMiddleware(videosCustomGetValidator('all')), asyncMiddleware(videosCustomGetValidator('all')),
asyncMiddleware(videoController) asyncMiddleware(videoController)
) )
......
...@@ -2,12 +2,12 @@ import * as express from 'express' ...@@ -2,12 +2,12 @@ import * as express from 'express'
import { StatsManager } from '@server/lib/stat-manager' import { StatsManager } from '@server/lib/stat-manager'
import { ROUTE_CACHE_LIFETIME } from '../../../initializers/constants' import { ROUTE_CACHE_LIFETIME } from '../../../initializers/constants'
import { asyncMiddleware } from '../../../middlewares' import { asyncMiddleware } from '../../../middlewares'
import { cacheRoute } from '../../../middlewares/cache' import { cacheRoute } from '../../../middlewares/cache/cache'
const statsRouter = express.Router() const statsRouter = express.Router()
statsRouter.get('/stats', statsRouter.get('/stats',
asyncMiddleware(cacheRoute()(ROUTE_CACHE_LIFETIME.STATS)), cacheRoute(ROUTE_CACHE_LIFETIME.STATS),
asyncMiddleware(getStats) asyncMiddleware(getStats)
) )
......
...@@ -5,7 +5,7 @@ import { SitemapStream, streamToPromise } from 'sitemap' ...@@ -5,7 +5,7 @@ import { SitemapStream, streamToPromise } from 'sitemap'
import { VideoModel } from '../models/video/video' import { VideoModel } from '../models/video/video'
import { VideoChannelModel } from '../models/video/video-channel' import { VideoChannelModel } from '../models/video/video-channel'
import { AccountModel } from '../models/account/account' import { AccountModel } from '../models/account/account'
import { cacheRoute } from '../middlewares/cache' import { cacheRoute } from '../middlewares/cache/cache'
import { buildNSFWFilter } from '../helpers/express-utils' import { buildNSFWFilter } from '../helpers/express-utils'
import { truncate } from 'lodash' import { truncate } from 'lodash'
...@@ -14,7 +14,7 @@ const botsRouter = express.Router() ...@@ -14,7 +14,7 @@ const botsRouter = express.Router()
// Special route that add OpenGraph and oEmbed tags // Special route that add OpenGraph and oEmbed tags
// Do not use a template engine for a so little thing // Do not use a template engine for a so little thing
botsRouter.use('/sitemap.xml', botsRouter.use('/sitemap.xml',
asyncMiddleware(cacheRoute()(ROUTE_CACHE_LIFETIME.SITEMAP)), cacheRoute(ROUTE_CACHE_LIFETIME.SITEMAP),
asyncMiddleware(getSitemap) asyncMiddleware(getSitemap)
) )
......
...@@ -16,20 +16,20 @@ import { ...@@ -16,20 +16,20 @@ import {
videosSortValidator, videosSortValidator,
videoSubscriptionFeedsValidator videoSubscriptionFeedsValidator
} from '../middlewares' } from '../middlewares'
import { cacheRoute } from '../middlewares/cache' import { cacheRouteFactory } from '../middlewares/cache/cache'
import { VideoModel } from '../models/video/video' import { VideoModel } from '../models/video/video'
import { VideoCommentModel } from '../models/video/video-comment' import { VideoCommentModel } from '../models/video/video-comment'
const feedsRouter = express.Router() const feedsRouter = express.Router()
const cacheRoute = cacheRouteFactory({
headerBlacklist: [ 'Content-Type' ]
})
feedsRouter.get('/feeds/video-comments.:format', feedsRouter.get('/feeds/video-comments.:format',
feedsFormatValidator, feedsFormatValidator,
setFeedFormatContentType, setFeedFormatContentType,
asyncMiddleware(cacheRoute({ cacheRoute(ROUTE_CACHE_LIFETIME.FEEDS),
headerBlacklist: [
'Content-Type'
]
})(ROUTE_CACHE_LIFETIME.FEEDS)),
asyncMiddleware(videoFeedsValidator), asyncMiddleware(videoFeedsValidator),
asyncMiddleware(videoCommentsFeedsValidator), asyncMiddleware(videoCommentsFeedsValidator),
asyncMiddleware(generateVideoCommentsFeed) asyncMiddleware(generateVideoCommentsFeed)
...@@ -40,11 +40,7 @@ feedsRouter.get('/feeds/videos.:format', ...@@ -40,11 +40,7 @@ feedsRouter.get('/feeds/videos.:format',
setDefaultVideosSort, setDefaultVideosSort,
feedsFormatValidator, feedsFormatValidator,
setFeedFormatContentType, setFeedFormatContentType,
asyncMiddleware(cacheRoute({ cacheRoute(ROUTE_CACHE_LIFETIME.FEEDS),
headerBlacklist: [
'Content-Type'
]
})(ROUTE_CACHE_LIFETIME.FEEDS)),
commonVideosFiltersValidator, commonVideosFiltersValidator,
asyncMiddleware(videoFeedsValidator), asyncMiddleware(videoFeedsValidator),
asyncMiddleware(generateVideoFeed) asyncMiddleware(generateVideoFeed)
...@@ -55,11 +51,7 @@ feedsRouter.get('/feeds/subscriptions.:format', ...@@ -55,11 +51,7 @@ feedsRouter.get('/feeds/subscriptions.:format',
setDefaultVideosSort, setDefaultVideosSort,
feedsFormatValidator, feedsFormatValidator,
setFeedFormatContentType, setFeedFormatContentType,
asyncMiddleware(cacheRoute({ cacheRoute(ROUTE_CACHE_LIFETIME.FEEDS),
headerBlacklist: [
'Content-Type'
]
})(ROUTE_CACHE_LIFETIME.FEEDS)),
commonVideosFiltersValidator, commonVideosFiltersValidator,
asyncMiddleware(videoSubscriptionFeedsValidator), asyncMiddleware(videoSubscriptionFeedsValidator),
asyncMiddleware(generateVideoFeedForSubscriptions) asyncMiddleware(generateVideoFeedForSubscriptions)
......
...@@ -19,7 +19,7 @@ import { ...@@ -19,7 +19,7 @@ import {
} from '../initializers/constants' } from '../initializers/constants'
import { getThemeOrDefault } from '../lib/plugins/theme-utils' import { getThemeOrDefault } from '../lib/plugins/theme-utils'
import { asyncMiddleware } from '../middlewares' import { asyncMiddleware } from '../middlewares'
import { cacheRoute } from '../middlewares/cache' import { cacheRoute } from '../middlewares/cache/cache'
import { UserModel } from '../models/user/user' import { UserModel } from '../models/user/user'
import { VideoModel } from '../models/video/video' import { VideoModel } from '../models/video/video'
import { VideoCommentModel } from '../models/video/video-comment' import { VideoCommentModel } from '../models/video/video-comment'
...@@ -66,7 +66,7 @@ staticRouter.use( ...@@ -66,7 +66,7 @@ staticRouter.use(
// robots.txt service // robots.txt service
staticRouter.get('/robots.txt', staticRouter.get('/robots.txt',
asyncMiddleware(cacheRoute()(ROUTE_CACHE_LIFETIME.ROBOTS)), cacheRoute(ROUTE_CACHE_LIFETIME.ROBOTS),
(_, res: express.Response) => { (_, res: express.Response) => {
res.type('text/plain') res.type('text/plain')
return res.send(CONFIG.INSTANCE.ROBOTS) return res.send(CONFIG.INSTANCE.ROBOTS)
...@@ -86,7 +86,7 @@ staticRouter.get('/security.txt', ...@@ -86,7 +86,7 @@ staticRouter.get('/security.txt',
) )
staticRouter.get('/.well-known/security.txt', staticRouter.get('/.well-known/security.txt',
asyncMiddleware(cacheRoute()(ROUTE_CACHE_LIFETIME.SECURITYTXT)), cacheRoute(ROUTE_CACHE_LIFETIME.SECURITYTXT),
(_, res: express.Response) => { (_, res: express.Response) => {
res.type('text/plain') res.type('text/plain')
return res.send(CONFIG.INSTANCE.SECURITYTXT + CONFIG.INSTANCE.SECURITYTXT_CONTACT) return res.send(CONFIG.INSTANCE.SECURITYTXT + CONFIG.INSTANCE.SECURITYTXT_CONTACT)
...@@ -95,7 +95,7 @@ staticRouter.get('/.well-known/security.txt', ...@@ -95,7 +95,7 @@ staticRouter.get('/.well-known/security.txt',
// nodeinfo service // nodeinfo service
staticRouter.use('/.well-known/nodeinfo', staticRouter.use('/.well-known/nodeinfo',
asyncMiddleware(cacheRoute()(ROUTE_CACHE_LIFETIME.NODEINFO)), cacheRoute(ROUTE_CACHE_LIFETIME.NODEINFO),
(_, res: express.Response) => { (_, res: express.Response) => {
return res.json({ return res.json({
links: [ links: [
...@@ -108,13 +108,13 @@ staticRouter.use('/.well-known/nodeinfo', ...@@ -108,13 +108,13 @@ staticRouter.use('/.well-known/nodeinfo',
} }
) )
staticRouter.use('/nodeinfo/:version.json', staticRouter.use('/nodeinfo/:version.json',
asyncMiddleware(cacheRoute()(ROUTE_CACHE_LIFETIME.NODEINFO)), cacheRoute(ROUTE_CACHE_LIFETIME.NODEINFO),
asyncMiddleware(generateNodeinfo) asyncMiddleware(generateNodeinfo)
) )
// dnt-policy.txt service (see https://www.eff.org/dnt-policy) // dnt-policy.txt service (see https://www.eff.org/dnt-policy)
staticRouter.use('/.well-known/dnt-policy.txt', staticRouter.use('/.well-known/dnt-policy.txt',
asyncMiddleware(cacheRoute()(ROUTE_CACHE_LIFETIME.DNT_POLICY)), cacheRoute(ROUTE_CACHE_LIFETIME.DNT_POLICY),
(_, res: express.Response) => { (_, res: express.Response) => {
res.type('text/plain') res.type('text/plain')
......
import * as apicache from 'apicache'
import { HttpStatusCode } from '../../shared/models/http/http-error-codes'
import { Redis } from '../lib/redis'
// Ensure Redis is initialized
Redis.Instance.init()
const defaultOptions = {
redisClient: Redis.Instance.getClient(),
appendKey: () => Redis.Instance.getPrefix(),
statusCodes: {
exclude: [
HttpStatusCode.FORBIDDEN_403,
HttpStatusCode.NOT_FOUND_404
]
}
}
const cacheRoute = (extraOptions = {}) => apicache.options({
...defaultOptions,
...extraOptions
}).middleware
// ---------------------------------------------------------------------------
export {
cacheRoute
}
import { HttpStatusCode } from '../../../shared/models/http/http-error-codes'
import { Redis } from '../../lib/redis'
import { ApiCache, APICacheOptions } from './shared'
// Ensure Redis is initialized
Redis.Instance.init()
const defaultOptions: APICacheOptions = {
excludeStatus: [
HttpStatusCode.FORBIDDEN_403,
HttpStatusCode.NOT_FOUND_404
]
}
function cacheRoute (duration: string) {
const instance = new ApiCache(defaultOptions)
return instance.buildMiddleware(duration)
}
function cacheRouteFactory (options: APICacheOptions) {
const instance = new ApiCache({ ...defaultOptions, ...options })
return instance.buildMiddleware.bind(instance)
}
// ---------------------------------------------------------------------------
export {
cacheRoute,
cacheRouteFactory
}
// Thanks: https://github.com/kwhitley/apicache
// We duplicated the library because it is unmaintened and prevent us to upgrade to recent NodeJS versions
import * as express from 'express'
import { OutgoingHttpHeaders } from 'http'
import { isTestInstance, parseDurationToMs } from '@server/helpers/core-utils'
import { logger } from '@server/helpers/logger'
import { Redis } from '@server/lib/redis'
import { HttpStatusCode } from '@shared/models'
export interface APICacheOptions {
headerBlacklist?: string[]
excludeStatus?: HttpStatusCode[]
}
interface CacheObject {
status: number
headers: OutgoingHttpHeaders
data: any
encoding: BufferEncoding
timestamp: number
}
export class ApiCache {
private readonly options: APICacheOptions
private readonly timers: { [ id: string ]: NodeJS.Timeout } = {}
private index: { all: string[] } = { all: [] }
constructor (options: APICacheOptions) {
this.options = {
headerBlacklist: [],
excludeStatus: [],
...options
}
}
buildMiddleware (strDuration: string) {
const duration = parseDurationToMs(strDuration)
return (req: express.Request, res: express.Response, next: express.NextFunction) => {
const key = Redis.Instance.getPrefix() + 'api-cache-' + req.originalUrl
const redis = Redis.Instance.getClient()
if (!redis.connected) return this.makeResponseCacheable(res, next, key, duration)
try {
redis.hgetall(key, (err, obj) => {
if (!err && obj && obj.response) {
return this.sendCachedResponse(req, res, JSON.parse(obj.response), duration)
}
return this.makeResponseCacheable(res, next, key, duration)
})
} catch (err) {
return this.makeResponseCacheable(res, next, key, duration)
}
}
}
private shouldCacheResponse (response: express.Response) {
if (!response) return false
if (this.options.excludeStatus.includes(response.statusCode)) return false
return true
}
private addIndexEntries (key: string) {
this.index.all.unshift(key)
}
private filterBlacklistedHeaders (headers: OutgoingHttpHeaders) {
return Object.keys(headers)
.filter(key => !this.options.headerBlacklist.includes(key))
.reduce((acc, header) => {
acc[header] = headers[header]
return acc
}, {})
}
private createCacheObject (status: number, headers: OutgoingHttpHeaders, data: any, encoding: BufferEncoding) {
return {
status,
headers: this.filterBlacklistedHeaders(headers),
data,
encoding,
// Seconds since epoch, used to properly decrement max-age headers in cached responses.
timestamp: new Date().getTime() / 1000
} as CacheObject
}
private cacheResponse (key: string, value: object, duration: number) {
const redis = Redis.Instance.getClient()
if (redis.connected) {
try {
redis.hset(key, 'response', JSON.stringify(value))
redis.hset(key, 'duration', duration + '')
redis.expire(key, duration / 1000)
} catch (err) {
logger.error('Cannot set cache in redis.', { err })
}
}
// add automatic cache clearing from duration, includes max limit on setTimeout
this.timers[key] = setTimeout(() => this.clear(key), Math.min(duration, 2147483647))
}
private accumulateContent (res: express.Response, content: any) {
if (!content) return
if (typeof content === 'string') {
res.locals.apicache.content = (res.locals.apicache.content || '') + content
return
}
if (Buffer.isBuffer(content)) {
let oldContent = res.locals.apicache.content
if (typeof oldContent === 'string') {
oldContent = Buffer.from(oldContent)
}
if (!oldContent) {
oldContent = Buffer.alloc(0)
}
res.locals.apicache.content = Buffer.concat(
[ oldContent, content ],
oldContent.length + content.length
)
return
}
res.locals.apicache.content = content
}
private makeResponseCacheable (res: express.Response, next: express.NextFunction, key: string, duration: number) {
const self = this
res.locals.apicache = {
write: res.write,
writeHead: res.writeHead,
end: res.end,
cacheable: true,
content: undefined,
headers: {}
}
// Patch express
res.writeHead = function () {
if (self.shouldCacheResponse(res)) {
res.setHeader('cache-control', 'max-age=' + (duration / 1000).toFixed(0))
} else {
res.setHeader('cache-control', 'no-cache, no-store, must-revalidate')
}
res.locals.apicache.headers = Object.assign({}, res.getHeaders())
return res.locals.apicache.writeHead.apply(this, arguments as any)
}
res.write = function (chunk: any) {
self.accumulateContent(res, chunk)
return res.locals.apicache.write.apply(this, arguments as any)
}
res.end = function (content: any, encoding: BufferEncoding) {
if (self.shouldCacheResponse(res)) {
self.accumulateContent(res, content)
if (res.locals.apicache.cacheable && res.locals.apicache.content) {
self.addIndexEntries(key)
const headers = res.locals.apicache.headers || res.getHeaders()
const cacheObject = self.createCacheObject(
res.statusCode,
headers,
res.locals.apicache.content,
encoding
)
self.cacheResponse(key, cacheObject, duration)
}
}
res.locals.apicache.end.apply(this, arguments as any)
} as any
next()
}
private sendCachedResponse (request: express.Request, response: express.Response, cacheObject: CacheObject, duration: number) {
const headers = response.getHeaders()
if (isTestInstance()) {
Object.assign(headers, {
'x-api-cache-cached': 'true'
})
}
Object.assign(headers, this.filterBlacklistedHeaders(cacheObject.headers || {}), {
// Set properly decremented max-age header
// This ensures that max-age is in sync with the cache expiration
'cache-control':
'max-age=' +
Math.max(
0,
(duration / 1000 - (new Date().getTime() / 1000 - cacheObject.timestamp))
).toFixed(0)
})
// unstringify buffers
let data = cacheObject.data
if (data && data.type === 'Buffer') {
data = typeof data.data === 'number'
? Buffer.alloc(data.data)
: Buffer.from(data.data)
}
// Test Etag against If-None-Match for 304
const cachedEtag = cacheObject.headers.etag
const requestEtag = request.headers['if-none-match']
if (requestEtag && cachedEtag === requestEtag) {
response.writeHead(304, headers)
return response.end()
}
response.writeHead(cacheObject.status || 200, headers)
return response.end(data, cacheObject.encoding)
}
private clear (target: string) {
const redis = Redis.Instance.getClient()
if (target) {
clearTimeout(this.timers[target])
delete this.timers[target]
try {
redis.del(target)
} catch (err) {
logger.error('Cannot delete %s in redis cache.', target, { err })
}
this.index.all = this.index.all.filter(key => key !== target)
} else {
for (const key of this.index.all) {
clearTimeout(this.timers[key])
delete this.timers[key]
try {
redis.del(key)
} catch (err) {
logger.error('Cannot delete %s in redis cache.', key, { err })
}
}
this.index.all = []
}
return this.index
}
}
export * from './validators' export * from './validators'
export * from './cache'
export * from './activitypub' export * from './activitypub'
export * from './async' export * from './async'
export * from './auth' export * from './auth'
......
...@@ -8,6 +8,7 @@ import { ...@@ -8,6 +8,7 @@ import {
createMultipleServers, createMultipleServers,
createSingleServer, createSingleServer,
doubleFollow, doubleFollow,
makeGetRequest,