Verified Commit be1df674 authored by Chocobozzz's avatar Chocobozzz
Browse files

Cleanup proxy PR

parent d23f8ef6
This diff is collapsed.
import { logger } from "./logger"
function getProxy () {
return process.env.HTTPS_PROXY ||
process.env.HTTP_PROXY ||
undefined
}
// read environment variable
const OS_HTTP_PROXY = process.env.HTTP_PROXY
const OS_HTTPS_PROXY = process.env.HTTPS_PROXY
function isProxyEnabled () {
return !!getProxy()
}
function getProxy() {
let proxyAddress
/* istanbul ignore else */
if (typeof OS_HTTPS_PROXY !== "undefined") {
logger.debug("Using OS HTTPS_PROXY Environment variable.")
proxyAddress = OS_HTTPS_PROXY
}
/* istanbul ignore else */
if (typeof OS_HTTP_PROXY !== "undefined") {
logger.debug("Using OS HTTP_PROXY Environment variable.")
proxyAddress = OS_HTTP_PROXY
}
return proxyAddress
export {
getProxy,
isProxyEnabled
}
module.exports = getProxy
import { createWriteStream, remove } from 'fs-extra'
import got, { CancelableRequest, Options as GotOptions, RequestError } from 'got'
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'
import { join } from 'path'
import { CONFIG } from '../initializers/config'
import { ACTIVITY_PUB, PEERTUBE_VERSION, REQUEST_TIMEOUT, WEBSERVER } from '../initializers/constants'
import { pipelinePromise } from './core-utils'
import { processImage } from './image-utils'
import { logger } from './logger'
const { HttpsProxyAgent } = require("hpagent")
const isProxyEnabled = require("./proxy")
import { getProxy, isProxyEnabled } from './proxy'
const httpSignature = require('http-signature')
export interface PeerTubeRequestError extends Error {
statusCode?: number
responseBody?: any
}
const httpSignature = require('http-signature')
type PeerTubeRequestOptions = {
activityPub?: boolean
bodyKBLimit?: number // 1MB
......@@ -31,80 +30,81 @@ type PeerTubeRequestOptions = {
jsonResponse?: boolean
} & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
const peertubeGot = got.extend({
agent: getAgentIfNeeded(),
headers: {
'user-agent': getUserAgent()
},
handlers: [
(options, next) => {
const promiseOrStream = next(options) as CancelableRequest<any>
const bodyKBLimit = options.context?.bodyKBLimit as number
if (!bodyKBLimit) throw new Error('No KB limit for this request')
const bodyLimit = bodyKBLimit * 1000
/* eslint-disable @typescript-eslint/no-floating-promises */
promiseOrStream.on('downloadProgress', progress => {
if (progress.transferred > bodyLimit && progress.percent !== 1) {
const message = `Exceeded the download limit of ${bodyLimit} B`
logger.warn(message)
// CancelableRequest
if (promiseOrStream.cancel) {
promiseOrStream.cancel()
return
}
// Stream
(promiseOrStream as any).destroy()
}
})
return promiseOrStream
}
],
hooks: {
beforeRequest: [
options => {
const headers = options.headers || {}
headers['host'] = options.url.host
},
options => {
const httpSignatureOptions = options.context?.httpSignature
if (httpSignatureOptions) {
const method = options.method ?? 'GET'
const path = options.path ?? options.url.pathname
if (!method || !path) {
throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
}
httpSignature.signRequest({
getHeader: function (header) {
return options.headers[header]
},
setHeader: function (header, value) {
options.headers[header] = value
},
method,
path
}, httpSignatureOptions)
const peertubeGot = got.extend({
...getAgent(),
headers: {
'user-agent': getUserAgent()
},
handlers: [
(options, next) => {
const promiseOrStream = next(options) as CancelableRequest<any>
const bodyKBLimit = options.context?.bodyKBLimit as number
if (!bodyKBLimit) throw new Error('No KB limit for this request')
const bodyLimit = bodyKBLimit * 1000
/* eslint-disable @typescript-eslint/no-floating-promises */
promiseOrStream.on('downloadProgress', progress => {
if (progress.transferred > bodyLimit && progress.percent !== 1) {
const message = `Exceeded the download limit of ${bodyLimit} B`
logger.warn(message)
// CancelableRequest
if (promiseOrStream.cancel) {
promiseOrStream.cancel()
return
}
},
(options: GotOptions) => {
options.timeout = REQUEST_TIMEOUT
// Stream
(promiseOrStream as any).destroy()
}
]
})
return promiseOrStream
}
})
],
hooks: {
beforeRequest: [
options => {
const headers = options.headers || {}
headers['host'] = options.url.host
},
options => {
const httpSignatureOptions = options.context?.httpSignature
if (httpSignatureOptions) {
const method = options.method ?? 'GET'
const path = options.path ?? options.url.pathname
if (!method || !path) {
throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
}
httpSignature.signRequest({
getHeader: function (header) {
return options.headers[header]
},
setHeader: function (header, value) {
options.headers[header] = value
},
method,
path
}, httpSignatureOptions)
}
},
(options: GotOptions) => {
options.timeout = REQUEST_TIMEOUT
}
]
}
})
function doRequest (url: string, options: PeerTubeRequestOptions = {}) {
const gotOptions = buildGotOptions(options)
......@@ -157,22 +157,27 @@ async function downloadImage (url: string, destDir: string, destName: string, si
}
}
function getAgentIfNeeded () {
if (!isProxyEnabled())
{
logger.debug("Without proxy")
return {}
function getAgent () {
if (!isProxyEnabled()) return {}
const proxy = getProxy()
logger.info('Using proxy %s.', proxy)
const proxyAgentOptions = {
keepAlive: true,
keepAliveMsecs: 1000,
maxSockets: 256,
maxFreeSockets: 256,
scheduling: 'lifo' as 'lifo',
proxy
}
return {
https: new HttpsProxyAgent({
keepAlive: true,
keepAliveMsecs: 1000,
maxSockets: 256,
maxFreeSockets: 256,
scheduling: "lifo",
proxy: isProxyEnabled()
})
agent: {
http: new HttpProxyAgent(proxyAgentOptions),
https: new HttpsProxyAgent(proxyAgentOptions)
}
}
}
......
......@@ -15,3 +15,4 @@ import './stats'
import './tracker'
import './no-client'
import './plugins'
import './proxy'
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import * as chai from 'chai'
import { cleanupTests, createMultipleServers, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils'
import { MockProxy } from '@shared/extra-utils/mock-servers/mock-proxy'
const expect = chai.expect
describe('Test proxy', function () {
let servers: PeerTubeServer[] = []
let proxy: MockProxy
const goodEnv = { HTTP_PROXY: '' }
const badEnv = { HTTP_PROXY: 'http://localhost:9000' }
before(async function () {
this.timeout(120000)
proxy = new MockProxy()
const proxyPort = await proxy.initialize()
servers = await createMultipleServers(2)
goodEnv.HTTP_PROXY = 'http://localhost:' + proxyPort
await setAccessTokensToServers(servers)
await doubleFollow(servers[0], servers[1])
})
it('Should succeed federation with the appropriate proxy config', async function () {
await servers[0].kill()
await servers[0].run({}, { env: goodEnv })
await servers[0].videos.quickUpload({ name: 'video 1' })
await waitJobs(servers)
for (const server of servers) {
const { total, data } = await server.videos.list()
expect(total).to.equal(1)
expect(data).to.have.lengthOf(1)
}
})
it('Should fail federation with a wrong proxy config', async function () {
await servers[0].kill()
await servers[0].run({}, { env: badEnv })
await servers[0].videos.quickUpload({ name: 'video 2' })
await waitJobs(servers)
{
const { total, data } = await servers[0].videos.list()
expect(total).to.equal(2)
expect(data).to.have.lengthOf(2)
}
{
const { total, data } = await servers[1].videos.list()
expect(total).to.equal(1)
expect(data).to.have.lengthOf(1)
}
})
after(async function () {
proxy.terminate()
await cleanupTests(servers)
})
})
const expect = require("chai").expect
const assert = require("chai").assert
const proxyTest = require("../middlewares/proxy")
// read environment variable
const httpProxy = process.env.HTTP_PROXY
const httpsProxy = process.env.HTTPS_PROXY
describe("The Proxy function with HTTP_PROXY or HTTPS_PROXY", function () {
if (proxyTest()) {
if (typeof httpsProxy || typeof httpProxy !== "undefined") {
it("value from proxy is not null", function () {
const result = proxyTest()
expect(result).to.equal(httpProxy || httpsProxy)
})
it("value from proxy is not undefined", function () {
const result = proxyTest()
expect(result).not.to.equal(undefined)
})
it("value from proxy type string", function () {
const result = proxyTest()
assert.typeOf(result, "string")
})
}
}
})
describe("The Proxy function with no value", function () {
if (!proxyTest()) {
it("value from proxy is null", function () {
const result = proxyTest()
expect(result).to.equal(null)
})
it("value in not equal to HTTP_PROXY and HTTPS_PROXY", function () {
const result = proxyTest()
expect(result).to.not.equal(httpProxy || httpsProxy)
})
}
})
import { createServer, Server } from 'http'
import * as proxy from 'proxy'
import { randomInt } from '@shared/core-utils'
class MockProxy {
private server: Server
initialize () {
return new Promise<number>(res => {
const port = 42501 + randomInt(1, 100)
this.server = proxy(createServer())
this.server.listen(port, () => res(port))
})
}
terminate () {
if (this.server) this.server.close()
}
}
// ---------------------------------------------------------------------------
export {
MockProxy
}
......@@ -51,7 +51,6 @@ services:
networks:
default:
ipv4_address: 172.18.0.42
env_file:
- .env
......
......@@ -2002,6 +2002,16 @@ argparse@^2.0.1:
resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38"
integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==
args@5.0.1:
version "5.0.1"
resolved "https://registry.yarnpkg.com/args/-/args-5.0.1.tgz#4bf298df90a4799a09521362c579278cc2fdd761"
integrity sha512-1kqmFCFsPffavQFGt8OxJdIcETti99kySRUPMpOhaGjL6mRJn8HFU1OxKY5bMqfZKUwTQc1mZkAjmGYaVOHFtQ==
dependencies:
camelcase "5.0.0"
chalk "2.4.2"
leven "2.1.0"
mri "1.1.4"
array-differ@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/array-differ/-/array-differ-3.0.0.tgz#3cbb3d0f316810eafcc47624734237d6aee4ae6b"
......@@ -2200,6 +2210,11 @@ basic-auth-connect@^1.0.0:
resolved "https://registry.yarnpkg.com/basic-auth-connect/-/basic-auth-connect-1.0.0.tgz#fdb0b43962ca7b40456a7c2bb48fe173da2d2122"
integrity sha1-/bC0OWLKe0BFanwrtI/hc9otISI=
basic-auth-parser@0.0.2:
version "0.0.2"
resolved "https://registry.yarnpkg.com/basic-auth-parser/-/basic-auth-parser-0.0.2.tgz#ce9e71a77f23c1279eecd2659b2a46244c156e41"
integrity sha1-zp5xp38jwSee7NJlmypGJEwVbkE=
basic-auth@2.0.1, basic-auth@~2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/basic-auth/-/basic-auth-2.0.1.tgz#b998279bf47ce38344b4f3cf916d4679bbf51e3a"
......@@ -2616,6 +2631,11 @@ camelcase-keys@^4.0.0:
map-obj "^2.0.0"
quick-lru "^1.0.0"
camelcase@5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.0.0.tgz#03295527d58bd3cd4aa75363f35b2e8d97be2f42"
integrity sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==
camelcase@^4.1.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-4.1.0.tgz#d545635be1e33c542649c69173e5de6acfae34dd"
......@@ -2663,7 +2683,7 @@ chai@^4.1.1:
pathval "^1.1.1"
type-detect "^4.0.5"
chalk@^2.0.0, chalk@^2.4.2:
chalk@2.4.2, chalk@^2.0.0, chalk@^2.4.2:
version "2.4.2"
resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==
......@@ -5553,6 +5573,11 @@ latest-version@^5.0.0:
dependencies:
package-json "^6.3.0"
leven@2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/leven/-/leven-2.1.0.tgz#c2e7a9f772094dee9d34202ae8acce4687875580"
integrity sha1-wuep93IJTe6dNCAq6KzORoeHVYA=
levn@^0.4.1:
version "0.4.1"
resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade"
......@@ -6201,6 +6226,11 @@ mp4-stream@^3.0.0:
queue-microtask "^1.2.2"
readable-stream "^3.0.6"
mri@1.1.4:
version "1.1.4"
resolved "https://registry.yarnpkg.com/mri/-/mri-1.1.4.tgz#7cb1dd1b9b40905f1fac053abe25b6720f44744a"
integrity sha512-6y7IjGPm8AzlvoUrwAaw1tLnUBudaS3752vcd8JtrpGGQn+rXIe63LFVHm/YMwtqAuh+LJPCFdlLYPWM1nYn6w==
ms@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
......@@ -7136,6 +7166,15 @@ proxy-addr@~2.0.5:
forwarded "0.2.0"
ipaddr.js "1.9.1"
proxy@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/proxy/-/proxy-1.0.2.tgz#e0cfbe11c0a7a8b238fd2d7134de4e2867578e7f"
integrity sha512-KNac2ueWRpjbUh77OAFPZuNdfEqNynm9DD4xHT14CccGpW8wKZwEkN0yjlb7X9G9Z9F55N0Q+1z+WfgAhwYdzQ==
dependencies:
args "5.0.1"
basic-auth-parser "0.0.2"
debug "^4.1.1"
pseudomap@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment