Verified Commit 1896bca0 authored by Chocobozzz's avatar Chocobozzz
Browse files

Support transcoding options/encoders by plugins

parent 529b3752
......@@ -223,11 +223,20 @@ user:
# Please, do not disable transcoding since many uploaded videos will not work
transcoding:
enabled: true
# Allow your users to upload .mkv, .mov, .avi, .wmv, .flv, .f4v, .3g2, .3gp, .mts, m2ts, .mxf, .nut videos
allow_additional_extensions: true
# If a user uploads an audio file, PeerTube will create a video by merging the preview file and the audio file
allow_audio_files: true
threads: 1
# Choose the transcoding profile
# New profiles can be added by plugins
# Available in core PeerTube: 'default'
profile: 'default'
resolutions: # Only created if the original video has a higher resolution, uses more storage!
0p: false # audio-only (creates mp4 without video stream, always created when enabled)
240p: false
......@@ -283,6 +292,11 @@ live:
enabled: true
threads: 2
# Choose the transcoding profile
# New profiles can be added by plugins
# Available in core PeerTube: 'default'
profile: 'default'
resolutions:
240p: false
360p: false
......
......@@ -236,11 +236,20 @@ user:
# Please, do not disable transcoding since many uploaded videos will not work
transcoding:
enabled: true
# Allow your users to upload .mkv, .mov, .avi, .wmv, .flv, .f4v, .3g2, .3gp, .mts, m2ts, .mxf, .nut videos
allow_additional_extensions: true
# If a user uploads an audio file, PeerTube will create a video by merging the preview file and the audio file
allow_audio_files: true
threads: 1
# Choose the transcoding profile
# New profiles can be added by plugins
# Available in core PeerTube: 'default'
profile: 'default'
resolutions: # Only created if the original video has a higher resolution, uses more storage!
0p: false # audio-only (creates mp4 without video stream, always created when enabled)
240p: false
......@@ -270,7 +279,7 @@ live:
enabled: false
# Limit lives duration
# Set null to disable duration limit
# -1 == unlimited
max_duration: -1 # For example: '5 hours'
# Limit max number of live videos created on your instance
......@@ -296,6 +305,11 @@ live:
enabled: true
threads: 2
# Choose the transcoding profile
# New profiles can be added by plugins
# Available in core PeerTube: 'default'
profile: 'default'
resolutions:
240p: false
360p: false
......
......@@ -18,6 +18,7 @@ import { PluginManager } from '../../lib/plugins/plugin-manager'
import { getThemeOrDefault } from '../../lib/plugins/theme-utils'
import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares'
import { customConfigUpdateValidator } from '../../middlewares/validators/config'
import { VideoTranscodingProfilesManager } from '@server/lib/video-transcoding-profiles'
const configRouter = express.Router()
......@@ -114,7 +115,9 @@ async function getConfig (req: express.Request, res: express.Response) {
webtorrent: {
enabled: CONFIG.TRANSCODING.WEBTORRENT.ENABLED
},
enabledResolutions: getEnabledResolutions('vod')
enabledResolutions: getEnabledResolutions('vod'),
profile: CONFIG.TRANSCODING.PROFILE,
availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('vod')
},
live: {
enabled: CONFIG.LIVE.ENABLED,
......@@ -126,7 +129,9 @@ async function getConfig (req: express.Request, res: express.Response) {
transcoding: {
enabled: CONFIG.LIVE.TRANSCODING.ENABLED,
enabledResolutions: getEnabledResolutions('live')
enabledResolutions: getEnabledResolutions('live'),
profile: CONFIG.LIVE.TRANSCODING.PROFILE,
availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('live')
},
rtmp: {
......@@ -412,6 +417,7 @@ function customConfig (): CustomConfig {
allowAdditionalExtensions: CONFIG.TRANSCODING.ALLOW_ADDITIONAL_EXTENSIONS,
allowAudioFiles: CONFIG.TRANSCODING.ALLOW_AUDIO_FILES,
threads: CONFIG.TRANSCODING.THREADS,
profile: CONFIG.TRANSCODING.PROFILE,
resolutions: {
'0p': CONFIG.TRANSCODING.RESOLUTIONS['0p'],
'240p': CONFIG.TRANSCODING.RESOLUTIONS['240p'],
......@@ -438,6 +444,7 @@ function customConfig (): CustomConfig {
transcoding: {
enabled: CONFIG.LIVE.TRANSCODING.ENABLED,
threads: CONFIG.LIVE.TRANSCODING.THREADS,
profile: CONFIG.LIVE.TRANSCODING.PROFILE,
resolutions: {
'240p': CONFIG.LIVE.TRANSCODING.RESOLUTIONS['240p'],
'360p': CONFIG.LIVE.TRANSCODING.RESOLUTIONS['360p'],
......
......@@ -3,9 +3,9 @@ import * as ffmpeg from 'fluent-ffmpeg'
import { readFile, remove, writeFile } from 'fs-extra'
import { dirname, join } from 'path'
import { FFMPEG_NICE, VIDEO_LIVE } from '@server/initializers/constants'
import { VideoResolution } from '../../shared/models/videos'
import { checkFFmpegEncoders } from '../initializers/checker-before-init'
import { AvailableEncoders, EncoderOptionsBuilder, EncoderProfile, VideoResolution } from '../../shared/models/videos'
import { CONFIG } from '../initializers/config'
import { promisify0 } from './core-utils'
import { computeFPS, getAudioStream, getVideoFileFPS } from './ffprobe-utils'
import { processImage } from './image-utils'
import { logger } from './logger'
......@@ -21,46 +21,45 @@ import { logger } from './logger'
// Encoder options
// ---------------------------------------------------------------------------
// Options builders
export type EncoderOptionsBuilder = (params: {
input: string
resolution: VideoResolution
fps?: number
streamNum?: number
}) => Promise<EncoderOptions> | EncoderOptions
type StreamType = 'audio' | 'video'
// Options types
// ---------------------------------------------------------------------------
// Encoders support
// ---------------------------------------------------------------------------
export interface EncoderOptions {
copy?: boolean
outputOptions: string[]
}
// Detect supported encoders by ffmpeg
let supportedEncoders: Map<string, boolean>
async function checkFFmpegEncoders (peertubeAvailableEncoders: AvailableEncoders): Promise<Map<string, boolean>> {
if (supportedEncoders !== undefined) {
return supportedEncoders
}
// All our encoders
const getAvailableEncodersPromise = promisify0(ffmpeg.getAvailableEncoders)
const availableFFmpegEncoders = await getAvailableEncodersPromise()
export interface EncoderProfile <T> {
[ profile: string ]: T
const searchEncoders = new Set<string>()
for (const type of [ 'live', 'vod' ]) {
for (const streamType of [ 'audio', 'video' ]) {
for (const encoder of peertubeAvailableEncoders.encodersToTry[type][streamType]) {
searchEncoders.add(encoder)
}
}
}
default: T
}
supportedEncoders = new Map<string, boolean>()
export type AvailableEncoders = {
live: {
[ encoder: string ]: EncoderProfile<EncoderOptionsBuilder>
for (const searchEncoder of searchEncoders) {
supportedEncoders.set(searchEncoder, availableFFmpegEncoders[searchEncoder] !== undefined)
}
vod: {
[ encoder: string ]: EncoderProfile<EncoderOptionsBuilder>
}
logger.info('Built supported ffmpeg encoders.', { supportedEncoders, searchEncoders })
encodersToTry: {
video: string[]
audio: string[]
}
return supportedEncoders
}
type StreamType = 'audio' | 'video'
function resetSupportedEncoders () {
supportedEncoders = undefined
}
// ---------------------------------------------------------------------------
// Image manipulation
......@@ -275,7 +274,7 @@ async function getLiveTranscodingCommand (options: {
addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps: resolutionFPS, streamNum: i })
logger.debug('Apply ffmpeg live video params from %s.', builderResult.encoder, builderResult)
logger.debug('Apply ffmpeg live video params from %s using %s profile.', builderResult.encoder, profile, builderResult)
command.outputOption(`${buildStreamSuffix('-c:v', i)} ${builderResult.encoder}`)
command.addOutputOptions(builderResult.result.outputOptions)
......@@ -292,7 +291,7 @@ async function getLiveTranscodingCommand (options: {
addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps: resolutionFPS, streamNum: i })
logger.debug('Apply ffmpeg live audio params from %s.', builderResult.encoder, builderResult)
logger.debug('Apply ffmpeg live audio params from %s using %s profile.', builderResult.encoder, profile, builderResult)
command.outputOption(`${buildStreamSuffix('-c:a', i)} ${builderResult.encoder}`)
command.addOutputOptions(builderResult.result.outputOptions)
......@@ -513,11 +512,19 @@ async function getEncoderBuilderResult (options: {
}) {
const { availableEncoders, input, profile, resolution, streamType, fps, streamNum, videoType } = options
const encodersToTry = availableEncoders.encodersToTry[streamType]
const encoders = availableEncoders[videoType]
const encodersToTry = availableEncoders.encodersToTry[videoType][streamType]
const encoders = availableEncoders.available[videoType]
for (const encoder of encodersToTry) {
if (!(await checkFFmpegEncoders()).get(encoder) || !encoders[encoder]) continue
if (!(await checkFFmpegEncoders(availableEncoders)).get(encoder)) {
logger.debug('Encoder %s not available in ffmpeg, skipping.', encoder)
continue
}
if (!encoders[encoder]) {
logger.debug('Encoder %s not available in peertube encoders, skipping.', encoder)
continue
}
// An object containing available profiles for this encoder
const builderProfiles: EncoderProfile<EncoderOptionsBuilder> = encoders[encoder]
......@@ -567,7 +574,7 @@ async function presetVideo (
if (!parsedAudio.audioStream) {
localCommand = localCommand.noAudio()
streamsToProcess = [ 'audio' ]
streamsToProcess = [ 'video' ]
}
for (const streamType of streamsToProcess) {
......@@ -587,7 +594,10 @@ async function presetVideo (
throw new Error('No available encoder found for stream ' + streamType)
}
logger.debug('Apply ffmpeg params from %s.', builderResult.encoder, builderResult)
logger.debug(
'Apply ffmpeg params from %s for %s stream of input %s using %s profile.',
builderResult.encoder, streamType, input, profile, builderResult
)
if (streamType === 'video') {
localCommand.videoCodec(builderResult.encoder)
......@@ -679,6 +689,8 @@ export {
transcode,
runCommand,
resetSupportedEncoders,
// builders
buildx264VODCommand
}
......@@ -27,6 +27,14 @@ function getLoggerReplacer () {
seen.add(value)
}
if (value instanceof Set) {
return Array.from(value)
}
if (value instanceof Map) {
return Array.from(value.entries())
}
if (value instanceof Error) {
const error = {}
......
......@@ -97,34 +97,6 @@ async function checkFFmpeg (CONFIG: { TRANSCODING: { ENABLED: boolean } }) {
throw new Error('Unavailable encode codec ' + codec + ' in FFmpeg')
}
}
return checkFFmpegEncoders()
}
// Detect supported encoders by ffmpeg
let supportedEncoders: Map<string, boolean>
async function checkFFmpegEncoders (): Promise<Map<string, boolean>> {
if (supportedEncoders !== undefined) {
return supportedEncoders
}
const Ffmpeg = require('fluent-ffmpeg')
const getAvailableEncodersPromise = promisify0(Ffmpeg.getAvailableEncoders)
const availableEncoders = await getAvailableEncodersPromise()
const searchEncoders = [
'aac',
'libfdk_aac',
'libx264'
]
supportedEncoders = new Map<string, boolean>()
for (const searchEncoder of searchEncoders) {
supportedEncoders.set(searchEncoder, availableEncoders[searchEncoder] !== undefined)
}
return supportedEncoders
}
function checkNodeVersion () {
......@@ -143,7 +115,6 @@ function checkNodeVersion () {
export {
checkFFmpeg,
checkFFmpegEncoders,
checkMissedConfig,
checkNodeVersion
}
......@@ -188,6 +188,7 @@ const CONFIG = {
get ALLOW_ADDITIONAL_EXTENSIONS () { return config.get<boolean>('transcoding.allow_additional_extensions') },
get ALLOW_AUDIO_FILES () { return config.get<boolean>('transcoding.allow_audio_files') },
get THREADS () { return config.get<number>('transcoding.threads') },
get PROFILE () { return config.get<string>('transcoding.profile') },
RESOLUTIONS: {
get '0p' () { return config.get<boolean>('transcoding.resolutions.0p') },
get '240p' () { return config.get<boolean>('transcoding.resolutions.240p') },
......@@ -221,6 +222,7 @@ const CONFIG = {
TRANSCODING: {
get ENABLED () { return config.get<boolean>('live.transcoding.enabled') },
get THREADS () { return config.get<number>('live.transcoding.threads') },
get PROFILE () { return config.get<string>('live.transcoding.profile') },
RESOLUTIONS: {
get '240p' () { return config.get<boolean>('live.transcoding.resolutions.240p') },
......
......@@ -338,7 +338,7 @@ class LiveManager {
resolutions: allResolutions,
fps,
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: 'default'
profile: CONFIG.LIVE.TRANSCODING.PROFILE
})
: getLiveMuxingCommand(rtmpUrl, outPath)
......
......@@ -20,7 +20,7 @@ import { PLUGIN_GLOBAL_CSS_PATH } from '../../initializers/constants'
import { PluginModel } from '../../models/server/plugin'
import { PluginLibrary, RegisterServerAuthExternalOptions, RegisterServerAuthPassOptions, RegisterServerOptions } from '../../types/plugins'
import { ClientHtml } from '../client-html'
import { RegisterHelpersStore } from './register-helpers-store'
import { RegisterHelpers } from './register-helpers'
import { installNpmPlugin, installNpmPluginFromDisk, removeNpmPlugin } from './yarn'
export interface RegisteredPlugin {
......@@ -40,7 +40,7 @@ export interface RegisteredPlugin {
css: string[]
// Only if this is a plugin
registerHelpersStore?: RegisterHelpersStore
registerHelpers?: RegisterHelpers
unregister?: Function
}
......@@ -109,7 +109,7 @@ export class PluginManager implements ServerHook {
npmName: p.npmName,
name: p.name,
version: p.version,
idAndPassAuths: p.registerHelpersStore.getIdAndPassAuths()
idAndPassAuths: p.registerHelpers.getIdAndPassAuths()
}))
.filter(v => v.idAndPassAuths.length !== 0)
}
......@@ -120,7 +120,7 @@ export class PluginManager implements ServerHook {
npmName: p.npmName,
name: p.name,
version: p.version,
externalAuths: p.registerHelpersStore.getExternalAuths()
externalAuths: p.registerHelpers.getExternalAuths()
}))
.filter(v => v.externalAuths.length !== 0)
}
......@@ -129,14 +129,14 @@ export class PluginManager implements ServerHook {
const result = this.getRegisteredPluginOrTheme(npmName)
if (!result || result.type !== PluginType.PLUGIN) return []
return result.registerHelpersStore.getSettings()
return result.registerHelpers.getSettings()
}
getRouter (npmName: string) {
const result = this.getRegisteredPluginOrTheme(npmName)
if (!result || result.type !== PluginType.PLUGIN) return null
return result.registerHelpersStore.getRouter()
return result.registerHelpers.getRouter()
}
getTranslations (locale: string) {
......@@ -194,7 +194,7 @@ export class PluginManager implements ServerHook {
logger.error('Cannot find plugin %s to call on settings changed.', name)
}
for (const cb of registered.registerHelpersStore.getOnSettingsChangedCallbacks()) {
for (const cb of registered.registerHelpers.getOnSettingsChangedCallbacks()) {
try {
cb(settings)
} catch (err) {
......@@ -268,8 +268,9 @@ export class PluginManager implements ServerHook {
this.hooks[key] = this.hooks[key].filter(h => h.npmName !== npmName)
}
const store = plugin.registerHelpersStore
const store = plugin.registerHelpers
store.reinitVideoConstants(plugin.npmName)
store.reinitTranscodingProfilesAndEncoders(plugin.npmName)
logger.info('Regenerating registered plugin CSS to global file.')
await this.regeneratePluginGlobalCSS()
......@@ -375,11 +376,11 @@ export class PluginManager implements ServerHook {
this.sanitizeAndCheckPackageJSONOrThrow(packageJSON, plugin.type)
let library: PluginLibrary
let registerHelpersStore: RegisterHelpersStore
let registerHelpers: RegisterHelpers
if (plugin.type === PluginType.PLUGIN) {
const result = await this.registerPlugin(plugin, pluginPath, packageJSON)
library = result.library
registerHelpersStore = result.registerStore
registerHelpers = result.registerStore
}
const clientScripts: { [id: string]: ClientScript } = {}
......@@ -398,7 +399,7 @@ export class PluginManager implements ServerHook {
staticDirs: packageJSON.staticDirs,
clientScripts,
css: packageJSON.css,
registerHelpersStore: registerHelpersStore || undefined,
registerHelpers: registerHelpers || undefined,
unregister: library ? library.unregister : undefined
}
......@@ -512,8 +513,8 @@ export class PluginManager implements ServerHook {
const plugin = this.getRegisteredPluginOrTheme(npmName)
if (!plugin || plugin.type !== PluginType.PLUGIN) return null
let auths: (RegisterServerAuthPassOptions | RegisterServerAuthExternalOptions)[] = plugin.registerHelpersStore.getIdAndPassAuths()
auths = auths.concat(plugin.registerHelpersStore.getExternalAuths())
let auths: (RegisterServerAuthPassOptions | RegisterServerAuthExternalOptions)[] = plugin.registerHelpers.getIdAndPassAuths()
auths = auths.concat(plugin.registerHelpers.getExternalAuths())
return auths.find(a => a.authName === authName)
}
......@@ -538,7 +539,7 @@ export class PluginManager implements ServerHook {
private getRegisterHelpers (
npmName: string,
plugin: PluginModel
): { registerStore: RegisterHelpersStore, registerOptions: RegisterServerOptions } {
): { registerStore: RegisterHelpers, registerOptions: RegisterServerOptions } {
const onHookAdded = (options: RegisterServerHookOptions) => {
if (!this.hooks[options.target]) this.hooks[options.target] = []
......@@ -550,11 +551,11 @@ export class PluginManager implements ServerHook {
})
}
const registerHelpersStore = new RegisterHelpersStore(npmName, plugin, onHookAdded.bind(this))
const registerHelpers = new RegisterHelpers(npmName, plugin, onHookAdded.bind(this))
return {
registerStore: registerHelpersStore,
registerOptions: registerHelpersStore.buildRegisterHelpers()
registerStore: registerHelpers,
registerOptions: registerHelpers.buildRegisterHelpers()
}
}
......
......@@ -17,6 +17,7 @@ import {
RegisterServerOptions
} from '@server/types/plugins'
import {
EncoderOptionsBuilder,
PluginPlaylistPrivacyManager,
PluginSettingsManager,
PluginStorageManager,
......@@ -28,7 +29,8 @@ import {
RegisterServerSettingOptions
} from '@shared/models'
import { serverHookObject } from '@shared/models/plugins/server-hook.model'
import { buildPluginHelpers } from './plugin-helpers'
import { VideoTranscodingProfilesManager } from '../video-transcoding-profiles'
import { buildPluginHelpers } from './plugin-helpers-builder'
type AlterableVideoConstant = 'language' | 'licence' | 'category' | 'privacy' | 'playlistPrivacy'
type VideoConstant = { [key in number | string]: string }
......@@ -40,7 +42,7 @@ type UpdatedVideoConstant = {
}
}
export class RegisterHelpersStore {
export class RegisterHelpers {
private readonly updatedVideoConstants: UpdatedVideoConstant = {
playlistPrivacy: { added: [], deleted: [] },
privacy: { added: [], deleted: [] },
......@@ -49,6 +51,23 @@ export class RegisterHelpersStore {
category: { added: [], deleted: [] }
}
private readonly transcodingProfiles: {
[ npmName: string ]: {
type: 'vod' | 'live'
encoder: string
profile: string
}[]
} = {}
private readonly transcodingEncoders: {
[ npmName: string ]: {
type: 'vod' | 'live'
streamType: 'audio' | 'video'
encoder: string
priority: number
}[]
} = {}
private readonly settings: RegisterServerSettingOptions[] = []
private idAndPassAuths: RegisterServerAuthPassOptions[] = []
......@@ -83,6 +102,8 @@ export class RegisterHelpersStore {
const videoPrivacyManager = this.buildVideoPrivacyManager()
const playlistPrivacyManager = this.buildPlaylistPrivacyManager()
const transcodingManager = this.buildTranscodingManager()
const registerIdAndPassAuth = this.buildRegisterIdAndPassAuth()
const registerExternalAuth = this.buildRegisterExternalAuth()
const unregisterIdAndPassAuth = this.buildUnregisterIdAndPassAuth()
......@@ -106,6 +127,8 @@ export class RegisterHelpersStore {
videoPrivacyManager,
playlistPrivacyManager,
transcodingManager,
registerIdAndPassAuth,
registerExternalAuth,
unregisterIdAndPassAuth,
......@@ -141,6 +164,22 @@ export class RegisterHelpersStore {
}
}
reinitTranscodingProfilesAndEncoders (npmName: string) {
const profiles = this.transcodingProfiles[npmName]
if (Array.isArray(profiles)) {
for (const profile of profiles) {
VideoTranscodingProfilesManager.Instance.removeProfile(profile)
}
}
const encoders = this.transcodingEncoders[npmName]
if (Array.isArray(encoders)) {
for (const o of encoders) {
VideoTranscodingProfilesManager.Instance.removeEncoderPriority(o.type, o.streamType, o.encoder, o.priority)
}
}
}
getSettings () {
return this.settings
}
......@@ -354,4 +393,52 @@ export class RegisterHelpersStore {
return true
}
private buildTranscodingManager () {
const self = this
function addProfile (type: 'live' | 'vod', encoder: string, profile: string, builder: EncoderOptionsBuilder) {
if (profile === 'default') {
logger.error('A plugin cannot add a default live transcoding profile')
return false
}
VideoTranscodingProfilesManager.Instance.addProfile({
type,
encoder,
profile,
builder
})
if (!self.transcodingProfiles[self.npmName]) self.transcodingProfiles[self.npmName] = []
self.transcodingProfiles[self.npmName].push({ type, encoder, profile })
return true
}
function addEncoderPriority (type: 'live' | 'vod', streamType: 'audio' | 'video', encoder: string, priority: number) {
VideoTranscodingProfilesManager.Instance.addEncoderPriority(type, streamType, encoder, priority)
if (!self.transcodingEncoders[self.npmName]) self.transcodingEncoders[self.npmName] = []
self.transcodingEncoders[self.npmName].push({ type, streamType, encoder, priority })
}