Verified Commit 94a5ff8a authored by Chocobozzz's avatar Chocobozzz
Browse files

Move job queue to redis

We'll use it as cache in the future.

/!\ You'll loose your old jobs (pending jobs too) so upgrade only when
you don't have pending job anymore.
parent d765fafc
......@@ -131,6 +131,7 @@ BitTorrent) inside the web browser, as of today.
* nginx
* PostgreSQL
* Redis
* **NodeJS >= 8.x**
* yarn
* OpenSSL (cli)
......
<div class="admin-sub-header">
<div class="admin-sub-title">Jobs list</div>
<div class="peertube-select-container">
<select [(ngModel)]="jobState" (ngModelChange)="onJobStateChanged()">
<option *ngFor="let state of jobStates" [value]="state">{{ state }}</option>
</select>
</div>
</div>
<p-dataTable
[value]="jobs" [lazy]="true" [paginator]="true" [totalRecords]="totalRecords" [rows]="rowsPerPage"
sortField="createdAt" (onLazyLoad)="loadLazy($event)" [scrollable]="true" [virtualScroll]="true" [scrollHeight]="scrollHeight"
>
<p-column field="id" header="ID" [style]="{ width: '60px' }"></p-column>
<p-column field="category" header="Category" [style]="{ width: '130px' }"></p-column>
<p-column field="handlerName" header="Handler name" [style]="{ width: '210px' }"></p-column>
<p-column header="Input data">
<p-column field="type" header="Type" [style]="{ width: '210px' }"></p-column>
<p-column field="state" header="State" [style]="{ width: '130px' }"></p-column>
<p-column header="Payload">
<ng-template pTemplate="body" let-job="rowData">
<pre>{{ job.handlerInputData }}</pre>
<pre>{{ job.data }}</pre>
</ng-template>
</p-column>
<p-column field="state" header="State" [style]="{ width: '100px' }"></p-column>
<p-column field="createdAt" header="Created date" [sortable]="true" [style]="{ width: '250px' }"></p-column>
<p-column field="updatedAt" header="Updated date" [style]="{ width: '250px' }"></p-column>
</p-dataTable>
@import '_variables';
@import '_mixins';
.peertube-select-container {
@include peertube-select-container(auto);
}
pre {
font-size: 11px;
}
......@@ -2,6 +2,7 @@ import { Component, OnInit } from '@angular/core'
import { NotificationsService } from 'angular2-notifications'
import { SortMeta } from 'primeng/primeng'
import { Job } from '../../../../../../shared/index'
import { JobState } from '../../../../../../shared/models'
import { RestPagination, RestTable } from '../../../shared'
import { viewportHeight } from '../../../shared/misc/utils'
import { JobService } from '../shared'
......@@ -13,10 +14,12 @@ import { RestExtractor } from '../../../shared/rest/rest-extractor.service'
styleUrls: [ './jobs-list.component.scss' ]
})
export class JobsListComponent extends RestTable implements OnInit {
jobState: JobState = 'inactive'
jobStates: JobState[] = [ 'active', 'complete', 'failed', 'inactive', 'delayed' ]
jobs: Job[] = []
totalRecords = 0
rowsPerPage = 20
sort: SortMeta = { field: 'createdAt', order: 1 }
sort: SortMeta = { field: 'createdAt', order: -1 }
pagination: RestPagination = { count: this.rowsPerPage, start: 0 }
scrollHeight = ''
......@@ -33,9 +36,13 @@ export class JobsListComponent extends RestTable implements OnInit {
this.scrollHeight = (viewportHeight() - 380) + 'px'
}
onJobStateChanged () {
this.loadData()
}
protected loadData () {
this.jobsService
.getJobs(this.pagination, this.sort)
.getJobs(this.jobState, this.pagination, this.sort)
.subscribe(
resultList => {
this.jobs = resultList.data
......
......@@ -5,6 +5,7 @@ import 'rxjs/add/operator/catch'
import 'rxjs/add/operator/map'
import { Observable } from 'rxjs/Observable'
import { ResultList } from '../../../../../../shared'
import { JobState } from '../../../../../../shared/models'
import { Job } from '../../../../../../shared/models/job.model'
import { environment } from '../../../../environments/environment'
import { RestExtractor, RestPagination, RestService } from '../../../shared'
......@@ -19,19 +20,19 @@ export class JobService {
private restExtractor: RestExtractor
) {}
getJobs (pagination: RestPagination, sort: SortMeta): Observable<ResultList<Job>> {
getJobs (state: JobState, pagination: RestPagination, sort: SortMeta): Observable<ResultList<Job>> {
let params = new HttpParams()
params = this.restService.addRestGetParams(params, pagination, sort)
return this.authHttp.get<ResultList<Job>>(JobService.BASE_JOB_URL, { params })
return this.authHttp.get<ResultList<Job>>(JobService.BASE_JOB_URL + '/' + state, { params })
.map(res => this.restExtractor.convertResultListDateToHuman(res))
.map(res => this.restExtractor.applyToResultListData(res, this.prettyPrintData))
.catch(err => this.restExtractor.handleError(err))
}
private prettyPrintData (obj: Job) {
const handlerInputData = JSON.stringify(obj.handlerInputData, null, 2)
const data = JSON.stringify(obj.data, null, 2)
return Object.assign(obj, { handlerInputData })
return Object.assign(obj, { data })
}
}
......@@ -14,6 +14,11 @@ database:
username: 'peertube'
password: 'peertube'
redis:
hostname: 'localhost'
port: 6379
auth: null
# From the project root directory
storage:
avatars: 'storage/avatars/'
......
......@@ -15,6 +15,11 @@ database:
username: 'peertube'
password: 'peertube'
redis:
hostname: 'localhost'
port: 6379
auth: null
# From the project root directory
storage:
avatars: '/var/www/peertube/storage/avatars/'
......
......@@ -6,4 +6,5 @@ for i in $(seq 1 6); do
rm -f "./config/local-test.json"
rm -f "./config/local-test-$i.json"
createdb "peertube_test$i"
redis-cli KEYS "q-localhost:900$i*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
done
......@@ -2,16 +2,34 @@ import { createReadStream } from 'fs'
import { join } from 'path'
import { createInterface } from 'readline'
import * as winston from 'winston'
import { labelFormatter, loggerFormat, timestampFormatter } from '../server/helpers/logger'
import { labelFormatter } from '../server/helpers/logger'
import { CONFIG } from '../server/initializers/constants'
const excludedKeys = {
level: true,
message: true,
splat: true,
timestamp: true,
label: true
}
function keysExcluder (key, value) {
return excludedKeys[key] === true ? undefined : value
}
const loggerFormat = winston.format.printf((info) => {
let additionalInfos = JSON.stringify(info, keysExcluder, 2)
if (additionalInfos === '{}') additionalInfos = ''
else additionalInfos = ' ' + additionalInfos
return `[${info.label}] ${new Date(info.timestamp).toISOString()} ${info.level}: ${info.message}${additionalInfos}`
})
const logger = new winston.createLogger({
transports: [
new winston.transports.Console({
level: 'debug',
stderrLevels: [],
format: winston.format.combine(
timestampFormatter,
winston.format.splat(),
labelFormatter,
winston.format.colorize(),
......
......@@ -53,10 +53,11 @@ migrate()
// ----------- PeerTube modules -----------
import { installApplication } from './server/initializers'
import { activitypubHttpJobScheduler, transcodingJobScheduler } from './server/lib/jobs'
import { JobQueue } from './server/lib/job-queue'
import { VideosPreviewCache } from './server/lib/cache'
import { apiRouter, clientsRouter, staticRouter, servicesRouter, webfingerRouter, activityPubRouter } from './server/controllers'
import { BadActorFollowScheduler } from './server/lib/schedulers/bad-actor-follow-scheduler'
import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-scheduler'
// ----------- Command line -----------
......@@ -170,9 +171,8 @@ function onDatabaseInitDone () {
server.listen(port, () => {
VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE)
BadActorFollowScheduler.Instance.enable()
activitypubHttpJobScheduler.activate()
transcodingJobScheduler.activate()
RemoveOldJobsScheduler.Instance.enable()
JobQueue.Instance.init()
logger.info('Server listening on port %d', port)
logger.info('Web server: %s', CONFIG.WEBSERVER.URL)
......
import * as express from 'express'
import { ResultList } from '../../../shared'
import { Job, JobType, JobState } from '../../../shared/models'
import { UserRight } from '../../../shared/models/users'
import { getFormattedObjects } from '../../helpers/utils'
import { JobQueue } from '../../lib/job-queue'
import {
asyncMiddleware, authenticate, ensureUserHasRight, jobsSortValidator, setDefaultPagination,
asyncMiddleware,
authenticate,
ensureUserHasRight,
jobsSortValidator,
setDefaultPagination,
setDefaultSort
} from '../../middlewares'
import { paginationValidator } from '../../middlewares/validators'
import { JobModel } from '../../models/job/job'
import { listJobsValidator } from '../../middlewares/validators/jobs'
const jobsRouter = express.Router()
jobsRouter.get('/',
jobsRouter.get('/:state',
authenticate,
ensureUserHasRight(UserRight.MANAGE_JOBS),
paginationValidator,
jobsSortValidator,
setDefaultSort,
setDefaultPagination,
asyncMiddleware(listJobsValidator),
asyncMiddleware(listJobs)
)
......@@ -29,7 +36,26 @@ export {
// ---------------------------------------------------------------------------
async function listJobs (req: express.Request, res: express.Response, next: express.NextFunction) {
const resultList = await JobModel.listForApi(req.query.start, req.query.count, req.query.sort)
const sort = req.query.sort === 'createdAt' ? 'asc' : 'desc'
const jobs = await JobQueue.Instance.listForApi(req.params.state, req.query.start, req.query.count, sort)
const total = await JobQueue.Instance.count(req.params.state)
const result: ResultList<any> = {
total,
data: jobs.map(j => formatJob(j.toJSON()))
}
return res.json(result)
}
return res.json(getFormattedObjects(resultList.data, resultList.total))
function formatJob (job: any): Job {
return {
id: job.id,
state: job.state as JobState,
type: job.type as JobType,
data: job.data,
error: job.error,
createdAt: new Date(parseInt(job.created_at, 10)),
updatedAt: new Date(parseInt(job.updated_at, 10))
}
}
......@@ -123,7 +123,7 @@ function follow (fromActor: ActorModel, targetActor: ActorModel) {
actorFollow.ActorFollower = fromActor
// Send a notification to remote server
await sendFollow(actorFollow, t)
await sendFollow(actorFollow)
})
}
......
......@@ -12,7 +12,7 @@ import {
} from '../../../initializers'
import { fetchRemoteVideoDescription, getVideoActivityPubUrl, shareVideoByServerAndChannel } from '../../../lib/activitypub'
import { sendCreateVideo, sendCreateViewToOrigin, sendCreateViewToVideoFollowers, sendUpdateVideo } from '../../../lib/activitypub/send'
import { transcodingJobScheduler } from '../../../lib/jobs/transcoding-job-scheduler'
import { JobQueue } from '../../../lib/job-queue'
import {
asyncMiddleware, authenticate, paginationValidator, setDefaultSort, setDefaultPagination, videosAddValidator, videosGetValidator,
videosRemoveValidator, videosSearchValidator, videosSortValidator, videosUpdateValidator
......@@ -176,18 +176,9 @@ async function addVideo (req: express.Request, res: express.Response, videoPhysi
)
await Promise.all(tasks)
return sequelizeTypescript.transaction(async t => {
const videoCreated = await sequelizeTypescript.transaction(async t => {
const sequelizeOptions = { transaction: t }
if (CONFIG.TRANSCODING.ENABLED === true) {
// Put uuid because we don't have id auto incremented for now
const dataInput = {
videoUUID: video.uuid
}
await transcodingJobScheduler.createJob(t, 'videoFileOptimizer', dataInput)
}
const videoCreated = await video.save(sequelizeOptions)
// Do not forget to add video channel information to the created video
videoCreated.VideoChannel = res.locals.videoChannel
......@@ -216,6 +207,17 @@ async function addVideo (req: express.Request, res: express.Response, videoPhysi
return videoCreated
})
if (CONFIG.TRANSCODING.ENABLED === true) {
// Put uuid because we don't have id auto incremented for now
const dataInput = {
videoUUID: videoCreated.uuid
}
await JobQueue.Instance.createJob({ type: 'video-file', payload: dataInput })
}
return videoCreated
}
async function updateVideoRetryWrapper (req: express.Request, res: express.Response, next: express.NextFunction) {
......
import { JobState } from '../../../shared/models'
import { exists } from './misc'
const jobStates: JobState[] = [ 'active', 'complete', 'failed', 'inactive', 'delayed' ]
function isValidJobState (value: JobState) {
return exists(value) && jobStates.indexOf(value) !== -1
}
// ---------------------------------------------------------------------------
export {
isValidJobState
}
......@@ -16,6 +16,7 @@ function retryTransactionWrapper <T> (
.catch(err => callback(err))
})
.catch(err => {
console.error(err)
logger.error(options.errorMessage, err)
throw err
})
......
import { IConfig } from 'config'
import { dirname, join } from 'path'
import { JobCategory, JobState, VideoRateType } from '../../shared/models'
import { JobType, VideoRateType } from '../../shared/models'
import { ActivityPubActorType } from '../../shared/models/activitypub'
import { FollowState } from '../../shared/models/actors'
import { VideoPrivacy } from '../../shared/models/videos'
......@@ -12,7 +12,7 @@ let config: IConfig = require('config')
// ---------------------------------------------------------------------------
const LAST_MIGRATION_VERSION = 175
const LAST_MIGRATION_VERSION = 180
// ---------------------------------------------------------------------------
......@@ -26,7 +26,7 @@ const PAGINATION_COUNT_DEFAULT = 15
const SORTABLE_COLUMNS = {
USERS: [ 'id', 'username', 'createdAt' ],
ACCOUNTS: [ 'createdAt' ],
JOBS: [ 'id', 'createdAt' ],
JOBS: [ 'createdAt' ],
VIDEO_ABUSES: [ 'id', 'createdAt' ],
VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ],
VIDEOS: [ 'name', 'duration', 'createdAt', 'views', 'likes' ],
......@@ -61,23 +61,20 @@ const REMOTE_SCHEME = {
WS: 'wss'
}
const JOB_STATES: { [ id: string ]: JobState } = {
PENDING: 'pending',
PROCESSING: 'processing',
ERROR: 'error',
SUCCESS: 'success'
}
const JOB_CATEGORIES: { [ id: string ]: JobCategory } = {
TRANSCODING: 'transcoding',
ACTIVITYPUB_HTTP: 'activitypub-http'
const JOB_ATTEMPTS: { [ id in JobType ]: number } = {
'activitypub-http-broadcast': 5,
'activitypub-http-unicast': 5,
'activitypub-http-fetcher': 5,
'video-file': 1
}
// How many maximum jobs we fetch from the database per cycle
const JOBS_FETCH_LIMIT_PER_CYCLE = {
transcoding: 10,
httpRequest: 20
const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
'activitypub-http-broadcast': 1,
'activitypub-http-unicast': 5,
'activitypub-http-fetcher': 1,
'video-file': 1
}
// 1 minutes
let JOBS_FETCHING_INTERVAL = 60000
// 2 days
const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2
// 1 hour
let SCHEDULER_INTERVAL = 60000 * 60
......@@ -96,6 +93,11 @@ const CONFIG = {
USERNAME: config.get<string>('database.username'),
PASSWORD: config.get<string>('database.password')
},
REDIS: {
HOSTNAME: config.get<string>('redis.hostname'),
PORT: config.get<string>('redis.port'),
AUTH: config.get<string>('redis.auth')
},
STORAGE: {
AVATARS_DIR: buildPath(config.get<string>('storage.avatars')),
LOG_DIR: buildPath(config.get<string>('storage.logs')),
......@@ -284,7 +286,6 @@ const ACTIVITY_PUB = {
PUBLIC: 'https://www.w3.org/ns/activitystreams#Public',
COLLECTION_ITEMS_PER_PAGE: 10,
FETCH_PAGE_LIMIT: 100,
MAX_HTTP_ATTEMPT: 5,
URL_MIME_TYPES: {
VIDEO: Object.keys(VIDEO_MIMETYPE_EXT),
TORRENT: [ 'application/x-bittorrent' ],
......@@ -358,7 +359,6 @@ const OPENGRAPH_AND_OEMBED_COMMENT = '<!-- open graph and oembed tags -->'
// Special constants for a test instance
if (isTestInstance() === true) {
ACTOR_FOLLOW_SCORE.BASE = 20
JOBS_FETCHING_INTERVAL = 1000
REMOTE_SCHEME.HTTP = 'http'
REMOTE_SCHEME.WS = 'ws'
STATIC_MAX_AGE = '0'
......@@ -381,10 +381,8 @@ export {
CONFIG,
CONSTRAINTS_FIELDS,
EMBED_SIZE,
JOB_STATES,
JOBS_FETCH_LIMIT_PER_CYCLE,
JOBS_FETCHING_INTERVAL,
JOB_CATEGORIES,
JOB_CONCURRENCY,
JOB_ATTEMPTS,
LAST_MIGRATION_VERSION,
OAUTH_LIFETIME,
OPENGRAPH_AND_OEMBED_COMMENT,
......@@ -408,7 +406,8 @@ export {
VIDEO_RATE_TYPES,
VIDEO_MIMETYPE_EXT,
AVATAR_MIMETYPE_EXT,
SCHEDULER_INTERVAL
SCHEDULER_INTERVAL,
JOB_COMPLETED_LIFETIME
}
// ---------------------------------------------------------------------------
......
......@@ -9,7 +9,6 @@ import { ActorModel } from '../models/activitypub/actor'
import { ActorFollowModel } from '../models/activitypub/actor-follow'
import { ApplicationModel } from '../models/application/application'
import { AvatarModel } from '../models/avatar/avatar'
import { JobModel } from '../models/job/job'
import { OAuthClientModel } from '../models/oauth/oauth-client'
import { OAuthTokenModel } from '../models/oauth/oauth-token'
import { ServerModel } from '../models/server/server'
......@@ -61,7 +60,6 @@ async function initDatabaseModels (silent: boolean) {
ActorFollowModel,
AvatarModel,
AccountModel,
JobModel,
OAuthClientModel,
OAuthTokenModel,
ServerModel,
......
import { values } from 'lodash'
import * as Sequelize from 'sequelize'
import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
import { shareVideoByServerAndChannel } from '../../lib/activitypub/share'
import { getVideoActivityPubUrl, getVideoChannelActivityPubUrl } from '../../lib/activitypub/url'
import { createLocalAccountWithoutKeys } from '../../lib/user'
import { ApplicationModel } from '../../models/application/application'
import { JOB_CATEGORIES, SERVER_ACTOR_NAME } from '../constants'
import { SERVER_ACTOR_NAME } from '../constants'
async function up (utils: {
transaction: Sequelize.Transaction,
......@@ -161,7 +160,7 @@ async function up (utils: {
{
const data = {
type: Sequelize.ENUM(values(JOB_CATEGORIES)),
type: Sequelize.ENUM('transcoding', 'activitypub-http'),
defaultValue: 'transcoding',
allowNull: false
}
......
import * as Sequelize from 'sequelize'
async function up (utils: {
transaction: Sequelize.Transaction,
queryInterface: Sequelize.QueryInterface,
sequelize: Sequelize.Sequelize
}): Promise<void> {
await utils.queryInterface.dropTable('job')
}
function down (options) {
throw new Error('Not implemented.')
}
export {
up,
down
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment