Verified Commit bd3c2daa authored by Chocobozzz's avatar Chocobozzz
Browse files

Add more tests for resumable uploads

parent eb9dbaab
......@@ -13,28 +13,36 @@ import { resolveUrl, UploaderX } from 'ngx-uploadx'
* };
*/
export class UploaderXFormData extends UploaderX {
async getFileUrl (): Promise<string> {
const headers = {
'X-Upload-Content-Length': this.size.toString(),
'X-Upload-Content-Type': this.file.type || 'application/octet-stream'
}
const previewfile = this.metadata.previewfile as any as File
delete this.metadata.previewfile
const data = objectToFormData(this.metadata)
if (previewfile !== undefined) {
data.append('previewfile', previewfile, previewfile.name)
data.append('thumbnailfile', previewfile, previewfile.name)
}
await this.request({
method: 'POST',
body: data,
url: this.endpoint,
headers
})
const location = this.getValueFromResponse('location')
if (!location) {
throw new Error('Invalid or missing Location header')
}
this.offset = this.responseStatus === 201 ? 0 : undefined
return resolveUrl(location, this.endpoint)
}
}
<div *ngIf="!isUploadingVideo" class="upload-video-container" dragDrop (fileDropped)="setVideoFile($event)">
<div *ngIf="!isUploadingVideo" class="upload-video-container" dragDrop (fileDropped)="onFileDropped($event)">
<div class="first-step-block">
<my-global-icon class="upload-icon" iconName="upload" aria-hidden="true"></my-global-icon>
......
......@@ -31,8 +31,6 @@ export class VideoUploadComponent extends VideoSend implements OnInit, OnDestroy
isUploadingAudioFile = false
isUploadingVideo = false
isUploadingPreviewFile = false
isUpdatingVideo = false
videoUploaded = false
videoUploadPercents = 0
......@@ -47,12 +45,13 @@ export class VideoUploadComponent extends VideoSend implements OnInit, OnDestroy
error: string
enableRetryAfterError: boolean
uploadxOptions: UploadxOptions
// So that it can be accessed in the template
protected readonly DEFAULT_VIDEO_PRIVACY = VideoPrivacy.PUBLIC
protected readonly BASE_VIDEO_UPLOAD_URL = VideoService.BASE_VIDEO_URL + 'upload-resumable'
private uploadxOptions: UploadxOptions
private isUpdatingVideo = false
constructor (
protected formValidatorService: FormValidatorService,
protected loadingBar: LoadingBarService,
......@@ -65,27 +64,13 @@ export class VideoUploadComponent extends VideoSend implements OnInit, OnDestroy
private router: Router,
private hooks: HooksService,
private resumableUploadService: UploadxService
) {
) {
super()
const self = this
this.uploadxOptions = {
endpoint: this.BASE_VIDEO_UPLOAD_URL,
multiple: false,
token: this.authService.getAccessToken(),
metadata: {
waitTranscoding: true,
commentsEnabled: true,
downloadEnabled: true,
get channelId () {
return self.firstStepChannelId
},
get nsfw () {
return self.serverConfig.instance.isNSFW
},
privacy: VideoPrivacy.PRIVATE.toString()
},
uploaderClass: UploaderXFormData
}
}
......@@ -109,6 +94,7 @@ export class VideoUploadComponent extends VideoSend implements OnInit, OnDestroy
url: state.url
})
break
case 'cancelled':
this.isUploadingVideo = false
this.videoUploadPercents = 0
......@@ -117,15 +103,19 @@ export class VideoUploadComponent extends VideoSend implements OnInit, OnDestroy
this.enableRetryAfterError = false
this.error = ''
break
case 'queue':
this.closeFirstStep(state.name)
break
case 'uploading':
this.videoUploadPercents = state.progress
break
case 'paused':
this.notifier.info($localize`Upload cancelled`)
break
case 'complete':
this.videoUploaded = true
this.videoUploadPercents = 100
......@@ -145,9 +135,7 @@ export class VideoUploadComponent extends VideoSend implements OnInit, OnDestroy
})
this.resumableUploadService.events
.subscribe(
state => this.onUploadVideoOngoing(state)
)
.subscribe(state => this.onUploadVideoOngoing(state))
}
ngAfterViewInit () {
......@@ -175,37 +163,14 @@ export class VideoUploadComponent extends VideoSend implements OnInit, OnDestroy
}
}
getVideoFile () {
return this.videofileInput.nativeElement.files[0]
}
getAudioUploadLabel () {
const videofile = this.getVideoFile()
if (!videofile) return $localize`Upload`
return $localize`Upload ${videofile.name}`
}
retryUpload () {
this.enableRetryAfterError = false
this.error = ''
}
cancelUpload () {
this.resumableUploadService.control({ action: 'cancel' })
}
uploadAudio () {
this.uploadFile(this.getVideoFile(), this.previewfileUpload)
}
setVideoFile (files: FileList) {
onFileDropped (files: FileList) {
this.videofileInput.nativeElement.files = files
this.onFileChange({ target: this.videofileInput.nativeElement })
}
onFileChange (event: { target: any }) {
const file = event.target.files[0]
onFileChange (event: Event | { target: HTMLInputElement }) {
const file = (event.target as HTMLInputElement).files[0]
if (!file) return
......@@ -221,6 +186,19 @@ export class VideoUploadComponent extends VideoSend implements OnInit, OnDestroy
this.uploadFile(file)
}
uploadAudio () {
this.uploadFile(this.getInputVideoFile(), this.previewfileUpload)
}
retryUpload () {
this.enableRetryAfterError = false
this.error = ''
}
cancelUpload () {
this.resumableUploadService.control({ action: 'cancel' })
}
isPublishingButtonDisabled () {
return !this.form.valid ||
this.isUpdatingVideo === true ||
......@@ -228,6 +206,13 @@ export class VideoUploadComponent extends VideoSend implements OnInit, OnDestroy
!this.videoUploadedIds.id
}
getAudioUploadLabel () {
const videofile = this.getInputVideoFile()
if (!videofile) return $localize`Upload`
return $localize`Upload ${videofile.name}`
}
updateSecondStep () {
if (this.isPublishingButtonDisabled() || !this.checkForm()) {
return
......@@ -258,16 +243,27 @@ export class VideoUploadComponent extends VideoSend implements OnInit, OnDestroy
)
}
private getInputVideoFile () {
return this.videofileInput.nativeElement.files[0]
}
private uploadFile (file: File, previewfile?: File) {
const metadata = {
waitTranscoding: true,
commentsEnabled: true,
downloadEnabled: true,
channelId: this.firstStepChannelId,
nsfw: this.serverConfig.instance.isNSFW,
privacy: VideoPrivacy.PRIVATE.toString(),
filename: file.name,
previewfile: previewfile as any
}
this.resumableUploadService.handleFiles(file, {
...this.uploadxOptions,
metadata: {
...this.uploadxOptions.metadata,
filename: file.name,
previewfile: previewfile as any
}
metadata
})
this.isUploadingPreviewFile = false
this.isUploadingVideo = true
}
......@@ -313,8 +309,7 @@ export class VideoUploadComponent extends VideoSend implements OnInit, OnDestroy
const videoQuotaUsedBytes = bytePipes.transform(this.userVideoQuotaUsed, 0)
const videoQuotaBytes = bytePipes.transform(videoQuota, 0)
const msg = $localize`Your video quota is exceeded with this video (
video size: ${videoSizeBytes}, used: ${videoQuotaUsedBytes}, quota: ${videoQuotaBytes})`
const msg = $localize`Your video quota is exceeded with this video (video size: ${videoSizeBytes}, used: ${videoQuotaUsedBytes}, quota: ${videoQuotaBytes})`
this.notifier.error(msg)
return false
......
......@@ -2,6 +2,7 @@ import * as express from 'express'
import { move } from 'fs-extra'
import { extname } from 'path'
import toInt from 'validator/lib/toInt'
import { getResumableUploadPath } from '@server/helpers/upload'
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
import { changeVideoChannelShare } from '@server/lib/activitypub/share'
import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
......@@ -10,15 +11,15 @@ import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnail
import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths'
import { getServerActor } from '@server/models/application/application'
import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
import { DiskStorageOptions, Metadata, uploadx } from '@uploadx/core'
import { uploadx } from '@uploadx/core'
import { VideoCreate, VideosCommonQuery, VideoState, VideoUpdate } from '../../../../shared'
import { HttpMethod, HttpStatusCode } from '../../../../shared/core-utils/miscs'
import { HttpStatusCode } from '../../../../shared/core-utils/miscs'
import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils'
import { buildNSFWFilter, createReqFiles, getCountVideos } from '../../../helpers/express-utils'
import { getMetadataFromFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils'
import { logger, loggerTagsFactory } from '../../../helpers/logger'
import { getFormattedObjects, getResumableUploadPath } from '../../../helpers/utils'
import { getFormattedObjects } from '../../../helpers/utils'
import { CONFIG } from '../../../initializers/config'
import {
DEFAULT_AUDIO_RESOLUTION,
......@@ -43,8 +44,6 @@ import {
authenticate,
checkVideoFollowConstraints,
commonVideosFiltersValidator,
executeIfPOST,
onlyAllowMethods,
optionalAuthenticate,
paginationValidator,
setDefaultPagination,
......@@ -74,7 +73,7 @@ import { watchingRouter } from './watching'
const lTags = loggerTagsFactory('api', 'video')
const auditLogger = auditLoggerFactory('videos')
const videosRouter = express.Router()
const uploadxOptions = { directory: getResumableUploadPath() } as DiskStorageOptions
const uploadxMiddleware = uploadx.upload({ directory: getResumableUploadPath() })
const reqVideoFileAdd = createReqFiles(
[ 'videofile', 'thumbnailfile', 'previewfile' ],
......@@ -135,12 +134,21 @@ videosRouter.post('/upload',
asyncRetryTransactionMiddleware(addVideoLegacy)
)
videosRouter.all('/upload-resumable',
onlyAllowMethods([ HttpMethod.DELETE, HttpMethod.POST, HttpMethod.PUT ]), // uploadx also allows GET and PATCH
videosRouter.post('/upload-resumable',
authenticate,
executeIfPOST(reqVideoFileAddResumable),
executeIfPOST(asyncMiddleware(videosAddResumableInitValidator)),
uploadx.upload(uploadxOptions), // uploadx doesn't use call next() before the file upload completes
reqVideoFileAddResumable,
asyncMiddleware(videosAddResumableInitValidator),
uploadxMiddleware
)
videosRouter.delete('/upload-resumable',
authenticate,
uploadxMiddleware
)
videosRouter.put('/upload-resumable',
authenticate,
uploadxMiddleware, // uploadx doesn't use call next() before the file upload completes
asyncMiddleware(videosAddResumableValidator),
asyncMiddleware(addVideoResumable)
)
......@@ -185,19 +193,19 @@ export {
// ---------------------------------------------------------------------------
function listVideoCategories (req: express.Request, res: express.Response) {
function listVideoCategories (_req: express.Request, res: express.Response) {
res.json(VIDEO_CATEGORIES)
}
function listVideoLicences (req: express.Request, res: express.Response) {
function listVideoLicences (_req: express.Request, res: express.Response) {
res.json(VIDEO_LICENCES)
}
function listVideoLanguages (req: express.Request, res: express.Response) {
function listVideoLanguages (_req: express.Request, res: express.Response) {
res.json(VIDEO_LANGUAGES)
}
function listVideoPrivacies (req: express.Request, res: express.Response) {
function listVideoPrivacies (_req: express.Request, res: express.Response) {
res.json(VIDEO_PRIVACIES)
}
......@@ -213,29 +221,34 @@ async function addVideoLegacy (req: express.Request, res: express.Response) {
const videoInfo: VideoCreate = req.body
const files = req.files
return addVideo(req, res, { videoPhysicalFile, videoInfo, files })
return addVideo({ res, videoPhysicalFile, videoInfo, files })
}
async function addVideoResumable (req: express.Request, res: express.Response) {
async function addVideoResumable (_req: express.Request, res: express.Response) {
const videoPhysicalFile = res.locals.videoFileResumable
const videoInfo = videoPhysicalFile.metadata
const files = { previewfile: videoInfo.previewfile as Express.Multer.File[] }
const files = { previewfile: videoInfo.previewfile }
return addVideo(req, res, { videoPhysicalFile, videoInfo, files })
return addVideo({ res, videoPhysicalFile, videoInfo, files })
}
async function addVideo (req: express.Request, res: express.Response, parameters: {
videoPhysicalFile: { duration: number, filename: string, size: number, path: string }
videoInfo: VideoCreate | Metadata
files: UploadVideoFiles | Express.Multer.File[]
async function addVideo (options: {
res: express.Response
videoPhysicalFile: express.VideoUploadFile
videoInfo: VideoCreate
files: express.UploadFiles
}) {
const { videoPhysicalFile, videoInfo, files } = parameters
const { res, videoPhysicalFile, videoInfo, files } = options
const videoChannel = res.locals.videoChannel
const user = res.locals.oauth.token.User
const videoData = buildLocalVideoFromReq(videoInfo as VideoCreate, videoChannel.id)
videoData.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
videoData.duration = videoPhysicalFile['duration'] // duration was added by a previous middleware
const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id)
videoData.state = CONFIG.TRANSCODING.ENABLED
? VideoState.TO_TRANSCODE
: VideoState.PUBLISHED
videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware
const video = new VideoModel(videoData) as MVideoFullLight
video.VideoChannel = videoChannel
......
import 'multer'
import validator from 'validator'
import { UploadFilesForCheck } from 'express'
import { sep } from 'path'
import { FileUploadMetadata } from 'express'
import validator from 'validator'
function exists (value: any) {
return value !== undefined && value !== null
......@@ -109,7 +109,7 @@ function isFileFieldValid (
}
function isFileMimeTypeValid (
files: { [ fieldname: string ]: FileUploadMetadata[] } | FileUploadMetadata[],
files: UploadFilesForCheck,
mimeTypeRegex: string,
field: string,
optional = false
......
import { UploadFilesForCheck } from 'express'
import { values } from 'lodash'
import * as magnetUtil from 'magnet-uri'
import validator from 'validator'
import { VideoFilter, VideoPrivacy, VideoRateType } from '../../../shared'
import {
......@@ -6,14 +8,12 @@ import {
MIMETYPES,
VIDEO_CATEGORIES,
VIDEO_LICENCES,
VIDEO_LIVE,
VIDEO_PRIVACIES,
VIDEO_RATE_TYPES,
VIDEO_STATES,
VIDEO_LIVE
VIDEO_STATES
} from '../../initializers/constants'
import { exists, isArray, isDateValid, isFileMimeTypeValid, isFileValid } from './misc'
import * as magnetUtil from 'magnet-uri'
import { FileUploadMetadata } from 'express'
const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS
......@@ -82,7 +82,7 @@ function isVideoFileExtnameValid (value: string) {
return exists(value) && (value === VIDEO_LIVE.EXTENSION || MIMETYPES.VIDEO.EXT_MIMETYPE[value] !== undefined)
}
function isVideoFileMimeTypeValid (files: { [ fieldname: string ]: FileUploadMetadata[] } | FileUploadMetadata[]) {
function isVideoFileMimeTypeValid (files: UploadFilesForCheck) {
return isFileMimeTypeValid(files, MIMETYPES.VIDEO.MIMETYPES_REGEX, 'videofile')
}
......
......@@ -2,7 +2,7 @@ import * as express from 'express'
import * as multer from 'multer'
import { REMOTE_SCHEME } from '../initializers/constants'
import { logger } from './logger'
import { deleteFileAsync, generateRandomString } from './utils'
import { deleteFileAndCatch, generateRandomString } from './utils'
import { extname } from 'path'
import { isArray } from './custom-validators/misc'
import { CONFIG } from '../initializers/config'
......@@ -36,15 +36,15 @@ function cleanUpReqFiles (req: { files: { [fieldname: string]: Express.Multer.Fi
if (!files) return
if (isArray(files)) {
(files as Express.Multer.File[]).forEach(f => deleteFileAsync(f.path))
(files as Express.Multer.File[]).forEach(f => deleteFileAndCatch(f.path))
return
}
for (const key of Object.keys(files)) {
const file = files[key]
if (isArray(file)) file.forEach(f => deleteFileAsync(f.path))
else deleteFileAsync(file.path)
if (isArray(file)) file.forEach(f => deleteFileAndCatch(f.path))
else deleteFileAndCatch(file.path)
}
}
......
import { join } from 'path'
import { RESUMABLE_UPLOAD_DIRECTORY } from '../initializers/constants'
function getResumableUploadPath (filename?: string) {
if (filename) return join(RESUMABLE_UPLOAD_DIRECTORY, filename)
return RESUMABLE_UPLOAD_DIRECTORY
}
// ---------------------------------------------------------------------------
export {
getResumableUploadPath
}
......@@ -7,12 +7,7 @@ import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-ut
import { logger } from './logger'
import { RESUMABLE_UPLOAD_DIRECTORY } from '../initializers/constants'
function deleteFile (path: string) {
return remove(path)
.catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err }))
}
function deleteFileAsync (path: string) {
function deleteFileAndCatch (path: string) {
remove(path)
.catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err }))
}
......@@ -86,20 +81,14 @@ function getUUIDFromFilename (filename: string) {
return result[0]
}
function getResumableUploadPath (filename?: string) {
return filename ? join(RESUMABLE_UPLOAD_DIRECTORY, filename) : RESUMABLE_UPLOAD_DIRECTORY
}
// ---------------------------------------------------------------------------
export {
deleteFile,
deleteFileAsync,
deleteFileAndCatch,
generateRandomString,
getFormattedObjects,
getSecureTorrentName,
getServerCommit,
generateVideoImportTmpPath,
getUUIDFromFilename,
getResumableUploadPath
getUUIDFromFilename
}
import { VideoUploadFile } from 'express'
import { PathLike } from 'fs-extra'
import { Transaction } from 'sequelize/types'
import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger'
import { afterCommitIfTransaction } from '@server/helpers/database-utils'
import { logger } from '@server/helpers/logger'
import { AbuseModel } from '@server/models/abuse/abuse'
import { VideoAbuseModel } from '@server/models/abuse/video-abuse'
......@@ -28,7 +30,6 @@ import { VideoModel } from '../models/video/video'
import { VideoCommentModel } from '../models/video/video-comment'
import { sendAbuse } from './activitypub/send/send-flag'
import { Notifier } from './notifier'
import { afterCommitIfTransaction } from '@server/helpers/database-utils'
export type AcceptResult = {
accepted: boolean
......@@ -38,7 +39,7 @@ export type AcceptResult = {
// Can be filtered by plugins
function isLocalVideoAccepted (object: {
videoBody: VideoCreate
videoFile: Express.Multer.File & { duration?: number }
videoFile: VideoUploadFile
user: UserModel
}): AcceptResult {
return { accepted: true }
......
import { readdir, stat, Stats } from 'fs-extra'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { deleteFile, getResumableUploadPath } from '@server/helpers/utils'
import { getResumableUploadPath } from '@server/helpers/upload'
import { deleteFileAndCatch } from '@server/helpers/utils'
import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants'
import { AbstractScheduler } from './abstract-scheduler'
import { METAFILE_EXTNAME } from '@uploadx/core'
import { AbstractScheduler } from './abstract-scheduler'
const lTags = loggerTagsFactory('scheduler', 'resumable-upload', 'cleaner')
......@@ -32,8 +33,8 @@ export class RemoveDanglingResumableUploadsScheduler extends AbstractScheduler {
if (statResult.ctimeMs < limit) {
await Promise.all([
deleteFile(filePath),
deleteFile(filePath + METAFILE_EXTNAME) // also delete the .META file, which was not updated since the initial POST request
deleteFileAndCatch(filePath),
deleteFileAndCatch(filePath + METAFILE_EXTNAME) // also delete the .META file, which was not updated since the initial POST request
])
}
}
......
import { UploadFiles } from 'express'
import { Transaction } from 'sequelize/types'
import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants'
import { sequelizeTypescript } from '@server/initializers/database'
......@@ -10,7 +11,6 @@ import { federateVideoIfNeeded } from './activitypub/videos'