Verified Commit e2600d8b authored by Chocobozzz's avatar Chocobozzz
Browse files

Add avatar to prune script

parent 5bb2eb56
Pipeline #157707 passed with stage
in 61 minutes and 50 seconds
......@@ -25,7 +25,7 @@ run()
async function run () {
await initDatabaseModels(true)
const video = await VideoModel.loadByUUIDWithFile(program['video'])
const video = await VideoModel.loadByUUID(program['video'])
if (!video) throw new Error('Video not found.')
if (video.isOwned() === false) throw new Error('Cannot import files of a non owned video.')
......
......@@ -29,7 +29,7 @@ run()
async function run () {
await initDatabaseModels(true)
const video = await VideoModel.loadByUUIDWithFile(program['video'])
const video = await VideoModel.loadByUUID(program['video'])
if (!video) throw new Error('Video not found.')
const dataInput: VideoTranscodingPayload = program.resolution !== undefined
......
......@@ -3,9 +3,12 @@ import { join } from 'path'
import { CONFIG } from '../server/initializers/config'
import { VideoModel } from '../server/models/video/video'
import { initDatabaseModels } from '../server/initializers'
import { remove, readdir } from 'fs-extra'
import { readdir, remove } from 'fs-extra'
import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
import * as Bluebird from 'bluebird'
import { getUUIDFromFilename } from '../server/helpers/utils'
import { ThumbnailModel } from '../server/models/video/thumbnail'
import { AvatarModel } from '../server/models/avatar/avatar'
run()
.then(() => process.exit(0))
......@@ -17,25 +20,19 @@ run()
async function run () {
await initDatabaseModels(true)
const storageOnlyOwnedToPrune = [
CONFIG.STORAGE.VIDEOS_DIR,
CONFIG.STORAGE.TORRENTS_DIR,
CONFIG.STORAGE.REDUNDANCY_DIR
]
let toDelete: string[] = []
const storageForAllToPrune = [
CONFIG.STORAGE.PREVIEWS_DIR,
CONFIG.STORAGE.THUMBNAILS_DIR
]
toDelete = toDelete.concat(
await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesVideoExist(true)),
await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesVideoExist(true)),
let toDelete: string[] = []
for (const directory of storageOnlyOwnedToPrune) {
toDelete = toDelete.concat(await pruneDirectory(directory, true))
}
await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
for (const directory of storageForAllToPrune) {
toDelete = toDelete.concat(await pruneDirectory(directory, false))
}
await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true)),
await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false)),
await pruneDirectory(CONFIG.STORAGE.AVATARS_DIR, doesAvatarExist)
)
const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
......@@ -61,30 +58,79 @@ async function run () {
}
}
async function pruneDirectory (directory: string, onlyOwned = false) {
type ExistFun = (file: string) => Promise<boolean>
async function pruneDirectory (directory: string, existFun: ExistFun) {
const files = await readdir(directory)
const toDelete: string[] = []
for (const file of files) {
await Bluebird.map(files, async file => {
if (await existFun(file) !== true) {
toDelete.push(join(directory, file))
}
}, { concurrency: 20 })
return toDelete
}
function doesVideoExist (keepOnlyOwned: boolean) {
return async (file: string) => {
const uuid = getUUIDFromFilename(file)
let video: VideoModel
let localRedundancy: boolean
const video = await VideoModel.loadByUUID(uuid)
if (uuid) {
video = await VideoModel.loadByUUIDWithFile(uuid)
localRedundancy = await VideoRedundancyModel.isLocalByVideoUUIDExists(uuid)
}
return video && (keepOnlyOwned === false || video.isOwned())
}
}
if (
!uuid ||
!video ||
(onlyOwned === true && (video.isOwned() === false && localRedundancy === false))
) {
toDelete.push(join(directory, file))
function doesThumbnailExist (keepOnlyOwned: boolean) {
return async (file: string) => {
const thumbnail = await ThumbnailModel.loadByName(file)
if (!thumbnail) return false
if (keepOnlyOwned) {
const video = await VideoModel.load(thumbnail.videoId)
if (video.isOwned() === false) return false
}
return true
}
}
return toDelete
async function doesAvatarExist (file: string) {
const avatar = await AvatarModel.loadByName(file)
return !!avatar
}
async function doesRedundancyExist (file: string) {
const uuid = getUUIDFromFilename(file)
const video = await VideoModel.loadWithFiles(uuid)
if (!video) return false
const isPlaylist = file.includes('.') === false
if (isPlaylist) {
const p = video.getHLSPlaylist()
if (!p) return false
const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
return !!redundancy
}
const resolution = parseInt(file.split('-')[5], 10)
if (isNaN(resolution)) {
console.error('Cannot prune %s because we cannot guess guess the resolution.', file)
return true
}
const videoFile = video.getFile(resolution)
if (!videoFile) {
console.error('Cannot find file of video %s - %d', video.url, resolution)
return true
}
const redundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id)
return !!redundancy
}
async function askConfirmation () {
......
......@@ -49,7 +49,12 @@ async function getAvatar (req: express.Request, res: express.Response) {
logger.info('Lazy serve remote avatar image %s.', avatar.fileUrl)
await pushAvatarProcessInQueue({ filename: avatar.filename, fileUrl: avatar.fileUrl })
try {
await pushAvatarProcessInQueue({ filename: avatar.filename, fileUrl: avatar.fileUrl })
} catch (err) {
logger.warn('Cannot process remote avatar %s.', avatar.fileUrl, { err })
return res.sendStatus(404)
}
avatar.onDisk = true
avatar.save()
......
......@@ -18,7 +18,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
}
async getFilePathImpl (videoUUID: string) {
const video = await VideoModel.loadByUUIDWithFile(videoUUID)
const video = await VideoModel.loadByUUID(videoUUID)
if (!video) return undefined
if (video.isOwned()) return { isOwned: true, path: video.getPreview().getPath() }
......
......@@ -100,6 +100,16 @@ export class ThumbnailModel extends Model<ThumbnailModel> {
.catch(err => logger.error('Cannot remove thumbnail file %s.', instance.filename, err))
}
static loadByName (filename: string) {
const query = {
where: {
filename
}
}
return ThumbnailModel.findOne(query)
}
static generateDefaultPreviewName (videoUUID: string) {
return videoUUID + '.jpg'
}
......
......@@ -119,6 +119,7 @@ import { CONFIG } from '../../initializers/config'
import { ThumbnailModel } from './thumbnail'
import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
import { createTorrentPromise } from '../../helpers/webtorrent'
import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
const indexes: (ModelIndexesOptions & { where?: WhereOptions })[] = [
......@@ -1422,15 +1423,23 @@ export class VideoModel extends Model<VideoModel> {
return VideoModel.scope(ScopeNames.WITH_THUMBNAILS).findOne(options)
}
static loadWithFiles (id: number, t?: Transaction, logging?: boolean) {
static loadWithFiles (id: number | string, t?: Transaction, logging?: boolean) {
const where = buildWhereIdOrUUID(id)
const query = {
where,
transaction: t,
logging
}
return VideoModel.scope([
ScopeNames.WITH_FILES,
ScopeNames.WITH_STREAMING_PLAYLISTS,
ScopeNames.WITH_THUMBNAILS
]).findByPk(id, { transaction: t, logging })
]).findOne(query)
}
static loadByUUIDWithFile (uuid: string) {
static loadByUUID (uuid: string) {
const options = {
where: {
uuid
......@@ -1754,7 +1763,7 @@ export class VideoModel extends Model<VideoModel> {
return maxBy(this.VideoFiles, file => file.resolution)
}
getFile (resolution: VideoResolution) {
getFile (resolution: number) {
if (Array.isArray(this.VideoFiles) === false) return undefined
return this.VideoFiles.find(f => f.resolution === resolution)
......@@ -1893,6 +1902,12 @@ export class VideoModel extends Model<VideoModel> {
return `/api/${API_VERSION}/videos/${this.uuid}/description`
}
getHLSPlaylist () {
if (!this.VideoStreamingPlaylists) return undefined
return this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
}
removeFile (videoFile: VideoFileModel, isRedundancy = false) {
const baseDir = isRedundancy ? CONFIG.STORAGE.REDUNDANCY_DIR : CONFIG.STORAGE.VIDEOS_DIR
......
......@@ -4,5 +4,6 @@ import './create-transcoding-job'
import './optimize-old-videos'
import './peertube'
import './plugins'
import './prune-storage'
import './reset-password'
import './update-host'
/* tslint:disable:no-unused-expression */
import 'mocha'
import * as chai from 'chai'
import { waitJobs } from '../../../shared/extra-utils/server/jobs'
import {
buildServerDirectory,
cleanupTests,
createVideoPlaylist,
doubleFollow,
execCLI,
flushAndRunMultipleServers,
getAccount,
getEnvCli,
ServerInfo,
setAccessTokensToServers, setDefaultVideoChannel,
updateMyAvatar,
uploadVideo,
wait
} from '../../../shared/extra-utils'
import { Account, VideoPlaylistPrivacy } from '../../../shared/models'
import { createFile, readdir } from 'fs-extra'
import * as uuidv4 from 'uuid/v4'
import { join } from 'path'
import * as request from 'supertest'
const expect = chai.expect
async function countFiles (internalServerNumber: number, directory: string) {
const files = await readdir(buildServerDirectory(internalServerNumber, directory))
return files.length
}
async function assertNotExists (internalServerNumber: number, directory: string, substring: string) {
const files = await readdir(buildServerDirectory(internalServerNumber, directory))
for (const f of files) {
expect(f).to.not.contain(substring)
}
}
async function assertCountAreOkay (servers: ServerInfo[]) {
for (const server of servers) {
const videosCount = await countFiles(server.internalServerNumber, 'videos')
expect(videosCount).to.equal(8)
const torrentsCount = await countFiles(server.internalServerNumber, 'torrents')
expect(torrentsCount).to.equal(8)
const previewsCount = await countFiles(server.internalServerNumber, 'previews')
expect(previewsCount).to.equal(2)
const thumbnailsCount = await countFiles(server.internalServerNumber, 'thumbnails')
expect(thumbnailsCount).to.equal(6)
const avatarsCount = await countFiles(server.internalServerNumber, 'avatars')
expect(avatarsCount).to.equal(2)
}
}
describe('Test prune storage scripts', function () {
let servers: ServerInfo[]
const badNames: { [ directory: string ]: string[] } = {}
before(async function () {
this.timeout(120000)
servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true } })
await setAccessTokensToServers(servers)
await setDefaultVideoChannel(servers)
for (const server of servers) {
await uploadVideo(server.url, server.accessToken, { name: 'video 1' })
await uploadVideo(server.url, server.accessToken, { name: 'video 2' })
await updateMyAvatar({ url: server.url, accessToken: server.accessToken, fixture: 'avatar.png' })
await createVideoPlaylist({
url: server.url,
token: server.accessToken,
playlistAttrs: {
displayName: 'playlist',
privacy: VideoPlaylistPrivacy.PUBLIC,
videoChannelId: server.videoChannel.id,
thumbnailfile: 'thumbnail.jpg'
}
})
}
await doubleFollow(servers[0], servers[1])
// Lazy load the remote avatar
{
const res = await getAccount(servers[ 0 ].url, 'root@localhost:' + servers[ 1 ].port)
const account: Account = res.body
await request('http://localhost:' + servers[ 0 ].port).get(account.avatar.path).expect(200)
}
{
const res = await getAccount(servers[ 1 ].url, 'root@localhost:' + servers[ 0 ].port)
const account: Account = res.body
await request('http://localhost:' + servers[ 1 ].port).get(account.avatar.path).expect(200)
}
await wait(1000)
await waitJobs(servers)
})
it('Should have the files on the disk', async function () {
await assertCountAreOkay(servers)
})
it('Should create some dirty files', async function () {
for (let i = 0; i < 2; i++) {
{
const base = buildServerDirectory(servers[0].internalServerNumber, 'videos')
const n1 = uuidv4() + '.mp4'
const n2 = uuidv4() + '.webm'
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['videos'] = [ n1, n2 ]
}
{
const base = buildServerDirectory(servers[0].internalServerNumber, 'torrents')
const n1 = uuidv4() + '-240.torrent'
const n2 = uuidv4() + '-480.torrent'
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['torrents'] = [ n1, n2 ]
}
{
const base = buildServerDirectory(servers[0].internalServerNumber, 'thumbnails')
const n1 = uuidv4() + '.jpg'
const n2 = uuidv4() + '.jpg'
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['thumbnails'] = [ n1, n2 ]
}
{
const base = buildServerDirectory(servers[0].internalServerNumber, 'previews')
const n1 = uuidv4() + '.jpg'
const n2 = uuidv4() + '.jpg'
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['previews'] = [ n1, n2 ]
}
{
const base = buildServerDirectory(servers[0].internalServerNumber, 'avatars')
const n1 = uuidv4() + '.png'
const n2 = uuidv4() + '.jpg'
await createFile(join(base, n1))
await createFile(join(base, n2))
badNames['avatars'] = [ n1, n2 ]
}
}
})
it('Should run prune storage', async function () {
this.timeout(30000)
const env = getEnvCli(servers[0])
await execCLI(`echo y | ${env} npm run prune-storage`)
})
it('Should have removed files', async function () {
await assertCountAreOkay(servers)
for (const directory of Object.keys(badNames)) {
for (const name of badNames[directory]) {
await assertNotExists(servers[0].internalServerNumber, directory, name)
}
}
})
after(async function () {
await cleanupTests(servers)
})
})
......@@ -44,6 +44,10 @@ function root () {
return root
}
function buildServerDirectory (internalServerNumber: number, directory: string) {
return join(root(), 'test' + internalServerNumber, directory)
}
async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') {
const res = await request(url)
.get(imagePath)
......@@ -105,6 +109,7 @@ async function generateHighBitrateVideo () {
export {
dateIsValid,
wait,
buildServerDirectory,
webtorrentAdd,
immutableAssign,
testImage,
......
......@@ -19,7 +19,7 @@ import {
import * as validator from 'validator'
import { VideoDetails, VideoPrivacy } from '../../models/videos'
import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, loadLanguages, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants'
import { dateIsValid, webtorrentAdd } from '../miscs/miscs'
import { dateIsValid, webtorrentAdd, buildServerDirectory } from '../miscs/miscs'
loadLanguages()
......@@ -308,10 +308,8 @@ async function checkVideoFilesWereRemoved (
join('redundancy', 'hls')
]
) {
const testDirectory = 'test' + serverNumber
for (const directory of directories) {
const directoryPath = join(root(), testDirectory, directory)
const directoryPath = buildServerDirectory(serverNumber, directory)
const directoryExists = await pathExists(directoryPath)
if (directoryExists === false) continue
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment