Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Framasoft
PeerTube
PeerTube
Commits
e2600d8b
Verified
Commit
e2600d8b
authored
Aug 09, 2019
by
Chocobozzz
Browse files
Add avatar to prune script
parent
5bb2eb56
Pipeline
#157707
passed with stage
in 61 minutes and 50 seconds
Changes
11
Pipelines
3
Hide whitespace changes
Inline
Side-by-side
scripts/create-import-video-file-job.ts
View file @
e2600d8b
...
...
@@ -25,7 +25,7 @@ run()
async
function
run
()
{
await
initDatabaseModels
(
true
)
const
video
=
await
VideoModel
.
loadByUUID
WithFile
(
program
[
'
video
'
])
const
video
=
await
VideoModel
.
loadByUUID
(
program
[
'
video
'
])
if
(
!
video
)
throw
new
Error
(
'
Video not found.
'
)
if
(
video
.
isOwned
()
===
false
)
throw
new
Error
(
'
Cannot import files of a non owned video.
'
)
...
...
scripts/create-transcoding-job.ts
View file @
e2600d8b
...
...
@@ -29,7 +29,7 @@ run()
async
function
run
()
{
await
initDatabaseModels
(
true
)
const
video
=
await
VideoModel
.
loadByUUID
WithFile
(
program
[
'
video
'
])
const
video
=
await
VideoModel
.
loadByUUID
(
program
[
'
video
'
])
if
(
!
video
)
throw
new
Error
(
'
Video not found.
'
)
const
dataInput
:
VideoTranscodingPayload
=
program
.
resolution
!==
undefined
...
...
scripts/prune-storage.ts
View file @
e2600d8b
...
...
@@ -3,9 +3,12 @@ import { join } from 'path'
import
{
CONFIG
}
from
'
../server/initializers/config
'
import
{
VideoModel
}
from
'
../server/models/video/video
'
import
{
initDatabaseModels
}
from
'
../server/initializers
'
import
{
re
move
,
readdir
}
from
'
fs-extra
'
import
{
re
addir
,
remove
}
from
'
fs-extra
'
import
{
VideoRedundancyModel
}
from
'
../server/models/redundancy/video-redundancy
'
import
*
as
Bluebird
from
'
bluebird
'
import
{
getUUIDFromFilename
}
from
'
../server/helpers/utils
'
import
{
ThumbnailModel
}
from
'
../server/models/video/thumbnail
'
import
{
AvatarModel
}
from
'
../server/models/avatar/avatar
'
run
()
.
then
(()
=>
process
.
exit
(
0
))
...
...
@@ -17,25 +20,19 @@ run()
async
function
run
()
{
await
initDatabaseModels
(
true
)
const
storageOnlyOwnedToPrune
=
[
CONFIG
.
STORAGE
.
VIDEOS_DIR
,
CONFIG
.
STORAGE
.
TORRENTS_DIR
,
CONFIG
.
STORAGE
.
REDUNDANCY_DIR
]
let
toDelete
:
string
[]
=
[]
const
storageForAllToPrune
=
[
CONFIG
.
STORAGE
.
PREVIEWS_DIR
,
CONFIG
.
STORAGE
.
THUMBNAILS_DIR
]
toDelete
=
toDelete
.
concat
(
await
pruneDirectory
(
CONFIG
.
STORAGE
.
VIDEOS_DIR
,
doesVideoExist
(
true
)),
await
pruneDirectory
(
CONFIG
.
STORAGE
.
TORRENTS_DIR
,
doesVideoExist
(
true
)),
let
toDelete
:
string
[]
=
[]
for
(
const
directory
of
storageOnlyOwnedToPrune
)
{
toDelete
=
toDelete
.
concat
(
await
pruneDirectory
(
directory
,
true
))
}
await
pruneDirectory
(
CONFIG
.
STORAGE
.
REDUNDANCY_DIR
,
doesRedundancyExist
),
for
(
const
directory
of
storageForAllToPrune
)
{
toDelete
=
toDelete
.
concat
(
await
pruneDirectory
(
directory
,
false
))
}
await
pruneDirectory
(
CONFIG
.
STORAGE
.
PREVIEWS_DIR
,
doesThumbnailExist
(
true
)),
await
pruneDirectory
(
CONFIG
.
STORAGE
.
THUMBNAILS_DIR
,
doesThumbnailExist
(
false
)),
await
pruneDirectory
(
CONFIG
.
STORAGE
.
AVATARS_DIR
,
doesAvatarExist
)
)
const
tmpFiles
=
await
readdir
(
CONFIG
.
STORAGE
.
TMP_DIR
)
toDelete
=
toDelete
.
concat
(
tmpFiles
.
map
(
t
=>
join
(
CONFIG
.
STORAGE
.
TMP_DIR
,
t
)))
...
...
@@ -61,30 +58,79 @@ async function run () {
}
}
async
function
pruneDirectory
(
directory
:
string
,
onlyOwned
=
false
)
{
type
ExistFun
=
(
file
:
string
)
=>
Promise
<
boolean
>
async
function
pruneDirectory
(
directory
:
string
,
existFun
:
ExistFun
)
{
const
files
=
await
readdir
(
directory
)
const
toDelete
:
string
[]
=
[]
for
(
const
file
of
files
)
{
await
Bluebird
.
map
(
files
,
async
file
=>
{
if
(
await
existFun
(
file
)
!==
true
)
{
toDelete
.
push
(
join
(
directory
,
file
))
}
},
{
concurrency
:
20
})
return
toDelete
}
function
doesVideoExist
(
keepOnlyOwned
:
boolean
)
{
return
async
(
file
:
string
)
=>
{
const
uuid
=
getUUIDFromFilename
(
file
)
let
video
:
VideoModel
let
localRedundancy
:
boolean
const
video
=
await
VideoModel
.
loadByUUID
(
uuid
)
if
(
uuid
)
{
video
=
await
VideoModel
.
loadByUUIDWithFile
(
uuid
)
localRedundancy
=
await
VideoRedundancyModel
.
isLocalByVideoUUIDExists
(
uuid
)
}
return
video
&&
(
keepOnlyOwned
===
false
||
video
.
isOwned
())
}
}
if
(
!
uuid
||
!
video
||
(
onlyOwned
===
true
&&
(
video
.
isOwned
()
===
false
&&
localRedundancy
===
false
))
)
{
toDelete
.
push
(
join
(
directory
,
file
))
function
doesThumbnailExist
(
keepOnlyOwned
:
boolean
)
{
return
async
(
file
:
string
)
=>
{
const
thumbnail
=
await
ThumbnailModel
.
loadByName
(
file
)
if
(
!
thumbnail
)
return
false
if
(
keepOnlyOwned
)
{
const
video
=
await
VideoModel
.
load
(
thumbnail
.
videoId
)
if
(
video
.
isOwned
()
===
false
)
return
false
}
return
true
}
}
return
toDelete
async
function
doesAvatarExist
(
file
:
string
)
{
const
avatar
=
await
AvatarModel
.
loadByName
(
file
)
return
!!
avatar
}
async
function
doesRedundancyExist
(
file
:
string
)
{
const
uuid
=
getUUIDFromFilename
(
file
)
const
video
=
await
VideoModel
.
loadWithFiles
(
uuid
)
if
(
!
video
)
return
false
const
isPlaylist
=
file
.
includes
(
'
.
'
)
===
false
if
(
isPlaylist
)
{
const
p
=
video
.
getHLSPlaylist
()
if
(
!
p
)
return
false
const
redundancy
=
await
VideoRedundancyModel
.
loadLocalByStreamingPlaylistId
(
p
.
id
)
return
!!
redundancy
}
const
resolution
=
parseInt
(
file
.
split
(
'
-
'
)[
5
],
10
)
if
(
isNaN
(
resolution
))
{
console
.
error
(
'
Cannot prune %s because we cannot guess guess the resolution.
'
,
file
)
return
true
}
const
videoFile
=
video
.
getFile
(
resolution
)
if
(
!
videoFile
)
{
console
.
error
(
'
Cannot find file of video %s - %d
'
,
video
.
url
,
resolution
)
return
true
}
const
redundancy
=
await
VideoRedundancyModel
.
loadLocalByFileId
(
videoFile
.
id
)
return
!!
redundancy
}
async
function
askConfirmation
()
{
...
...
server/controllers/lazy-static.ts
View file @
e2600d8b
...
...
@@ -49,7 +49,12 @@ async function getAvatar (req: express.Request, res: express.Response) {
logger
.
info
(
'
Lazy serve remote avatar image %s.
'
,
avatar
.
fileUrl
)
await
pushAvatarProcessInQueue
({
filename
:
avatar
.
filename
,
fileUrl
:
avatar
.
fileUrl
})
try
{
await
pushAvatarProcessInQueue
({
filename
:
avatar
.
filename
,
fileUrl
:
avatar
.
fileUrl
})
}
catch
(
err
)
{
logger
.
warn
(
'
Cannot process remote avatar %s.
'
,
avatar
.
fileUrl
,
{
err
})
return
res
.
sendStatus
(
404
)
}
avatar
.
onDisk
=
true
avatar
.
save
()
...
...
server/lib/files-cache/videos-preview-cache.ts
View file @
e2600d8b
...
...
@@ -18,7 +18,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
}
async
getFilePathImpl
(
videoUUID
:
string
)
{
const
video
=
await
VideoModel
.
loadByUUID
WithFile
(
videoUUID
)
const
video
=
await
VideoModel
.
loadByUUID
(
videoUUID
)
if
(
!
video
)
return
undefined
if
(
video
.
isOwned
())
return
{
isOwned
:
true
,
path
:
video
.
getPreview
().
getPath
()
}
...
...
server/models/video/thumbnail.ts
View file @
e2600d8b
...
...
@@ -100,6 +100,16 @@ export class ThumbnailModel extends Model<ThumbnailModel> {
.
catch
(
err
=>
logger
.
error
(
'
Cannot remove thumbnail file %s.
'
,
instance
.
filename
,
err
))
}
static
loadByName
(
filename
:
string
)
{
const
query
=
{
where
:
{
filename
}
}
return
ThumbnailModel
.
findOne
(
query
)
}
static
generateDefaultPreviewName
(
videoUUID
:
string
)
{
return
videoUUID
+
'
.jpg
'
}
...
...
server/models/video/video.ts
View file @
e2600d8b
...
...
@@ -119,6 +119,7 @@ import { CONFIG } from '../../initializers/config'
import
{
ThumbnailModel
}
from
'
./thumbnail
'
import
{
ThumbnailType
}
from
'
../../../shared/models/videos/thumbnail.type
'
import
{
createTorrentPromise
}
from
'
../../helpers/webtorrent
'
import
{
VideoStreamingPlaylistType
}
from
'
../../../shared/models/videos/video-streaming-playlist.type
'
// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
const
indexes
:
(
ModelIndexesOptions
&
{
where
?:
WhereOptions
})[]
=
[
...
...
@@ -1422,15 +1423,23 @@ export class VideoModel extends Model<VideoModel> {
return
VideoModel
.
scope
(
ScopeNames
.
WITH_THUMBNAILS
).
findOne
(
options
)
}
static
loadWithFiles
(
id
:
number
,
t
?:
Transaction
,
logging
?:
boolean
)
{
static
loadWithFiles
(
id
:
number
|
string
,
t
?:
Transaction
,
logging
?:
boolean
)
{
const
where
=
buildWhereIdOrUUID
(
id
)
const
query
=
{
where
,
transaction
:
t
,
logging
}
return
VideoModel
.
scope
([
ScopeNames
.
WITH_FILES
,
ScopeNames
.
WITH_STREAMING_PLAYLISTS
,
ScopeNames
.
WITH_THUMBNAILS
]).
find
ByPk
(
id
,
{
transaction
:
t
,
logging
}
)
]).
find
One
(
query
)
}
static
loadByUUID
WithFile
(
uuid
:
string
)
{
static
loadByUUID
(
uuid
:
string
)
{
const
options
=
{
where
:
{
uuid
...
...
@@ -1754,7 +1763,7 @@ export class VideoModel extends Model<VideoModel> {
return
maxBy
(
this
.
VideoFiles
,
file
=>
file
.
resolution
)
}
getFile
(
resolution
:
VideoResolution
)
{
getFile
(
resolution
:
number
)
{
if
(
Array
.
isArray
(
this
.
VideoFiles
)
===
false
)
return
undefined
return
this
.
VideoFiles
.
find
(
f
=>
f
.
resolution
===
resolution
)
...
...
@@ -1893,6 +1902,12 @@ export class VideoModel extends Model<VideoModel> {
return
`/api/
${
API_VERSION
}
/videos/
${
this
.
uuid
}
/description`
}
getHLSPlaylist
()
{
if
(
!
this
.
VideoStreamingPlaylists
)
return
undefined
return
this
.
VideoStreamingPlaylists
.
find
(
p
=>
p
.
type
===
VideoStreamingPlaylistType
.
HLS
)
}
removeFile
(
videoFile
:
VideoFileModel
,
isRedundancy
=
false
)
{
const
baseDir
=
isRedundancy
?
CONFIG
.
STORAGE
.
REDUNDANCY_DIR
:
CONFIG
.
STORAGE
.
VIDEOS_DIR
...
...
server/tests/cli/index.ts
View file @
e2600d8b
...
...
@@ -4,5 +4,6 @@ import './create-transcoding-job'
import
'
./optimize-old-videos
'
import
'
./peertube
'
import
'
./plugins
'
import
'
./prune-storage
'
import
'
./reset-password
'
import
'
./update-host
'
server/tests/cli/prune-storage.ts
0 → 100644
View file @
e2600d8b
/* tslint:disable:no-unused-expression */
import
'
mocha
'
import
*
as
chai
from
'
chai
'
import
{
waitJobs
}
from
'
../../../shared/extra-utils/server/jobs
'
import
{
buildServerDirectory
,
cleanupTests
,
createVideoPlaylist
,
doubleFollow
,
execCLI
,
flushAndRunMultipleServers
,
getAccount
,
getEnvCli
,
ServerInfo
,
setAccessTokensToServers
,
setDefaultVideoChannel
,
updateMyAvatar
,
uploadVideo
,
wait
}
from
'
../../../shared/extra-utils
'
import
{
Account
,
VideoPlaylistPrivacy
}
from
'
../../../shared/models
'
import
{
createFile
,
readdir
}
from
'
fs-extra
'
import
*
as
uuidv4
from
'
uuid/v4
'
import
{
join
}
from
'
path
'
import
*
as
request
from
'
supertest
'
const
expect
=
chai
.
expect
async
function
countFiles
(
internalServerNumber
:
number
,
directory
:
string
)
{
const
files
=
await
readdir
(
buildServerDirectory
(
internalServerNumber
,
directory
))
return
files
.
length
}
async
function
assertNotExists
(
internalServerNumber
:
number
,
directory
:
string
,
substring
:
string
)
{
const
files
=
await
readdir
(
buildServerDirectory
(
internalServerNumber
,
directory
))
for
(
const
f
of
files
)
{
expect
(
f
).
to
.
not
.
contain
(
substring
)
}
}
async
function
assertCountAreOkay
(
servers
:
ServerInfo
[])
{
for
(
const
server
of
servers
)
{
const
videosCount
=
await
countFiles
(
server
.
internalServerNumber
,
'
videos
'
)
expect
(
videosCount
).
to
.
equal
(
8
)
const
torrentsCount
=
await
countFiles
(
server
.
internalServerNumber
,
'
torrents
'
)
expect
(
torrentsCount
).
to
.
equal
(
8
)
const
previewsCount
=
await
countFiles
(
server
.
internalServerNumber
,
'
previews
'
)
expect
(
previewsCount
).
to
.
equal
(
2
)
const
thumbnailsCount
=
await
countFiles
(
server
.
internalServerNumber
,
'
thumbnails
'
)
expect
(
thumbnailsCount
).
to
.
equal
(
6
)
const
avatarsCount
=
await
countFiles
(
server
.
internalServerNumber
,
'
avatars
'
)
expect
(
avatarsCount
).
to
.
equal
(
2
)
}
}
describe
(
'
Test prune storage scripts
'
,
function
()
{
let
servers
:
ServerInfo
[]
const
badNames
:
{
[
directory
:
string
]:
string
[]
}
=
{}
before
(
async
function
()
{
this
.
timeout
(
120000
)
servers
=
await
flushAndRunMultipleServers
(
2
,
{
transcoding
:
{
enabled
:
true
}
})
await
setAccessTokensToServers
(
servers
)
await
setDefaultVideoChannel
(
servers
)
for
(
const
server
of
servers
)
{
await
uploadVideo
(
server
.
url
,
server
.
accessToken
,
{
name
:
'
video 1
'
})
await
uploadVideo
(
server
.
url
,
server
.
accessToken
,
{
name
:
'
video 2
'
})
await
updateMyAvatar
({
url
:
server
.
url
,
accessToken
:
server
.
accessToken
,
fixture
:
'
avatar.png
'
})
await
createVideoPlaylist
({
url
:
server
.
url
,
token
:
server
.
accessToken
,
playlistAttrs
:
{
displayName
:
'
playlist
'
,
privacy
:
VideoPlaylistPrivacy
.
PUBLIC
,
videoChannelId
:
server
.
videoChannel
.
id
,
thumbnailfile
:
'
thumbnail.jpg
'
}
})
}
await
doubleFollow
(
servers
[
0
],
servers
[
1
])
// Lazy load the remote avatar
{
const
res
=
await
getAccount
(
servers
[
0
].
url
,
'
root@localhost:
'
+
servers
[
1
].
port
)
const
account
:
Account
=
res
.
body
await
request
(
'
http://localhost:
'
+
servers
[
0
].
port
).
get
(
account
.
avatar
.
path
).
expect
(
200
)
}
{
const
res
=
await
getAccount
(
servers
[
1
].
url
,
'
root@localhost:
'
+
servers
[
0
].
port
)
const
account
:
Account
=
res
.
body
await
request
(
'
http://localhost:
'
+
servers
[
1
].
port
).
get
(
account
.
avatar
.
path
).
expect
(
200
)
}
await
wait
(
1000
)
await
waitJobs
(
servers
)
})
it
(
'
Should have the files on the disk
'
,
async
function
()
{
await
assertCountAreOkay
(
servers
)
})
it
(
'
Should create some dirty files
'
,
async
function
()
{
for
(
let
i
=
0
;
i
<
2
;
i
++
)
{
{
const
base
=
buildServerDirectory
(
servers
[
0
].
internalServerNumber
,
'
videos
'
)
const
n1
=
uuidv4
()
+
'
.mp4
'
const
n2
=
uuidv4
()
+
'
.webm
'
await
createFile
(
join
(
base
,
n1
))
await
createFile
(
join
(
base
,
n2
))
badNames
[
'
videos
'
]
=
[
n1
,
n2
]
}
{
const
base
=
buildServerDirectory
(
servers
[
0
].
internalServerNumber
,
'
torrents
'
)
const
n1
=
uuidv4
()
+
'
-240.torrent
'
const
n2
=
uuidv4
()
+
'
-480.torrent
'
await
createFile
(
join
(
base
,
n1
))
await
createFile
(
join
(
base
,
n2
))
badNames
[
'
torrents
'
]
=
[
n1
,
n2
]
}
{
const
base
=
buildServerDirectory
(
servers
[
0
].
internalServerNumber
,
'
thumbnails
'
)
const
n1
=
uuidv4
()
+
'
.jpg
'
const
n2
=
uuidv4
()
+
'
.jpg
'
await
createFile
(
join
(
base
,
n1
))
await
createFile
(
join
(
base
,
n2
))
badNames
[
'
thumbnails
'
]
=
[
n1
,
n2
]
}
{
const
base
=
buildServerDirectory
(
servers
[
0
].
internalServerNumber
,
'
previews
'
)
const
n1
=
uuidv4
()
+
'
.jpg
'
const
n2
=
uuidv4
()
+
'
.jpg
'
await
createFile
(
join
(
base
,
n1
))
await
createFile
(
join
(
base
,
n2
))
badNames
[
'
previews
'
]
=
[
n1
,
n2
]
}
{
const
base
=
buildServerDirectory
(
servers
[
0
].
internalServerNumber
,
'
avatars
'
)
const
n1
=
uuidv4
()
+
'
.png
'
const
n2
=
uuidv4
()
+
'
.jpg
'
await
createFile
(
join
(
base
,
n1
))
await
createFile
(
join
(
base
,
n2
))
badNames
[
'
avatars
'
]
=
[
n1
,
n2
]
}
}
})
it
(
'
Should run prune storage
'
,
async
function
()
{
this
.
timeout
(
30000
)
const
env
=
getEnvCli
(
servers
[
0
])
await
execCLI
(
`echo y |
${
env
}
npm run prune-storage`
)
})
it
(
'
Should have removed files
'
,
async
function
()
{
await
assertCountAreOkay
(
servers
)
for
(
const
directory
of
Object
.
keys
(
badNames
))
{
for
(
const
name
of
badNames
[
directory
])
{
await
assertNotExists
(
servers
[
0
].
internalServerNumber
,
directory
,
name
)
}
}
})
after
(
async
function
()
{
await
cleanupTests
(
servers
)
})
})
shared/extra-utils/miscs/miscs.ts
View file @
e2600d8b
...
...
@@ -44,6 +44,10 @@ function root () {
return
root
}
function
buildServerDirectory
(
internalServerNumber
:
number
,
directory
:
string
)
{
return
join
(
root
(),
'
test
'
+
internalServerNumber
,
directory
)
}
async
function
testImage
(
url
:
string
,
imageName
:
string
,
imagePath
:
string
,
extension
=
'
.jpg
'
)
{
const
res
=
await
request
(
url
)
.
get
(
imagePath
)
...
...
@@ -105,6 +109,7 @@ async function generateHighBitrateVideo () {
export
{
dateIsValid
,
wait
,
buildServerDirectory
,
webtorrentAdd
,
immutableAssign
,
testImage
,
...
...
shared/extra-utils/videos/videos.ts
View file @
e2600d8b
...
...
@@ -19,7 +19,7 @@ import {
import
*
as
validator
from
'
validator
'
import
{
VideoDetails
,
VideoPrivacy
}
from
'
../../models/videos
'
import
{
VIDEO_CATEGORIES
,
VIDEO_LANGUAGES
,
loadLanguages
,
VIDEO_LICENCES
,
VIDEO_PRIVACIES
}
from
'
../../../server/initializers/constants
'
import
{
dateIsValid
,
webtorrentAdd
}
from
'
../miscs/miscs
'
import
{
dateIsValid
,
webtorrentAdd
,
buildServerDirectory
}
from
'
../miscs/miscs
'
loadLanguages
()
...
...
@@ -308,10 +308,8 @@ async function checkVideoFilesWereRemoved (
join
(
'
redundancy
'
,
'
hls
'
)
]
)
{
const
testDirectory
=
'
test
'
+
serverNumber
for
(
const
directory
of
directories
)
{
const
directoryPath
=
join
(
root
(),
testDirectory
,
directory
)
const
directoryPath
=
buildServerDirectory
(
serverNumber
,
directory
)
const
directoryExists
=
await
pathExists
(
directoryPath
)
if
(
directoryExists
===
false
)
continue
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment