Verified Commit 49af5ac8 authored by Chocobozzz's avatar Chocobozzz
Browse files

Refactor AP playlists

parent 9777fe9e
import { CacheFileObject } from '../../../shared/index'
import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
import { Transaction } from 'sequelize'
import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
import { MActorId, MVideoRedundancy, MVideoWithAllFiles } from '@server/types/models'
import { CacheFileObject } from '../../../shared/index'
import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId) {
if (cacheFileObject.url.mediaType === 'application/x-mpegURL') {
const url = cacheFileObject.url
const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS)
if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url)
async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) {
const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t)
return {
expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
url: cacheFileObject.id,
fileUrl: url.href,
strategy: null,
videoStreamingPlaylistId: playlist.id,
actorId: byActor.id
}
if (redundancyModel) {
return updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t)
}
const url = cacheFileObject.url
const videoFile = video.VideoFiles.find(f => {
return f.resolution === url.height && f.fps === url.fps
})
if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`)
return {
expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
url: cacheFileObject.id,
fileUrl: url.href,
strategy: null,
videoFileId: videoFile.id,
actorId: byActor.id
}
return createCacheFile(cacheFileObject, video, byActor, t)
}
async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) {
const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t)
// ---------------------------------------------------------------------------
if (!redundancyModel) {
await createCacheFile(cacheFileObject, video, byActor, t)
} else {
await updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t)
}
export {
createOrUpdateCacheFile
}
// ---------------------------------------------------------------------------
function createCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) {
const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
......@@ -74,9 +47,37 @@ function updateCacheFile (
return redundancyModel.save({ transaction: t })
}
export {
createOrUpdateCacheFile,
createCacheFile,
updateCacheFile,
cacheFileActivityObjectToDBAttributes
function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId) {
if (cacheFileObject.url.mediaType === 'application/x-mpegURL') {
const url = cacheFileObject.url
const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS)
if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url)
return {
expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
url: cacheFileObject.id,
fileUrl: url.href,
strategy: null,
videoStreamingPlaylistId: playlist.id,
actorId: byActor.id
}
}
const url = cacheFileObject.url
const videoFile = video.VideoFiles.find(f => {
return f.resolution === url.height && f.fps === url.fps
})
if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`)
return {
expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
url: cacheFileObject.id,
fileUrl: url.href,
strategy: null,
videoFileId: videoFile.id,
actorId: byActor.id
}
}
import * as Bluebird from 'bluebird'
import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
import { checkUrlsSameHost } from '../../helpers/activitypub'
import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
import { isArray } from '../../helpers/custom-validators/misc'
import { logger } from '../../helpers/logger'
import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests'
import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
import { sequelizeTypescript } from '../../initializers/database'
import { VideoPlaylistModel } from '../../models/video/video-playlist'
import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
import { MAccountDefault, MAccountId, MVideoId } from '../../types/models'
import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../types/models/video/video-playlist'
import { FilteredModelAttributes } from '../../types/sequelize'
import { createPlaylistMiniatureFromUrl } from '../thumbnail'
import { getOrCreateActorAndServerAndModel } from './actor'
import { crawlCollectionPage } from './crawl'
import { getOrCreateAPVideo } from './videos'
function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
? VideoPlaylistPrivacy.PUBLIC
: VideoPlaylistPrivacy.UNLISTED
return {
name: playlistObject.name,
description: playlistObject.content,
privacy,
url: playlistObject.id,
uuid: playlistObject.uuid,
ownerAccountId: byAccount.id,
videoChannelId: null,
createdAt: new Date(playlistObject.published),
updatedAt: new Date(playlistObject.updated)
}
}
function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) {
return {
position: elementObject.position,
url: elementObject.id,
startTimestamp: elementObject.startTimestamp || null,
stopTimestamp: elementObject.stopTimestamp || null,
videoPlaylistId: videoPlaylist.id,
videoId: video.id
}
}
async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) {
await Bluebird.map(playlistUrls, async playlistUrl => {
try {
const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
if (exists === true) return
// Fetch url
const { body } = await doJSONRequest<PlaylistObject>(playlistUrl, { activityPub: true })
if (!isPlaylistObjectValid(body)) {
throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`)
}
if (!isArray(body.to)) {
throw new Error('Playlist does not have an audience.')
}
return createOrUpdateVideoPlaylist(body, account, body.to)
} catch (err) {
logger.warn('Cannot add playlist element %s.', playlistUrl, { err })
}
}, { concurrency: CRAWL_REQUEST_CONCURRENCY })
}
async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) {
const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
if (actor.VideoChannel) {
playlistAttributes.videoChannelId = actor.VideoChannel.id
} else {
logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject })
}
}
const [ playlist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true })
let accItems: string[] = []
await crawlCollectionPage<string>(playlistObject.id, items => {
accItems = accItems.concat(items)
return Promise.resolve()
})
const refreshedPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(playlist.id, null)
if (playlistObject.icon) {
try {
const thumbnailModel = await createPlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist: refreshedPlaylist })
await refreshedPlaylist.setAndSaveThumbnail(thumbnailModel, undefined)
} catch (err) {
logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err })
}
} else if (refreshedPlaylist.hasThumbnail()) {
await refreshedPlaylist.Thumbnail.destroy()
refreshedPlaylist.Thumbnail = null
}
return resetVideoPlaylistElements(accItems, refreshedPlaylist)
}
async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
if (!videoPlaylist.isOutdated()) return videoPlaylist
try {
const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
if (playlistObject === undefined) {
logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
await videoPlaylist.setAsRefreshed()
return videoPlaylist
}
const byAccount = videoPlaylist.OwnerAccount
await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
return videoPlaylist
} catch (err) {
if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
await videoPlaylist.destroy()
return undefined
}
logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
await videoPlaylist.setAsRefreshed()
return videoPlaylist
}
}
// ---------------------------------------------------------------------------
export {
createAccountPlaylists,
playlistObjectToDBAttributes,
playlistElementObjectToDBAttributes,
createOrUpdateVideoPlaylist,
refreshVideoPlaylistIfNeeded
}
// ---------------------------------------------------------------------------
async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
await Bluebird.map(elementUrls, async elementUrl => {
try {
const { body } = await doJSONRequest<PlaylistElementObject>(elementUrl, { activityPub: true })
if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`)
if (checkUrlsSameHost(body.id, elementUrl) !== true) {
throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
}
const { video } = await getOrCreateAPVideo({ videoObject: { id: body.url }, fetchType: 'only-video' })
elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video))
} catch (err) {
logger.warn('Cannot add playlist element %s.', elementUrl, { err })
}
}, { concurrency: CRAWL_REQUEST_CONCURRENCY })
await sequelizeTypescript.transaction(async t => {
await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
for (const element of elementsToCreate) {
await VideoPlaylistElementModel.create(element, { transaction: t })
}
})
logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length)
return undefined
}
async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
logger.info('Fetching remote playlist %s.', playlistUrl)
const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true })
if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
logger.debug('Remote video playlist JSON is not valid.', { body })
return { statusCode, playlistObject: undefined }
}
return { statusCode, playlistObject: body }
}
import { isArray } from '@server/helpers/custom-validators/misc'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants'
import { sequelizeTypescript } from '@server/initializers/database'
import { createPlaylistMiniatureFromUrl } from '@server/lib/thumbnail'
import { VideoPlaylistModel } from '@server/models/video/video-playlist'
import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
import { FilteredModelAttributes } from '@server/types'
import { MAccountDefault, MAccountId, MVideoPlaylist, MVideoPlaylistFull } from '@server/types/models'
import { AttributesOnly } from '@shared/core-utils'
import { PlaylistObject } from '@shared/models'
import { getOrCreateActorAndServerAndModel } from '../actor'
import { crawlCollectionPage } from '../crawl'
import { getOrCreateAPVideo } from '../videos'
import {
fetchRemotePlaylistElement,
fetchRemoteVideoPlaylist,
playlistElementObjectToDBAttributes,
playlistObjectToDBAttributes
} from './shared'
import Bluebird = require('bluebird')
const lTags = loggerTagsFactory('ap', 'video-playlist')
async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) {
await Bluebird.map(playlistUrls, async playlistUrl => {
try {
const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
if (exists === true) return
const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl)
if (playlistObject === undefined) {
throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`)
}
return createOrUpdateVideoPlaylist(playlistObject, account, playlistObject.to)
} catch (err) {
logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) })
}
}, { concurrency: CRAWL_REQUEST_CONCURRENCY })
}
async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
await setVideoChannelIfNeeded(playlistObject, playlistAttributes)
const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true })
const playlistElementUrls = await fetchElementUrls(playlistObject)
// Refetch playlist from DB since elements fetching could be long in time
const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null)
try {
await updatePlaylistThumbnail(playlistObject, playlist)
} catch (err) {
logger.warn('Cannot update thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) })
}
return rebuildVideoPlaylistElements(playlistElementUrls, playlist)
}
// ---------------------------------------------------------------------------
export {
createAccountPlaylists,
createOrUpdateVideoPlaylist
}
// ---------------------------------------------------------------------------
async function setVideoChannelIfNeeded (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) {
if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) return
const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
if (!actor.VideoChannel) {
logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) })
return
}
playlistAttributes.videoChannelId = actor.VideoChannel.id
}
async function fetchElementUrls (playlistObject: PlaylistObject) {
let accItems: string[] = []
await crawlCollectionPage<string>(playlistObject.id, items => {
accItems = accItems.concat(items)
return Promise.resolve()
})
return accItems
}
async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) {
if (playlistObject.icon) {
const thumbnailModel = await createPlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist })
await playlist.setAndSaveThumbnail(thumbnailModel, undefined)
return
}
// Playlist does not have an icon, destroy existing one
if (playlist.hasThumbnail()) {
await playlist.Thumbnail.destroy()
playlist.Thumbnail = null
}
}
async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist)
await sequelizeTypescript.transaction(async t => {
await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
for (const element of elementsToCreate) {
await VideoPlaylistElementModel.create(element, { transaction: t })
}
})
logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url))
return undefined
}
async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) {
const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
await Bluebird.map(elementUrls, async elementUrl => {
try {
const { elementObject } = await fetchRemotePlaylistElement(elementUrl)
const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' })
elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video))
} catch (err) {
logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) })
}
}, { concurrency: CRAWL_REQUEST_CONCURRENCY })
return elementsToCreate
}
export * from './create-update'
export * from './refresh'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { PeerTubeRequestError } from '@server/helpers/requests'
import { MVideoPlaylistOwner } from '@server/types/models'
import { HttpStatusCode } from '@shared/core-utils'
import { createOrUpdateVideoPlaylist } from './create-update'
import { fetchRemoteVideoPlaylist } from './shared'
async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
if (!videoPlaylist.isOutdated()) return videoPlaylist
const lTags = loggerTagsFactory('ap', 'video-playlist', 'refresh', videoPlaylist.uuid, videoPlaylist.url)
try {
const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
if (playlistObject === undefined) {
logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url, lTags())
await videoPlaylist.setAsRefreshed()
return videoPlaylist
}
const byAccount = videoPlaylist.OwnerAccount
await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
return videoPlaylist
} catch (err) {
if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
logger.info('Cannot refresh not existing playlist %s. Deleting it.', videoPlaylist.url, lTags())
await videoPlaylist.destroy()
return undefined
}
logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err, ...lTags() })
await videoPlaylist.setAsRefreshed()
return videoPlaylist
}
}
export {
refreshVideoPlaylistIfNeeded
}
export * from './object-to-model-attributes'
export * from './url-to-object'
import { ACTIVITY_PUB } from '@server/initializers/constants'
import { VideoPlaylistModel } from '@server/models/video/video-playlist'
import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element'
import { MAccountId, MVideoId, MVideoPlaylistId } from '@server/types/models'
import { AttributesOnly } from '@shared/core-utils'
import { PlaylistElementObject, PlaylistObject, VideoPlaylistPrivacy } from '@shared/models'
function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
? VideoPlaylistPrivacy.PUBLIC
: VideoPlaylistPrivacy.UNLISTED
return {
name: playlistObject.name,
description: playlistObject.content,
privacy,
url: playlistObject.id,
uuid: playlistObject.uuid,
ownerAccountId: byAccount.id,
videoChannelId: null,
createdAt: new Date(playlistObject.published),
updatedAt: new Date(playlistObject.updated)
} as AttributesOnly<VideoPlaylistModel>
}
function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) {
return {
position: elementObject.position,
url: elementObject.id,
startTimestamp: elementObject.startTimestamp || null,
stopTimestamp: elementObject.stopTimestamp || null,
videoPlaylistId: videoPlaylist.id,
videoId: video.id
} as AttributesOnly<VideoPlaylistElementModel>
}
export {
playlistObjectToDBAttributes,
playlistElementObjectToDBAttributes
}
import { isArray } from 'lodash'
import { checkUrlsSameHost } from '@server/helpers/activitypub'
import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '@server/helpers/custom-validators/activitypub/playlist'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { doJSONRequest } from '@server/helpers/requests'
import { PlaylistElementObject, PlaylistObject } from '@shared/models'
async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
const lTags = loggerTagsFactory('ap', 'video-playlist', playlistUrl)
logger.info('Fetching remote playlist %s.', playlistUrl, lTags())
const { body, statusCode } = await doJSONRequest<any>(playlistUrl, { activityPub: true })
if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
logger.debug('Remote video playlist JSON is not valid.', { body, ...lTags() })
return { statusCode, playlistObject: undefined }
}