Verified Commit 2284f202 authored by Chocobozzz's avatar Chocobozzz
Browse files

Add gitlab ci support

parent 112be80e
Pipeline #154733 canceled with stage
in 59 minutes and 31 seconds
image: chocobozzz/peertube-ci:10
stages:
- build-and-lint
- test
- nightly
before_script:
- 'sed -i -z "s/database:\n hostname: ''localhost''/database:\n hostname: ''postgres''/" config/test.yaml'
- 'sed -i -z "s/redis:\n hostname: ''localhost''/redis:\n hostname: ''redis''/" config/test.yaml'
- if [[ $CI_JOB_STAGE == "test" ]]; then psql -c "create user peertube with password 'peertube';"; fi
- NOCLIENT=1 yarn install --pure-lockfile --cache-folder .yarn-cache
cache:
key: yarn
paths:
- .yarn-cache
- cached-fixtures
###
## Jobs templates
#
#.build-and-lint: &build-and-lint
# stage: build-and-lint
#
#.tests: &tests
# stage: test
# dependencies:
# - build-server
# services:
# - name: postgres:9.6
# alias: postgres
# - name: redis:latest
# alias: redis
# variables:
# PGHOST: postgres
# PGUSER: postgres
# REDIS_HOST: redis
# artifacts:
# expire_in: 1 day
# paths:
# - test*/logs
# when: always
#
####
### Build and lint
##
#build-server:
# <<: *build-and-lint
# artifacts:
# expire_in: 5h
# paths:
# - dist/
# script:
# - npm run build:server
#
#lint:
# <<: *build-and-lint
# script:
# - yarn install --pure-lockfile --cache-folder .yarn-cache
# - npm run ci -- "lint"
#
####
### Tests
#
#test-misc:
# <<: *tests
# script:
# - yarn install --pure-lockfile --cache-folder .yarn-cache
# - npm run ci -- "misc"
#
#test-cli:
# <<: *tests
# retry:
# max: 1
# script:
# - npm run ci -- "cli"
#
#api:
# <<: *tests
# parallel: 4
# retry:
# max: 1
# script:
# - NODE_PENDING_JOB_WAIT=1000 npm run ci -- api-$CI_NODE_INDEX
build-nightly:
stage: nightly
only:
- schedules
script:
- yarn install --pure-lockfile --cache-folder .yarn-cache
- npm run nightly
- mkdir "${HOME}/.ssh"
- chmod 700 "${HOME}/.ssh"
- if [ ! -z ${DEPLOYEMENT_KNOWN_HOSTS+x} ]; then echo -e "${DEPLOYEMENT_KNOWN_HOSTS}" > ${HOME}/.ssh/known_hosts; fi
- eval `ssh-agent -s`
- if [ ! -z ${DEPLOYEMENT_KEY+x} ]; then ssh-add <(echo "${DEPLOYEMENT_KEY}"); fi
- if [ ! -z ${DEPLOYEMENT_KEY+x} ]; then scp ./peertube-nightly-* ${DEPLOYEMENT_USER}@${DEPLOYEMENT_HOST}:../../web/nightly; fi
......@@ -47,7 +47,7 @@ matrix:
- env: TEST_SUITE=lint
script:
- NODE_PENDING_JOB_WAIT=2000 travis_retry npm run travis -- "$TEST_SUITE"
- NODE_PENDING_JOB_WAIT=2000 travis_retry npm run ci -- "$TEST_SUITE"
after_failure:
- cat test1/logs/peertube.log
......
......@@ -45,8 +45,8 @@ Be part of a network of multiple small federated, interoperable video hosting pr
<br />
<a href="https://travis-ci.org/Chocobozzz/PeerTube">
<img src="https://travis-ci.org/Chocobozzz/PeerTube.svg?branch=develop" alt="Build Status" />
<a href="https://framagit.org/framasoft/peertube/PeerTube/commits/develop">
<img alt="pipeline status" src="https://framagit.org/framasoft/peertube/PeerTube/badges/develop/pipeline.svg" />
</a>
<a href="https://david-dm.org/Chocobozzz/PeerTube">
......
......@@ -20,6 +20,9 @@ database:
hostname: 'localhost'
port: 5432
redis:
hostname: 'localhost'
smtp:
hostname: null
port: 1025
......
......@@ -60,17 +60,19 @@
"postinstall": "test -n \"$NOCLIENT\" || (cd client && yarn install --pure-lockfile)",
"tsc": "tsc",
"commander": "commander",
"lint": "npm run travis -- lint",
"lint": "npm run ci -- lint",
"ng": "ng",
"nodemon": "nodemon",
"ts-node": "ts-node",
"tslint": "tslint",
"concurrently": "concurrently",
"mocha-parallel-tests": "mocha-parallel-tests",
"sasslint": "sass-lint --verbose --no-exit",
"sasslint:fix": "sass-lint-auto-fix -c .sass-lint.yml --verbose",
"mocha": "mocha",
"travis": "scripty",
"ci": "scripty",
"release": "scripty",
"nightly": "scripty",
"client-report": "scripty"
},
"husky": {
......@@ -93,7 +95,7 @@
]
},
"resolutions": {
"@types/bluebird": "3.5.21"
"@types/bluebird": "3.5.27"
},
"dependencies": {
"apicache": "^1.4.0",
......@@ -206,7 +208,7 @@
"maildev": "^1.0.0-rc3",
"marked-man": "^0.6.0",
"mocha": "^6.0.0",
"mocha-parallel-tests": "^2.1.0",
"mocha-parallel-tests": "^2.2.1",
"nodemon": "^1.18.6",
"sass-lint": "^1.12.1",
"source-map-support": "^0.5.0",
......
......@@ -22,16 +22,16 @@ elif [ "$1" = "cli" ]; then
mocha --timeout 5000 --exit --require ts-node/register --bail server/tests/cli/index.ts
elif [ "$1" = "api-1" ]; then
npm run build:server
sh ./server/tests/api/travis-1.sh 2
sh ./server/tests/api/ci-1.sh 2
elif [ "$1" = "api-2" ]; then
npm run build:server
sh ./server/tests/api/travis-2.sh 2
sh ./server/tests/api/ci-2.sh 2
elif [ "$1" = "api-3" ]; then
npm run build:server
sh ./server/tests/api/travis-3.sh 2
sh ./server/tests/api/ci-3.sh 2
elif [ "$1" = "api-4" ]; then
npm run build:server
sh ./server/tests/api/travis-4.sh 2
sh ./server/tests/api/ci-4.sh 2
elif [ "$1" = "lint" ]; then
npm run tslint -- --project ./tsconfig.json -c ./tslint.json server.ts "server/**/*.ts" "shared/**/*.ts"
......
......@@ -5,7 +5,7 @@ set -eu
recreateDB () {
dbname="peertube_test$1"
dropdb --if-exists "$dbname"
dropdb --if-exists "$dbname" 2>&1
createdb -O peertube "$dbname"
psql -c "CREATE EXTENSION pg_trgm;" "$dbname" &
......@@ -18,10 +18,15 @@ removeFiles () {
dropRedis () {
port=$((9000+$1))
host="localhost"
redis-cli KEYS "bull-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
redis-cli KEYS "redis-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
redis-cli KEYS "*redis-localhost:$port-" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
if [ ! -z ${GITLAB_CI+x} ]; then
host="redis"
fi
redis-cli -h "$host" KEYS "bull-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli -h "$host" DEL
redis-cli -h "$host" KEYS "redis-localhost:$port*" | grep -v empty | xargs --no-run-if-empty redis-cli -h "$host" DEL
redis-cli -h "$host" KEYS "*redis-localhost:$port-" | grep -v empty | xargs --no-run-if-empty redis-cli -h "$host" DEL
}
seq=$(seq 1 6)
......
#!/bin/bash
set -eu
shutdown() {
# Get our process group id
# shellcheck disable=SC2009
PGID=$(ps -o pgid= $$ | grep -o "[0-9]*")
# Kill it in a new new process group
setsid kill -- -"$PGID"
exit 0
}
trap "shutdown" SIGINT SIGTERM
today=$(date '+%F')
directory_name="peertube-nightly-$today"
tar_name="peertube-nightly-$today.tar.xz"
npm run build
# Creating the archives
(
# local variables
directories_to_archive=("$directory_name/CREDITS.md" "$directory_name/FAQ.md" \
"$directory_name/LICENSE" "$directory_name/README.md" \
"$directory_name/client/dist/" "$directory_name/client/yarn.lock" \
"$directory_name/client/package.json" "$directory_name/config" \
"$directory_name/dist" "$directory_name/package.json" \
"$directory_name/scripts" "$directory_name/support" \
"$directory_name/tsconfig.json" "$directory_name/yarn.lock")
# temporary setup
cd ..
ln -s "PeerTube" "$directory_name"
XZ_OPT=-e9 tar cfJ "PeerTube/$tar_name" "${directories_to_archive[@]}"
# temporary setup destruction
rm "$directory_name"
)
......@@ -5,6 +5,6 @@ set -eu
npm run build:server
npm run setup:cli
npm run travis -- lint
npm run ci -- lint
mocha --exit --require ts-node/register/type-check --bail server/tests/index.ts
......@@ -113,16 +113,16 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) {
// We send the video abuse to the origin server
if (videoInstance.isOwned() === false) {
await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance)
await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance, t)
}
Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance)
auditLogger.create(reporterAccount.Actor.getIdentifier(), new VideoAbuseAuditView(videoAbuseInstance.toFormattedJSON()))
return videoAbuseInstance
})
Notifier.Instance.notifyOnNewVideoAbuse(videoAbuse)
logger.info('Abuse report for video %s created.', videoInstance.name)
return res.json({ videoAbuse: videoAbuse.toFormattedJSON() }).end()
......
......@@ -2,6 +2,7 @@ import * as retry from 'async/retry'
import * as Bluebird from 'bluebird'
import { Model } from 'sequelize-typescript'
import { logger } from './logger'
import { Transaction } from 'sequelize'
function retryTransactionWrapper <T, A, B, C> (
functionToRetry: (arg1: A, arg2: B, arg3: C) => Promise<T> | Bluebird<T>,
......@@ -72,11 +73,18 @@ function resetSequelizeInstance (instance: Model<any>, savedFields: object) {
})
}
function afterCommitIfTransaction (t: Transaction, fn: Function) {
if (t) return t.afterCommit(() => fn())
return fn()
}
// ---------------------------------------------------------------------------
export {
resetSequelizeInstance,
retryTransactionWrapper,
transactionRetryer,
updateInstanceWithAnother
updateInstanceWithAnother,
afterCommitIfTransaction
}
......@@ -31,7 +31,7 @@ async function processCreateVideoAbuse (activity: ActivityCreate | ActivityFlag,
const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: flag.object })
return sequelizeTypescript.transaction(async t => {
const videoAbuse = await sequelizeTypescript.transaction(async t => {
const videoAbuseData = {
reporterAccountId: account.id,
reason: flag.content,
......@@ -42,8 +42,10 @@ async function processCreateVideoAbuse (activity: ActivityCreate | ActivityFlag,
const videoAbuseInstance = await VideoAbuseModel.create(videoAbuseData, { transaction: t })
videoAbuseInstance.Video = video
Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance)
logger.info('Remote abuse for video uuid %s created', flag.object)
return videoAbuseInstance
})
Notifier.Instance.notifyOnNewVideoAbuse(videoAbuse)
}
......@@ -11,6 +11,7 @@ import { VideoRedundancyModel } from '../../../models/redundancy/video-redundanc
import { VideoPlaylistModel } from '../../../models/video/video-playlist'
import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model'
import { getServerActor } from '../../../helpers/utils'
import * as Bluebird from 'bluebird'
async function sendCreateVideo (video: VideoModel, t: Transaction) {
if (video.privacy === VideoPrivacy.PRIVATE) return undefined
......@@ -82,7 +83,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
// This was a reply, send it to the parent actors
const actorsException = [ byActor ]
await broadcastToActors(createActivity, byActor, parentsCommentActors, actorsException)
await broadcastToActors(createActivity, byActor, parentsCommentActors, t, actorsException)
// Broadcast to our followers
await broadcastToFollowers(createActivity, byActor, [ byActor ], t)
......@@ -91,7 +92,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
if (isOrigin) return broadcastToFollowers(createActivity, byActor, actorsInvolvedInComment, t, actorsException)
// Send to origin
return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
t.afterCommit(() => unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl))
}
function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate {
......
......@@ -59,7 +59,7 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans
// This was a reply, send it to the parent actors
const actorsException = [ byActor ]
await broadcastToActors(activity, byActor, threadParentComments.map(c => c.Account.Actor), actorsException)
await broadcastToActors(activity, byActor, threadParentComments.map(c => c.Account.Actor), t, actorsException)
// Broadcast to our followers
await broadcastToFollowers(activity, byActor, [ byActor ], t)
......@@ -68,7 +68,7 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans
if (isVideoOrigin) return broadcastToFollowers(activity, byActor, actorsInvolvedInComment, t, actorsException)
// Send to origin
return unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
t.afterCommit(() => unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl))
}
async function sendDeleteVideoPlaylist (videoPlaylist: VideoPlaylistModel, t: Transaction) {
......
......@@ -6,8 +6,9 @@ import { unicastTo } from './utils'
import { logger } from '../../../helpers/logger'
import { ActivityAudience, ActivityFlag } from '../../../../shared/models/activitypub'
import { audiencify, getAudience } from '../audience'
import { Transaction } from 'sequelize'
async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) {
async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel, t: Transaction) {
if (!video.VideoChannel.Account.Actor.serverId) return // Local user
const url = getVideoAbuseActivityPubUrl(videoAbuse)
......@@ -18,7 +19,7 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel,
const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] }
const flagActivity = buildFlagActivity(url, byActor, videoAbuse, audience)
return unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
t.afterCommit(() => unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl))
}
function buildFlagActivity (url: string, byActor: ActorModel, videoAbuse: VideoAbuseModel, audience: ActivityAudience): ActivityFlag {
......
......@@ -4,8 +4,9 @@ import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
import { getActorFollowActivityPubUrl } from '../url'
import { unicastTo } from './utils'
import { logger } from '../../../helpers/logger'
import { Transaction } from 'sequelize'
function sendFollow (actorFollow: ActorFollowModel) {
function sendFollow (actorFollow: ActorFollowModel, t: Transaction) {
const me = actorFollow.ActorFollower
const following = actorFollow.ActorFollowing
......@@ -17,7 +18,7 @@ function sendFollow (actorFollow: ActorFollowModel) {
const url = getActorFollowActivityPubUrl(me, following)
const data = buildFollowActivity(url, me, following)
return unicastTo(data, me, following.inboxUrl)
t.afterCommit(() => unicastTo(data, me, following.inboxUrl))
}
function buildFollowActivity (url: string, byActor: ActorModel, targetActor: ActorModel): ActivityFollow {
......
......@@ -37,7 +37,7 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) {
const followActivity = buildFollowActivity(followUrl, me, following)
const undoActivity = undoActivityData(undoUrl, me, followActivity)
return unicastTo(undoActivity, me, following.inboxUrl)
t.afterCommit(() => unicastTo(undoActivity, me, following.inboxUrl))
}
async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
......
......@@ -26,7 +26,9 @@ async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByAct
const url = getUpdateActivityPubUrl(video.url, video.updatedAt.toISOString())
// Needed to build the AP object
if (!video.VideoCaptions) video.VideoCaptions = await video.$get('VideoCaptions') as VideoCaptionModel[]
if (!video.VideoCaptions) {
video.VideoCaptions = await video.$get('VideoCaptions', { transaction: t }) as VideoCaptionModel[]
}
const videoObject = video.toActivityPubObject()
const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC)
......
......@@ -7,6 +7,7 @@ import { JobQueue } from '../../job-queue'
import { VideoModel } from '../../../models/video/video'
import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience'
import { getServerActor } from '../../../helpers/utils'
import { afterCommitIfTransaction } from '../../../helpers/database-utils'
async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: {
byActor: ActorModel,
......@@ -20,7 +21,9 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud
const audience = getRemoteVideoAudience(options.video, actorsInvolvedInVideo)
const activity = activityBuilder(audience)
return unicastTo(activity, options.byActor, options.video.VideoChannel.Account.Actor.sharedInboxUrl)
return afterCommitIfTransaction(options.transaction, () => {
return unicastTo(activity, options.byActor, options.video.VideoChannel.Account.Actor.sharedInboxUrl)
})
}
// Send to followers
......@@ -28,6 +31,7 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud
const activity = activityBuilder(audience)
const actorsException = [ options.byActor ]
return broadcastToFollowers(activity, options.byActor, actorsInvolvedInVideo, options.transaction, actorsException)
}
......@@ -76,7 +80,7 @@ async function forwardActivity (
uris,
body: activity
}
return JobQueue.Instance.createJob({ type: 'activitypub-http-broadcast', payload })
return afterCommitIfTransaction(t, () => JobQueue.Instance.createJob({ type: 'activitypub-http-broadcast', payload }))
}
async function broadcastToFollowers (
......@@ -87,20 +91,22 @@ async function broadcastToFollowers (
actorsException: ActorModel[] = []
) {
const uris = await computeFollowerUris(toFollowersOf, actorsException, t)
return broadcastTo(uris, data, byActor)
return afterCommitIfTransaction(t, () => broadcastTo(uris, data, byActor))
}
async function broadcastToActors (
data: any,
byActor: ActorModel,
toActors: ActorModel[],
t?: Transaction,
actorsException: ActorModel[] = []
) {
const uris = await computeUris(toActors, actorsException)
return broadcastTo(uris, data, byActor)
return afterCommitIfTransaction(t, () => broadcastTo(uris, data, byActor))
}
async function broadcastTo (uris: string[], data: any, byActor: ActorModel) {
function broadcastTo (uris: string[], data: any, byActor: ActorModel) {
if (uris.length === 0) return undefined
logger.debug('Creating broadcast job.', { uris })
......@@ -114,7 +120,7 @@ async function broadcastTo (uris: string[], data: any, byActor: ActorModel) {
return JobQueue.Instance.createJob({ type: 'activitypub-http-broadcast', payload })
}
async function unicastTo (data: any, byActor: ActorModel, toActorUrl: string) {
function unicastTo (data: any, byActor: ActorModel, toActorUrl: string) {
logger.debug('Creating unicast job.', { uri: toActorUrl })
const payload = {
......@@ -123,7 +129,7 @@ async function unicastTo (data: any, byActor: ActorModel, toActorUrl: string) {
body: data
}
return JobQueue.Instance.createJob({ type: 'activitypub-http-unicast', payload })
JobQueue.Instance.createJob({ type: 'activitypub-http-unicast', payload })
}
// ---------------------------------------------------------------------------
......
......@@ -69,7 +69,7 @@ async function follow (fromActor: ActorModel, targetActor: ActorModel) {
actorFollow.ActorFollower = fromActor
// Send a notification to remote server if our follow is not already accepted
if (actorFollow.state !== 'accepted') await sendFollow(actorFollow)
if (actorFollow.state !== 'accepted') sendFollow(actorFollow, t)
return actorFollow
})
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment