Verified Commit c795e196 authored by Chocobozzz's avatar Chocobozzz
Browse files

Automatically rebuild native modules on ABI change

parent fd59208e
......@@ -138,6 +138,7 @@ import { ServerConfigManager } from '@server/lib/server-config-manager'
import { VideoViewsManager } from '@server/lib/views/video-views-manager'
import { isTestOrDevInstance } from './server/helpers/core-utils'
import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics'
import { ApplicationModel } from '@server/models/application/application'
// ----------- Command line -----------
......@@ -330,12 +331,17 @@ async function startApplication () {
server.listen(port, hostname, async () => {
if (cliOptions.plugins) {
try {
await PluginManager.Instance.rebuildNativePluginsIfNeeded()
await PluginManager.Instance.registerPluginsAndThemes()
} catch (err) {
logger.error('Cannot register plugins and themes.', { err })
}
}
ApplicationModel.updateNodeVersions()
.catch(err => logger.error('Cannot update node versions.', { err }))
logger.info('HTTP server listening on %s:%d', hostname, port)
logger.info('Web server: %s', WEBSERVER.URL)
......
......@@ -4,7 +4,7 @@ import { join } from 'path'
import { sha256 } from '@shared/extra-utils'
import { ResultList } from '@shared/models'
import { CONFIG } from '../initializers/config'
import { execPromise, execPromise2, randomBytesPromise } from './core-utils'
import { randomBytesPromise } from './core-utils'
import { logger } from './logger'
function deleteFileAndCatch (path: string) {
......@@ -44,29 +44,6 @@ function getSecureTorrentName (originalName: string) {
return sha256(originalName) + '.torrent'
}
async function getServerCommit () {
try {
const tag = await execPromise2(
'[ ! -d .git ] || git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null || true',
{ stdio: [ 0, 1, 2 ] }
)
if (tag) return tag.replace(/^v/, '')
} catch (err) {
logger.debug('Cannot get version from git tags.', { err })
}
try {
const version = await execPromise('[ ! -d .git ] || git rev-parse --short HEAD')
if (version) return version.toString().trim()
} catch (err) {
logger.debug('Cannot get version from git HEAD.', { err })
}
return ''
}
/**
* From a filename like "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3.mp4", returns
* only the "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3" part. If the filename does
......@@ -88,7 +65,6 @@ export {
generateRandomString,
getFormattedObjects,
getSecureTorrentName,
getServerCommit,
generateVideoImportTmpPath,
getUUIDFromFilename
}
import { execPromise, execPromise2 } from './core-utils'
import { logger } from './logger'
async function getServerCommit () {
try {
const tag = await execPromise2(
'[ ! -d .git ] || git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null || true',
{ stdio: [ 0, 1, 2 ] }
)
if (tag) return tag.replace(/^v/, '')
} catch (err) {
logger.debug('Cannot get version from git tags.', { err })
}
try {
const version = await execPromise('[ ! -d .git ] || git rev-parse --short HEAD')
if (version) return version.toString().trim()
} catch (err) {
logger.debug('Cannot get version from git HEAD.', { err })
}
return ''
}
function getNodeABIVersion () {
const version = process.versions.modules
return parseInt(version)
}
export {
getServerCommit,
getNodeABIVersion
}
......@@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
// ---------------------------------------------------------------------------
const LAST_MIGRATION_VERSION = 720
const LAST_MIGRATION_VERSION = 725
// ---------------------------------------------------------------------------
......
import { ensureDir, readdir, remove } from 'fs-extra'
import passwordGenerator from 'password-generator'
import { join } from 'path'
import { isTestOrDevInstance } from '@server/helpers/core-utils'
import { getNodeABIVersion } from '@server/helpers/version'
import { UserRole } from '@shared/models'
import { logger } from '../helpers/logger'
import { buildUser, createApplicationActor, createUserAccountAndChannelAndPlaylist } from '../lib/user'
......@@ -10,7 +12,6 @@ import { applicationExist, clientsExist, usersExist } from './checker-after-init
import { CONFIG } from './config'
import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION, RESUMABLE_UPLOAD_DIRECTORY } from './constants'
import { sequelizeTypescript } from './database'
import { isTestOrDevInstance } from '@server/helpers/core-utils'
async function installApplication () {
try {
......@@ -175,7 +176,9 @@ async function createApplicationIfNotExist () {
logger.info('Creating application account.')
const application = await ApplicationModel.create({
migrationVersion: LAST_MIGRATION_VERSION
migrationVersion: LAST_MIGRATION_VERSION,
nodeVersion: process.version,
nodeABIVersion: getNodeABIVersion()
})
return createApplicationActor(application.id)
......
import * as Sequelize from 'sequelize'
async function up (utils: {
transaction: Sequelize.Transaction
queryInterface: Sequelize.QueryInterface
sequelize: Sequelize.Sequelize
db: any
}): Promise<void> {
const { transaction } = utils
{
const data = {
type: Sequelize.STRING,
defaultValue: null,
allowNull: true
}
await utils.queryInterface.addColumn('application', 'nodeVersion', data, { transaction })
}
{
const data = {
type: Sequelize.STRING,
defaultValue: null,
allowNull: true
}
await utils.queryInterface.addColumn('application', 'nodeABIVersion', data, { transaction })
}
{
const query = `UPDATE "application" SET "nodeVersion" = '${process.version}'`
await utils.sequelize.query(query, { transaction })
}
{
const nodeABIVersion = parseInt(process.versions.modules)
const query = `UPDATE "application" SET "nodeABIVersion" = ${nodeABIVersion}`
await utils.sequelize.query(query, { transaction })
}
{
const data = {
type: Sequelize.STRING,
defaultValue: null,
allowNull: false
}
await utils.queryInterface.changeColumn('application', 'nodeVersion', data, { transaction })
}
{
const data = {
type: Sequelize.STRING,
defaultValue: null,
allowNull: false
}
await utils.queryInterface.changeColumn('application', 'nodeABIVersion', data, { transaction })
}
}
function down (options) {
throw new Error('Not implemented.')
}
export {
up,
down
}
......@@ -3,6 +3,7 @@ import { createReadStream, createWriteStream } from 'fs'
import { ensureDir, outputFile, readJSON } from 'fs-extra'
import { basename, join } from 'path'
import { decachePlugin } from '@server/helpers/decache'
import { ApplicationModel } from '@server/models/application/application'
import { MOAuthTokenUser, MUser } from '@server/types/models'
import { getCompleteLocale } from '@shared/core-utils'
import {
......@@ -23,7 +24,7 @@ import { PluginModel } from '../../models/server/plugin'
import { PluginLibrary, RegisterServerAuthExternalOptions, RegisterServerAuthPassOptions, RegisterServerOptions } from '../../types/plugins'
import { ClientHtml } from '../client-html'
import { RegisterHelpers } from './register-helpers'
import { installNpmPlugin, installNpmPluginFromDisk, removeNpmPlugin } from './yarn'
import { installNpmPlugin, installNpmPluginFromDisk, rebuildNativePlugins, removeNpmPlugin } from './yarn'
export interface RegisteredPlugin {
npmName: string
......@@ -384,6 +385,12 @@ export class PluginManager implements ServerHook {
logger.info('Plugin %s uninstalled.', npmName)
}
async rebuildNativePluginsIfNeeded () {
if (!await ApplicationModel.nodeABIChanged()) return
return rebuildNativePlugins()
}
// ###################### Private register ######################
private async registerPluginOrTheme (plugin: PluginModel) {
......
......@@ -31,11 +31,16 @@ async function removeNpmPlugin (name: string) {
await execYarn('remove ' + name)
}
async function rebuildNativePlugins () {
await execYarn('install --pure-lockfile')
}
// ############################################################################
export {
installNpmPlugin,
installNpmPluginFromDisk,
rebuildNativePlugins,
removeNpmPlugin
}
......
import { getServerCommit } from '@server/helpers/utils'
import { getServerCommit } from '@server/helpers/version'
import { CONFIG, isEmailEnabled } from '@server/initializers/config'
import { CONSTRAINTS_FIELDS, DEFAULT_THEME_NAME, PEERTUBE_VERSION } from '@server/initializers/constants'
import { isSignupAllowed, isSignupAllowedForCurrentIP } from '@server/lib/signup'
......
import memoizee from 'memoizee'
import { AllowNull, Column, Default, DefaultScope, HasOne, IsInt, Model, Table } from 'sequelize-typescript'
import { getNodeABIVersion } from '@server/helpers/version'
import { AttributesOnly } from '@shared/typescript-utils'
import { AccountModel } from '../account/account'
......@@ -37,6 +38,14 @@ export class ApplicationModel extends Model<Partial<AttributesOnly<ApplicationMo
@Column
latestPeerTubeVersion: string
@AllowNull(false)
@Column
nodeVersion: string
@AllowNull(false)
@Column
nodeABIVersion: number
@HasOne(() => AccountModel, {
foreignKey: {
allowNull: true
......@@ -52,4 +61,17 @@ export class ApplicationModel extends Model<Partial<AttributesOnly<ApplicationMo
static load () {
return ApplicationModel.findOne()
}
static async nodeABIChanged () {
const application = await this.load()
return application.nodeABIVersion !== getNodeABIVersion()
}
static async updateNodeVersions () {
const application = await this.load()
application.nodeABIVersion = getNodeABIVersion()
application.nodeVersion = process.version
}
}
......@@ -2,6 +2,8 @@
import 'mocha'
import * as chai from 'chai'
import { pathExists, remove } from 'fs-extra'
import { join } from 'path'
import { testHelloWorldRegisteredSettings } from '@server/tests/shared'
import { wait } from '@shared/core-utils'
import { HttpStatusCode, PluginType } from '@shared/models'
......@@ -9,6 +11,7 @@ import {
cleanupTests,
createSingleServer,
killallServers,
makeGetRequest,
PeerTubeServer,
PluginsCommand,
setAccessTokensToServers
......@@ -349,6 +352,35 @@ describe('Test plugins', function () {
await check()
})
it('Should rebuild native modules on Node ABI change', async function () {
await command.install({ path: PluginsCommand.getPluginTestPath('-native') })
await makeGetRequest({
url: server.url,
path: '/plugins/test-native/router',
expectedStatus: HttpStatusCode.NO_CONTENT_204
})
const query = `UPDATE "application" SET "nodeABIVersion" = 1`
await server.sql.updateQuery(query)
const baseNativeModule = server.servers.buildDirectory(join('plugins', 'node_modules', 'a-native-example'))
await remove(join(baseNativeModule, 'build'))
await remove(join(baseNativeModule, 'prebuilds'))
await server.kill()
await server.run()
await pathExists(join(baseNativeModule, 'build'))
await pathExists(join(baseNativeModule, 'prebuilds'))
await makeGetRequest({
url: server.url,
path: '/plugins/test-native/router',
expectedStatus: HttpStatusCode.NO_CONTENT_204
})
})
after(async function () {
await cleanupTests([ server ])
})
......
const print = require('a-native-example')
async function register ({ getRouter }) {
print('hello world')
const router = getRouter()
router.get('/', (req, res) => {
print('hello world')
res.sendStatus(204)
})
}
async function unregister () {
return
}
module.exports = {
register,
unregister
}
{
"name": "peertube-plugin-test-native",
"version": "0.0.1",
"description": "Plugin test-native",
"engine": {
"peertube": ">=4.3.0"
},
"keywords": [
"peertube",
"plugin"
],
"homepage": "https://github.com/Chocobozzz/PeerTube",
"author": "Chocobozzz",
"bugs": "https://github.com/Chocobozzz/PeerTube/issues",
"library": "./main.js",
"staticDirs": {},
"css": [],
"clientScripts": [],
"translations": {},
"dependencies": {
"a-native-example": "^1.0.0"
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment