From 1261b0bb8fb061f78dd2c275733106c28d02b498 Mon Sep 17 00:00:00 2001 From: foxriver76 Date: Thu, 17 Oct 2024 09:45:51 +0200 Subject: [PATCH 1/3] allow to install scoped packages --- packages/cli/src/lib/setup/setupInstall.ts | 127 +++++++-------------- packages/cli/src/lib/setup/utils.ts | 12 +- packages/types-dev/objects.d.ts | 4 + 3 files changed, 53 insertions(+), 90 deletions(-) diff --git a/packages/cli/src/lib/setup/setupInstall.ts b/packages/cli/src/lib/setup/setupInstall.ts index 0ad6b7f50..c68fdf485 100644 --- a/packages/cli/src/lib/setup/setupInstall.ts +++ b/packages/cli/src/lib/setup/setupInstall.ts @@ -148,29 +148,28 @@ export class Install { /** * Download given packet * - * @param repoUrl - * @param packetName + * @param repoUrlOrRepo repository url or already the repository object + * @param packetName name of the package to install * @param options options.stopDb will stop the db before upgrade ONLY use it for controller upgrade - db is gone afterwards, does not work with stoppedList - * @param stoppedList + * @param stoppedList list of stopped instances (as instance objects) */ async downloadPacket( - repoUrl: string | undefined | Record, + repoUrlOrRepo: string | undefined | Record, packetName: string, options?: CLIDownloadPacketOptions, stoppedList?: ioBroker.InstanceObject[], ): Promise { - let url; if (!options || typeof options !== 'object') { options = {}; } stoppedList = stoppedList || []; - let sources: Record; + let sources: Record; - if (!repoUrl || !tools.isObject(repoUrl)) { - sources = await getRepository({ repoName: repoUrl, objects: this.objects }); + if (!repoUrlOrRepo || !tools.isObject(repoUrlOrRepo)) { + sources = await getRepository({ repoName: repoUrlOrRepo, objects: this.objects }); } else { - sources = repoUrl; + sources = repoUrlOrRepo; } if (options.stopDb && stoppedList.length) { @@ -194,97 +193,55 @@ export class Install { version = ''; } } - options.packetName = packetName; - options.unsafePerm = sources[packetName]?.unsafePerm; + const source = sources[packetName]; + + options.packetName = packetName; + options.unsafePerm = source?.unsafePerm; // Check if flag stopBeforeUpdate is true or on windows we stop because of issue #1436 - if ((sources[packetName]?.stopBeforeUpdate || osPlatform === 'win32') && !stoppedList.length) { + if ((source?.stopBeforeUpdate || osPlatform === 'win32') && !stoppedList.length) { stoppedList = await this._getInstancesOfAdapter(packetName); await this.enableInstances(stoppedList, false); } - // try to extract the information from local sources-dist.json - if (!sources[packetName]) { - try { - const sourcesDist = fs.readJsonSync(`${tools.getControllerDir()}/conf/sources-dist.json`); - sources[packetName] = sourcesDist[packetName]; - } catch { - // OK - } + if (!source) { + console.error( + `host.${hostname} Unknown packet name ${packetName}. Please install packages from outside the repository using "${tools.appNameLowerCase} url "!`, + ); + throw new IoBrokerError({ + code: EXIT_CODES.UNKNOWN_PACKET_NAME, + message: `Unknown packetName ${packetName}. Please install packages from outside the repository using npm!`, + }); } - if (sources[packetName]) { - url = sources[packetName].url; - - if ( - url && - packetName === 'js-controller' && - fs.pathExistsSync( - `${tools.getControllerDir()}/../../node_modules/${tools.appName.toLowerCase()}.js-controller`, - ) - ) { - url = null; + if (options.stopDb) { + if (this.objects.destroy) { + await this.objects.destroy(); + console.log('Stopped Objects DB'); } - - if (!url && packetName !== 'example') { - if (options.stopDb) { - if (this.objects.destroy) { - await this.objects.destroy(); - console.log('Stopped Objects DB'); - } - if (this.states.destroy) { - await this.states.destroy(); - console.log('Stopped States DB'); - } - } - - // Install node modules - await this._npmInstallWithCheck( - `${tools.appName.toLowerCase()}.${packetName}${version ? `@${version}` : ''}`, - options, - debug, - ); - - return { packetName, stoppedList }; - } else if (url && url.match(this.tarballRegex)) { - if (options.stopDb) { - if (this.objects.destroy) { - await this.objects.destroy(); - console.log('Stopped Objects DB'); - } - if (this.states.destroy) { - await this.states.destroy(); - console.log('Stopped States DB'); - } - } - - // Install node modules - await this._npmInstallWithCheck(url, options, debug); - return { packetName, stoppedList }; - } else if (!url) { - // Adapter - console.warn( - `host.${hostname} Adapter "${packetName}" can be updated only together with ${tools.appName.toLowerCase()}.js-controller`, - ); - return { packetName, stoppedList }; + if (this.states.destroy) { + await this.states.destroy(); + console.log('Stopped States DB'); } } - console.error( - `host.${hostname} Unknown packet name ${packetName}. Please install packages from outside the repository using "${tools.appNameLowerCase} url "!`, - ); - throw new IoBrokerError({ - code: EXIT_CODES.UNKNOWN_PACKET_NAME, - message: `Unknown packetName ${packetName}. Please install packages from outside the repository using npm!`, - }); + // vendor packages could be scoped and thus differ in the package name + const npmPacketName = source.packetName + ? `${tools.appName.toLowerCase()}.${packetName}@npm:${source.packetName}` + : `${tools.appName.toLowerCase()}.${packetName}`; + + // Install node modules + await this._npmInstallWithCheck(`${npmPacketName}${version ? `@${version}` : ''}`, options, debug); + + return { packetName, stoppedList }; } /** * Install npm module from url * - * @param npmUrl - * @param options + * @param npmUrl parameter passed to `npm install ` + * @param options additional packet download options * @param debug if debug output should be printed */ private async _npmInstallWithCheck( @@ -337,8 +294,8 @@ export class Install { try { return await this._npmInstall({ npmUrl, options, debug, isRetry: false }); - } catch (err) { - console.error(`Could not install ${npmUrl}: ${err.message}`); + } catch (e) { + console.error(`Could not install ${npmUrl}: ${e.message}`); } } @@ -379,7 +336,7 @@ export class Install { const { npmUrl, debug, isRetry } = installOptions; let { options } = installOptions; - if (typeof options !== 'object') { + if (!tools.isObject(options)) { options = {}; } diff --git a/packages/cli/src/lib/setup/utils.ts b/packages/cli/src/lib/setup/utils.ts index ce271639a..88e9c3f2d 100644 --- a/packages/cli/src/lib/setup/utils.ts +++ b/packages/cli/src/lib/setup/utils.ts @@ -16,21 +16,23 @@ interface GetRepositoryOptions { * * @param options Repository specific options */ -export async function getRepository(options: GetRepositoryOptions): Promise> { +export async function getRepository( + options: GetRepositoryOptions, +): Promise> { const { objects } = options; const { repoName } = options; let repoNameOrArray: string | string[] | undefined = repoName; if (!repoName || repoName === 'auto') { - const systemConfig = await objects.getObjectAsync('system.config'); + const systemConfig = await objects.getObject('system.config'); repoNameOrArray = systemConfig!.common.activeRepo; } const repoArr = !Array.isArray(repoNameOrArray) ? [repoNameOrArray!] : repoNameOrArray; - const systemRepos = (await objects.getObjectAsync('system.repositories'))!; + const systemRepos = (await objects.getObject('system.repositories'))!; - const allSources = {}; + const allSources: Record = {}; let changed = false; let anyFound = false; for (const repoUrl of repoArr) { @@ -62,7 +64,7 @@ export async function getRepository(options: GetRepositoryOptions): Promise Date: Thu, 17 Oct 2024 10:06:09 +0200 Subject: [PATCH 2/3] optimize code --- packages/cli/src/lib/setup/setupInstall.ts | 23 +++++++++++----------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/packages/cli/src/lib/setup/setupInstall.ts b/packages/cli/src/lib/setup/setupInstall.ts index c68fdf485..a8890804a 100644 --- a/packages/cli/src/lib/setup/setupInstall.ts +++ b/packages/cli/src/lib/setup/setupInstall.ts @@ -196,25 +196,24 @@ export class Install { const source = sources[packetName]; + if (!source) { + const errMessage = `Unknown packet name ${packetName}. Please install packages from outside the repository using "${tools.appNameLowerCase} url "!`; + console.error(`host.${hostname} ${errMessage}`); + throw new IoBrokerError({ + code: EXIT_CODES.UNKNOWN_PACKET_NAME, + message: errMessage, + }); + } + options.packetName = packetName; - options.unsafePerm = source?.unsafePerm; + options.unsafePerm = source.unsafePerm; // Check if flag stopBeforeUpdate is true or on windows we stop because of issue #1436 - if ((source?.stopBeforeUpdate || osPlatform === 'win32') && !stoppedList.length) { + if ((source.stopBeforeUpdate || osPlatform === 'win32') && !stoppedList.length) { stoppedList = await this._getInstancesOfAdapter(packetName); await this.enableInstances(stoppedList, false); } - if (!source) { - console.error( - `host.${hostname} Unknown packet name ${packetName}. Please install packages from outside the repository using "${tools.appNameLowerCase} url "!`, - ); - throw new IoBrokerError({ - code: EXIT_CODES.UNKNOWN_PACKET_NAME, - message: `Unknown packetName ${packetName}. Please install packages from outside the repository using npm!`, - }); - } - if (options.stopDb) { if (this.objects.destroy) { await this.objects.destroy(); From e825836af83acc9b144eab701e581b0ade86782d Mon Sep 17 00:00:00 2001 From: foxriver76 Date: Thu, 31 Oct 2024 11:09:15 +0100 Subject: [PATCH 3/3] added documentation --- README.md | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/README.md b/README.md index 25580c048..7b6133be4 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ The main configuration is stored in `iobroker-data/iobroker.json`. Normally, the - [js-controller Host Messages](#js-controller-host-messages) - [Adapter Development](#adapter-development) - [Environment Variables](#environment-variables) +- [Vendor Packages Workflow](#vendor-packages-workflow) ### Admin UI **Feature status:** stable @@ -1318,6 +1319,74 @@ However, on upgrades of Node.js these get lost. If js-controller detects a Node. In some scenarios, e.g. during development it may be useful to deactivate this feature. You can do so by settings the `IOB_NO_SETCAP` environment variable to `true`. +### Vendor Packages Workflow +Feature status: New in 7.0.0 + +This feature is only of interest for vendors which aim to provide a package which is published to a private package registry (e.g. GitHub Packages). +This may be desirable if the adapter is only relevant for a specific customer and/or contains business logic which needs to be kept secret. + +In the following, information is provided how private packages can be installed into the ioBroker ecosystem. +The information is tested with GitHub packages. However, it should work in a similar fashion with other registries, like GitLab Packages. + +#### Package Registry +You can use e.g. the GitHub package registry. Simply scope your adapter to your organization or personal scope by changing the package name in the `package.json` +and configuring the `publishConfig`: + +```json +{ + "name": "@org/vendorAdapter", + "publishConfig": { + "registry": "https://npm.pkg.github.com" + } +} +``` + +Note, that you need to configure `npm` to authenticate via your registry. +Find more information in the [documentation](https://docs.npmjs.com/cli/v9/configuring-npm/npmrc#auth-related-configuration). + +Example `.npmrc` file (can be in your project or in the users home directory): + +``` +//npm.pkg.github.com/:_authToken= +@org:registry=https://npm.pkg.github.com +``` + +Where `YOUR_TOKEN` is an access token which has the permissions to write packages. + +If you then execute `npm publish`, the package will be published to your custom registry instead of the `npm` registry. + +#### Vendor Repository +In your vendor-specific repository, each adapter can have a separate field called `packetName`. +This represents the real name of the npm packet. E.g. + +```json +{ + "vendorAdapter": { + "version": "1.0.0", + "name": "vendorAdapter", + "packetName": "@org/vendorAdapter" + } +} +``` + +The js-controller will alias the package name to the adapter name on installation. +This has one drawback, which is normally not relevant for vendor setups. You can not install the adapter via the `npm url` command, meaning no installation from GitHub or local tarballs. + +#### Token setup +On the customers ioBroker host, create a `.npmrc` file inside of `/home/iobroker/`. +It should look like: + +``` +//npm.pkg.github.com/:_authToken= +@org:registry=https://npm.pkg.github.com +``` + +Where `YOUR_TOKEN` is an access token which has the permissions to read packages. +A best practice working with multiple customers is, to create an organization for each customer instead of using your personal scope. +Hence, you can scope them to not have access to packages of other customers or your own. + +Find more information in the [documentation](https://docs.npmjs.com/cli/v9/configuring-npm/npmrc#auth-related-configuration). + ## Release cycle and Development process overview The goal is to release an update for the js-controller roughly all 6 months (April/September). The main reasons for this are shorter iterations and fewer changes that can be problematic for the users (and getting fast feedback) and also trying to stay up-to-date with the dependencies.