From 0f090b97bf29f378a3ab07a496439e2fdc69674f Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Mon, 22 Apr 2024 09:07:56 -0500 Subject: [PATCH] start custom exclusions --- packages/jupyterlab-lsp/package.json | 3 +- .../jupyterlab-lsp/schema/transclusions.json | 101 ++++++++ .../jupyterlab-lsp/src/extractors/regexp.ts | 10 +- packages/jupyterlab-lsp/src/index.ts | 2 + .../src/transclusions/settings/index.ts | 26 +++ .../src/transclusions/settings/manager.ts | 215 ++++++++++++++++++ .../src/transclusions/settings/tokens.ts | 34 +++ 7 files changed, 389 insertions(+), 2 deletions(-) create mode 100644 packages/jupyterlab-lsp/schema/transclusions.json create mode 100644 packages/jupyterlab-lsp/src/transclusions/settings/index.ts create mode 100644 packages/jupyterlab-lsp/src/transclusions/settings/manager.ts create mode 100644 packages/jupyterlab-lsp/src/transclusions/settings/tokens.ts diff --git a/packages/jupyterlab-lsp/package.json b/packages/jupyterlab-lsp/package.json index 32ebe148d..48e12c6ed 100644 --- a/packages/jupyterlab-lsp/package.json +++ b/packages/jupyterlab-lsp/package.json @@ -34,7 +34,7 @@ "build:labextension:dev": "jupyter labextension build --development True .", "build:lib": "tsc", "build:prod": "jlpm run build:lib && jlpm run build:labextension", - "build:schema": "jlpm build:schema-backend && jlpm build:schema-completion && jlpm build:schema-hover && jlpm build:schema-diagnostics && jlpm build:schema-syntax_highlighting && jlpm build:schema-jump_to && jlpm build:schema-signature && jlpm build:schema-highlights && jlpm build:schema-plugin && jlpm build:schema-rename && jlpm build:schema-symbol", + "build:schema": "jlpm build:schema-backend && jlpm build:schema-completion && jlpm build:schema-hover && jlpm build:schema-diagnostics && jlpm build:schema-syntax_highlighting && jlpm build:schema-jump_to && jlpm build:schema-signature && jlpm build:schema-highlights && jlpm build:schema-plugin && jlpm build:schema-rename && jlpm build:schema-symbol && jlpm build:schema-transclusions", "build:schema-backend": "json2ts ../../python_packages/jupyter_lsp/jupyter_lsp/schema/schema.json --unreachableDefinitions | prettier --stdin-filepath _schema.d.ts > src/_schema.ts", "build:schema-plugin": "json2ts schema/plugin.json | prettier --stdin-filepath _plugin.d.ts > src/_plugin.ts", "build:schema-completion": "json2ts schema/completion.json | prettier --stdin-filepath _completion.d.ts > src/_completion.ts ", @@ -46,6 +46,7 @@ "build:schema-rename": "json2ts schema/rename.json | prettier --stdin-filepath _rename.d.ts > src/_rename.ts", "build:schema-signature": "json2ts schema/signature.json | prettier --stdin-filepath _signature.d.ts > src/_signature.ts", "build:schema-symbol": "json2ts schema/symbol.json | prettier --stdin-filepath _symbol.d.ts > src/_symbol.ts", + "build:schema-transclusions": "json2ts schema/transclusions.json | prettier --stdin-filepath _transclusions.d.ts > src/transclusions/settings/_transclusions.ts", "bundle": "npm pack .", "clean": "jlpm run clean:lib", "clean:all": "jlpm run clean:lib && jlpm run clean:labextension", diff --git a/packages/jupyterlab-lsp/schema/transclusions.json b/packages/jupyterlab-lsp/schema/transclusions.json new file mode 100644 index 000000000..e758b0462 --- /dev/null +++ b/packages/jupyterlab-lsp/schema/transclusions.json @@ -0,0 +1,101 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "jupyter.lab.setting-icon": "lsp:transclusions", + "jupyter.lab.setting-icon-label": "Language integration", + "title": "Code Extractors", + "description": "Simple declarative patterns for describing \"foreign languages\" embedded within a host language document", + "type": "object", + "properties": { + "enabled": { + "title": "Enable Custom Code Extractors", + "type": "boolean", + "description": "Whether settings-based extractors will be used to find embedded 'foreign' language fragments within a 'host' language.", + "default": true + }, + "codeExtractors": { + "title": "Code Extractors", + "description": "A named set of patterns that find language fragments embedded in other languages", + "additionalProperties": { + "$ref": "#/definitions/a-code-extractor" + } + } + }, + "definitions": { + "a-language-name": { + "anyOf": [ + { + "title": "Well-known language", + "$ref": "#/definitions/a-well-known-language" + }, + { + "title": "Custom Language", + "type": "string" + } + ] + }, + "a-well-known-language": { + "type": "string", + "enum": ["javascript", "ipythongfm"] + }, + "a-code-extractor": { + "type": "object", + "required": [ + "pattern", + "fileExtension", + "isStandalone", + "hostLanguage", + "foreignLanguage", + "foreignCaptureGroups", + "cellTypes" + ], + "properties": { + "pattern": { + "title": "Pattern", + "type": "string", + "description": "Regular expression to test cells for the foreign language presence." + }, + "hostLanguage": { + "title": "Host Language", + "description": "Name of the language in which to look for foreign language fragments.", + "type": "string", + "$ref": "#/definitions/a-language-name" + }, + "foreignLanguage": { + "title": "Foreign Language", + "description": "Name of the language for an embedded fragment", + "type": "string", + "$ref": "#/definitions/a-language-name" + }, + "foreignCaptureGroups": { + "type": "array", + "description": "Array of numbers specifying match groups to be extracted from the regular expression match, for the use in virtual document of the foreign language", + "items": { + "type": "number", + "minimum": 0 + }, + "minItems": 1 + }, + "isStandalone": { + "type": "boolean", + "description": "Should the foreign code be appended (False) to the previously established virtual document of the same language, or is it standalone snippet which requires separate connection?", + "default": false + }, + "cellTypes": { + "type": "array", + "description": "Types of Notebook cells in which to look for guest language fragments", + "uniqueItems": true, + "minItems": 1, + "items": { + "type": "string", + "enum": ["code", "markdown", "raw"] + } + }, + "fileExtension": { + "type": "string", + "description": "The extension (without a leading period)", + "pattern": "^[^\\.].+" + } + } + } + } +} diff --git a/packages/jupyterlab-lsp/src/extractors/regexp.ts b/packages/jupyterlab-lsp/src/extractors/regexp.ts index 5f702c3e2..e725caf28 100644 --- a/packages/jupyterlab-lsp/src/extractors/regexp.ts +++ b/packages/jupyterlab-lsp/src/extractors/regexp.ts @@ -67,6 +67,7 @@ export class RegExpForeignCodeExtractor implements IForeignCodeExtractor { this.expression = new RegExp(options.pattern); this.standalone = this.options.isStandalone; this.fileExtension = this.options.fileExtension; + this.cellType = this.options.cellTypes || ['code']; } hasForeignCode(code: string): boolean { @@ -186,7 +187,7 @@ export class RegExpForeignCodeExtractor implements IForeignCodeExtractor { } } -namespace RegExpForeignCodeExtractor { +export namespace RegExpForeignCodeExtractor { export interface IOptions { /** * The foreign language. @@ -249,6 +250,13 @@ namespace RegExpForeignCodeExtractor { * or is it standalone snippet which requires separate connection? */ isStandalone: boolean; + /** + * The file extension (without a leading `.`) to create for extracted virtual documents + */ fileExtension: string; + /** + * Cell types in which this extractor should be checked for presence. + */ + cellTypes?: ('code' | 'markdown' | 'raw')[]; } } diff --git a/packages/jupyterlab-lsp/src/index.ts b/packages/jupyterlab-lsp/src/index.ts index 196a4e414..59c36839e 100644 --- a/packages/jupyterlab-lsp/src/index.ts +++ b/packages/jupyterlab-lsp/src/index.ts @@ -54,6 +54,7 @@ import { TLanguageServerConfigurations } from './tokens'; import { DEFAULT_TRANSCLUSIONS } from './transclusions/defaults'; +import { SETTINGS_TRANSCLUSIONS } from './transclusions/settings'; import { LOG_CONSOLE } from './virtual/console'; const PLUGIN_ID = PLUGIN_ID_BASE + ':plugin'; @@ -258,6 +259,7 @@ const plugins: JupyterFrontEndPlugin[] = [ NOTEBOOK_ADAPTER_PLUGIN, FILEEDITOR_ADAPTER_PLUGIN, plugin, + SETTINGS_TRANSCLUSIONS, ...DEFAULT_TRANSCLUSIONS, ...DEFAULT_FEATURES, COMPLETION_FALLBACK_PLUGIN diff --git a/packages/jupyterlab-lsp/src/transclusions/settings/index.ts b/packages/jupyterlab-lsp/src/transclusions/settings/index.ts new file mode 100644 index 000000000..56af7c970 --- /dev/null +++ b/packages/jupyterlab-lsp/src/transclusions/settings/index.ts @@ -0,0 +1,26 @@ +import { + JupyterFrontEnd, + JupyterFrontEndPlugin +} from '@jupyterlab/application'; +import { ILSPCodeExtractorsManager } from '@jupyterlab/lsp'; +import { ISettingRegistry } from '@jupyterlab/settingregistry'; + +import { CustomTransclusionsManager } from './manager'; +import { PLUGIN_ID, ILSPCustomTransclusionsManager } from './tokens'; + +export const SETTINGS_TRANSCLUSIONS: JupyterFrontEndPlugin = + { + id: PLUGIN_ID, + autoStart: true, + requires: [ISettingRegistry, ILSPCodeExtractorsManager], + activate: ( + app: JupyterFrontEnd, + settingsRegistry: ISettingRegistry, + extractorsManager: ILSPCodeExtractorsManager + ): ILSPCustomTransclusionsManager => { + const options = { settingsRegistry, extractorsManager }; + const manager = new CustomTransclusionsManager(options); + void manager.initialize().catch(console.warn); + return manager; + } + }; diff --git a/packages/jupyterlab-lsp/src/transclusions/settings/manager.ts b/packages/jupyterlab-lsp/src/transclusions/settings/manager.ts new file mode 100644 index 000000000..7e473b066 --- /dev/null +++ b/packages/jupyterlab-lsp/src/transclusions/settings/manager.ts @@ -0,0 +1,215 @@ +import { + IForeignCodeExtractor, + ILSPCodeExtractorsManager +} from '@jupyterlab/lsp'; +import { ISettingRegistry } from '@jupyterlab/settingregistry'; +import { PromiseDelegate } from '@lumino/coreutils'; +import { Debouncer } from '@lumino/polling'; +import { Signal, ISignal } from '@lumino/signaling'; + +import { RegExpForeignCodeExtractor } from '../../api'; + +import * as SCHEMA from './_transclusions'; +import { ILSPCustomTransclusionsManager, PLUGIN_ID } from './tokens'; + +const CELL_TYPES: CustomTransclusionsManager.TCellType[] = [ + 'code', + 'markdown', + 'raw' +]; + +/** + * Manage declarative patterns of host/foreign language transclusions + */ +export class CustomTransclusionsManager + implements ILSPCustomTransclusionsManager +{ + extractorsManager: ILSPCodeExtractorsManager; + settingsRegistry: ISettingRegistry; + + constructor(options: CustomTransclusionsManager.IOptions) { + this.extractorsManager = options.extractorsManager; + this.settingsRegistry = options.settingsRegistry; + } + + /** + * A signal that emits when extractors change and are ready to be (re)used. + */ + get extractorsChanged(): ISignal { + return this._extractorsChanged; + } + + /** + * A promise that resolves when the manager is initialized. + */ + get ready() { + return this._ready.promise; + } + + /** + * Request the settings and initialize. + */ + async initialize() { + this._settings = await this.settingsRegistry.load(PLUGIN_ID); + this._settings.changed.connect(this.onSettings, this); + this._onSettings(); + this._ready.resolve(void 0); + } + + protected onSettings(): Promise { + return this._debouncedOnSettings.invoke(); + } + + /** + * Update the upstream extractors based on settings. + */ + protected _onSettings(): void { + this.cleanExtractors(); + + if (this.enabled) { + for (const [key, model] of Object.entries(this.extractorModels)) { + this.ensureOneExtractor(key, model); + } + } + + this._extractorsChanged.emit(void 0); + } + + /** + * Whether settings-based extractors should be used. + */ + protected get enabled() { + return this._settings.composite['enabled']; + } + + /** + * Typed getter for settings. + */ + protected get extractorModels(): CustomTransclusionsManager.TModels { + return this._settings.composite['codeExtractors'] as any; + } + + /** + * Remove all of the extractors added by this plugin. + */ + protected cleanExtractors() { + this.forAllExtractors(this.cleanExtractor); + this._extractorsByHost = new Map(); + } + + /** + * Clean a single extractor. + */ + protected cleanExtractor( + options: CustomTransclusionsManager.IForAllOptions + ): void { + const { hostLanguage, cellType, extractor } = options; + const managed = this.extractorsManager.getExtractors( + cellType, + hostLanguage + ); + if (managed.length && managed.includes(extractor)) { + managed.splice(managed.indexOf(extractor), 1); + } + } + + /** + * Convenience method for running functions for all extractors + */ + protected forAllExtractors(fn: CustomTransclusionsManager.IForAllBack) { + for (const [ + hostLanguage, + hostExtractors + ] of this._extractorsByHost.entries()) { + for (const extractor of hostExtractors.values()) { + for (const cellType of CELL_TYPES) { + fn({ hostLanguage, extractor, cellType }); + } + } + } + } + + /** + * Register a new extractor. + */ + protected ensureOneExtractor( + key: string, + model: SCHEMA.ACodeExtractor + ): void { + let _hostExtractors = this.hostExtractors(model.hostLanguage); + const extractor = this.createNew(model); + _hostExtractors.set(key, extractor); + this.extractorsManager.register(extractor, model.hostLanguage); + } + + /** + * Get the known host extractors for host language. + */ + protected hostExtractors( + hostLanguage: string + ): CustomTransclusionsManager.THostExtractorMap { + let _hostExtractors = this._extractorsByHost.get(hostLanguage); + if (_hostExtractors == null) { + _hostExtractors = new Map(); + this._extractorsByHost.set(hostLanguage, _hostExtractors); + } + return _hostExtractors; + } + + /** Create a new extractor from a model */ + protected createNew( + model: SCHEMA.ACodeExtractor + ): RegExpForeignCodeExtractor { + const options = this.schemaToOptions(model); + return new RegExpForeignCodeExtractor(options); + } + + /** + * Transform a settings model into the constructor options for an extractor + */ + protected schemaToOptions( + model: SCHEMA.ACodeExtractor + ): RegExpForeignCodeExtractor.IOptions { + return { + pattern: model.pattern, + cellTypes: model.cellTypes || ['code'], + fileExtension: model.fileExtension, + foreignCaptureGroups: model.foreignCaptureGroups, + isStandalone: model.isStandalone, + language: model.foreignLanguage + }; + } + + protected _ready = new PromiseDelegate(); + protected _extractorsByHost = new Map< + string, + CustomTransclusionsManager.THostExtractorMap + >(); + protected _settings: ISettingRegistry.ISettings; + protected _extractorsChanged = new Signal( + this + ); + protected _debouncedOnSettings = new Debouncer(this._onSettings); +} + +/** + * A namespace for settings-based extractors + */ +export namespace CustomTransclusionsManager { + /** Constructor options for `CustomTransclusionsManager` */ + export interface IOptions { + extractorsManager: ILSPCodeExtractorsManager; + settingsRegistry: ISettingRegistry; + } + export type THostExtractorMap = Map; + export type TModels = Record; + export type TCellType = 'code' | 'markdown' | 'raw'; + export interface IForAllOptions { + extractor: IForeignCodeExtractor; + hostLanguage: string; + cellType: TCellType; + } + export interface IForAllBack { + (options: IForAllOptions): void; + } +} diff --git a/packages/jupyterlab-lsp/src/transclusions/settings/tokens.ts b/packages/jupyterlab-lsp/src/transclusions/settings/tokens.ts new file mode 100644 index 000000000..fb5de7519 --- /dev/null +++ b/packages/jupyterlab-lsp/src/transclusions/settings/tokens.ts @@ -0,0 +1,34 @@ +import { Token } from '@lumino/coreutils'; +import { ISignal } from '@lumino/signaling'; + +import { PLUGIN_ID as PLUGIN_ID_BASE } from '../../tokens'; + +export const PLUGIN_ID = `${PLUGIN_ID_BASE}:transclusions`; + +/** + * An interface for settings-based extractors + */ +export interface ILSPCustomTransclusionsManager { + /** + * Fetch the settings and initialize extractors. + */ + initialize(): Promise; + + /** + * A promise that resolves when the manager is initialized. + */ + ready: Promise; + + /** + * A signal that emits when extractors change and are ready to be (re)used. + */ + extractorsChanged: ISignal; +} + +/** + * The dependency injection token for requiring the settings-based extractor manager. + */ +export const ILSPCustomTransclusionsManager = + new Token( + `${PLUGIN_ID}:ILSPCustomTransclusionsManager` + );