diff --git a/api/.dockerignore b/api/.dockerignore index 6b61b97e..b55f7aab 100644 --- a/api/.dockerignore +++ b/api/.dockerignore @@ -10,3 +10,7 @@ node_modules coverage* README.md test +*.spec.ts +*.mock.ts +__mock__ +__test__ diff --git a/api/src/app.module.ts b/api/src/app.module.ts index 20b8cb60..6fe1bf55 100644 --- a/api/src/app.module.ts +++ b/api/src/app.module.ts @@ -8,13 +8,15 @@ import path from 'path'; +// eslint-disable-next-line import/order +import { MailerModule } from '@nestjs-modules/mailer'; +// eslint-disable-next-line import/order +import { MjmlAdapter } from '@nestjs-modules/mailer/dist/adapters/mjml.adapter'; import { CacheModule } from '@nestjs/cache-manager'; import { Module } from '@nestjs/common'; import { APP_GUARD } from '@nestjs/core'; import { EventEmitterModule } from '@nestjs/event-emitter'; import { MongooseModule } from '@nestjs/mongoose'; -import { MailerModule } from '@nestjs-modules/mailer'; -import { MjmlAdapter } from '@nestjs-modules/mailer/dist/adapters/mjml.adapter'; import { CsrfGuard, CsrfModule } from '@tekuconcept/nestjs-csrf'; import { AcceptLanguageResolver, @@ -31,6 +33,7 @@ import { ChannelModule } from './channel/channel.module'; import { ChatModule } from './chat/chat.module'; import { CmsModule } from './cms/cms.module'; import { config } from './config'; +import { HelperModule } from './helper/helper.module'; import { I18nModule } from './i18n/i18n.module'; import { LoggerModule } from './logger/logger.module'; import { NlpModule } from './nlp/nlp.module'; @@ -99,6 +102,7 @@ const i18nOptions: I18nOptions = { ChatModule, ChannelModule, PluginsModule, + HelperModule, LoggerModule, WebsocketModule, EventEmitterModule.forRoot({ diff --git a/api/src/channel/channel.module.ts b/api/src/channel/channel.module.ts index c4924735..c5732dcb 100644 --- a/api/src/channel/channel.module.ts +++ b/api/src/channel/channel.module.ts @@ -13,7 +13,6 @@ import { InjectDynamicProviders } from 'nestjs-dynamic-providers'; import { AttachmentModule } from '@/attachment/attachment.module'; import { ChatModule } from '@/chat/chat.module'; import { CmsModule } from '@/cms/cms.module'; -import { NlpModule } from '@/nlp/nlp.module'; import { ChannelController } from './channel.controller'; import { ChannelMiddleware } from './channel.middleware'; @@ -29,7 +28,7 @@ export interface ChannelModuleOptions { controllers: [WebhookController, ChannelController], providers: [ChannelService], exports: [ChannelService], - imports: [NlpModule, ChatModule, AttachmentModule, CmsModule, HttpModule], + imports: [ChatModule, AttachmentModule, CmsModule, HttpModule], }) export class ChannelModule { configure(consumer: MiddlewareConsumer) { diff --git a/api/src/channel/lib/EventWrapper.ts b/api/src/channel/lib/EventWrapper.ts index fbb98cb9..0c7f8f23 100644 --- a/api/src/channel/lib/EventWrapper.ts +++ b/api/src/channel/lib/EventWrapper.ts @@ -17,7 +17,7 @@ import { StdIncomingMessage, } from '@/chat/schemas/types/message'; import { Payload } from '@/chat/schemas/types/quick-reply'; -import { Nlp } from '@/nlp/lib/types'; +import { Nlp } from '@/helper/types'; import ChannelHandler from './Handler'; diff --git a/api/src/channel/lib/Handler.ts b/api/src/channel/lib/Handler.ts index 174c047d..44d13b7e 100644 --- a/api/src/channel/lib/Handler.ts +++ b/api/src/channel/lib/Handler.ts @@ -16,8 +16,6 @@ import { StdOutgoingMessage, } from '@/chat/schemas/types/message'; import { LoggerService } from '@/logger/logger.service'; -import BaseNlpHelper from '@/nlp/lib/BaseNlpHelper'; -import { NlpService } from '@/nlp/services/nlp.service'; import { SettingService } from '@/setting/services/setting.service'; import { hyphenToUnderscore } from '@/utils/helpers/misc'; import { SocketRequest } from '@/websocket/utils/socket-request'; @@ -34,14 +32,11 @@ export default abstract class ChannelHandler { private readonly settings: ChannelSetting[]; - protected NLP: BaseNlpHelper; - constructor( name: N, settings: ChannelSetting[], protected readonly settingService: SettingService, private readonly channelService: ChannelService, - protected readonly nlpService: NlpService, protected readonly logger: LoggerService, ) { this.name = name; @@ -56,10 +51,6 @@ export default abstract class ChannelHandler { this.setup(); } - protected getGroup() { - return hyphenToUnderscore(this.getChannel()) as ChannelSetting['group']; - } - async setup() { await this.settingService.seedIfNotExist( this.getChannel(), @@ -68,19 +59,9 @@ export default abstract class ChannelHandler { weight: i + 1, })), ); - const nlp = this.nlpService.getNLP(); - this.setNLP(nlp); this.init(); } - setNLP(nlp: BaseNlpHelper) { - this.NLP = nlp; - } - - getNLP() { - return this.NLP; - } - /** * Returns the channel's name * @returns Channel's name @@ -89,6 +70,14 @@ export default abstract class ChannelHandler { return this.name; } + /** + * Returns the channel's group + * @returns Channel's group + */ + protected getGroup() { + return hyphenToUnderscore(this.getChannel()) as ChannelSetting['group']; + } + /** * Returns the channel's settings * @returns Channel's settings diff --git a/api/src/chat/chat.module.ts b/api/src/chat/chat.module.ts index 424f55e6..f2c284f8 100644 --- a/api/src/chat/chat.module.ts +++ b/api/src/chat/chat.module.ts @@ -13,7 +13,6 @@ import { MongooseModule } from '@nestjs/mongoose'; import { AttachmentModule } from '@/attachment/attachment.module'; import { ChannelModule } from '@/channel/channel.module'; import { CmsModule } from '@/cms/cms.module'; -import { NlpModule } from '@/nlp/nlp.module'; import { UserModule } from '@/user/user.module'; import { BlockController } from './controllers/block.controller'; @@ -63,7 +62,6 @@ import { SubscriberService } from './services/subscriber.service'; forwardRef(() => ChannelModule), CmsModule, AttachmentModule, - NlpModule, EventEmitter2, UserModule, ], diff --git a/api/src/chat/repositories/message.repository.ts b/api/src/chat/repositories/message.repository.ts index bfcbc927..64bd99f4 100644 --- a/api/src/chat/repositories/message.repository.ts +++ b/api/src/chat/repositories/message.repository.ts @@ -6,16 +6,11 @@ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). */ -import { Injectable, Optional } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { InjectModel } from '@nestjs/mongoose'; import { Model } from 'mongoose'; -import { LanguageService } from '@/i18n/services/language.service'; -import { LoggerService } from '@/logger/logger.service'; -import { NlpSampleCreateDto } from '@/nlp/dto/nlp-sample.dto'; -import { NlpSampleState } from '@/nlp/schemas/types'; -import { NlpSampleService } from '@/nlp/services/nlp-sample.service'; import { BaseRepository } from '@/utils/generics/base-repository'; import { @@ -33,18 +28,9 @@ export class MessageRepository extends BaseRepository< MessagePopulate, MessageFull > { - private readonly nlpSampleService: NlpSampleService; - - private readonly logger: LoggerService; - - private readonly languageService: LanguageService; - constructor( readonly eventEmitter: EventEmitter2, @InjectModel(Message.name) readonly model: Model, - @Optional() nlpSampleService?: NlpSampleService, - @Optional() logger?: LoggerService, - @Optional() languageService?: LanguageService, ) { super( eventEmitter, @@ -53,9 +39,6 @@ export class MessageRepository extends BaseRepository< MESSAGE_POPULATE, MessageFull, ); - this.logger = logger; - this.nlpSampleService = nlpSampleService; - this.languageService = languageService; } /** @@ -69,35 +52,8 @@ export class MessageRepository extends BaseRepository< async preCreate(_doc: AnyMessage): Promise { if (_doc) { if (!('sender' in _doc) && !('recipient' in _doc)) { - this.logger.error('Either sender or recipient must be provided!', _doc); throw new Error('Either sender or recipient must be provided!'); } - // If message is sent by the user then add it as an inbox sample - if ( - 'sender' in _doc && - _doc.sender && - 'message' in _doc && - 'text' in _doc.message - ) { - const defaultLang = await this.languageService?.getDefaultLanguage(); - const record: NlpSampleCreateDto = { - text: _doc.message.text, - type: NlpSampleState.inbox, - trained: false, - // @TODO : We need to define the language in the message entity - language: defaultLang.id, - }; - try { - await this.nlpSampleService.findOneOrCreate(record, record); - this.logger.debug('User message saved as a inbox sample !'); - } catch (err) { - this.logger.error( - 'Unable to add message as a new inbox sample!', - err, - ); - throw err; - } - } } } diff --git a/api/src/chat/schemas/types/context.ts b/api/src/chat/schemas/types/context.ts index 8d2fc30d..7fdc08e9 100644 --- a/api/src/chat/schemas/types/context.ts +++ b/api/src/chat/schemas/types/context.ts @@ -6,7 +6,7 @@ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). */ -import { Nlp } from '@/nlp/lib/types'; +import { Nlp } from '@/helper/types'; import { Subscriber } from '../subscriber.schema'; diff --git a/api/src/chat/services/block.service.ts b/api/src/chat/services/block.service.ts index 952a6af6..73ffa57a 100644 --- a/api/src/chat/services/block.service.ts +++ b/api/src/chat/services/block.service.ts @@ -12,10 +12,10 @@ import { Attachment } from '@/attachment/schemas/attachment.schema'; import { AttachmentService } from '@/attachment/services/attachment.service'; import EventWrapper from '@/channel/lib/EventWrapper'; import { ContentService } from '@/cms/services/content.service'; +import { Nlp } from '@/helper/types'; import { I18nService } from '@/i18n/services/i18n.service'; import { LanguageService } from '@/i18n/services/language.service'; import { LoggerService } from '@/logger/logger.service'; -import { Nlp } from '@/nlp/lib/types'; import { PluginService } from '@/plugins/plugins.service'; import { PluginType } from '@/plugins/types'; import { SettingService } from '@/setting/services/setting.service'; diff --git a/api/src/chat/services/bot.service.spec.ts b/api/src/chat/services/bot.service.spec.ts index be816bce..3134452d 100644 --- a/api/src/chat/services/bot.service.spec.ts +++ b/api/src/chat/services/bot.service.spec.ts @@ -27,24 +27,12 @@ import { MenuService } from '@/cms/services/menu.service'; import { offlineEventText } from '@/extensions/channels/offline/__test__/events.mock'; import OfflineHandler from '@/extensions/channels/offline/index.channel'; import OfflineEventWrapper from '@/extensions/channels/offline/wrapper'; +import { HelperService } from '@/helper/helper.service'; import { LanguageRepository } from '@/i18n/repositories/language.repository'; import { LanguageModel } from '@/i18n/schemas/language.schema'; import { I18nService } from '@/i18n/services/i18n.service'; import { LanguageService } from '@/i18n/services/language.service'; import { LoggerService } from '@/logger/logger.service'; -import { NlpEntityRepository } from '@/nlp/repositories/nlp-entity.repository'; -import { NlpSampleEntityRepository } from '@/nlp/repositories/nlp-sample-entity.repository'; -import { NlpSampleRepository } from '@/nlp/repositories/nlp-sample.repository'; -import { NlpValueRepository } from '@/nlp/repositories/nlp-value.repository'; -import { NlpEntityModel } from '@/nlp/schemas/nlp-entity.schema'; -import { NlpSampleEntityModel } from '@/nlp/schemas/nlp-sample-entity.schema'; -import { NlpSampleModel } from '@/nlp/schemas/nlp-sample.schema'; -import { NlpValueModel } from '@/nlp/schemas/nlp-value.schema'; -import { NlpEntityService } from '@/nlp/services/nlp-entity.service'; -import { NlpSampleEntityService } from '@/nlp/services/nlp-sample-entity.service'; -import { NlpSampleService } from '@/nlp/services/nlp-sample.service'; -import { NlpValueService } from '@/nlp/services/nlp-value.service'; -import { NlpService } from '@/nlp/services/nlp.service'; import { PluginService } from '@/plugins/plugins.service'; import { SettingService } from '@/setting/services/setting.service'; import { installBlockFixtures } from '@/utils/test/fixtures/block'; @@ -109,10 +97,6 @@ describe('BlockService', () => { SubscriberModel, MessageModel, MenuModel, - NlpValueModel, - NlpEntityModel, - NlpSampleEntityModel, - NlpSampleModel, ContextVarModel, LanguageModel, ]), @@ -130,10 +114,6 @@ describe('BlockService', () => { SubscriberRepository, MessageRepository, MenuRepository, - NlpValueRepository, - NlpEntityRepository, - NlpSampleEntityRepository, - NlpSampleRepository, LanguageRepository, BlockService, CategoryService, @@ -147,14 +127,13 @@ describe('BlockService', () => { MessageService, MenuService, OfflineHandler, - NlpValueService, - NlpEntityService, - NlpSampleEntityService, - NlpSampleService, - NlpService, ContextVarService, ContextVarRepository, LanguageService, + { + provide: HelperService, + useValue: {}, + }, { provide: PluginService, useValue: {}, diff --git a/api/src/chat/services/chat.service.ts b/api/src/chat/services/chat.service.ts index e7229ec1..d3e93551 100644 --- a/api/src/chat/services/chat.service.ts +++ b/api/src/chat/services/chat.service.ts @@ -11,8 +11,8 @@ import { EventEmitter2, OnEvent } from '@nestjs/event-emitter'; import EventWrapper from '@/channel/lib/EventWrapper'; import { config } from '@/config'; +import { HelperService } from '@/helper/helper.service'; import { LoggerService } from '@/logger/logger.service'; -import { NlpService } from '@/nlp/services/nlp.service'; import { WebsocketGateway } from '@/websocket/websocket.gateway'; import { MessageCreateDto } from '../dto/message.dto'; @@ -35,7 +35,7 @@ export class ChatService { private readonly subscriberService: SubscriberService, private readonly botService: BotService, private readonly websocketGateway: WebsocketGateway, - private readonly nlpService: NlpService, + private readonly helperService: HelperService, ) {} /** @@ -268,9 +268,9 @@ export class ChatService { } if (event.getText() && !event.getNLP()) { - const nlpAdapter = this.nlpService.getNLP(); try { - const nlp = await nlpAdapter.parse(event.getText()); + const helper = await this.helperService.getDefaultNluHelper(); + const nlp = await helper.predict(event.getText()); event.setNLP(nlp); } catch (err) { this.logger.error('Unable to perform NLP parse', err); diff --git a/api/src/extensions/channels/live-chat-tester/index.channel.ts b/api/src/extensions/channels/live-chat-tester/index.channel.ts index 6f8868bd..0acddbac 100644 --- a/api/src/extensions/channels/live-chat-tester/index.channel.ts +++ b/api/src/extensions/channels/live-chat-tester/index.channel.ts @@ -16,7 +16,6 @@ import { SubscriberService } from '@/chat/services/subscriber.service'; import { MenuService } from '@/cms/services/menu.service'; import { I18nService } from '@/i18n/services/i18n.service'; import { LoggerService } from '@/logger/logger.service'; -import { NlpService } from '@/nlp/services/nlp.service'; import { SettingService } from '@/setting/services/setting.service'; import { WebsocketGateway } from '@/websocket/websocket.gateway'; @@ -34,7 +33,6 @@ export default class LiveChatTesterHandler extends BaseWebChannelHandler< constructor( settingService: SettingService, channelService: ChannelService, - nlpService: NlpService, logger: LoggerService, eventEmitter: EventEmitter2, i18n: I18nService, @@ -49,7 +47,6 @@ export default class LiveChatTesterHandler extends BaseWebChannelHandler< DEFAULT_LIVE_CHAT_TEST_SETTINGS, settingService, channelService, - nlpService, logger, eventEmitter, i18n, diff --git a/api/src/extensions/channels/offline/__test__/index.spec.ts b/api/src/extensions/channels/offline/__test__/index.spec.ts index 78f8df7e..b93b16cf 100644 --- a/api/src/extensions/channels/offline/__test__/index.spec.ts +++ b/api/src/extensions/channels/offline/__test__/index.spec.ts @@ -36,7 +36,6 @@ import { MenuModel } from '@/cms/schemas/menu.schema'; import { MenuService } from '@/cms/services/menu.service'; import { I18nService } from '@/i18n/services/i18n.service'; import { LoggerService } from '@/logger/logger.service'; -import { NlpService } from '@/nlp/services/nlp.service'; import { SettingService } from '@/setting/services/setting.service'; import { UserModel } from '@/user/schemas/user.schema'; import { installMessageFixtures } from '@/utils/test/fixtures/message'; @@ -92,12 +91,6 @@ describe('Offline Handler', () => { })), }, }, - { - provide: NlpService, - useValue: { - getNLP: jest.fn(() => undefined), - }, - }, ChannelService, WebsocketGateway, SocketEventDispatcherService, diff --git a/api/src/extensions/channels/offline/base-web-channel.ts b/api/src/extensions/channels/offline/base-web-channel.ts index 81561705..65d0c4d0 100644 --- a/api/src/extensions/channels/offline/base-web-channel.ts +++ b/api/src/extensions/channels/offline/base-web-channel.ts @@ -53,7 +53,6 @@ import { MenuService } from '@/cms/services/menu.service'; import { config } from '@/config'; import { I18nService } from '@/i18n/services/i18n.service'; import { LoggerService } from '@/logger/logger.service'; -import { NlpService } from '@/nlp/services/nlp.service'; import { SettingService } from '@/setting/services/setting.service'; import { SocketRequest } from '@/websocket/utils/socket-request'; import { SocketResponse } from '@/websocket/utils/socket-response'; @@ -72,7 +71,6 @@ export default class BaseWebChannelHandler< settings: ChannelSetting[], settingService: SettingService, channelService: ChannelService, - nlpService: NlpService, logger: LoggerService, protected readonly eventEmitter: EventEmitter2, protected readonly i18n: I18nService, @@ -82,7 +80,7 @@ export default class BaseWebChannelHandler< protected readonly menuService: MenuService, private readonly websocketGateway: WebsocketGateway, ) { - super(name, settings, settingService, channelService, nlpService, logger); + super(name, settings, settingService, channelService, logger); } /** diff --git a/api/src/extensions/channels/offline/index.channel.ts b/api/src/extensions/channels/offline/index.channel.ts index 3b71989c..e7be89ae 100644 --- a/api/src/extensions/channels/offline/index.channel.ts +++ b/api/src/extensions/channels/offline/index.channel.ts @@ -16,7 +16,6 @@ import { SubscriberService } from '@/chat/services/subscriber.service'; import { MenuService } from '@/cms/services/menu.service'; import { I18nService } from '@/i18n/services/i18n.service'; import { LoggerService } from '@/logger/logger.service'; -import { NlpService } from '@/nlp/services/nlp.service'; import { SettingService } from '@/setting/services/setting.service'; import { WebsocketGateway } from '@/websocket/websocket.gateway'; @@ -30,7 +29,6 @@ export default class OfflineHandler extends BaseWebChannelHandler< constructor( settingService: SettingService, channelService: ChannelService, - nlpService: NlpService, logger: LoggerService, eventEmitter: EventEmitter2, i18n: I18nService, @@ -45,7 +43,6 @@ export default class OfflineHandler extends BaseWebChannelHandler< DEFAULT_OFFLINE_SETTINGS, settingService, channelService, - nlpService, logger, eventEmitter, i18n, diff --git a/api/src/extensions/helpers/nlp/default/__test__/__mock__/base.mock.ts b/api/src/extensions/helpers/core-nlu/__test__/__mock__/base.mock.ts similarity index 100% rename from api/src/extensions/helpers/nlp/default/__test__/__mock__/base.mock.ts rename to api/src/extensions/helpers/core-nlu/__test__/__mock__/base.mock.ts diff --git a/api/src/extensions/helpers/nlp/default/__test__/index.mock.ts b/api/src/extensions/helpers/core-nlu/__test__/index.mock.ts similarity index 92% rename from api/src/extensions/helpers/nlp/default/__test__/index.mock.ts rename to api/src/extensions/helpers/core-nlu/__test__/index.mock.ts index dd8458b6..617710c3 100644 --- a/api/src/extensions/helpers/nlp/default/__test__/index.mock.ts +++ b/api/src/extensions/helpers/core-nlu/__test__/index.mock.ts @@ -6,11 +6,11 @@ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). */ -import { Nlp } from '@/nlp/lib/types'; +import { Nlp } from '@/helper/types'; -import { DatasetType, NlpParseResultType } from '../types'; +import { NlpParseResultType, RasaNlu } from '../types'; -export const nlpEmptyFormated: DatasetType = { +export const nlpEmptyFormated: RasaNlu.Dataset = { common_examples: [], regex_features: [], lookup_tables: [ @@ -35,7 +35,7 @@ export const nlpEmptyFormated: DatasetType = { ], }; -export const nlpFormatted: DatasetType = { +export const nlpFormatted: RasaNlu.Dataset = { common_examples: [ { text: 'Hello', diff --git a/api/src/extensions/helpers/nlp/default/__test__/index.spec.ts b/api/src/extensions/helpers/core-nlu/__test__/index.spec.ts similarity index 52% rename from api/src/extensions/helpers/nlp/default/__test__/index.spec.ts rename to api/src/extensions/helpers/core-nlu/__test__/index.spec.ts index 12e4f02e..9870b8a5 100644 --- a/api/src/extensions/helpers/nlp/default/__test__/index.spec.ts +++ b/api/src/extensions/helpers/core-nlu/__test__/index.spec.ts @@ -12,31 +12,19 @@ import { EventEmitter2 } from '@nestjs/event-emitter'; import { MongooseModule } from '@nestjs/mongoose'; import { Test, TestingModule } from '@nestjs/testing'; +import { HelperService } from '@/helper/helper.service'; import { LanguageRepository } from '@/i18n/repositories/language.repository'; import { LanguageModel } from '@/i18n/schemas/language.schema'; import { LanguageService } from '@/i18n/services/language.service'; import { LoggerService } from '@/logger/logger.service'; -import { NlpEntityRepository } from '@/nlp/repositories/nlp-entity.repository'; -import { NlpSampleEntityRepository } from '@/nlp/repositories/nlp-sample-entity.repository'; -import { NlpSampleRepository } from '@/nlp/repositories/nlp-sample.repository'; -import { NlpValueRepository } from '@/nlp/repositories/nlp-value.repository'; -import { NlpEntityModel } from '@/nlp/schemas/nlp-entity.schema'; -import { NlpSampleEntityModel } from '@/nlp/schemas/nlp-sample-entity.schema'; -import { NlpSampleModel } from '@/nlp/schemas/nlp-sample.schema'; -import { NlpValueModel } from '@/nlp/schemas/nlp-value.schema'; -import { NlpEntityService } from '@/nlp/services/nlp-entity.service'; -import { NlpSampleEntityService } from '@/nlp/services/nlp-sample-entity.service'; -import { NlpSampleService } from '@/nlp/services/nlp-sample.service'; -import { NlpValueService } from '@/nlp/services/nlp-value.service'; -import { NlpService } from '@/nlp/services/nlp.service'; import { SettingService } from '@/setting/services/setting.service'; -import { installNlpSampleEntityFixtures } from '@/utils/test/fixtures/nlpsampleentity'; +import { installLanguageFixtures } from '@/utils/test/fixtures/language'; import { closeInMongodConnection, rootMongooseTestModule, } from '@/utils/test/test'; -import DefaultNlpHelper from '../index.nlp.helper'; +import CoreNluHelper from '../index.helper'; import { entitiesMock, samplesMock } from './__mock__/base.mock'; import { @@ -46,45 +34,31 @@ import { nlpParseResult, } from './index.mock'; -describe('NLP Default Helper', () => { +describe('Core NLU Helper', () => { let settingService: SettingService; - let nlpService: NlpService; - let defaultNlpHelper: DefaultNlpHelper; + let defaultNlpHelper: CoreNluHelper; beforeAll(async () => { const module: TestingModule = await Test.createTestingModule({ imports: [ - rootMongooseTestModule(installNlpSampleEntityFixtures), - MongooseModule.forFeature([ - NlpEntityModel, - NlpValueModel, - NlpSampleModel, - NlpSampleEntityModel, - LanguageModel, - ]), + rootMongooseTestModule(async () => { + await installLanguageFixtures(); + }), + MongooseModule.forFeature([LanguageModel]), HttpModule, ], providers: [ - NlpService, - NlpSampleService, - NlpSampleRepository, - NlpEntityService, - NlpEntityRepository, - NlpValueService, - NlpValueRepository, - NlpSampleEntityService, - NlpSampleEntityRepository, LanguageService, LanguageRepository, EventEmitter2, - DefaultNlpHelper, + HelperService, + CoreNluHelper, LoggerService, { provide: SettingService, useValue: { getSettings: jest.fn(() => ({ - nlp_settings: { - provider: 'default', + core_nlu: { endpoint: 'path', token: 'token', threshold: '0.5', @@ -103,56 +77,51 @@ describe('NLP Default Helper', () => { ], }).compile(); settingService = module.get(SettingService); - nlpService = module.get(NlpService); - defaultNlpHelper = module.get(DefaultNlpHelper); - nlpService.setHelper('default', defaultNlpHelper); - nlpService.initNLP(); + defaultNlpHelper = module.get(CoreNluHelper); }); afterAll(closeInMongodConnection); - it('should init() properly', () => { - const nlp = nlpService.getNLP(); - expect(nlp).toBeDefined(); - }); - it('should format empty training set properly', async () => { - const nlp = nlpService.getNLP(); - const results = await nlp.format([], entitiesMock); + const results = await defaultNlpHelper.format([], entitiesMock); expect(results).toEqual(nlpEmptyFormated); }); it('should format training set properly', async () => { - const nlp = nlpService.getNLP(); - const results = await nlp.format(samplesMock, entitiesMock); + const results = await defaultNlpHelper.format(samplesMock, entitiesMock); expect(results).toEqual(nlpFormatted); }); - it('should return best guess from empty parse results', () => { - const nlp = nlpService.getNLP(); - const results = nlp.bestGuess( + it('should return best guess from empty parse results', async () => { + const results = await defaultNlpHelper.filterEntitiesByConfidence( { entities: [], - intent: {}, + intent: { name: 'greeting', confidence: 0 }, intent_ranking: [], text: 'test', }, false, ); - expect(results).toEqual({ entities: [] }); + expect(results).toEqual({ + entities: [{ entity: 'intent', value: 'greeting', confidence: 0 }], + }); }); - it('should return best guess from parse results', () => { - const nlp = nlpService.getNLP(); - const results = nlp.bestGuess(nlpParseResult, false); + it('should return best guess from parse results', async () => { + const results = await defaultNlpHelper.filterEntitiesByConfidence( + nlpParseResult, + false, + ); expect(results).toEqual(nlpBestGuess); }); it('should return best guess from parse results with threshold', async () => { - const nlp = nlpService.getNLP(); - const results = nlp.bestGuess(nlpParseResult, true); + const results = await defaultNlpHelper.filterEntitiesByConfidence( + nlpParseResult, + true, + ); const settings = await settingService.getSettings(); - const threshold = settings.nlp_settings.threshold; + const threshold = settings.core_nlu.threshold; const thresholdGuess = { entities: nlpBestGuess.entities.filter( (g) => diff --git a/api/src/extensions/helpers/core-nlu/i18n/en/help.json b/api/src/extensions/helpers/core-nlu/i18n/en/help.json new file mode 100644 index 00000000..14b318e9 --- /dev/null +++ b/api/src/extensions/helpers/core-nlu/i18n/en/help.json @@ -0,0 +1,5 @@ +{ + "endpoint": "Enter the endpoint URL for the Core NLU API where requests will be sent.", + "token": "Provide the API token for authenticating requests to the Core NLU API.", + "threshold": "Set the minimum confidence score for predictions to be considered valid." +} diff --git a/api/src/extensions/helpers/core-nlu/i18n/en/label.json b/api/src/extensions/helpers/core-nlu/i18n/en/label.json new file mode 100644 index 00000000..fc71fe5c --- /dev/null +++ b/api/src/extensions/helpers/core-nlu/i18n/en/label.json @@ -0,0 +1,5 @@ +{ + "endpoint": "Core NLU API", + "token": "API Token", + "threshold": "Confidence Threshold" +} diff --git a/api/src/extensions/helpers/core-nlu/i18n/en/title.json b/api/src/extensions/helpers/core-nlu/i18n/en/title.json new file mode 100644 index 00000000..70534a3d --- /dev/null +++ b/api/src/extensions/helpers/core-nlu/i18n/en/title.json @@ -0,0 +1,3 @@ +{ + "core_nlu": "Core NLU Engine" +} diff --git a/api/src/extensions/helpers/core-nlu/i18n/fr/help.json b/api/src/extensions/helpers/core-nlu/i18n/fr/help.json new file mode 100644 index 00000000..559fc7cc --- /dev/null +++ b/api/src/extensions/helpers/core-nlu/i18n/fr/help.json @@ -0,0 +1,5 @@ +{ + "endpoint": "Entrez l'URL de point de terminaison pour l'API NLU Core où les requêtes seront envoyées.", + "token": "Fournissez le jeton d'API pour authentifier les requêtes à l'API NLU Core.", + "threshold": "Définissez le score de confiance minimum pour que les prédictions soient considérées comme valides." +} diff --git a/api/src/extensions/helpers/core-nlu/i18n/fr/label.json b/api/src/extensions/helpers/core-nlu/i18n/fr/label.json new file mode 100644 index 00000000..b12a1eb5 --- /dev/null +++ b/api/src/extensions/helpers/core-nlu/i18n/fr/label.json @@ -0,0 +1,5 @@ +{ + "endpoint": "API NLU Core", + "token": "Jeton d'API", + "threshold": "Seuil de Confiance" +} diff --git a/api/src/extensions/helpers/core-nlu/i18n/fr/title.json b/api/src/extensions/helpers/core-nlu/i18n/fr/title.json new file mode 100644 index 00000000..70534a3d --- /dev/null +++ b/api/src/extensions/helpers/core-nlu/i18n/fr/title.json @@ -0,0 +1,3 @@ +{ + "core_nlu": "Core NLU Engine" +} diff --git a/api/src/extensions/helpers/core-nlu/index.d.ts b/api/src/extensions/helpers/core-nlu/index.d.ts new file mode 100644 index 00000000..00c0d3c8 --- /dev/null +++ b/api/src/extensions/helpers/core-nlu/index.d.ts @@ -0,0 +1,14 @@ +import { CORE_NLU_HELPER_GROUP, CORE_NLU_HELPER_SETTINGS } from './settings'; + +declare global { + interface Settings extends SettingTree {} +} + +declare module '@nestjs/event-emitter' { + interface IHookExtensionsOperationMap { + [CORE_NLU_HELPER_GROUP]: TDefinition< + object, + SettingMapByType + >; + } +} diff --git a/api/src/extensions/helpers/core-nlu/index.helper.ts b/api/src/extensions/helpers/core-nlu/index.helper.ts new file mode 100644 index 00000000..58327ef1 --- /dev/null +++ b/api/src/extensions/helpers/core-nlu/index.helper.ts @@ -0,0 +1,283 @@ +/* + * Copyright © 2024 Hexastack. All rights reserved. + * + * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: + * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. + * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). + */ + +import { HttpService } from '@nestjs/axios'; +import { Injectable } from '@nestjs/common'; + +import { HelperService } from '@/helper/helper.service'; +import BaseNlpHelper from '@/helper/lib/base-nlp-helper'; +import { Nlp } from '@/helper/types'; +import { LanguageService } from '@/i18n/services/language.service'; +import { LoggerService } from '@/logger/logger.service'; +import { NlpEntity, NlpEntityFull } from '@/nlp/schemas/nlp-entity.schema'; +import { NlpSampleFull } from '@/nlp/schemas/nlp-sample.schema'; +import { NlpValue } from '@/nlp/schemas/nlp-value.schema'; +import { SettingService } from '@/setting/services/setting.service'; +import { buildURL } from '@/utils/helpers/URL'; + +import { CORE_NLU_HELPER_NAME, CORE_NLU_HELPER_SETTINGS } from './settings'; +import { NlpParseResultType, RasaNlu } from './types'; + +@Injectable() +export default class CoreNluHelper extends BaseNlpHelper< + typeof CORE_NLU_HELPER_NAME +> { + constructor( + settingService: SettingService, + helperService: HelperService, + logger: LoggerService, + private readonly httpService: HttpService, + private readonly languageService: LanguageService, + ) { + super( + CORE_NLU_HELPER_NAME, + CORE_NLU_HELPER_SETTINGS, + settingService, + helperService, + logger, + ); + } + + /** + * Formats a set of NLP samples into the Rasa NLU-compatible training dataset format. + * + * @param samples - The NLP samples to format. + * @param entities - The NLP entities available in the dataset. + * + * @returns The formatted Rasa NLU training dataset. + */ + async format( + samples: NlpSampleFull[], + entities: NlpEntityFull[], + ): Promise { + const entityMap = NlpEntity.getEntityMap(entities); + const valueMap = NlpValue.getValueMap( + NlpValue.getValuesFromEntities(entities), + ); + + const common_examples: RasaNlu.CommonExample[] = samples + .filter((s) => s.entities.length > 0) + .map((s) => { + const intent = s.entities.find( + (e) => entityMap[e.entity].name === 'intent', + ); + if (!intent) { + throw new Error('Unable to find the `intent` nlp entity.'); + } + const sampleEntities: RasaNlu.ExampleEntity[] = s.entities + .filter((e) => entityMap[e.entity].name !== 'intent') + .map((e) => { + const res: RasaNlu.ExampleEntity = { + entity: entityMap[e.entity].name, + value: valueMap[e.value].value, + }; + if ('start' in e && 'end' in e) { + Object.assign(res, { + start: e.start, + end: e.end, + }); + } + return res; + }) + // TODO : place language at the same level as the intent + .concat({ + entity: 'language', + value: s.language.code, + }); + + return { + text: s.text, + intent: valueMap[intent.value].value, + entities: sampleEntities, + }; + }); + + const languages = await this.languageService.getLanguages(); + const lookup_tables: RasaNlu.LookupTable[] = entities + .map((e) => { + return { + name: e.name, + elements: e.values.map((v) => { + return v.value; + }), + }; + }) + .concat({ + name: 'language', + elements: Object.keys(languages), + }); + const entity_synonyms = entities + .reduce((acc, e) => { + const synonyms = e.values.map((v) => { + return { + value: v.value, + synonyms: v.expressions, + }; + }); + return acc.concat(synonyms); + }, [] as RasaNlu.EntitySynonym[]) + .filter((s) => { + return s.synonyms.length > 0; + }); + return { + common_examples, + regex_features: [], + lookup_tables, + entity_synonyms, + }; + } + + /** + * Perform a training request + * + * @param samples - Samples to train + * @param entities - All available entities + * @returns The training result + */ + async train( + samples: NlpSampleFull[], + entities: NlpEntityFull[], + ): Promise { + const nluData: RasaNlu.Dataset = await this.format(samples, entities); + const settings = await this.getSettings(); + // Train samples + return await this.httpService.axiosRef.post( + buildURL(settings.endpoint, `/train`), + nluData, + { + params: { + token: settings.token, + }, + }, + ); + } + + /** + * Perform evaluation request + * + * @param samples - Samples to evaluate + * @param entities - All available entities + * @returns Evaluation results + */ + async evaluate( + samples: NlpSampleFull[], + entities: NlpEntityFull[], + ): Promise { + const settings = await this.getSettings(); + const nluTestData: RasaNlu.Dataset = await this.format(samples, entities); + // Evaluate model with test samples + return await this.httpService.axiosRef.post( + buildURL(settings.endpoint, `/evaluate`), + nluTestData, + { + params: { + token: settings.token, + }, + }, + ); + } + + /** + * Returns only the entities that have strong confidence (> than the threshold), can return an empty result + * + * @param nlp - The nlp returned result + * @param threshold - Whenever to apply threshold filter or not + * + * @returns The parsed entities + */ + async filterEntitiesByConfidence( + nlp: NlpParseResultType, + threshold: boolean, + ): Promise { + try { + let minConfidence = 0; + const guess: Nlp.ParseEntities = { + entities: nlp.entities.slice(), + }; + if (threshold) { + const settings = await this.getSettings(); + const threshold = settings.threshold; + minConfidence = + typeof threshold === 'string' + ? Number.parseFloat(threshold) + : threshold; + guess.entities = guess.entities + .map((e) => { + e.confidence = + typeof e.confidence === 'string' + ? Number.parseFloat(e.confidence) + : e.confidence; + return e; + }) + .filter((e) => e.confidence >= minConfidence); + // Get past threshold and the highest confidence for the same entity + // .filter((e, idx, self) => { + // const sameEntities = self.filter((s) => s.entity === e.entity); + // const max = Math.max.apply(Math, sameEntities.map((e) => { return e.confidence; })); + // return e.confidence === max; + // }); + } + + ['intent', 'language'].forEach((trait) => { + if (trait in nlp && (nlp as any)[trait].confidence >= minConfidence) { + guess.entities.push({ + entity: trait, + value: (nlp as any)[trait].name, + confidence: (nlp as any)[trait].confidence, + }); + } + }); + return guess; + } catch (e) { + this.logger.error( + 'Core NLU Helper : Unable to parse nlp result to extract best guess!', + e, + ); + return { + entities: [], + }; + } + } + + /** + * Returns only the entities that have strong confidence (> than the threshold), can return an empty result + * + * @param text - The text to parse + * @param threshold - Whenever to apply threshold filter or not + * @param project - Whenever to request a specific model + * + * @returns The prediction + */ + async predict( + text: string, + threshold: boolean, + project: string = 'current', + ): Promise { + try { + const settings = await this.getSettings(); + const { data: nlp } = + await this.httpService.axiosRef.post( + buildURL(settings.endpoint, '/parse'), + { + q: text, + project, + }, + { + params: { + token: settings.token, + }, + }, + ); + + return this.filterEntitiesByConfidence(nlp, threshold); + } catch (err) { + this.logger.error('Core NLU Helper : Unable to parse nlp', err); + throw err; + } + } +} diff --git a/api/src/extensions/helpers/core-nlu/package.json b/api/src/extensions/helpers/core-nlu/package.json new file mode 100644 index 00000000..02413a2d --- /dev/null +++ b/api/src/extensions/helpers/core-nlu/package.json @@ -0,0 +1,8 @@ +{ + "name": "hexabot-core-nlu", + "version": "2.0.0", + "description": "The Core NLU Helper Extension for Hexabot Chatbot / Agent Builder to enable the Intent Classification and Language Detection", + "dependencies": {}, + "author": "Hexastack", + "license": "AGPL-3.0-only" +} diff --git a/api/src/extensions/helpers/core-nlu/settings.ts b/api/src/extensions/helpers/core-nlu/settings.ts new file mode 100644 index 00000000..3a5f3d83 --- /dev/null +++ b/api/src/extensions/helpers/core-nlu/settings.ts @@ -0,0 +1,32 @@ +import { HelperSetting } from '@/helper/types'; +import { SettingType } from '@/setting/schemas/types'; + +export const CORE_NLU_HELPER_NAME = 'core-nlu'; + +export const CORE_NLU_HELPER_GROUP = 'core_nlu'; + +export const CORE_NLU_HELPER_SETTINGS = [ + { + group: CORE_NLU_HELPER_GROUP, + label: 'endpoint', + value: 'http://nlu-api:5000/', + type: SettingType.text, + }, + { + group: CORE_NLU_HELPER_GROUP, + label: 'token', + value: 'token123', + type: SettingType.text, + }, + { + group: CORE_NLU_HELPER_GROUP, + label: 'threshold', + value: 0.1, + type: SettingType.number, + config: { + min: 0, + max: 1, + step: 0.01, + }, + }, +] as const satisfies HelperSetting[]; diff --git a/api/src/extensions/helpers/nlp/default/types.ts b/api/src/extensions/helpers/core-nlu/types.ts similarity index 62% rename from api/src/extensions/helpers/nlp/default/types.ts rename to api/src/extensions/helpers/core-nlu/types.ts index 77afc5bf..495097af 100644 --- a/api/src/extensions/helpers/nlp/default/types.ts +++ b/api/src/extensions/helpers/core-nlu/types.ts @@ -6,34 +6,36 @@ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). */ -export interface ExampleEntity { - entity: string; - value: string; - start?: number; - end?: number; -} +export namespace RasaNlu { + export interface ExampleEntity { + entity: string; + value: string; + start?: number; + end?: number; + } -export interface CommonExample { - text: string; - intent: string; - entities: ExampleEntity[]; -} + export interface CommonExample { + text: string; + intent: string; + entities: ExampleEntity[]; + } -export interface LookupTable { - name: string; - elements: string[]; -} + export interface LookupTable { + name: string; + elements: string[]; + } -export interface EntitySynonym { - value: string; - synonyms: string[]; -} + export interface EntitySynonym { + value: string; + synonyms: string[]; + } -export interface DatasetType { - common_examples: CommonExample[]; - regex_features: any[]; - lookup_tables: LookupTable[]; - entity_synonyms: EntitySynonym[]; + export interface Dataset { + common_examples: CommonExample[]; + regex_features: any[]; + lookup_tables: LookupTable[]; + entity_synonyms: EntitySynonym[]; + } } export interface ParseEntity { diff --git a/api/src/extensions/helpers/nlp/default/index.nlp.helper.ts b/api/src/extensions/helpers/nlp/default/index.nlp.helper.ts deleted file mode 100644 index b605126f..00000000 --- a/api/src/extensions/helpers/nlp/default/index.nlp.helper.ts +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Copyright © 2024 Hexastack. All rights reserved. - * - * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: - * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. - * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). - */ - -import { HttpService } from '@nestjs/axios'; -import { Injectable } from '@nestjs/common'; - -import { LoggerService } from '@/logger/logger.service'; -import BaseNlpHelper from '@/nlp/lib/BaseNlpHelper'; -import { Nlp } from '@/nlp/lib/types'; -import { NlpEntityFull } from '@/nlp/schemas/nlp-entity.schema'; -import { NlpSampleFull } from '@/nlp/schemas/nlp-sample.schema'; -import { NlpEntityService } from '@/nlp/services/nlp-entity.service'; -import { NlpSampleService } from '@/nlp/services/nlp-sample.service'; -import { NlpService } from '@/nlp/services/nlp.service'; -import { buildURL } from '@/utils/helpers/URL'; - -import { DatasetType, NlpParseResultType } from './types'; - -@Injectable() -export default class DefaultNlpHelper extends BaseNlpHelper { - /** - * Instantiate a nlp helper - * - * @param settings - NLP settings - */ - constructor( - logger: LoggerService, - nlpService: NlpService, - nlpSampleService: NlpSampleService, - nlpEntityService: NlpEntityService, - protected readonly httpService: HttpService, - ) { - super(logger, nlpService, nlpSampleService, nlpEntityService); - } - - onModuleInit() { - this.nlpService.setHelper(this.getName(), this); - } - - getName() { - return 'default'; - } - - /** - * Return training dataset in compatible format - * - * @param samples - Sample to train - * @param entities - All available entities - * @returns {DatasetType} - The formatted RASA training set - */ - async format( - samples: NlpSampleFull[], - entities: NlpEntityFull[], - ): Promise { - const nluData = await this.nlpSampleService.formatRasaNlu( - samples, - entities, - ); - - return nluData; - } - - /** - * Perform Rasa training request - * - * @param samples - Samples to train - * @param entities - All available entities - * @returns {Promise} - Rasa training result - */ - async train( - samples: NlpSampleFull[], - entities: NlpEntityFull[], - ): Promise { - const self = this; - const nluData: DatasetType = await self.format(samples, entities); - // Train samples - const result = await this.httpService.axiosRef.post( - buildURL(this.settings.endpoint, `/train`), - nluData, - { - params: { - token: this.settings.token, - }, - }, - ); - // Mark samples as trained - await this.nlpSampleService.updateMany( - { type: 'train' }, - { trained: true }, - ); - return result; - } - - /** - * Perform evaluation request - * - * @param samples - Samples to evaluate - * @param entities - All available entities - * @returns {Promise} - Evaluation results - */ - async evaluate( - samples: NlpSampleFull[], - entities: NlpEntityFull[], - ): Promise { - const self = this; - const nluTestData: DatasetType = await self.format(samples, entities); - // Evaluate model with test samples - return await this.httpService.axiosRef.post( - buildURL(this.settings.endpoint, `/evaluate`), - nluTestData, - { - params: { - token: this.settings.token, - }, - }, - ); - } - - /** - * Returns only the entities that have strong confidence (> than the threshold), can return an empty result - * - * @param nlp - The nlp returned result - * @param threshold - Whenever to apply threshold filter or not - * @returns {Nlp.ParseEntities} - */ - bestGuess(nlp: NlpParseResultType, threshold: boolean): Nlp.ParseEntities { - try { - let minConfidence = 0; - const guess: Nlp.ParseEntities = { - entities: nlp.entities.slice(), - }; - if (threshold) { - const threshold = this.settings.threshold; - minConfidence = - typeof threshold === 'string' - ? Number.parseFloat(threshold) - : threshold; - guess.entities = guess.entities - .map((e) => { - e.confidence = - typeof e.confidence === 'string' - ? Number.parseFloat(e.confidence) - : e.confidence; - return e; - }) - .filter((e) => e.confidence >= minConfidence); - // Get past threshold and the highest confidence for the same entity - // .filter((e, idx, self) => { - // const sameEntities = self.filter((s) => s.entity === e.entity); - // const max = Math.max.apply(Math, sameEntities.map((e) => { return e.confidence; })); - // return e.confidence === max; - // }); - } - - ['intent', 'language'].forEach((trait) => { - if (trait in nlp && (nlp as any)[trait].confidence >= minConfidence) { - guess.entities.push({ - entity: trait, - value: (nlp as any)[trait].name, - confidence: (nlp as any)[trait].confidence, - }); - } - }); - return guess; - } catch (e) { - this.logger.error( - 'NLP RasaAdapter : Unable to parse nlp result to extract best guess!', - e, - ); - return { - entities: [], - }; - } - } - - /** - * Returns only the entities that have strong confidence (> than the threshold), can return an empty result - * - * @param text - The text to parse - * @param threshold - Whenever to apply threshold filter or not - * @param project - Whenever to request a specific model - * @returns {Promise} - */ - async parse( - text: string, - threshold: boolean, - project: string = 'current', - ): Promise { - try { - const { data: nlp } = - await this.httpService.axiosRef.post( - buildURL(this.settings.endpoint, '/parse'), - { - q: text, - project, - }, - { - params: { - token: this.settings.token, - }, - }, - ); - - return this.bestGuess(nlp, threshold); - } catch (err) { - this.logger.error('NLP RasaAdapter : Unable to parse nlp', err); - throw err; - } - } -} diff --git a/api/src/nlp/controllers/nlp.controller.ts b/api/src/helper/helper.controller.ts similarity index 57% rename from api/src/nlp/controllers/nlp.controller.ts rename to api/src/helper/helper.controller.ts index 0622ce50..b0a8b2e9 100644 --- a/api/src/nlp/controllers/nlp.controller.ts +++ b/api/src/helper/helper.controller.ts @@ -6,22 +6,26 @@ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). */ -import { Controller, Get } from '@nestjs/common'; +import { Controller, Get, Param } from '@nestjs/common'; -import { NlpService } from '../services/nlp.service'; +import { Roles } from '@/utils/decorators/roles.decorator'; -@Controller('nlp') -export class NlpController { - constructor(private readonly nlpService: NlpService) {} +import { HelperService } from './helper.service'; +import { HelperType } from './types'; + +@Controller('helper') +export class HelperController { + constructor(private readonly helperService: HelperService) {} /** - * Retrieves a list of NLP helpers. + * Retrieves a list of helpers. * * @returns An array of objects containing the name of each NLP helper. */ - @Get() - getNlpHelpers(): { name: string }[] { - return this.nlpService.getAll().map((helper) => { + @Roles('public') + @Get(':type') + getHelpers(@Param('type') type: HelperType) { + return this.helperService.getAllByType(type).map((helper) => { return { name: helper.getName(), }; diff --git a/api/src/helper/helper.module.ts b/api/src/helper/helper.module.ts new file mode 100644 index 00000000..5cb84f2b --- /dev/null +++ b/api/src/helper/helper.module.ts @@ -0,0 +1,24 @@ +/* + * Copyright © 2024 Hexastack. All rights reserved. + * + * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: + * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. + * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). + */ + +import { HttpModule } from '@nestjs/axios'; +import { Global, Module } from '@nestjs/common'; +import { InjectDynamicProviders } from 'nestjs-dynamic-providers'; + +import { HelperController } from './helper.controller'; +import { HelperService } from './helper.service'; + +@Global() +@InjectDynamicProviders('dist/extensions/**/*.helper.js') +@Module({ + imports: [HttpModule], + controllers: [HelperController], + providers: [HelperService], + exports: [HelperService], +}) +export class HelperModule {} diff --git a/api/src/helper/helper.service.ts b/api/src/helper/helper.service.ts new file mode 100644 index 00000000..8b8e53d7 --- /dev/null +++ b/api/src/helper/helper.service.ts @@ -0,0 +1,89 @@ +/* + * Copyright © 2024 Hexastack. All rights reserved. + * + * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: + * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. + * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). + */ + +import { Injectable } from '@nestjs/common'; + +import { LoggerService } from '@/logger/logger.service'; +import { SettingService } from '@/setting/services/setting.service'; + +import BaseHelper from './lib/base-helper'; +import { HelperRegistry, HelperType, TypeOfHelper } from './types'; + +@Injectable() +export class HelperService { + private registry: HelperRegistry = new Map(); + + constructor( + private readonly settingService: SettingService, + private readonly logger: LoggerService, + ) { + // Init empty registry + Object.values(HelperType).forEach((type: HelperType) => { + this.registry.set(type, new Map()); + }); + } + + /** + * Registers a helper. + * + * @param name - The helper to be registered. + */ + public register>(helper: H) { + const helpers = this.registry.get(helper.getType()); + helpers.set(helper.getName(), helper); + this.logger.log(`Helper "${helper.getName()}" has been registered!`); + } + + /** + * Get a helper by name and type. + * + * @param type - The type of helper. + * @param name - The helper's name. + * + * @returns - The helper + */ + public get(type: T, name: string) { + const helpers = this.registry.get(type); + + if (!helpers.has(name)) { + throw new Error('Uknown type of helpers'); + } + return helpers.get(name) as TypeOfHelper; + } + + /** + * Get all helpers by type. + * + * @returns - The helpers + */ + public getAllByType(type: T) { + const helpers = this.registry.get(type) as Map>; + + return Array.from(helpers.values()); + } + + /** + * Get default NLU helper. + * + * @returns - The helper + */ + async getDefaultNluHelper() { + const settings = await this.settingService.getSettings(); + + const defaultHelper = this.get( + HelperType.NLU, + settings.chatbot_settings.default_nlu_helper, + ); + + if (!defaultHelper) { + throw new Error(`Unable to find default NLU helper`); + } + + return defaultHelper; + } +} diff --git a/api/src/helper/lib/base-helper.ts b/api/src/helper/lib/base-helper.ts new file mode 100644 index 00000000..2597b175 --- /dev/null +++ b/api/src/helper/lib/base-helper.ts @@ -0,0 +1,86 @@ +/* + * Copyright © 2024 Hexastack. All rights reserved. + * + * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms: + * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission. + * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). + */ + +import { LoggerService } from '@nestjs/common'; + +import { SettingService } from '@/setting/services/setting.service'; +import { hyphenToUnderscore } from '@/utils/helpers/misc'; + +import { HelperService } from '../helper.service'; +import { HelperSetting, HelperType } from '../types'; + +export default abstract class BaseHelper { + protected readonly name: N; + + protected readonly settings: HelperSetting[] = []; + + protected abstract type: HelperType; + + constructor( + name: N, + settings: HelperSetting[], + protected readonly settingService: SettingService, + protected readonly helperService: HelperService, + protected readonly logger: LoggerService, + ) { + this.name = name; + this.settings = settings; + } + + onModuleInit() { + this.helperService.register(this); + this.setup(); + } + + async setup() { + await this.settingService.seedIfNotExist( + this.getName(), + this.settings.map((s, i) => ({ + ...s, + weight: i + 1, + })), + ); + } + + /** + * Returns the helper's name + * + * @returns Helper's name + */ + public getName() { + return this.name; + } + + /** + * Returns the helper's group + * @returns Helper's group + */ + protected getGroup() { + return hyphenToUnderscore(this.getName()) as HelperSetting['group']; + } + + /** + * Get the helper's type + * + * @returns Helper's type + */ + public getType() { + return this.type; + } + + /** + * Get the helper's settings + * + * @returns Helper's settings + */ + async getSettings>() { + const settings = await this.settingService.getSettings(); + // @ts-expect-error workaround typing + return settings[this.getGroup() as keyof Settings] as Settings[S]; + } +} diff --git a/api/src/nlp/lib/BaseNlpHelper.ts b/api/src/helper/lib/base-nlp-helper.ts similarity index 79% rename from api/src/nlp/lib/BaseNlpHelper.ts rename to api/src/helper/lib/base-nlp-helper.ts index 805973e2..cf07c5f9 100644 --- a/api/src/nlp/lib/BaseNlpHelper.ts +++ b/api/src/helper/lib/base-nlp-helper.ts @@ -6,17 +6,6 @@ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). */ -/** - * @file NlpAdapter is an abstract class for define an NLP provider adapter - * @author Hexastack - */ - -/** - * @module Services/NLP - * - * NlpAdapter is an abstract class from which each NLP provider adapter should extend from. - */ - import { v4 as uuidv4 } from 'uuid'; import { LoggerService } from '@/logger/logger.service'; @@ -31,34 +20,29 @@ import { NlpValueDocument, NlpValueFull, } from '@/nlp/schemas/nlp-value.schema'; +import { SettingService } from '@/setting/services/setting.service'; -import { NlpEntityService } from '../services/nlp-entity.service'; -import { NlpSampleService } from '../services/nlp-sample.service'; -import { NlpService } from '../services/nlp.service'; +import { HelperService } from '../helper.service'; +import { HelperSetting, HelperType, Nlp } from '../types'; -import { Nlp } from './types'; +import BaseHelper from './base-helper'; -export default abstract class BaseNlpHelper { - protected settings: Settings['nlp_settings']; +// eslint-disable-next-line prettier/prettier +export default abstract class BaseNlpHelper< + N extends string, +> extends BaseHelper { + protected readonly type: HelperType = HelperType.NLU; constructor( - protected readonly logger: LoggerService, - protected readonly nlpService: NlpService, - protected readonly nlpSampleService: NlpSampleService, - protected readonly nlpEntityService: NlpEntityService, - ) {} - - setSettings(settings: Settings['nlp_settings']) { - this.settings = settings; + name: N, + settings: HelperSetting[], + settingService: SettingService, + helperService: HelperService, + logger: LoggerService, + ) { + super(name, settings, settingService, helperService, logger); } - /** - * Returns the helper's name - * - * @returns Helper's name - */ - abstract getName(): string; - /** * Updates an entity * @@ -183,7 +167,10 @@ export default abstract class BaseNlpHelper { * * @returns NLP Parsed entities */ - abstract bestGuess(nlp: any, threshold: boolean): Nlp.ParseEntities; + abstract filterEntitiesByConfidence( + nlp: any, + threshold: boolean, + ): Promise; /** * Returns only the entities that have strong confidence (> than the threshold), can return an empty result @@ -194,7 +181,7 @@ export default abstract class BaseNlpHelper { * * @returns NLP Parsed entities */ - abstract parse( + abstract predict( text: string, threshold?: boolean, project?: string, diff --git a/api/src/helper/types.ts b/api/src/helper/types.ts new file mode 100644 index 00000000..e8104742 --- /dev/null +++ b/api/src/helper/types.ts @@ -0,0 +1,44 @@ +import { SettingCreateDto } from '@/setting/dto/setting.dto'; + +import BaseHelper from './lib/base-helper'; +import BaseNlpHelper from './lib/base-nlp-helper'; + +export namespace Nlp { + export interface Config { + endpoint?: string; + token: string; + } + + export interface ParseEntity { + entity: string; // Entity name + value: string; // Value name + confidence: number; + start?: number; + end?: number; + } + + export interface ParseEntities { + entities: ParseEntity[]; + } +} + +export enum HelperType { + NLU = 'nlu', + UTIL = 'util', +} + +export type TypeOfHelper = T extends HelperType.NLU + ? BaseNlpHelper + : BaseHelper; + +export type HelperRegistry = Map< + HelperType, + Map +>; + +export type HelperSetting = Omit< + SettingCreateDto, + 'group' | 'weight' +> & { + group: HyphenToUnderscore; +}; diff --git a/api/src/nlp/controllers/nlp-sample.controller.spec.ts b/api/src/nlp/controllers/nlp-sample.controller.spec.ts index fd3c30fd..6b0fae31 100644 --- a/api/src/nlp/controllers/nlp-sample.controller.spec.ts +++ b/api/src/nlp/controllers/nlp-sample.controller.spec.ts @@ -17,6 +17,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { AttachmentRepository } from '@/attachment/repositories/attachment.repository'; import { AttachmentModel } from '@/attachment/schemas/attachment.schema'; import { AttachmentService } from '@/attachment/services/attachment.service'; +import { HelperService } from '@/helper/helper.service'; import { LanguageRepository } from '@/i18n/repositories/language.repository'; import { Language, LanguageModel } from '@/i18n/schemas/language.schema'; import { I18nService } from '@/i18n/services/i18n.service'; @@ -98,6 +99,7 @@ describe('NlpSampleController', () => { LanguageService, EventEmitter2, NlpService, + HelperService, SettingRepository, SettingService, SettingSeeder, diff --git a/api/src/nlp/controllers/nlp-sample.controller.ts b/api/src/nlp/controllers/nlp-sample.controller.ts index 940912cb..fc4f41b9 100644 --- a/api/src/nlp/controllers/nlp-sample.controller.ts +++ b/api/src/nlp/controllers/nlp-sample.controller.ts @@ -17,6 +17,7 @@ import { Delete, Get, HttpCode, + InternalServerErrorException, NotFoundException, Param, Patch, @@ -33,6 +34,7 @@ import Papa from 'papaparse'; import { AttachmentService } from '@/attachment/services/attachment.service'; import { config } from '@/config'; +import { HelperService } from '@/helper/helper.service'; import { LanguageService } from '@/i18n/services/language.service'; import { CsrfInterceptor } from '@/interceptors/csrf.interceptor'; import { LoggerService } from '@/logger/logger.service'; @@ -72,6 +74,7 @@ export class NlpSampleController extends BaseController< private readonly logger: LoggerService, private readonly nlpService: NlpService, private readonly languageService: LanguageService, + private readonly helperService: HelperService, ) { super(nlpSampleService); } @@ -93,7 +96,8 @@ export class NlpSampleController extends BaseController< type ? { type } : {}, ); const entities = await this.nlpEntityService.findAllAndPopulate(); - const result = await this.nlpSampleService.formatRasaNlu(samples, entities); + const helper = await this.helperService.getDefaultNluHelper(); + const result = helper.format(samples, entities); // Sending the JSON data as a file const buffer = Buffer.from(JSON.stringify(result)); @@ -171,7 +175,8 @@ export class NlpSampleController extends BaseController< */ @Get('message') async message(@Query('text') text: string) { - return this.nlpService.getNLP().parse(text); + const helper = await this.helperService.getDefaultNluHelper(); + return helper.predict(text); } /** @@ -201,7 +206,21 @@ export class NlpSampleController extends BaseController< const { samples, entities } = await this.getSamplesAndEntitiesByType('train'); - return await this.nlpService.getNLP().train(samples, entities); + try { + const helper = await this.helperService.getDefaultNluHelper(); + const response = await helper.train(samples, entities); + // Mark samples as trained + await this.nlpSampleService.updateMany( + { type: 'train' }, + { trained: true }, + ); + return response; + } catch (err) { + this.logger.error(err); + throw new InternalServerErrorException( + 'Unable to perform the train operation', + ); + } } /** @@ -214,7 +233,8 @@ export class NlpSampleController extends BaseController< const { samples, entities } = await this.getSamplesAndEntitiesByType('test'); - return await this.nlpService.getNLP().evaluate(samples, entities); + const helper = await this.helperService.getDefaultNluHelper(); + return await helper.evaluate(samples, entities); } /** diff --git a/api/src/nlp/nlp.module.ts b/api/src/nlp/nlp.module.ts index 05c7896b..8446276b 100644 --- a/api/src/nlp/nlp.module.ts +++ b/api/src/nlp/nlp.module.ts @@ -16,7 +16,6 @@ import { AttachmentModule } from '@/attachment/attachment.module'; import { NlpEntityController } from './controllers/nlp-entity.controller'; import { NlpSampleController } from './controllers/nlp-sample.controller'; import { NlpValueController } from './controllers/nlp-value.controller'; -import { NlpController } from './controllers/nlp.controller'; import { NlpEntityRepository } from './repositories/nlp-entity.repository'; import { NlpSampleEntityRepository } from './repositories/nlp-sample-entity.repository'; import { NlpSampleRepository } from './repositories/nlp-sample.repository'; @@ -45,12 +44,7 @@ import { NlpService } from './services/nlp.service'; AttachmentModule, HttpModule, ], - controllers: [ - NlpEntityController, - NlpValueController, - NlpSampleController, - NlpController, - ], + controllers: [NlpEntityController, NlpValueController, NlpSampleController], providers: [ NlpEntityRepository, NlpValueRepository, diff --git a/api/src/nlp/services/nlp-sample.service.spec.ts b/api/src/nlp/services/nlp-sample.service.spec.ts index 97cf5dd9..0a95e92b 100644 --- a/api/src/nlp/services/nlp-sample.service.spec.ts +++ b/api/src/nlp/services/nlp-sample.service.spec.ts @@ -14,6 +14,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { LanguageRepository } from '@/i18n/repositories/language.repository'; import { Language, LanguageModel } from '@/i18n/schemas/language.schema'; import { LanguageService } from '@/i18n/services/language.service'; +import { LoggerService } from '@/logger/logger.service'; import { nlpSampleFixtures } from '@/utils/test/fixtures/nlpsample'; import { installNlpSampleEntityFixtures } from '@/utils/test/fixtures/nlpsampleentity'; import { getPageQuery } from '@/utils/test/pagination'; @@ -28,10 +29,10 @@ import { NlpSampleRepository } from '../repositories/nlp-sample.repository'; import { NlpValueRepository } from '../repositories/nlp-value.repository'; import { NlpEntityModel } from '../schemas/nlp-entity.schema'; import { - NlpSampleEntityModel, NlpSampleEntity, + NlpSampleEntityModel, } from '../schemas/nlp-sample-entity.schema'; -import { NlpSampleModel, NlpSample } from '../schemas/nlp-sample.schema'; +import { NlpSample, NlpSampleModel } from '../schemas/nlp-sample.schema'; import { NlpValueModel } from '../schemas/nlp-value.schema'; import { NlpEntityService } from './nlp-entity.service'; @@ -72,6 +73,7 @@ describe('NlpSampleService', () => { NlpValueService, LanguageService, EventEmitter2, + LoggerService, { provide: CACHE_MANAGER, useValue: { diff --git a/api/src/nlp/services/nlp-sample.service.ts b/api/src/nlp/services/nlp-sample.service.ts index 6521af44..8d5366f8 100644 --- a/api/src/nlp/services/nlp-sample.service.ts +++ b/api/src/nlp/services/nlp-sample.service.ts @@ -9,25 +9,20 @@ import { Injectable } from '@nestjs/common'; import { OnEvent } from '@nestjs/event-emitter'; -import { - CommonExample, - DatasetType, - EntitySynonym, - ExampleEntity, - LookupTable, -} from '@/extensions/helpers/nlp/default/types'; +import { AnyMessage } from '@/chat/schemas/types/message'; import { Language } from '@/i18n/schemas/language.schema'; import { LanguageService } from '@/i18n/services/language.service'; +import { LoggerService } from '@/logger/logger.service'; import { BaseService } from '@/utils/generics/base-service'; +import { NlpSampleCreateDto } from '../dto/nlp-sample.dto'; import { NlpSampleRepository } from '../repositories/nlp-sample.repository'; -import { NlpEntity, NlpEntityFull } from '../schemas/nlp-entity.schema'; import { NlpSample, NlpSampleFull, NlpSamplePopulate, } from '../schemas/nlp-sample.schema'; -import { NlpValue } from '../schemas/nlp-value.schema'; +import { NlpSampleState } from '../schemas/types'; @Injectable() export class NlpSampleService extends BaseService< @@ -38,6 +33,7 @@ export class NlpSampleService extends BaseService< constructor( readonly repository: NlpSampleRepository, private readonly languageService: LanguageService, + private readonly logger: LoggerService, ) { super(repository); } @@ -53,95 +49,6 @@ export class NlpSampleService extends BaseService< return await this.repository.deleteOne(id); } - /** - * Formats a set of NLP samples into the Rasa NLU-compatible training dataset format. - * - * @param samples - The NLP samples to format. - * @param entities - The NLP entities available in the dataset. - * - * @returns The formatted Rasa NLU training dataset. - */ - async formatRasaNlu( - samples: NlpSampleFull[], - entities: NlpEntityFull[], - ): Promise { - const entityMap = NlpEntity.getEntityMap(entities); - const valueMap = NlpValue.getValueMap( - NlpValue.getValuesFromEntities(entities), - ); - - const common_examples: CommonExample[] = samples - .filter((s) => s.entities.length > 0) - .map((s) => { - const intent = s.entities.find( - (e) => entityMap[e.entity].name === 'intent', - ); - if (!intent) { - throw new Error('Unable to find the `intent` nlp entity.'); - } - const sampleEntities: ExampleEntity[] = s.entities - .filter((e) => entityMap[e.entity].name !== 'intent') - .map((e) => { - const res: ExampleEntity = { - entity: entityMap[e.entity].name, - value: valueMap[e.value].value, - }; - if ('start' in e && 'end' in e) { - Object.assign(res, { - start: e.start, - end: e.end, - }); - } - return res; - }) - // TODO : place language at the same level as the intent - .concat({ - entity: 'language', - value: s.language.code, - }); - - return { - text: s.text, - intent: valueMap[intent.value].value, - entities: sampleEntities, - }; - }); - - const languages = await this.languageService.getLanguages(); - const lookup_tables: LookupTable[] = entities - .map((e) => { - return { - name: e.name, - elements: e.values.map((v) => { - return v.value; - }), - }; - }) - .concat({ - name: 'language', - elements: Object.keys(languages), - }); - const entity_synonyms = entities - .reduce((acc, e) => { - const synonyms = e.values.map((v) => { - return { - value: v.value, - synonyms: v.expressions, - }; - }); - return acc.concat(synonyms); - }, [] as EntitySynonym[]) - .filter((s) => { - return s.synonyms.length > 0; - }); - return { - common_examples, - regex_features: [], - lookup_tables, - entity_synonyms, - }; - } - /** * When a language gets deleted, we need to set related samples to null * @@ -158,4 +65,31 @@ export class NlpSampleService extends BaseService< }, ); } + + @OnEvent('hook:message:preCreate') + async handleNewMessage(doc: AnyMessage) { + // If message is sent by the user then add it as an inbox sample + if ( + 'sender' in doc && + doc.sender && + 'message' in doc && + 'text' in doc.message + ) { + const defaultLang = await this.languageService.getDefaultLanguage(); + const record: NlpSampleCreateDto = { + text: doc.message.text, + type: NlpSampleState.inbox, + trained: false, + // @TODO : We need to define the language in the message entity + language: defaultLang.id, + }; + try { + await this.findOneOrCreate(record, record); + this.logger.debug('User message saved as a inbox sample !'); + } catch (err) { + this.logger.error('Unable to add message as a new inbox sample!', err); + throw err; + } + } + } } diff --git a/api/src/nlp/services/nlp.service.ts b/api/src/nlp/services/nlp.service.ts index 12f9f06c..e8fe92ad 100644 --- a/api/src/nlp/services/nlp.service.ts +++ b/api/src/nlp/services/nlp.service.ts @@ -6,13 +6,12 @@ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). */ -import { Injectable, OnApplicationBootstrap } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import { OnEvent } from '@nestjs/event-emitter'; +import { HelperService } from '@/helper/helper.service'; import { LoggerService } from '@/logger/logger.service'; -import { SettingService } from '@/setting/services/setting.service'; -import BaseNlpHelper from '../lib/BaseNlpHelper'; import { NlpEntity, NlpEntityDocument } from '../schemas/nlp-entity.schema'; import { NlpValue, NlpValueDocument } from '../schemas/nlp-value.schema'; @@ -21,93 +20,15 @@ import { NlpSampleService } from './nlp-sample.service'; import { NlpValueService } from './nlp-value.service'; @Injectable() -export class NlpService implements OnApplicationBootstrap { - private registry: Map = new Map(); - - private nlp: BaseNlpHelper; - +export class NlpService { constructor( - private readonly settingService: SettingService, private readonly logger: LoggerService, protected readonly nlpSampleService: NlpSampleService, protected readonly nlpEntityService: NlpEntityService, protected readonly nlpValueService: NlpValueService, + protected readonly helperService: HelperService, ) {} - onApplicationBootstrap() { - this.initNLP(); - } - - /** - * Registers a helper with a specific name in the registry. - * - * @param name - The name of the helper to register. - * @param helper - The NLP helper to be associated with the given name. - * @typeParam C - The type of the helper, which must extend `BaseNlpHelper`. - */ - public setHelper(name: string, helper: C) { - this.registry.set(name, helper); - } - - /** - * Retrieves all registered helpers. - * - * @returns An array of all helpers currently registered. - */ - public getAll() { - return Array.from(this.registry.values()); - } - - /** - * Retrieves the appropriate helper based on the helper name. - * - * @param helperName - The name of the helper (messenger, offline, ...). - * - * @returns The specified helper. - */ - public getHelper(name: string): C { - const handler = this.registry.get(name); - if (!handler) { - throw new Error(`NLP Helper ${name} not found`); - } - return handler as C; - } - - async initNLP() { - try { - const settings = await this.settingService.getSettings(); - const nlpSettings = settings.nlp_settings; - const helper = this.getHelper(nlpSettings.provider); - - if (helper) { - this.nlp = helper; - this.nlp.setSettings(nlpSettings); - } else { - throw new Error(`Undefined NLP Helper ${nlpSettings.provider}`); - } - } catch (e) { - this.logger.error('NLP Service : Unable to instantiate NLP Helper !', e); - // throw e; - } - } - - /** - * Retrieves the currently active NLP helper. - * - * @returns The current NLP helper. - */ - getNLP() { - return this.nlp; - } - - /** - * Handles the event triggered when NLP settings are updated. Re-initializes the NLP service. - */ - @OnEvent('hook:nlp_settings:*') - async handleSettingsUpdate() { - this.initNLP(); - } - /** * Handles the event triggered when a new NLP entity is created. Synchronizes the entity with the external NLP provider. * @@ -118,7 +39,8 @@ export class NlpService implements OnApplicationBootstrap { async handleEntityCreate(entity: NlpEntityDocument) { // Synchonize new entity with NLP try { - const foreignId = await this.getNLP().addEntity(entity); + const helper = await this.helperService.getDefaultNluHelper(); + const foreignId = await helper.addEntity(entity); this.logger.debug('New entity successfully synced!', foreignId); return await this.nlpEntityService.updateOne(entity._id, { foreign_id: foreignId, @@ -138,7 +60,8 @@ export class NlpService implements OnApplicationBootstrap { async handleEntityUpdate(entity: NlpEntity) { // Synchonize new entity with NLP provider try { - await this.getNLP().updateEntity(entity); + const helper = await this.helperService.getDefaultNluHelper(); + await helper.updateEntity(entity); this.logger.debug('Updated entity successfully synced!', entity); } catch (err) { this.logger.error('Unable to sync updated entity', err); @@ -154,7 +77,8 @@ export class NlpService implements OnApplicationBootstrap { async handleEntityDelete(entity: NlpEntity) { // Synchonize new entity with NLP provider try { - await this.getNLP().deleteEntity(entity.foreign_id); + const helper = await this.helperService.getDefaultNluHelper(); + await helper.deleteEntity(entity.foreign_id); this.logger.debug('Deleted entity successfully synced!', entity); } catch (err) { this.logger.error('Unable to sync deleted entity', err); @@ -172,7 +96,8 @@ export class NlpService implements OnApplicationBootstrap { async handleValueCreate(value: NlpValueDocument) { // Synchonize new value with NLP provider try { - const foreignId = await this.getNLP().addValue(value); + const helper = await this.helperService.getDefaultNluHelper(); + const foreignId = await helper.addValue(value); this.logger.debug('New value successfully synced!', foreignId); return await this.nlpValueService.updateOne(value._id, { foreign_id: foreignId, @@ -192,7 +117,8 @@ export class NlpService implements OnApplicationBootstrap { async handleValueUpdate(value: NlpValue) { // Synchonize new value with NLP provider try { - await this.getNLP().updateValue(value); + const helper = await this.helperService.getDefaultNluHelper(); + await helper.updateValue(value); this.logger.debug('Updated value successfully synced!', value); } catch (err) { this.logger.error('Unable to sync updated value', err); @@ -208,10 +134,11 @@ export class NlpService implements OnApplicationBootstrap { async handleValueDelete(value: NlpValue) { // Synchonize new value with NLP provider try { + const helper = await this.helperService.getDefaultNluHelper(); const populatedValue = await this.nlpValueService.findOneAndPopulate( value.id, ); - await this.getNLP().deleteValue(populatedValue); + await helper.deleteValue(populatedValue); this.logger.debug('Deleted value successfully synced!', value); } catch (err) { this.logger.error('Unable to sync deleted value', err); diff --git a/api/src/setting/seeds/setting.seed-model.ts b/api/src/setting/seeds/setting.seed-model.ts index be580afa..15816ca9 100644 --- a/api/src/setting/seeds/setting.seed-model.ts +++ b/api/src/setting/seeds/setting.seed-model.ts @@ -10,12 +10,33 @@ import { SettingCreateDto } from '../dto/setting.dto'; import { SettingType } from '../schemas/types'; export const DEFAULT_SETTINGS = [ + { + group: 'chatbot_settings', + label: 'default_nlu_helper', + value: 'core-nlu', + type: SettingType.select, + config: { + multiple: false, + allowCreate: false, + entity: 'Helper', + idKey: 'name', + labelKey: 'name', + }, + weight: 1, + }, { group: 'chatbot_settings', label: 'global_fallback', value: true, type: SettingType.checkbox, - weight: 1, + weight: 3, + }, + { + group: 'chatbot_settings', + label: 'global_fallback', + value: true, + type: SettingType.checkbox, + weight: 4, }, { group: 'chatbot_settings', @@ -26,11 +47,11 @@ export const DEFAULT_SETTINGS = [ config: { multiple: false, allowCreate: false, - source: '/Block/', - valueKey: 'id', + entity: 'Block', + idKey: 'id', labelKey: 'name', }, - weight: 2, + weight: 5, }, { group: 'chatbot_settings', @@ -40,41 +61,7 @@ export const DEFAULT_SETTINGS = [ "I'm really sorry but i don't quite understand what you are saying :(", ] as string[], type: SettingType.multiple_text, - weight: 3, - }, - { - group: 'nlp_settings', - label: 'provider', - value: 'default', - options: ['default'], - type: SettingType.select, - weight: 1, - }, - { - group: 'nlp_settings', - label: 'endpoint', - value: 'http://nlu-api:5000/', - type: SettingType.text, - weight: 2, - }, - { - group: 'nlp_settings', - label: 'token', - value: 'token123', - type: SettingType.text, - weight: 3, - }, - { - group: 'nlp_settings', - label: 'threshold', - value: 0.1, - type: SettingType.number, - config: { - min: 0, - max: 1, - step: 0.01, - }, - weight: 4, + weight: 6, }, { group: 'contact', diff --git a/api/src/utils/test/mocks/nlp.ts b/api/src/utils/test/mocks/nlp.ts index a1b33701..5fb31634 100644 --- a/api/src/utils/test/mocks/nlp.ts +++ b/api/src/utils/test/mocks/nlp.ts @@ -6,7 +6,7 @@ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). */ -import { Nlp } from '@/nlp/lib/types'; +import { Nlp } from '@/helper/types'; export const nlpEntitiesGreeting: Nlp.ParseEntities = { entities: [ diff --git a/frontend/public/locales/en/chatbot_settings.json b/frontend/public/locales/en/chatbot_settings.json index 466abd72..d99b21ee 100644 --- a/frontend/public/locales/en/chatbot_settings.json +++ b/frontend/public/locales/en/chatbot_settings.json @@ -4,10 +4,12 @@ }, "label": { "global_fallback": "Enable Global Fallback?", - "fallback_message": "Fallback Message" + "fallback_message": "Fallback Message", + "default_nlu_helper": "Default NLU Helper" }, "help": { "global_fallback": "Global fallback allows you to send custom messages when user entry does not match any of the block messages.", - "fallback_message": "If no fallback block is selected, then one of these messages will be sent." + "fallback_message": "If no fallback block is selected, then one of these messages will be sent.", + "default_nlu_helper": "The NLU helper is responsible for processing and understanding user inputs, including tasks like intent prediction, language detection, and entity recognition." } } diff --git a/frontend/public/locales/fr/chatbot_settings.json b/frontend/public/locales/fr/chatbot_settings.json index 7eaae1cc..cba90376 100644 --- a/frontend/public/locales/fr/chatbot_settings.json +++ b/frontend/public/locales/fr/chatbot_settings.json @@ -1,13 +1,15 @@ { "title": { - "chatbot_settings": "Chatbot" + "chatbot_settings": "Paramètres du Chatbot" }, "label": { - "global_fallback": "Activer le message de secours global?", - "fallback_message": "Message de secours" + "global_fallback": "Activer la réponse de secours globale ?", + "fallback_message": "Message de secours", + "default_nlu_helper": "Utilitaire NLU par défaut" }, "help": { - "global_fallback": "Le message de secours global vous permet d'envoyer des messages personnalisés lorsque le message de l'utilisateur ne déclenche aucun bloc de message.", - "fallback_message": "Si aucun bloc de secours n'est spécifié, alors de ces messages sera envoyé." + "global_fallback": "La réponse de secours globale vous permet d'envoyer des messages personnalisés lorsque l'entrée de l'utilisateur ne correspond à aucun des messages des blocs.", + "fallback_message": "Si aucun bloc de secours n'est sélectionné, l'un de ces messages sera envoyé.", + "default_nlu_helper": "Utilitaire du traitement et de la compréhension des entrées des utilisateurs, incluant des tâches telles que la prédiction d'intention, la détection de langue et la reconnaissance d'entités." } } diff --git a/frontend/src/app-components/inputs/AutoCompleteSelect.tsx b/frontend/src/app-components/inputs/AutoCompleteSelect.tsx index 3a62a223..fde6cf39 100644 --- a/frontend/src/app-components/inputs/AutoCompleteSelect.tsx +++ b/frontend/src/app-components/inputs/AutoCompleteSelect.tsx @@ -11,7 +11,7 @@ import Autocomplete, { AutocompleteProps, AutocompleteValue, } from "@mui/material/Autocomplete"; -import { useState, useCallback, useMemo, useEffect, forwardRef } from "react"; +import { forwardRef, useCallback, useEffect, useMemo, useState } from "react"; import { Input } from "@/app-components/inputs/Input"; diff --git a/frontend/src/components/settings/SettingInput.tsx b/frontend/src/components/settings/SettingInput.tsx index 11c02ce9..cdb6f018 100644 --- a/frontend/src/components/settings/SettingInput.tsx +++ b/frontend/src/components/settings/SettingInput.tsx @@ -19,6 +19,7 @@ import { PasswordInput } from "@/app-components/inputs/PasswordInput"; import { useTranslate } from "@/hooks/useTranslate"; import { EntityType, Format } from "@/services/types"; import { IBlock } from "@/types/block.types"; +import { IHelper } from "@/types/helper.types"; import { ISetting } from "@/types/setting.types"; import { MIME_TYPES } from "@/utils/attachment"; @@ -115,11 +116,29 @@ const SettingInput: React.FC = ({ format={Format.BASIC} labelKey="name" label={t("label.fallback_block")} + helperText={t("help.fallback_block")} multiple={false} onChange={(_e, selected, ..._) => onChange(selected?.id)} {...rest} /> ); + } else if (setting.label === "default_nlu_helper") { + const { onChange, ...rest } = field; + + return ( + + searchFields={["name"]} + entity={EntityType.NLU_HELPER} + format={Format.BASIC} + labelKey="name" + idKey="name" + label={t("label.default_nlu_helper")} + helperText={t("help.default_nlu_helper")} + multiple={false} + onChange={(_e, selected, ..._) => onChange(selected?.name)} + {...rest} + /> + ); } return ( diff --git a/frontend/src/services/api.class.ts b/frontend/src/services/api.class.ts index ea6bc568..f9e891f7 100644 --- a/frontend/src/services/api.class.ts +++ b/frontend/src/services/api.class.ts @@ -64,6 +64,8 @@ export const ROUTES = { [EntityType.TRANSLATION]: "/translation", [EntityType.ATTACHMENT]: "/attachment", [EntityType.CHANNEL]: "/channel", + [EntityType.HELPER]: "/helper", + [EntityType.NLU_HELPER]: "/helper/nlu", } as const; export class ApiClient { diff --git a/frontend/src/services/entities.ts b/frontend/src/services/entities.ts index 1074646f..595673ec 100644 --- a/frontend/src/services/entities.ts +++ b/frontend/src/services/entities.ts @@ -284,6 +284,19 @@ export const ChannelEntity = new schema.Entity(EntityType.CHANNEL, undefined, { idAttribute: ({ name }) => name, }); +export const HelperEntity = new schema.Entity(EntityType.HELPER, undefined, { + idAttribute: ({ name }) => name, +}); + +export const NluHelperEntity = new schema.Entity( + EntityType.NLU_HELPER, + undefined, + { + idAttribute: ({ name }) => name, + }, +); + + export const ENTITY_MAP = { [EntityType.SUBSCRIBER]: SubscriberEntity, [EntityType.LABEL]: LabelEntity, @@ -310,4 +323,6 @@ export const ENTITY_MAP = { [EntityType.CUSTOM_BLOCK]: CustomBlockEntity, [EntityType.CUSTOM_BLOCK_SETTINGS]: CustomBlockSettingEntity, [EntityType.CHANNEL]: ChannelEntity, + [EntityType.HELPER]: HelperEntity, + [EntityType.NLU_HELPER]: NluHelperEntity, } as const; diff --git a/frontend/src/services/types.ts b/frontend/src/services/types.ts index 1dc7b382..71a94fae 100644 --- a/frontend/src/services/types.ts +++ b/frontend/src/services/types.ts @@ -35,6 +35,8 @@ export enum EntityType { TRANSLATION = "Translation", ATTACHMENT = "Attachment", CHANNEL = "Channel", + HELPER = "Helper", + NLU_HELPER = "NluHelper", } export type NormalizedEntities = Record>; diff --git a/frontend/src/types/base.types.ts b/frontend/src/types/base.types.ts index 5b047d2d..58fea277 100644 --- a/frontend/src/types/base.types.ts +++ b/frontend/src/types/base.types.ts @@ -22,6 +22,7 @@ import { IChannel, IChannelAttributes } from "./channel.types"; import { IContentType, IContentTypeAttributes } from "./content-type.types"; import { IContent, IContentAttributes, IContentFull } from "./content.types"; import { IContextVar, IContextVarAttributes } from "./context-var.types"; +import { IHelper, IHelperAttributes } from "./helper.types"; import { ILabel, ILabelAttributes, ILabelFull } from "./label.types"; import { ILanguage, ILanguageAttributes } from "./language.types"; import { @@ -112,6 +113,8 @@ export const POPULATE_BY_TYPE = { [EntityType.CUSTOM_BLOCK]: [], [EntityType.CUSTOM_BLOCK_SETTINGS]: [], [EntityType.CHANNEL]: [], + [EntityType.HELPER]: [], + [EntityType.NLU_HELPER]: [], } as const; export type Populate = @@ -200,6 +203,8 @@ export interface IEntityMapTypes { IMessageFull >; [EntityType.CHANNEL]: IEntityTypes; + [EntityType.HELPER]: IEntityTypes; + [EntityType.NLU_HELPER]: IEntityTypes; } export type TType = diff --git a/api/src/nlp/lib/types.ts b/frontend/src/types/helper.types.ts similarity index 61% rename from api/src/nlp/lib/types.ts rename to frontend/src/types/helper.types.ts index 251ea230..5070085c 100644 --- a/api/src/nlp/lib/types.ts +++ b/frontend/src/types/helper.types.ts @@ -6,21 +6,11 @@ * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file). */ -export namespace Nlp { - export interface Config { - endpoint?: string; - token: string; - } +import { IBaseSchema } from "./base.types"; - export interface ParseEntity { - entity: string; // Entity name - value: string; // Value name - confidence: number; - start?: number; - end?: number; - } - - export interface ParseEntities { - entities: ParseEntity[]; - } +export interface IHelperAttributes { + name: string; } + +// @TODO: not all entities extend from IBaseSchema +export interface IHelper extends IHelperAttributes, IBaseSchema {}