Merge branch 'langgenius:main' into main

api/.env.example

@@ -20,6 +20,9 @@ FILES_URL=http://127.0.0.1:5001
 # The time in seconds after the signature is rejected
 FILES_ACCESS_TIMEOUT=300
 
+# Access token expiration time in minutes
+ACCESS_TOKEN_EXPIRE_MINUTES=60
+
 # celery configuration
 CELERY_BROKER_URL=redis://:difyai123456@localhost:6379/1
 
@@ -39,7 +42,7 @@ DB_DATABASE=dify
 
 # Storage configuration
 # use for store upload files, private keys...
-# storage type: local, s3, azure-blob, google-storage, tencent-cos, huawei-obs, volcengine-tos, baidu-obs
+# storage type: local, s3, azure-blob, google-storage, tencent-cos, huawei-obs, volcengine-tos, baidu-obs, supabase
 STORAGE_TYPE=local
 STORAGE_LOCAL_PATH=storage
 S3_USE_AWS_MANAGED_IAM=false
@@ -99,11 +102,16 @@ VOLCENGINE_TOS_ACCESS_KEY=your-access-key
 VOLCENGINE_TOS_SECRET_KEY=your-secret-key
 VOLCENGINE_TOS_REGION=your-region
 
+# Supabase Storage Configuration
+SUPABASE_BUCKET_NAME=your-bucket-name
+SUPABASE_API_KEY=your-access-key
+SUPABASE_URL=your-server-url
+
 # CORS configuration
 WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 
-# Vector database configuration, support: weaviate, qdrant, milvus, myscale, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector
+# Vector database configuration, support: weaviate, qdrant, milvus, myscale, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector, vikingdb
 VECTOR_STORE=weaviate
 
 # Weaviate configuration
@@ -203,6 +211,24 @@ OPENSEARCH_USER=admin
 OPENSEARCH_PASSWORD=admin
 OPENSEARCH_SECURE=true
 
+# Baidu configuration
+BAIDU_VECTOR_DB_ENDPOINT=http://127.0.0.1:5287
+BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS=30000
+BAIDU_VECTOR_DB_ACCOUNT=root
+BAIDU_VECTOR_DB_API_KEY=dify
+BAIDU_VECTOR_DB_DATABASE=dify
+BAIDU_VECTOR_DB_SHARD=1
+BAIDU_VECTOR_DB_REPLICAS=3
+
+# ViKingDB configuration
+VIKINGDB_ACCESS_KEY=your-ak
+VIKINGDB_SECRET_KEY=your-sk
+VIKINGDB_REGION=cn-shanghai
+VIKINGDB_HOST=api-vikingdb.xxx.volces.com
+VIKINGDB_SCHEMA=http
+VIKINGDB_CONNECTION_TIMEOUT=30
+VIKINGDB_SOCKET_TIMEOUT=30
+
 # Upload configuration
 UPLOAD_FILE_SIZE_LIMIT=15
 UPLOAD_FILE_BATCH_LIMIT=5

api/app.py

@@ -183,7 +183,7 @@ def load_user_from_request(request_from_flask_login):
  decoded = PassportService().verify(auth_token)
  user_id = decoded.get("user_id")
 
- logged_in_account = AccountService.load_logged_in_account(account_id=user_id, token=auth_token)
+ logged_in_account = AccountService.load_logged_in_account(account_id=user_id)
  if logged_in_account:
  contexts.tenant_id.set(logged_in_account.current_tenant_id)
  return logged_in_account

api/commands.py

@@ -347,6 +347,14 @@ def migrate_knowledge_vector_database():
  index_name = Dataset.gen_collection_name_by_id(dataset_id)
  index_struct_dict = {"type": "elasticsearch", "vector_store": {"class_prefix": index_name}}
  dataset.index_struct = json.dumps(index_struct_dict)
+ elif vector_type == VectorType.BAIDU:
+ dataset_id = dataset.id
+ collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+ index_struct_dict = {
+ "type": VectorType.BAIDU,
+ "vector_store": {"class_prefix": collection_name},
+ }
+ dataset.index_struct = json.dumps(index_struct_dict)
  else:
  raise ValueError(f"Vector store {vector_type} is not supported.")
 

api/configs/feature/__init__.py

@@ -360,9 +360,9 @@ class WorkflowConfig(BaseSettings):
  )
 
 
-class OAuthConfig(BaseSettings):
+class AuthConfig(BaseSettings):
  """
- Configuration for OAuth authentication
+ Configuration for authentication and OAuth
  """
 
  OAUTH_REDIRECT_PATH: str = Field(
@@ -371,7 +371,7 @@ class OAuthConfig(BaseSettings):
  )
 
  GITHUB_CLIENT_ID: Optional[str] = Field(
- description="GitHub OAuth client secret",
+ description="GitHub OAuth client ID",
  default=None,
  )
 
@@ -390,6 +390,11 @@ class OAuthConfig(BaseSettings):
  default=None,
  )
 
+ ACCESS_TOKEN_EXPIRE_MINUTES: PositiveInt = Field(
+ description="Expiration time for access tokens in minutes",
+ default=60,
+ )
+
 
 class ModerationConfig(BaseSettings):
  """
@@ -607,6 +612,7 @@ def POSITION_TOOL_EXCLUDES_SET(self) -> set[str]:
 class FeatureConfig(
  # place the configs in alphabet order
  AppExecutionConfig,
+ AuthConfig, # Changed from OAuthConfig to AuthConfig
  BillingConfig,
  CodeExecutionSandboxConfig,
  DataSetConfig,
@@ -621,14 +627,13 @@ class FeatureConfig(
  MailConfig,
  ModelLoadBalanceConfig,
  ModerationConfig,
- OAuthConfig,
+ PositionConfig,
  RagEtlConfig,
  SecurityConfig,
  ToolConfig,
  UpdateConfig,
  WorkflowConfig,
  WorkspaceConfig,
- PositionConfig,
  # hosted services config
  HostedServiceConfig,
  CeleryBeatConfig,

api/configs/middleware/__init__.py

@@ -12,6 +12,7 @@
 from configs.middleware.storage.google_cloud_storage_config import GoogleCloudStorageConfig
 from configs.middleware.storage.huawei_obs_storage_config import HuaweiCloudOBSStorageConfig
 from configs.middleware.storage.oci_storage_config import OCIStorageConfig
+from configs.middleware.storage.supabase_storage_config import SupabaseStorageConfig
 from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
 from configs.middleware.storage.volcengine_tos_storage_config import VolcengineTOSStorageConfig
 from configs.middleware.vdb.analyticdb_config import AnalyticdbConfig
@@ -27,6 +28,7 @@
 from configs.middleware.vdb.relyt_config import RelytConfig
 from configs.middleware.vdb.tencent_vector_config import TencentVectorDBConfig
 from configs.middleware.vdb.tidb_vector_config import TiDBVectorConfig
+from configs.middleware.vdb.vikingdb_config import VikingDBConfig
 from configs.middleware.vdb.weaviate_config import WeaviateConfig
 
 
@@ -222,6 +224,7 @@ class MiddlewareConfig(
  HuaweiCloudOBSStorageConfig,
  OCIStorageConfig,
  S3StorageConfig,
+ SupabaseStorageConfig,
  TencentCloudCOSStorageConfig,
  VolcengineTOSStorageConfig,
  # configs of vdb and vdb providers
@@ -241,5 +244,6 @@ class MiddlewareConfig(
  WeaviateConfig,
  ElasticsearchConfig,
  InternalTestConfig,
+ VikingDBConfig,
 ):
  pass

api/configs/middleware/storage/supabase_storage_config.py

@@ -0,0 +1,24 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class SupabaseStorageConfig(BaseModel):
+ """
+ Configuration settings for Supabase Object Storage Service
+ """
+
+ SUPABASE_BUCKET_NAME: Optional[str] = Field(
+ description="Name of the Supabase bucket to store and retrieve objects (e.g., 'dify-bucket')",
+ default=None,
+ )
+
+ SUPABASE_API_KEY: Optional[str] = Field(
+ description="API KEY for authenticating with Supabase",
+ default=None,
+ )
+
+ SUPABASE_URL: Optional[str] = Field(
+ description="URL of the Supabase",
+ default=None,
+ )

api/configs/middleware/vdb/baidu_vector_config.py

@@ -0,0 +1,45 @@
+from typing import Optional
+
+from pydantic import Field, NonNegativeInt, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class BaiduVectorDBConfig(BaseSettings):
+ """
+ Configuration settings for Baidu Vector Database
+ """
+
+ BAIDU_VECTOR_DB_ENDPOINT: Optional[str] = Field(
+ description="URL of the Baidu Vector Database service (e.g., 'http://vdb.bj.baidubce.com')",
+ default=None,
+ )
+
+ BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS: PositiveInt = Field(
+ description="Timeout in milliseconds for Baidu Vector Database operations (default is 30000 milliseconds)",
+ default=30000,
+ )
+
+ BAIDU_VECTOR_DB_ACCOUNT: Optional[str] = Field(
+ description="Account for authenticating with the Baidu Vector Database",
+ default=None,
+ )
+
+ BAIDU_VECTOR_DB_API_KEY: Optional[str] = Field(
+ description="API key for authenticating with the Baidu Vector Database service",
+ default=None,
+ )
+
+ BAIDU_VECTOR_DB_DATABASE: Optional[str] = Field(
+ description="Name of the specific Baidu Vector Database to connect to",
+ default=None,
+ )
+
+ BAIDU_VECTOR_DB_SHARD: PositiveInt = Field(
+ description="Number of shards for the Baidu Vector Database (default is 1)",
+ default=1,
+ )
+
+ BAIDU_VECTOR_DB_REPLICAS: NonNegativeInt = Field(
+ description="Number of replicas for the Baidu Vector Database (default is 3)",
+ default=3,
+ )

api/configs/middleware/vdb/vikingdb_config.py

@@ -0,0 +1,37 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class VikingDBConfig(BaseModel):
+ """
+ Configuration for connecting to Volcengine VikingDB.
+ Refer to the following documentation for details on obtaining credentials:
+ https://www.volcengine.com/docs/6291/65568
+ """
+
+ VIKINGDB_ACCESS_KEY: Optional[str] = Field(
+ default=None, description="The Access Key provided by Volcengine VikingDB for API authentication."
+ )
+ VIKINGDB_SECRET_KEY: Optional[str] = Field(
+ default=None, description="The Secret Key provided by Volcengine VikingDB for API authentication."
+ )
+ VIKINGDB_REGION: Optional[str] = Field(
+ default="cn-shanghai",
+ description="The region of the Volcengine VikingDB service.(e.g., 'cn-shanghai', 'cn-beijing').",
+ )
+ VIKINGDB_HOST: Optional[str] = Field(
+ default="api-vikingdb.mlp.cn-shanghai.volces.com",
+ description="The host of the Volcengine VikingDB service.(e.g., 'api-vikingdb.volces.com', \
+ 'api-vikingdb.mlp.cn-shanghai.volces.com')",
+ )
+ VIKINGDB_SCHEME: Optional[str] = Field(
+ default="http",
+ description="The scheme of the Volcengine VikingDB service.(e.g., 'http', 'https').",
+ )
+ VIKINGDB_CONNECTION_TIMEOUT: Optional[int] = Field(
+ default=30, description="The connection timeout of the Volcengine VikingDB service."
+ )
+ VIKINGDB_SOCKET_TIMEOUT: Optional[int] = Field(
+ default=30, description="The socket timeout of the Volcengine VikingDB service."
+ )

api/controllers/console/auth/login.py

@@ -7,7 +7,7 @@
 import services
 from controllers.console import api
 from controllers.console.setup import setup_required
-from libs.helper import email, get_remote_ip
+from libs.helper import email, extract_remote_ip
 from libs.password import valid_password
 from models.account import Account
 from services.account_service import AccountService, TenantService
@@ -40,17 +40,16 @@ def post(self):
  "data": "workspace not found, please contact system admin to invite you to join in a workspace",
  }
 
- token = AccountService.login(account, ip_address=get_remote_ip(request))
+ token_pair = AccountService.login(account=account, ip_address=extract_remote_ip(request))
 
- return {"result": "success", "data": token}
+ return {"result": "success", "data": token_pair.model_dump()}
 
 
 class LogoutApi(Resource):
  @setup_required
  def get(self):
  account = cast(Account, flask_login.current_user)
- token = request.headers.get("Authorization", "").split(" ")[1]
- AccountService.logout(account=account, token=token)
+ AccountService.logout(account=account)
  flask_login.logout_user()
  return {"result": "success"}
 
@@ -106,5 +105,19 @@ def get(self):
  return {"result": "success"}
 
 
+class RefreshTokenApi(Resource):
+ def post(self):
+ parser = reqparse.RequestParser()
+ parser.add_argument("refresh_token", type=str, required=True, location="json")
+ args = parser.parse_args()
+
+ try:
+ new_token_pair = AccountService.refresh_token(args["refresh_token"])
+ return {"result": "success", "data": new_token_pair.model_dump()}
+ except Exception as e:
+ return {"result": "fail", "data": str(e)}, 401
+
+
 api.add_resource(LoginApi, "/login")
 api.add_resource(LogoutApi, "/logout")
+api.add_resource(RefreshTokenApi, "/refresh-token")

api/controllers/console/auth/oauth.py

@@ -9,7 +9,7 @@
 from configs import dify_config
 from constants.languages import languages
 from extensions.ext_database import db
-from libs.helper import get_remote_ip
+from libs.helper import extract_remote_ip
 from libs.oauth import GitHubOAuth, GoogleOAuth, OAuthUserInfo
 from models.account import Account, AccountStatus
 from services.account_service import AccountService, RegisterService, TenantService
@@ -81,9 +81,14 @@ def get(self, provider: str):
 
  TenantService.create_owner_tenant_if_not_exist(account)
 
- token = AccountService.login(account, ip_address=get_remote_ip(request))
+ token_pair = AccountService.login(
+ account=account,
+ ip_address=extract_remote_ip(request),
+ )
 
- return redirect(f"{dify_config.CONSOLE_WEB_URL}?console_token={token}")
+ return redirect(
+ f"{dify_config.CONSOLE_WEB_URL}?access_token={token_pair.access_token}&refresh_token={token_pair.refresh_token}"
+ )
 
 
 def _get_account_by_openid_or_email(provider: str, user_info: OAuthUserInfo) -> Optional[Account]:

api/controllers/console/datasets/datasets.py

@@ -617,6 +617,8 @@ def get(self):
  | VectorType.CHROMA
  | VectorType.TENCENT
  | VectorType.PGVECTO_RS
+ | VectorType.BAIDU
+ | VectorType.VIKINGDB
  ):
  return {"retrieval_method": [RetrievalMethod.SEMANTIC_SEARCH.value]}
  case (
@@ -653,6 +655,8 @@ def get(self, vector_type):
  | VectorType.CHROMA
  | VectorType.TENCENT
  | VectorType.PGVECTO_RS
+ | VectorType.BAIDU
+ | VectorType.VIKINGDB
  ):
  return {"retrieval_method": [RetrievalMethod.SEMANTIC_SEARCH.value]}
  case (

api/controllers/console/setup.py

@@ -4,7 +4,7 @@
 from flask_restful import Resource, reqparse
 
 from configs import dify_config
-from libs.helper import StrLen, email, get_remote_ip
+from libs.helper import StrLen, email, extract_remote_ip
 from libs.password import valid_password
 from models.model import DifySetup
 from services.account_service import RegisterService, TenantService
@@ -46,7 +46,7 @@ def post(self):
 
  # setup
  RegisterService.setup(
- email=args["email"], name=args["name"], password=args["password"], ip_address=get_remote_ip(request)
+ email=args["email"], name=args["name"], password=args["password"], ip_address=extract_remote_ip(request)
  )
 
  return {"result": "success"}, 201