Feat/0.3.3 (#787)

.drone.yml

@@ -54,7 +54,11 @@ steps: # 定义流水线执行步骤，这些步骤将顺序执行
  from_secret: PY_NEXUS
  PROXY:
  from_secret: APT-GET
+ volumes: # 将容器内目录挂载到宿主机，仓库需要开启Trusted设置
+ - name: bisheng-cache
+ path: /app/build/
  commands:
+ - cp -r /app/build/taggers ./src/backend/
  - cd ./src/backend/
  - echo $REPO
  - REPO2=$(echo $REPO | sed 's/http:\\/\\///g')
@@ -74,6 +78,8 @@ steps: # 定义流水线执行步骤，这些步骤将顺序执行
  path: /var/cache/apt/archives # 将应用打包好的Jar和执行脚本挂载出来
  - name: socket
  path: /var/run/docker.sock
+ - name: bisheng-cache
+ path: /app/build/
  settings:
  registry: http://192.168.106.8:6082
  insecure: true
@@ -86,6 +92,7 @@ steps: # 定义流水线执行步骤，这些步骤将顺序执行
  from_secret: NEXUS_USER
  password:
  from_secret: NEXUS_PASSWORD
+
  - name: build_docker_frontend
  pull: if-not-exists
  image: plugins/docker
@@ -157,6 +164,7 @@ steps: # 定义流水线执行步骤，这些步骤将顺序执行
 trigger:
  branch:
  - release
+ # - feat/*
  event:
  - push
 

src/backend/Dockerfile

@@ -17,6 +17,9 @@ ENV PATH="${PATH}:/root/.local/bin"
 # Copy the rest of the application codes
 COPY ./ ./
 
+# Install NLTK data
+RUN mv -f ./taggers /root/nltk_data/
+
 RUN python -m pip install --upgrade pip && \
  pip install shapely==2.0.1 
 

src/backend/bisheng/api/errcode/base.py

@@ -20,3 +20,8 @@ def http_exception(cls, msg: str = None) -> Exception:
 class UnAuthorizedError(BaseErrorCode):
  Code: int = 403
  Msg: str = '暂无操作权限'
+
+
+class NotFoundError(BaseErrorCode):
+ Code: int = 404
+ Msg: str = '资源不存在'

src/backend/bisheng/api/errcode/tag.py

@@ -0,0 +1,12 @@
+from bisheng.api.errcode.base import BaseErrorCode
+
+
+# 标签模块相关的返回错误码，功能模块代码：107
+class TagExistError(BaseErrorCode):
+ Code: int = 10700
+ Msg: str = '标签已存在'
+
+
+class TagNotExistError(BaseErrorCode):
+ Code: int = 10700
+ Msg: str = '未找到对应的标签'

src/backend/bisheng/api/router.py

@@ -2,7 +2,8 @@
 from bisheng.api.v1 import (assistant_router, chat_router, component_router, endpoints_router,
  finetune_router, flows_router, group_router, knowledge_router,
  qa_router, report_router, server_router, skillcenter_router,
- user_router, validate_router, variable_router, audit_router, evaluation_router)
+ user_router, validate_router, variable_router, audit_router, evaluation_router,
+ tag_router)
 from bisheng.api.v2 import chat_router_rpc, knowledge_router_rpc, rpc_router_rpc, flow_router, assistant_router_rpc
 from fastapi import APIRouter
 
@@ -24,6 +25,7 @@
 router.include_router(group_router)
 router.include_router(audit_router)
 router.include_router(evaluation_router)
+router.include_router(tag_router)
 
 router_rpc = APIRouter(prefix='/api/v2', )
 router_rpc.include_router(knowledge_router_rpc)

src/backend/bisheng/api/services/assistant.py

@@ -3,6 +3,7 @@
 from uuid import UUID
 
 from fastapi import Request
+from loguru import logger
 
 from bisheng.api.errcode.assistant import (AssistantInitError, AssistantNameRepeatError,
  AssistantNotEditError, AssistantNotExistsError, ToolTypeRepeatError,
@@ -11,6 +12,7 @@
 from bisheng.api.services.assistant_agent import AssistantAgent
 from bisheng.api.services.assistant_base import AssistantUtils
 from bisheng.api.services.audit_log import AuditLogService
+from bisheng.api.services.base import BaseService
 from bisheng.api.services.user_service import UserPayload
 from bisheng.api.utils import get_request_ip
 from bisheng.api.v1.schemas import (AssistantInfo, AssistantSimpleInfo, AssistantUpdateReq,
@@ -23,28 +25,39 @@
 from bisheng.database.models.group_resource import GroupResourceDao, GroupResource, ResourceTypeEnum
 from bisheng.database.models.knowledge import KnowledgeDao
 from bisheng.database.models.role_access import AccessType, RoleAccessDao
+from bisheng.database.models.tag import TagDao
 from bisheng.database.models.user import UserDao
 from bisheng.database.models.user_group import UserGroupDao
 from bisheng.database.models.user_role import UserRoleDao
-from loguru import logger
 
 
-class AssistantService(AssistantUtils):
+class AssistantService(BaseService, AssistantUtils):
  UserCache: InMemoryCache = InMemoryCache()
 
  @classmethod
  def get_assistant(cls,
  user: UserPayload,
  name: str = None,
  status: int | None = None,
+ tag_id: int | None = None,
  page: int = 1,
  limit: int = 20) -> UnifiedResponseModel[List[AssistantSimpleInfo]]:
  """
  获取助手列表
  """
+ assistant_ids = []
+ if tag_id:
+ ret = TagDao.get_resources_by_tags([tag_id], ResourceTypeEnum.ASSISTANT)
+ assistant_ids = [UUID(one.resource_id) for one in ret]
+ if not assistant_ids:
+ return resp_200(data={
+ 'data': [],
+ 'total': 0
+ })
+
  data = []
  if user.is_admin():
- res, total = AssistantDao.get_all_assistants(name, page, limit)
+ res, total = AssistantDao.get_all_assistants(name, page, limit, assistant_ids)
  else:
  # 权限管理可见的助手信息
  assistant_ids_extra = []
@@ -54,12 +67,28 @@ def get_assistant(cls,
  role_access = RoleAccessDao.get_role_access(role_ids, AccessType.ASSISTANT_READ)
  if role_access:
  assistant_ids_extra = [UUID(access.third_id).hex for access in role_access]
- res, total = AssistantDao.get_assistants(user.user_id, name, assistant_ids_extra, status, page, limit)
+ res, total = AssistantDao.get_assistants(user.user_id, name, assistant_ids_extra, status, page, limit,
+ assistant_ids)
+
+ assistant_ids = [one.id.hex for one in res]
+ # 查询助手所属的分组
+ assistant_groups = GroupResourceDao.get_resources_group(ResourceTypeEnum.ASSISTANT, assistant_ids)
+ assistant_group_dict = {}
+ for one in assistant_groups:
+ if one.third_id not in assistant_group_dict:
+ assistant_group_dict[one.third_id] = []
+ assistant_group_dict[one.third_id].append(one.group_id)
+
+ # 获取助手关联的tag
+ flow_tags = TagDao.get_tags_by_resource(ResourceTypeEnum.ASSISTANT, assistant_ids)
 
  for one in res:
+ one.logo = cls.get_logo_share_link(one.logo)
  simple_assistant = cls.return_simple_assistant_info(one)
  if one.user_id == user.user_id or user.is_admin():
  simple_assistant.write = True
+ simple_assistant.group_ids = assistant_group_dict.get(one.id.hex, [])
+ simple_assistant.tags = flow_tags.get(one.id.hex, [])
  data.append(simple_assistant)
  return resp_200(data={'data': data, 'total': total})
 
@@ -99,6 +128,7 @@ def get_assistant_info(cls, assistant_id: UUID, login_user: UserPayload):
  logger.error(f'not expect link info: {one.dict()}')
  tool_list, flow_list, knowledge_list = cls.get_link_info(tool_list, flow_list,
  knowledge_list)
+ assistant.logo = cls.get_logo_share_link(assistant.logo)
  return resp_200(data=AssistantInfo(**assistant.dict(),
  tool_list=tool_list,
  flow_list=flow_list,
@@ -149,6 +179,9 @@ def create_assistant_hook(cls, request: Request, assistant: Assistant, user_payl
 
  # 写入审计日志
  AuditLogService.create_build_assistant(user_payload, get_request_ip(request), assistant.id.hex)
+
+ # 写入logo缓存
+ cls.get_logo_share_link(assistant.logo)
  return True
 
  # 删除助手
@@ -268,6 +301,9 @@ def update_assistant_hook(cls, request: Request, login_user: UserPayload, assist
 
  # 写入审计日志
  AuditLogService.update_build_assistant(login_user, get_request_ip(request), assistant.id.hex)
+
+ # 写入缓存
+ cls.get_logo_share_link(assistant.logo)
  return True
 
  @classmethod

src/backend/bisheng/api/services/base.py

@@ -0,0 +1,31 @@
+from bisheng.cache import InMemoryCache
+from bisheng.cache.redis import redis_client
+from bisheng.utils.minio_client import MinioClient
+
+
+class BaseService:
+
+ LogoMemoryCache = InMemoryCache(max_size=200, expiration_time=3600 * 24)
+
+ @classmethod
+ def get_logo_share_link(cls, logo_path: str):
+ if not logo_path:
+ return ''
+ cache_key = f'logo_cache:{logo_path}'
+ # 先从内存中获取
+ share_url = cls.LogoMemoryCache.get(cache_key)
+ if share_url:
+ return share_url
+
+ # 再从redis缓存中获取
+ share_url = redis_client.get(cache_key)
+ if share_url:
+ cls.LogoMemoryCache.set(cache_key, share_url)
+ return share_url
+
+ minio_client = MinioClient()
+ share_url = minio_client.get_share_link(logo_path)
+ # 缓存5天， 临时链接有效期为7天
+ redis_client.set(cache_key, share_url, 3600 * 120)
+ cls.LogoMemoryCache.set(cache_key, share_url)
+ return share_url

src/backend/bisheng/api/services/evaluation.py

@@ -137,7 +137,7 @@ def upload_file(cls, file: UploadFile):
 
  file_ext = os.path.basename(file.filename).split('.')[-1]
  file_path = f'evaluation/dataset/{file_id}.{file_ext}'
- minio_client.upload_minio_file(file_path, file.file, file.size, content_type=file.content_type)
+ minio_client.upload_minio_file(file_path, file.file, length=file.size, content_type=file.content_type)
  return file_name, file_path
 
  @classmethod

src/backend/bisheng/api/services/finetune.py

@@ -228,7 +228,7 @@ def delete_job_log(cls, finetune: Finetune):
  def upload_job_log(cls, finetune: Finetune, log_data: io.BytesIO, length: int) -> str:
  minio_client = MinioClient()
  log_path = f'finetune/log/{finetune.id.hex}'
- minio_client.upload_minio_file(log_path, log_data, length)
+ minio_client.upload_minio_file(log_path, log_data, length=length)
  return log_path
 
  @classmethod

src/backend/bisheng/api/services/finetune_file.py

@@ -45,7 +45,7 @@ def upload_file_to_minio(cls, files: List[UploadFile], file_root: str, user: Any
  file_info = PresetTrain(id=file_id, name=file.filename,
  url=f'{file_root}/{file_id}.{file_ext}',
  user_id=user.get('user_id'), user_name=user.get('user_name'))
- minio_client.upload_minio_file(file_info.url, file.file, file.size, content_type=file.content_type)
+ minio_client.upload_minio_file(file_info.url, file.file, length=file.size, content_type=file.content_type)
  ret.append(file_info)
  return ret
 

src/backend/bisheng/api/services/flow.py

@@ -7,11 +7,12 @@
 from fastapi import Request
 from loguru import logger
 
-from bisheng.api.errcode.base import UnAuthorizedError
+from bisheng.api.errcode.base import UnAuthorizedError, NotFoundError
 from bisheng.api.errcode.flow import NotFoundVersionError, CurVersionDelError, VersionNameExistsError, \
  NotFoundFlowError, \
  FlowOnlineEditError
 from bisheng.api.services.audit_log import AuditLogService
+from bisheng.api.services.base import BaseService
 from bisheng.api.services.user_service import UserPayload
 from bisheng.api.utils import get_L2_param_from_flow, get_request_ip
 from bisheng.api.v1.schemas import UnifiedResponseModel, resp_200, FlowVersionCreate, FlowCompareReq, resp_500, \
@@ -21,14 +22,15 @@
 from bisheng.database.models.flow_version import FlowVersionDao, FlowVersionRead, FlowVersion
 from bisheng.database.models.group_resource import GroupResourceDao, ResourceTypeEnum, GroupResource
 from bisheng.database.models.role_access import RoleAccessDao, AccessType
+from bisheng.database.models.tag import TagDao
 from bisheng.database.models.user import UserDao
 from bisheng.database.models.user_group import UserGroupDao
 from bisheng.database.models.user_role import UserRoleDao
 from bisheng.database.models.variable_value import VariableDao
 from bisheng.processing.process import process_graph_cached, process_tweaks
 
 
-class FlowService:
+class FlowService(BaseService):
 
  @classmethod
  def get_version_list_by_flow(cls, user: UserPayload, flow_id: str) -> UnifiedResponseModel[List[FlowVersionRead]]:
@@ -182,24 +184,48 @@ def update_version_info(cls, request: Request, user: UserPayload, version_id: in
  return resp_200(data=flow_version)
 
  @classmethod
- def get_all_flows(cls, user: UserPayload, name: str, status: int, page: int = 1, page_size: int = 10) -> \
- UnifiedResponseModel[List[Dict]]:
+ def get_one_flow(cls, login_user: UserPayload, flow_id: str) -> UnifiedResponseModel[Flow]:
+ """
+ 获取单个技能的详情
+ """
+ flow_info = FlowDao.get_flow_by_id(flow_id)
+ if not flow_info:
+ raise NotFoundFlowError.http_exception()
+ if not login_user.access_check(flow_info.user_id, flow_info.id.hex, AccessType.FLOW):
+ raise UnAuthorizedError.http_exception()
+ flow_info.logo = cls.get_logo_share_link(flow_info.logo)
+
+ return resp_200(data=flow_info)
+
+ @classmethod
+ def get_all_flows(cls, user: UserPayload, name: str, status: int, tag_id: int = 0, page: int = 1,
+ page_size: int = 10) -> UnifiedResponseModel[List[Dict]]:
  """
  获取所有技能
  """
+ flow_ids = []
+ if tag_id:
+ ret = TagDao.get_resources_by_tags([tag_id], ResourceTypeEnum.FLOW)
+ flow_ids = [UUID(one.resource_id) for one in ret]
+ assistant_ids = [UUID(one.resource_id) for one in ret]
+ if not assistant_ids:
+ return resp_200(data={
+ 'data': [],
+ 'total': 0
+ })
  # 获取用户可见的技能列表
  if user.is_admin():
- data = FlowDao.get_flows(user.user_id, "admin", name, status, page, page_size)
- total = FlowDao.count_flows(user.user_id, "admin", name, status)
+ data = FlowDao.get_flows(user.user_id, "admin", name, status, flow_ids, page, page_size)
+ total = FlowDao.count_flows(user.user_id, "admin", name, status, flow_ids)
  else:
  user_role = UserRoleDao.get_user_roles(user.user_id)
  role_ids = [role.role_id for role in user_role]
  role_access = RoleAccessDao.get_role_access(role_ids, AccessType.FLOW)
  flow_id_extra = []
  if role_access:
  flow_id_extra = [access.third_id for access in role_access]
- data = FlowDao.get_flows(user.user_id, flow_id_extra, name, status, page, page_size)
- total = FlowDao.count_flows(user.user_id, flow_id_extra, name, status)
+ data = FlowDao.get_flows(user.user_id, flow_id_extra, name, status, flow_ids, page, page_size)
+ total = FlowDao.count_flows(user.user_id, flow_id_extra, name, status, flow_ids)
 
  # 获取技能列表对应的用户信息和版本信息
  # 技能ID列表
@@ -221,13 +247,28 @@ def get_all_flows(cls, user: UserPayload, name: str, status: int, page: int = 1,
  flow_versions[one.flow_id] = []
  flow_versions[one.flow_id].append(jsonable_encoder(one))
 
+ # 获取技能所属的分组
+ flow_groups = GroupResourceDao.get_resources_group(ResourceTypeEnum.FLOW, flow_ids)
+ flow_group_dict = {}
+ for one in flow_groups:
+ if one.third_id not in flow_group_dict:
+ flow_group_dict[one.third_id] = []
+ flow_group_dict[one.third_id].append(one.group_id)
+
+ # 获取技能关联的tag
+ flow_tags = TagDao.get_tags_by_resource(ResourceTypeEnum.FLOW, flow_ids)
+
  # 重新拼接技能列表list信息
  res = []
  for one in data:
+ one.logo = cls.get_logo_share_link(one.logo)
  flow_info = jsonable_encoder(one)
  flow_info['user_name'] = user_dict.get(one.user_id, one.user_id)
  flow_info['write'] = True if user.is_admin() or user.user_id == one.user_id else False
  flow_info['version_list'] = flow_versions.get(one.id.hex, [])
+ flow_info['group_ids'] = flow_group_dict.get(one.id.hex, [])
+ flow_info['tags'] = flow_tags.get(one.id.hex, [])
+
  res.append(flow_info)
 
  return resp_200(data={
@@ -373,12 +414,18 @@ def create_flow_hook(cls, request: Request, login_user: UserPayload, flow_info:
  GroupResourceDao.insert_group_batch(batch_resource)
  # 写入审计日志
  AuditLogService.create_build_flow(login_user, get_request_ip(request), flow_info.id.hex)
+
+ # 写入logo缓存
+ cls.get_logo_share_link(flow_info.logo)
  return True
 
  @classmethod
  def update_flow_hook(cls, request: Request, login_user: UserPayload, flow_info: Flow) -> bool:
  # 写入审计日志
  AuditLogService.update_build_flow(login_user, get_request_ip(request), flow_info.id.hex)
+
+ # 写入logo缓存
+ cls.get_logo_share_link(flow_info.logo)
  return True
 
  @classmethod