2024-03-28 20:12:36 +08:00
import streamlit as st
import docker
import torch , os , sys , json
from loguru import logger
src_dir = os . path . join (
os . path . dirname ( os . path . dirname ( os . path . abspath ( __file__ ) ) )
)
sys . path . append ( src_dir )
from configs . default_config import *
import platform
system_name = platform . system ( )
VERSION = " v0.1.0 "
MODEL_WORKER_SETS = [
" ChatGLMWorker " , " MiniMaxWorker " , " XingHuoWorker " , " QianFanWorker " , " FangZhouWorker " ,
" QwenWorker " , " BaiChuanWorker " , " AzureWorker " , " TianGongWorker " , " ExampleWorker "
]
openai_models = [ " gpt-3.5-turbo " , " gpt-3.5-turbo-1106 " , " gpt-3.5-turbo-16k " , " gpt-4 " ]
embedding_models = [ " openai " ]
st . write ( " 启动配置页面! " )
st . write ( " 如果你要使用语言模型, 请将LLM放到 ~/Codefuse-chatbot/llm_models " )
st . write ( " 如果你要使用向量模型,请将向量模型放到 ~/Codefuse-chatbot/embedding_models " )
with st . container ( ) :
col1 , col2 = st . columns ( 2 )
with col1 . container ( ) :
llm_model_name = st . selectbox ( ' LLM Model Name ' , openai_models + [ i for i in os . listdir ( LOCAL_LLM_MODEL_DIR ) if os . path . isdir ( os . path . join ( LOCAL_LLM_MODEL_DIR , i ) ) ] )
llm_apikey = st . text_input ( ' 填写 LLM API KEY ' , ' EMPTY ' )
llm_apiurl = st . text_input ( ' 填写 LLM API URL ' , ' http://localhost:8888/v1 ' )
llm_engine = st . selectbox ( ' 选择哪个llm引擎 ' , [ " online " , " fastchat " , " fastchat-vllm " ] )
llm_model_port = st . text_input ( ' LLM Model Port, 非fastchat模式可无视 ' , ' 20006 ' )
llm_provider_option = st . selectbox ( ' 选择哪个online模型加载器, 非online可无视 ' , [ " openai " ] + MODEL_WORKER_SETS )
if llm_engine == " online " and llm_provider_option == " openai " :
try :
from zdatafront import OPENAI_API_BASE
except :
OPENAI_API_BASE = " https://api.openai.com/v1 "
llm_apiurl = OPENAI_API_BASE
device = " cuda " if torch . cuda . is_available ( ) else " mps " if torch . backends . mps . is_available ( ) else " cpu "
FSCHAT_MODEL_WORKERS = {
llm_model_name : {
' host ' : " 127.0.0.1 " , ' port ' : llm_model_port ,
" device " : device ,
# todo: 多卡加载需要配置的参数
" gpus " : None ,
" numgpus " : 1 , } ,
}
ONLINE_LLM_MODEL , llm_model_dict , VLLM_MODEL_DICT = { } , { } , { }
if llm_engine == " online " :
ONLINE_LLM_MODEL = {
llm_model_name : {
" model_name " : llm_model_name ,
" version " : llm_model_name ,
" api_base_url " : llm_apiurl , # "https://api.openai.com/v1",
" api_key " : llm_apikey ,
" openai_proxy " : " " ,
" provider " : llm_provider_option
} ,
}
if llm_engine == " fastchat " :
llm_model_dict = {
llm_model_name : {
" local_model_path " : llm_model_name ,
" api_base_url " : llm_apiurl , # "name"修改为fastchat服务中的"api_base_url"
" api_key " : llm_apikey
} }
if llm_engine == " fastchat-vllm " :
VLLM_MODEL_DICT = {
llm_model_name : {
" local_model_path " : llm_model_name ,
" api_base_url " : llm_apiurl , # "name"修改为fastchat服务中的"api_base_url"
" api_key " : llm_apikey
}
}
llm_model_dict = {
llm_model_name : {
" local_model_path " : llm_model_name ,
" api_base_url " : llm_apiurl , # "name"修改为fastchat服务中的"api_base_url"
" api_key " : llm_apikey
} }
with col2 . container ( ) :
em_model_name = st . selectbox ( ' Embedding Model Name ' , [ i for i in os . listdir ( LOCAL_EM_MODEL_DIR ) if os . path . isdir ( os . path . join ( LOCAL_EM_MODEL_DIR , i ) ) ] + embedding_models )
em_engine = st . selectbox ( ' 选择哪个embedding引擎 ' , [ " model " , " openai " ] )
device = " cuda " if torch . cuda . is_available ( ) else " mps " if torch . backends . mps . is_available ( ) else " cpu "
embedding_model_dict = { em_model_name : em_model_name }
# em_apikey = st.text_input('Embedding API KEY', '')
# em_apiurl = st.text_input('Embedding API URL', '')
#
try :
client = docker . from_env ( )
has_docker = True
except :
has_docker = False
if has_docker :
with st . container ( ) :
DOCKER_SERVICE = st . toggle ( ' DOCKER_SERVICE ' , True )
SANDBOX_DO_REMOTE = st . toggle ( ' SANDBOX_DO_REMOTE ' , True )
else :
DOCKER_SERVICE = False
SANDBOX_DO_REMOTE = False
with st . container ( ) :
cols = st . columns ( 3 )
if cols [ 0 ] . button (
" 重启服务,按前配置生效 " ,
use_container_width = True ,
) :
from start import start_main
from stop import stop_main
stop_main ( )
start_main ( )
if cols [ 1 ] . button (
" 停止服务 " ,
use_container_width = True ,
) :
from stop import stop_main
stop_main ( )
if cols [ 2 ] . button (
" 启动对话服务 " ,
use_container_width = True
) :
os . environ [ " API_BASE_URL " ] = llm_apiurl
os . environ [ " OPENAI_API_KEY " ] = llm_apikey
os . environ [ " EMBEDDING_ENGINE " ] = em_engine
os . environ [ " EMBEDDING_MODEL " ] = em_model_name
os . environ [ " LLM_MODEL " ] = llm_model_name
embedding_model_dict = { k : f " /home/user/chatbot/embedding_models/ { v } " if DOCKER_SERVICE else f " { LOCAL_EM_MODEL_DIR } / { v } " for k , v in embedding_model_dict . items ( ) }
os . environ [ " embedding_model_dict " ] = json . dumps ( embedding_model_dict )
os . environ [ " ONLINE_LLM_MODEL " ] = json . dumps ( ONLINE_LLM_MODEL )
# 模型路径重置
llm_model_dict_c = { }
for k , v in llm_model_dict . items ( ) :
v_c = { }
for kk , vv in v . items ( ) :
if k == " local_model_path " :
v_c [ kk ] = f " /home/user/chatbot/llm_models/ { vv } " if DOCKER_SERVICE else f " { LOCAL_LLM_MODEL_DIR } / { vv } "
else :
v_c [ kk ] = vv
llm_model_dict_c [ k ] = v_c
llm_model_dict = llm_model_dict_c
os . environ [ " llm_model_dict " ] = json . dumps ( llm_model_dict )
#
VLLM_MODEL_DICT_c = { }
for k , v in VLLM_MODEL_DICT . items ( ) :
VLLM_MODEL_DICT_c [ k ] = f " /home/user/chatbot/llm_models/ { v } " if DOCKER_SERVICE else f " { LOCAL_LLM_MODEL_DIR } / { v } "
VLLM_MODEL_DICT = VLLM_MODEL_DICT_c
os . environ [ " VLLM_MODEL_DICT " ] = json . dumps ( VLLM_MODEL_DICT )
# server config
os . environ [ " DOCKER_SERVICE " ] = json . dumps ( DOCKER_SERVICE )
os . environ [ " SANDBOX_DO_REMOTE " ] = json . dumps ( SANDBOX_DO_REMOTE )
os . environ [ " FSCHAT_MODEL_WORKERS " ] = json . dumps ( FSCHAT_MODEL_WORKERS )
update_json = {
" API_BASE_URL " : llm_apiurl ,
" OPENAI_API_KEY " : llm_apikey ,
" EMBEDDING_ENGINE " : em_engine ,
" EMBEDDING_MODEL " : em_model_name ,
" LLM_MODEL " : llm_model_name ,
2024-04-23 16:44:13 +08:00
" embedding_model_dict " : embedding_model_dict ,
" llm_model_dict " : llm_model_dict ,
" ONLINE_LLM_MODEL " : ONLINE_LLM_MODEL ,
" VLLM_MODEL_DICT " : VLLM_MODEL_DICT ,
" DOCKER_SERVICE " : DOCKER_SERVICE ,
" SANDBOX_DO_REMOTE " : SANDBOX_DO_REMOTE ,
" FSCHAT_MODEL_WORKERS " : FSCHAT_MODEL_WORKERS
2024-03-28 20:12:36 +08:00
}
with open ( os . path . join ( src_dir , " configs/local_config.json " ) , " w " ) as f :
json . dump ( update_json , f )
from start import start_main
from stop import stop_main
stop_main ( )
start_main ( )