209 lines
7.8 KiB
Python
209 lines
7.8 KiB
Python
|
import streamlit as st
|
|||
|
|
import docker
|
|||
|
|
import torch, os, sys, json
|
|||
|
|
from loguru import logger
|
|||
|
|
|
|||
|
|
src_dir = os.path.join(
|
|||
|
|
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|||
|
|
)
|
|||
|
|
sys.path.append(src_dir)
|
|||
|
|
from configs.default_config import *
|
|||
|
|
|
|||
|
|
import platform
|
|||
|
|
system_name = platform.system()
|
|||
|
|
|
|||
|
|
|
|||
|
|
VERSION = "v0.1.0"
|
|||
|
|
|
|||
|
|
MODEL_WORKER_SETS = [
|
|||
|
|
"ChatGLMWorker", "MiniMaxWorker", "XingHuoWorker", "QianFanWorker", "FangZhouWorker",
|
|||
|
|
"QwenWorker", "BaiChuanWorker", "AzureWorker", "TianGongWorker", "ExampleWorker"
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
openai_models = ["gpt-3.5-turbo", "gpt-3.5-turbo-1106", "gpt-3.5-turbo-16k", "gpt-4"]
|
|||
|
|
embedding_models = ["openai"]
|
|||
|
|
|
|||
|
|
|
|||
|
|
st.write("启动配置页面！")
|
|||
|
|
|
|||
|
|
st.write("如果你要使用语言模型，请将LLM放到 ~/Codefuse-chatbot/llm_models")
|
|||
|
|
|
|||
|
|
st.write("如果你要使用向量模型，请将向量模型放到 ~/Codefuse-chatbot/embedding_models")
|
|||
|
|
|
|||
|
|
with st.container():
|
|||
|
|
|
|||
|
|
col1, col2 = st.columns(2)
|
|||
|
|
with col1.container():
|
|||
|
|
llm_model_name = st.selectbox('LLM Model Name', openai_models + [i for i in os.listdir(LOCAL_LLM_MODEL_DIR) if os.path.isdir(os.path.join(LOCAL_LLM_MODEL_DIR, i))])
|
|||
|
|
|
|||
|
|
llm_apikey = st.text_input('填写 LLM API KEY', 'EMPTY')
|
|||
|
|
llm_apiurl = st.text_input('填写 LLM API URL', 'http://localhost:8888/v1')
|
|||
|
|
|
|||
|
|
llm_engine = st.selectbox('选择哪个llm引擎', ["online", "fastchat", "fastchat-vllm"])
|
|||
|
|
llm_model_port = st.text_input('LLM Model Port，非fastchat模式可无视', '20006')
|
|||
|
|
llm_provider_option = st.selectbox('选择哪个online模型加载器，非online可无视', ["openai"] + MODEL_WORKER_SETS)
|
|||
|
|
|
|||
|
|
if llm_engine == "online" and llm_provider_option == "openai":
|
|||
|
|
try:
|
|||
|
|
from zdatafront import OPENAI_API_BASE
|
|||
|
|
except:
|
|||
|
|
OPENAI_API_BASE = "https://api.openai.com/v1"
|
|||
|
|
llm_apiurl = OPENAI_API_BASE
|
|||
|
|
|
|||
|
|
device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
|
|||
|
|
|
|||
|
|
FSCHAT_MODEL_WORKERS = {
|
|||
|
|
llm_model_name: {
|
|||
|
|
'host': "127.0.0.1", 'port': llm_model_port,
|
|||
|
|
"device": device,
|
|||
|
|
# todo: 多卡加载需要配置的参数
|
|||
|
|
"gpus": None,
|
|||
|
|
"numgpus": 1,},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
ONLINE_LLM_MODEL, llm_model_dict, VLLM_MODEL_DICT = {}, {}, {}
|
|||
|
|
if llm_engine == "online":
|
|||
|
|
ONLINE_LLM_MODEL = {
|
|||
|
|
llm_model_name: {
|
|||
|
|
"model_name": llm_model_name,
|
|||
|
|
"version": llm_model_name,
|
|||
|
|
"api_base_url": llm_apiurl, # "https://api.openai.com/v1",
|
|||
|
|
"api_key": llm_apikey,
|
|||
|
|
"openai_proxy": "",
|
|||
|
|
"provider": llm_provider_option
|
|||
|
|
},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if llm_engine == "fastchat":
|
|||
|
|
llm_model_dict = {
|
|||
|
|
llm_model_name: {
|
|||
|
|
"local_model_path": llm_model_name,
|
|||
|
|
"api_base_url": llm_apiurl, # "name"修改为fastchat服务中的"api_base_url"
|
|||
|
|
"api_key": llm_apikey
|
|||
|
|
}}
|
|||
|
|
|
|||
|
|
|
|||
|
|
if llm_engine == "fastchat-vllm":
|
|||
|
|
VLLM_MODEL_DICT = {
|
|||
|
|
llm_model_name: {
|
|||
|
|
"local_model_path": llm_model_name,
|
|||
|
|
"api_base_url": llm_apiurl, # "name"修改为fastchat服务中的"api_base_url"
|
|||
|
|
"api_key": llm_apikey
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
llm_model_dict = {
|
|||
|
|
llm_model_name: {
|
|||
|
|
"local_model_path": llm_model_name,
|
|||
|
|
"api_base_url": llm_apiurl, # "name"修改为fastchat服务中的"api_base_url"
|
|||
|
|
"api_key": llm_apikey
|
|||
|
|
}}
|
|||
|
|
|
|||
|
|
|
|||
|
|
with col2.container():
|
|||
|
|
em_model_name = st.selectbox('Embedding Model Name', [i for i in os.listdir(LOCAL_EM_MODEL_DIR) if os.path.isdir(os.path.join(LOCAL_EM_MODEL_DIR, i))] + embedding_models)
|
|||
|
|
em_engine = st.selectbox('选择哪个embedding引擎', ["model", "openai"])
|
|||
|
|
device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
|
|||
|
|
embedding_model_dict = {em_model_name: em_model_name}
|
|||
|
|
# em_apikey = st.text_input('Embedding API KEY', '')
|
|||
|
|
# em_apiurl = st.text_input('Embedding API URL', '')
|
|||
|
|
|
|||
|
|
#
|
|||
|
|
try:
|
|||
|
|
client = docker.from_env()
|
|||
|
|
has_docker = True
|
|||
|
|
except:
|
|||
|
|
has_docker = False
|
|||
|
|
|
|||
|
|
if has_docker:
|
|||
|
|
with st.container():
|
|||
|
|
DOCKER_SERVICE = st.toggle('DOCKER_SERVICE', True)
|
|||
|
|
SANDBOX_DO_REMOTE = st.toggle('SANDBOX_DO_REMOTE', True)
|
|||
|
|
else:
|
|||
|
|
DOCKER_SERVICE = False
|
|||
|
|
SANDBOX_DO_REMOTE = False
|
|||
|
|
|
|||
|
|
|
|||
|
|
with st.container():
|
|||
|
|
cols = st.columns(3)
|
|||
|
|
|
|||
|
|
if cols[0].button(
|
|||
|
|
"重启服务，按前配置生效",
|
|||
|
|
use_container_width=True,
|
|||
|
|
):
|
|||
|
|
from start import start_main
|
|||
|
|
from stop import stop_main
|
|||
|
|
stop_main()
|
|||
|
|
start_main()
|
|||
|
|
if cols[1].button(
|
|||
|
|
"停止服务",
|
|||
|
|
use_container_width=True,
|
|||
|
|
):
|
|||
|
|
from stop import stop_main
|
|||
|
|
stop_main()
|
|||
|
|
|
|||
|
|
if cols[2].button(
|
|||
|
|
"启动对话服务",
|
|||
|
|
use_container_width=True
|
|||
|
|
):
|
|||
|
|
|
|||
|
|
os.environ["API_BASE_URL"] = llm_apiurl
|
|||
|
|
os.environ["OPENAI_API_KEY"] = llm_apikey
|
|||
|
|
|
|||
|
|
os.environ["EMBEDDING_ENGINE"] = em_engine
|
|||
|
|
os.environ["EMBEDDING_MODEL"] = em_model_name
|
|||
|
|
os.environ["LLM_MODEL"] = llm_model_name
|
|||
|
|
|
|||
|
|
embedding_model_dict = {k: f"/home/user/chatbot/embedding_models/{v}" if DOCKER_SERVICE else f"{LOCAL_EM_MODEL_DIR}/{v}" for k, v in embedding_model_dict.items()}
|
|||
|
|
os.environ["embedding_model_dict"] = json.dumps(embedding_model_dict)
|
|||
|
|
|
|||
|
|
os.environ["ONLINE_LLM_MODEL"] = json.dumps(ONLINE_LLM_MODEL)
|
|||
|
|
|
|||
|
|
# 模型路径重置
|
|||
|
|
llm_model_dict_c = {}
|
|||
|
|
for k, v in llm_model_dict.items():
|
|||
|
|
v_c = {}
|
|||
|
|
for kk, vv in v.items():
|
|||
|
|
if k=="local_model_path":
|
|||
|
|
v_c[kk] = f"/home/user/chatbot/llm_models/{vv}" if DOCKER_SERVICE else f"{LOCAL_LLM_MODEL_DIR}/{vv}"
|
|||
|
|
else:
|
|||
|
|
v_c[kk] = vv
|
|||
|
|
llm_model_dict_c[k] = v_c
|
|||
|
|
|
|||
|
|
llm_model_dict = llm_model_dict_c
|
|||
|
|
os.environ["llm_model_dict"] = json.dumps(llm_model_dict)
|
|||
|
|
#
|
|||
|
|
VLLM_MODEL_DICT_c = {}
|
|||
|
|
for k, v in VLLM_MODEL_DICT.items():
|
|||
|
|
VLLM_MODEL_DICT_c[k] = f"/home/user/chatbot/llm_models/{v}" if DOCKER_SERVICE else f"{LOCAL_LLM_MODEL_DIR}/{v}"
|
|||
|
|
VLLM_MODEL_DICT = VLLM_MODEL_DICT_c
|
|||
|
|
os.environ["VLLM_MODEL_DICT"] = json.dumps(VLLM_MODEL_DICT)
|
|||
|
|
|
|||
|
|
# server config
|
|||
|
|
os.environ["DOCKER_SERVICE"] = json.dumps(DOCKER_SERVICE)
|
|||
|
|
os.environ["SANDBOX_DO_REMOTE"] = json.dumps(SANDBOX_DO_REMOTE)
|
|||
|
|
os.environ["FSCHAT_MODEL_WORKERS"] = json.dumps(FSCHAT_MODEL_WORKERS)
|
|||
|
|
|
|||
|
|
update_json = {
|
|||
|
|
"API_BASE_URL": llm_apiurl,
|
|||
|
|
"OPENAI_API_KEY": llm_apikey,
|
|||
|
|
"EMBEDDING_ENGINE": em_engine,
|
|||
|
|
"EMBEDDING_MODEL": em_model_name,
|
|||
|
|
"LLM_MODEL": llm_model_name,
|
|||
|
|
"embedding_model_dict": json.dumps(embedding_model_dict),
|
|||
|
|
"llm_model_dict": json.dumps(llm_model_dict),
|
|||
|
|
"ONLINE_LLM_MODEL": json.dumps(ONLINE_LLM_MODEL),
|
|||
|
|
"VLLM_MODEL_DICT": json.dumps(VLLM_MODEL_DICT),
|
|||
|
|
"DOCKER_SERVICE": json.dumps(DOCKER_SERVICE),
|
|||
|
|
"SANDBOX_DO_REMOTE": json.dumps(SANDBOX_DO_REMOTE),
|
|||
|
|
"FSCHAT_MODEL_WORKERS": json.dumps(FSCHAT_MODEL_WORKERS)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
with open(os.path.join(src_dir, "configs/local_config.json"), "w") as f:
|
|||
|
|
json.dump(update_json, f)
|
|||
|
|
|
|||
|
|
from start import start_main
|
|||
|
|
from stop import stop_main
|
|||
|
|
stop_main()
|
|||
|
|
start_main()
|