112 lines
2.8 KiB
Plaintext
112 lines
2.8 KiB
Plaintext
|
from .model_config import LLM_MODEL, LLM_DEVICE
|
|||
|
|
|
|||
|
|
# API 是否开启跨域，默认为False，如果需要开启，请设置为True
|
|||
|
|
# is open cross domain
|
|||
|
|
OPEN_CROSS_DOMAIN = False
|
|||
|
|
|
|||
|
|
# 各服务器默认绑定host
|
|||
|
|
DEFAULT_BIND_HOST = "127.0.0.1"
|
|||
|
|
|
|||
|
|
# webui.py server
|
|||
|
|
WEBUI_SERVER = {
|
|||
|
|
"host": DEFAULT_BIND_HOST,
|
|||
|
|
"port": 8501,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# api.py server
|
|||
|
|
API_SERVER = {
|
|||
|
|
"host": DEFAULT_BIND_HOST,
|
|||
|
|
"port": 7861,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# fastchat openai_api server
|
|||
|
|
FSCHAT_OPENAI_API = {
|
|||
|
|
"host": DEFAULT_BIND_HOST,
|
|||
|
|
"port": 8888, # model_config.llm_model_dict中模型配置的api_base_url需要与这里一致。
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# sandbox api server
|
|||
|
|
CONTRAINER_NAME = "devopsgt_default"
|
|||
|
|
IMAGE_NAME = "devopsgpt:pypy38"
|
|||
|
|
SANDBOX_SERVER = {
|
|||
|
|
"host": DEFAULT_BIND_HOST,
|
|||
|
|
"port": 5050,
|
|||
|
|
"url": "http://localhost:5050",
|
|||
|
|
"do_remote": True,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
# fastchat model_worker server
|
|||
|
|
# 这些模型必须是在model_config.llm_model_dict中正确配置的。
|
|||
|
|
# 在启动startup.py时，可用通过`--model-worker --model-name xxxx`指定模型，不指定则为LLM_MODEL
|
|||
|
|
FSCHAT_MODEL_WORKERS = {
|
|||
|
|
LLM_MODEL: {
|
|||
|
|
"host": DEFAULT_BIND_HOST,
|
|||
|
|
"port": 20002,
|
|||
|
|
"device": LLM_DEVICE,
|
|||
|
|
# todo: 多卡加载需要配置的参数
|
|||
|
|
"gpus": None,
|
|||
|
|
"numgpus": 1,
|
|||
|
|
# 以下为非常用参数，可根据需要配置
|
|||
|
|
# "max_gpu_memory": "20GiB",
|
|||
|
|
# "load_8bit": False,
|
|||
|
|
# "cpu_offloading": None,
|
|||
|
|
# "gptq_ckpt": None,
|
|||
|
|
# "gptq_wbits": 16,
|
|||
|
|
# "gptq_groupsize": -1,
|
|||
|
|
# "gptq_act_order": False,
|
|||
|
|
# "awq_ckpt": None,
|
|||
|
|
# "awq_wbits": 16,
|
|||
|
|
# "awq_groupsize": -1,
|
|||
|
|
# "model_names": [LLM_MODEL],
|
|||
|
|
# "conv_template": None,
|
|||
|
|
# "limit_worker_concurrency": 5,
|
|||
|
|
# "stream_interval": 2,
|
|||
|
|
# "no_register": False,
|
|||
|
|
},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# fastchat multi model worker server
|
|||
|
|
FSCHAT_MULTI_MODEL_WORKERS = {
|
|||
|
|
# todo
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# fastchat controller server
|
|||
|
|
FSCHAT_CONTROLLER = {
|
|||
|
|
"host": DEFAULT_BIND_HOST,
|
|||
|
|
"port": 20001,
|
|||
|
|
"dispatch_method": "shortest_queue",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
# 以下不要更改
|
|||
|
|
def fschat_controller_address() -> str:
|
|||
|
|
host = FSCHAT_CONTROLLER["host"]
|
|||
|
|
port = FSCHAT_CONTROLLER["port"]
|
|||
|
|
return f"http://{host}:{port}"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def fschat_model_worker_address(model_name: str = LLM_MODEL) -> str:
|
|||
|
|
if model := FSCHAT_MODEL_WORKERS.get(model_name):
|
|||
|
|
host = model["host"]
|
|||
|
|
port = model["port"]
|
|||
|
|
return f"http://{host}:{port}"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def fschat_openai_api_address() -> str:
|
|||
|
|
host = FSCHAT_OPENAI_API["host"]
|
|||
|
|
port = FSCHAT_OPENAI_API["port"]
|
|||
|
|
return f"http://{host}:{port}"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def api_address() -> str:
|
|||
|
|
host = API_SERVER["host"]
|
|||
|
|
port = API_SERVER["port"]
|
|||
|
|
return f"http://{host}:{port}"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def webui_address() -> str:
|
|||
|
|
host = WEBUI_SERVER["host"]
|
|||
|
|
port = WEBUI_SERVER["port"]
|
|||
|
|
return f"http://{host}:{port}"
|