Use offline chat prompt config to set context window of loaded chat model

Previously you couldn't configure the n_ctx of the loaded offline chat
model. This made it hard to use good offline chat model (which these
days also have larger context) on machines with lower VRAM
This commit is contained in:
Debanjum Singh Solanky
2024-04-13 22:15:34 +05:30
parent 689202e00e
commit 4977b55106
8 changed files with 81 additions and 44 deletions

View File

@@ -78,6 +78,7 @@ dependencies = [
"phonenumbers == 8.13.27",
"markdownify ~= 0.11.6",
"websockets == 12.0",
"psutil >= 5.8.0",
]
dynamic = ["version"]
@@ -105,7 +106,6 @@ dev = [
"pytest-asyncio == 0.21.1",
"freezegun >= 1.2.0",
"factory-boy >= 3.2.1",
"psutil >= 5.8.0",
"mypy >= 1.0.1",
"black >= 23.1.0",
"pre-commit >= 3.0.4",