feat(llama-swap): add synthetic peer with per-model context windows
This commit is contained in:
@@ -7,6 +7,7 @@ let
|
||||
flatten
|
||||
listToAttrs
|
||||
nameValuePair
|
||||
optionalAttrs
|
||||
;
|
||||
in
|
||||
{
|
||||
@@ -54,10 +55,15 @@ in
|
||||
map
|
||||
(
|
||||
modelName:
|
||||
nameValuePair modelName {
|
||||
id = modelName;
|
||||
name = modelName;
|
||||
}
|
||||
nameValuePair modelName (
|
||||
{
|
||||
id = modelName;
|
||||
name = modelName;
|
||||
}
|
||||
// optionalAttrs (peer.contextWindows ? ${modelName}) {
|
||||
contextWindow = peer.contextWindows.${modelName};
|
||||
}
|
||||
)
|
||||
)
|
||||
peer.models
|
||||
)
|
||||
|
||||
@@ -836,4 +836,22 @@ in
|
||||
concurrent = "(go | g4 | q36a | q36b | q36ik | v180 | v145 | v75 | v50 | zi | qie | qi | cr) & (q4 | q9)";
|
||||
};
|
||||
};
|
||||
|
||||
peers = {
|
||||
synthetic = {
|
||||
proxy = "https://api.synthetic.new/openai/";
|
||||
contextWindows = {
|
||||
"hf:moonshotai/Kimi-K3" = 524288;
|
||||
"hf:Qwen/Qwen3.6-27B" = 262144;
|
||||
"hf:zai-org/GLM-4.7-Flash" = 196608;
|
||||
"hf:zai-org/GLM-5.2" = 524288;
|
||||
};
|
||||
models = [
|
||||
"hf:Qwen/Qwen3.6-27B"
|
||||
"hf:moonshotai/Kimi-K3"
|
||||
"hf:zai-org/GLM-4.7-Flash"
|
||||
"hf:zai-org/GLM-5.2"
|
||||
];
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
@@ -114,12 +114,16 @@ in
|
||||
|
||||
# Create Config
|
||||
sops = {
|
||||
secrets = listToAttrs (map (name: {
|
||||
secrets = (listToAttrs (map (name: {
|
||||
name = "llama_swap_api_keys/${name}";
|
||||
value = {
|
||||
sopsFile = lib.snowfall.fs.get-file "secrets/common/llama-swap.yaml";
|
||||
};
|
||||
}) apiKeys);
|
||||
}) apiKeys)) // {
|
||||
synthetic_apikey = {
|
||||
sopsFile = lib.snowfall.fs.get-file "secrets/common/systems.yaml";
|
||||
};
|
||||
};
|
||||
templates."llama-swap.json" = {
|
||||
restartUnits = [ "llama-swap.service" ];
|
||||
owner = "llama-swap";
|
||||
@@ -128,6 +132,7 @@ in
|
||||
content = builtins.toJSON (
|
||||
recursiveUpdate cfg.config {
|
||||
apiKeys = map (name: config.sops.placeholder."llama_swap_api_keys/${name}") apiKeys;
|
||||
peers.synthetic.apiKey = config.sops.placeholder.synthetic_apikey;
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user