Public/koboldcpp
1
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-09-19 09:15:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upstream' into concedo_experimental

Browse Source
This commit is contained in:
Concedo
2025-10-11 23:19:38 +08:00
parent e92f9fd422 a3cb04744f
commit 720fc30832
12 changed files with 149 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/public/index.html.gz
BIN
View File
Binary file not shown.
tools/server/server.cpp
+4 -10
View File
@@ -5401,15 +5401,6 @@ int main(int argc, char ** argv) {
const json body = json::parse(req.body);
// TODO: implement
//int top_n = 1;
//if (body.count("top_n") != 1) {
// top_n = body.at("top_n");
//} else {
// res_error(res, format_error_response("\"top_n\" must be provided", ERROR_TYPE_INVALID_REQUEST));
// return;
//}
// if true, use TEI API format, otherwise use Jina API format
// Jina: https://jina.ai/reranker/
// TEI: https://huggingface.github.io/text-embeddings-inference/#/Text%20Embeddings%20Inference/rerank
@@ -5434,6 +5425,8 @@ int main(int argc, char ** argv) {
return;
}
int top_n = json_value(body, "top_n", (int)documents.size());
// create and queue the task
json responses = json::array();
bool error = false;
@@ -5474,7 +5467,8 @@ int main(int argc, char ** argv) {
body,
responses,
is_tei_format,
documents);
documents,
top_n);
res_ok(res, root);
};
tools/server/tests/unit/test_rerank.py
+42
View File
@@ -102,3 +102,45 @@ def test_rerank_usage(query, doc1, doc2, n_tokens):
assert res.status_code == 200
assert res.body['usage']['prompt_tokens'] == res.body['usage']['total_tokens']
assert res.body['usage']['prompt_tokens'] == n_tokens
@pytest.mark.parametrize("top_n,expected_len", [
(None, len(TEST_DOCUMENTS)), # no top_n parameter
(2, 2),
(4, 4),
(99, len(TEST_DOCUMENTS)), # higher than available docs
])
def test_rerank_top_n(top_n, expected_len):
global server
server.start()
data = {
"query": "Machine learning is",
"documents": TEST_DOCUMENTS,
}
if top_n is not None:
data["top_n"] = top_n
res = server.make_request("POST", "/rerank", data=data)
assert res.status_code == 200
assert len(res.body["results"]) == expected_len
@pytest.mark.parametrize("top_n,expected_len", [
(None, len(TEST_DOCUMENTS)), # no top_n parameter
(2, 2),
(4, 4),
(99, len(TEST_DOCUMENTS)), # higher than available docs
])
def test_rerank_tei_top_n(top_n, expected_len):
global server
server.start()
data = {
"query": "Machine learning is",
"texts": TEST_DOCUMENTS,
}
if top_n is not None:
data["top_n"] = top_n
res = server.make_request("POST", "/rerank", data=data)
assert res.status_code == 200
assert len(res.body) == expected_len
tools/server/utils.hpp
+35 -38
View File
@@ -849,47 +849,44 @@ static json format_response_rerank(
const json & request,
const json & ranks,
bool is_tei_format,
std::vector<std::string> & texts) {
json res;
if (is_tei_format) {
// TEI response format
res = json::array();
bool return_text = json_value(request, "return_text", false);
for (const auto & rank : ranks) {
int index = json_value(rank, "index", 0);
json elem = json{
{"index", index},
{"score", json_value(rank, "score", 0.0)},
};
if (return_text) {
elem["text"] = std::move(texts[index]);
}
res.push_back(elem);
}
} else {
// Jina response format
json results = json::array();
int32_t n_tokens = 0;
for (const auto & rank : ranks) {
results.push_back(json{
{"index", json_value(rank, "index", 0)},
{"relevance_score", json_value(rank, "score", 0.0)},
});
n_tokens += json_value(rank, "tokens_evaluated", 0);
}
res = json{
{"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},
{"object", "list"},
{"usage", json{
{"prompt_tokens", n_tokens},
{"total_tokens", n_tokens}
}},
{"results", results}
std::vector<std::string> & texts,
int top_n) {
int32_t n_tokens = 0;
bool return_text = is_tei_format && json_value(request, "return_text", false);
std::vector<json> elements; // Temporary vector to hold unsorted elements
std::string score_label = is_tei_format ? "score" : "relevance_score";
for (const auto & rank : ranks) {
int index = json_value(rank, "index", 0);
json elem = json{
{"index", index},
{score_label, json_value(rank, "score", 0.0)},
};
n_tokens += json_value(rank, "tokens_evaluated", 0);
if (return_text) {
elem["text"] = std::move(texts[index]);
}
elements.push_back(elem);
}
std::sort(elements.begin(), elements.end(), [score_label](const json& a, const json& b) {
return json_value(a, score_label, 0.0) > json_value(b, score_label, 0.0);
});
elements.resize(std::min(top_n, (int)elements.size()));
json results = elements;
if (is_tei_format) return results;
json res = json{
{"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},
{"object", "list"},
{"usage", json{
{"prompt_tokens", n_tokens},
{"total_tokens", n_tokens}
}},
{"results", results}
};
return res;
}
tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessageUser.svelte
+15 -4
View File
@@ -2,8 +2,9 @@
import { Check, X } from '@lucide/svelte';
import { Card } from '$lib/components/ui/card';
import { Button } from '$lib/components/ui/button';
import { ChatAttachmentsList } from '$lib/components/app';
import { ChatAttachmentsList, MarkdownContent } from '$lib/components/app';
import { INPUT_CLASSES } from '$lib/constants/input-classes';
import { config } from '$lib/stores/settings.svelte';
import ChatMessageActions from './ChatMessageActions.svelte';
interface Props {
@@ -55,6 +56,7 @@
let isMultiline = $state(false);
let messageElement: HTMLElement | undefined = $state();
const currentConfig = config();
$effect(() => {
if (!messageElement || !message.content.trim()) return;
@@ -123,9 +125,18 @@
class="max-w-[80%] rounded-[1.125rem] bg-primary px-3.75 py-1.5 text-primary-foreground data-[multiline]:py-2.5"
data-multiline={isMultiline ? '' : undefined}
>
<span bind:this={messageElement} class="text-md whitespace-pre-wrap">
{message.content}
</span>
{#if currentConfig.renderUserContentAsMarkdown}
<div bind:this={messageElement} class="text-md">
<MarkdownContent
class="markdown-user-content text-primary-foreground"
content={message.content}
/>
</div>
{:else}
<span bind:this={messageElement} class="text-md whitespace-pre-wrap">
{message.content}
</span>
{/if}
</Card>
{/if}
tools/server/webui/src/lib/components/app/chat/ChatSettings/ChatSettingsDialog.svelte
+5
View File
@@ -80,6 +80,11 @@
key: 'showModelInfo',
label: 'Show model information',
type: 'checkbox'
},
{
key: 'renderUserContentAsMarkdown',
label: 'Render user content as Markdown',
type: 'checkbox'
}
]
},
tools/server/webui/src/lib/constants/settings-config.ts
+2
View File
@@ -12,6 +12,7 @@ export const SETTING_CONFIG_DEFAULT: Record<string, string | number | boolean> =
pasteLongTextToFileLen: 2500,
pdfAsImage: false,
showModelInfo: false,
renderUserContentAsMarkdown: false,
// make sure these default values are in sync with `common.h`
samplers: 'top_k;typ_p;top_p;min_p;temperature',
temperature: 0.8,
@@ -84,6 +85,7 @@ export const SETTING_CONFIG_INFO: Record<string, string> = {
'Ask for confirmation before automatically changing conversation title when editing the first message.',
pdfAsImage: 'Parse PDF as image instead of text (requires vision-capable model).',
showModelInfo: 'Display the model name used to generate each message below the message content.',
renderUserContentAsMarkdown: 'Render user messages using markdown formatting in the chat.',
pyInterpreterEnabled:
'Enable Python interpreter using Pyodide. Allows running Python code in markdown code blocks.'
};