mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-09-18 16:55:14 +02:00
Merge commit '6effcecd0bf3cb2209999cecfa297ed4d8523b5a' into concedo_experimental
# Conflicts: # .github/workflows/build-msys.yml # .pi/gg/SYSTEM.md # common/CMakeLists.txt # examples/speculative-simple/speculative-simple.cpp # ggml/src/ggml-cpu/kleidiai/kleidiai.cpp # ggml/src/ggml-sycl/ggml-sycl.cpp # ggml/src/ggml-sycl/mmvq.cpp # scripts/ui-assets.cmake # src/llama-model.cpp # tools/cli/cli.cpp # tools/imatrix/imatrix.cpp # tools/quantize/quantize.cpp # tools/server/README.md # tools/ui/tests/stories/a11y/ChatScreenForm.a11y.stories.svelte
This commit is contained in:
Concedo
+61
-36
@@ -5,62 +5,87 @@ Quantization reduces the precision of model weights (e.g., from 32-bit floats to
|
||||
This process however, may introduce some accuracy loss which is usually measured in [Perplexity](https://huggingface.co/docs/transformers/en/perplexity) (ppl) and/or [Kullback–Leibler Divergence](https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence) (kld).
|
||||
This can be minimized by using a suitable imatrix file.
|
||||
|
||||
You can also use the [GGUF-my-repo](https://huggingface.co/spaces/ggml-org/gguf-my-repo) space on Hugging Face to build your own quants without any setup.
|
||||
You can also use the [GGUF-my-repo](https://huggingface.co/spaces/ggml-org/gguf-my-repo) space on Hugging Face to build your own quants without any setup. It syncs from llama.cpp `main` every 6 hours.
|
||||
|
||||
Note: It is synced from llama.cpp `main` every 6 hours.
|
||||
## Overview
|
||||
|
||||
Example usage:
|
||||
Quantization is done in two phases:
|
||||
- Convert the original model to GGUF format.
|
||||
- Quantize the converted GGUF file.
|
||||
|
||||
```./llama-quantize [options] input-model-f32.gguf [output-model-quant.gguf] type [threads]```
|
||||
If the model supports multimodal inputs (images or audio), you also need to convert and quantize the multimodal encoders and projectors.
|
||||
|
||||
To perform these tasks, you need to install the Python requirements:
|
||||
|
||||
```bash
|
||||
# from Hugginface, obtain the official meta-llama/Llama-3.1-8B model weights and place them in ./models
|
||||
ls ./models
|
||||
config.json model-00001-of-00004.safetensors model-00004-of-00004.safetensors README.md tokenizer.json
|
||||
generation_config.json model-00002-of-00004.safetensors model.safetensors.index.json special_tokens_map.json USE_POLICY.md
|
||||
LICENSE model-00003-of-00004.safetensors original tokenizer_config.json
|
||||
|
||||
# [Optional] for PyTorch .bin models like Mistral-7B
|
||||
ls ./models
|
||||
<folder containing weights and tokenizer json>
|
||||
|
||||
# install Python dependencies
|
||||
python3 -m pip install -r requirements.txt
|
||||
|
||||
# convert the model to ggml FP16 format
|
||||
python3 convert_hf_to_gguf.py ./models/mymodel/
|
||||
|
||||
# quantize the model to 4-bits (using Q4_K_M method)
|
||||
./llama-quantize ./models/mymodel/ggml-model-f16.gguf ./models/mymodel/ggml-model-Q4_K_M.gguf Q4_K_M
|
||||
|
||||
# update the gguf filetype to current version if older version is now unsupported
|
||||
./llama-quantize ./models/mymodel/ggml-model-Q4_K_M.gguf ./models/mymodel/ggml-model-Q4_K_M-v2.gguf COPY
|
||||
```
|
||||
|
||||
Run the quantized model:
|
||||
Or if you use `uv`:
|
||||
|
||||
```bash
|
||||
# start inference on a gguf model
|
||||
./llama-cli -m ./models/mymodel/ggml-model-Q4_K_M.gguf -cnv -p "You are a helpful assistant"
|
||||
uv pip install -r requirements.txt --index-strategy unsafe-best-match
|
||||
```
|
||||
|
||||
## Prepare the input GGUF file
|
||||
|
||||
To convert a model from a Hugging Face repo, you can use a command like the following:
|
||||
|
||||
```
|
||||
python convert_hf_to_gguf.py --outfile gemma-4-E2B-it-bf16.gguf --outtype bf16 --remote google/gemma-4-E2B-it
|
||||
```
|
||||
|
||||
Notes:
|
||||
- In the usual case where the model is distributed in 16-bit format, `--outtype auto` (or omitting `--outtype` entirely) also works well.
|
||||
- If you have previously downloaded the model locally, specify the directory and remove the `--remote` flag.
|
||||
- For compatibility reasons, the Python requirements install transformers 4, but more and more models (like Gemma 4) require transformers 5. You can safely `pip install -U transformers` to get the latest version.
|
||||
|
||||
## Quantize the GGUF
|
||||
|
||||
After you have created a high-quality GGUF version of the model, you use `llama-quantize` to apply quantization. For example, quantize to `Q4_K_M` using a command like the following:
|
||||
|
||||
```bash
|
||||
./build/bin/llama-quantize gemma-4-E2B-it-bf16.gguf gemma-4-E2B-it-Q4_K_M.gguf Q4_K_M
|
||||
```
|
||||
|
||||
Various quantization methods are described [later in this document](#quantize).
|
||||
|
||||
Options:
|
||||
* `--allow-requantize` allows requantizing tensors that have already been quantized. Warning: This can severely reduce quality compared to quantizing from 16bit or 32bit
|
||||
* `--leave-output-tensor` will leave output.weight un(re)quantized. Increases model size but may also increase quality, especially when requantizing
|
||||
* `--pure` disables k-quant mixtures and quantizes all tensors to the same type
|
||||
* `--imatrix` uses data in file generated by `llama-imatrix` as importance matrix for quant optimizations (highly recommended)
|
||||
* `--include-weights` use an importance matrix for tensor(s) in the list. Cannot be used with `--exclude-weights`
|
||||
* `--exclude-weights` use an importance matrix for tensor(s) in the list. Cannot be used with `--include-weights`
|
||||
* `--allow-requantize` allow requantizing tensors that have already been quantized. Warning: This can severely reduce quality compared to quantizing from 16bit or 32bit
|
||||
* `--leave-output-tensor` leave output.weight un(re)quantized. Increases model size but may also increase quality, especially when requantizing
|
||||
* `--pure` disable k-quant mixtures and quantizes all tensors to the same type
|
||||
* `--imatrix file_name` use data in file_name as importance matrix for quant optimizations
|
||||
* `--include-weights tensor_name` use importance matrix for this tensor (can be specified multiple times)
|
||||
* `--exclude-weights tensor_name` use importance matrix for the tensors **not** specified (include/exclude cannot be mixed)
|
||||
* `--output-tensor-type` use a specific quant type for the output.weight tensor
|
||||
* `--token-embedding-type` use a specific quant type for the token embeddings tensor
|
||||
* `--keep-split` will generate the quantized model in the same shards as the input file otherwise it will produce a single quantized file
|
||||
* `--keep-split` generate the quantized model in the same shards as the input file instead of a single quantized file
|
||||
|
||||
Advanced options:
|
||||
* `--tensor-type` quantize specific tensor(s) to specific quant types. Supports regex syntax. May be specified multiple times.
|
||||
* `--prune-layers` prune (remove) the layers in the list
|
||||
* `--override-kv` option to override model metadata by key in the quantized model. May be specified multiple times
|
||||
* `--override-kv` option to override model metadata by key in the quantized model. May be specified multiple times.
|
||||
|
||||
Examples:
|
||||
## (Optional) Convert the multimodal components
|
||||
|
||||
llama.cpp will convert the LLM portion of the source model, which is enough for conversational applications. If the model accepts multimodal inputs and you wish to take advantage of them, you need to create a separate GGUF file. This file is generically known as `mmproj`, for "multimedia projector"; however, it may contain various components such as vision or audio encoders in addition to projections.
|
||||
|
||||
Multimodal components are usually much smaller than the LLMs they come with. In addition, their quality has a direct impact on the quality of LLM generations, because these components are in charge of preparing the inputs for the LLM: the closer inputs are to data seen during training, the better LLM results will be.
|
||||
|
||||
For these reasons, multimodal components are usually kept in a high-quality format such as bf16 or q8. The impact on speed and memory from using a smaller quant is negligible, but overall quality could be impacted.
|
||||
|
||||
```bash
|
||||
python convert_hf_to_gguf.py --mmproj --outfile mmproj-gemma-4-E2B-it-Q8_0.gguf --outtype q8_0 --remote google/gemma-4-E2B-it
|
||||
```
|
||||
|
||||
## Run the quantized model
|
||||
|
||||
|
||||
```bash
|
||||
./build/bin/llama cli -m ./gemma-4-E2B-it-Q4_K_M.gguf --mmproj ./mmproj-gemma-4-E2B-it-Q8_0.gguf --image <input_image> --prompt "Describe this image"
|
||||
```
|
||||
|
||||
## Quantization Examples
|
||||
|
||||
```bash
|
||||
# naive Q4_K_M quantization using default settings and 8 CPU threads. Output will be "ggml-model-Q4_K_M.gguf"
|
||||
|
||||
+60
-165
@@ -1,6 +1,7 @@
|
||||
#include "llama.h"
|
||||
#include "build-info.h"
|
||||
#include "common/common.h"
|
||||
#include "common/imatrix-loader.h"
|
||||
|
||||
#include "gguf.h"
|
||||
|
||||
@@ -13,7 +14,6 @@
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <map>
|
||||
#include <fstream>
|
||||
#include <filesystem>
|
||||
|
||||
@@ -77,11 +77,6 @@ static const char * const LLM_KV_QUANTIZE_IMATRIX_DATASET = "quantize.imatrix
|
||||
static const char * const LLM_KV_QUANTIZE_IMATRIX_N_ENTRIES = "quantize.imatrix.entries_count";
|
||||
static const char * const LLM_KV_QUANTIZE_IMATRIX_N_CHUNKS = "quantize.imatrix.chunks_count";
|
||||
|
||||
// TODO: share with imatrix.cpp
|
||||
static const char * const LLM_KV_IMATRIX_DATASETS = "imatrix.datasets";
|
||||
static const char * const LLM_KV_IMATRIX_CHUNK_COUNT = "imatrix.chunk_count";
|
||||
static const char * const LLM_KV_IMATRIX_CHUNK_SIZE = "imatrix.chunk_size";
|
||||
|
||||
static bool striequals(const char * a, const char * b) {
|
||||
while (*a && *b) {
|
||||
if (std::tolower(*a) != std::tolower(*b)) {
|
||||
@@ -180,184 +175,84 @@ static void usage(const char * executable) {
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static int load_legacy_imatrix(const std::string & imatrix_file, std::vector<std::string> & imatrix_datasets, std::unordered_map<std::string, std::vector<float>> & imatrix_data) {
|
||||
std::ifstream in(imatrix_file.c_str(), std::ios::binary);
|
||||
if (!in) {
|
||||
printf("%s: failed to open %s\n",__func__, imatrix_file.c_str());
|
||||
exit(1);
|
||||
}
|
||||
int n_entries;
|
||||
in.read((char *)&n_entries, sizeof(n_entries));
|
||||
if (in.fail() || n_entries < 1) {
|
||||
printf("%s: no data in file %s\n", __func__, imatrix_file.c_str());
|
||||
exit(1);
|
||||
}
|
||||
for (int i = 0; i < n_entries; ++i) {
|
||||
int len; in.read((char *)&len, sizeof(len));
|
||||
std::vector<char> name_as_vec(len+1);
|
||||
in.read((char *)name_as_vec.data(), len);
|
||||
if (in.fail()) {
|
||||
printf("%s: failed reading name for entry %d from %s\n", __func__, i+1, imatrix_file.c_str());
|
||||
exit(1);
|
||||
}
|
||||
name_as_vec[len] = 0;
|
||||
std::string name{name_as_vec.data()};
|
||||
auto & e = imatrix_data[name];
|
||||
int ncall;
|
||||
in.read((char *)&ncall, sizeof(ncall));
|
||||
int nval;
|
||||
in.read((char *)&nval, sizeof(nval));
|
||||
if (in.fail() || nval < 1) {
|
||||
printf("%s: failed reading number of values for entry %d\n", __func__, i);
|
||||
imatrix_data = {};
|
||||
exit(1);
|
||||
}
|
||||
e.resize(nval);
|
||||
in.read((char *)e.data(), nval*sizeof(float));
|
||||
if (in.fail()) {
|
||||
printf("%s: failed reading data for entry %d\n", __func__, i);
|
||||
imatrix_data = {};
|
||||
exit(1);
|
||||
}
|
||||
if (ncall > 0) {
|
||||
for (auto & v : e) {
|
||||
v /= ncall;
|
||||
}
|
||||
}
|
||||
|
||||
if (getenv("LLAMA_TRACE")) {
|
||||
printf("%s: loaded data (size = %6d, ncall = %6d) for '%s'\n", __func__, int(e.size()), ncall, name.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
// latest legacy imatrix version contains the dataset filename at the end of the file
|
||||
int m_last_call = 0;
|
||||
if (in.peek() != EOF) {
|
||||
in.read((char *)&m_last_call, sizeof(m_last_call));
|
||||
int dataset_len;
|
||||
in.read((char *)&dataset_len, sizeof(dataset_len));
|
||||
std::vector<char> dataset_as_vec(dataset_len);
|
||||
in.read(dataset_as_vec.data(), dataset_len);
|
||||
imatrix_datasets.resize(1);
|
||||
imatrix_datasets[0].assign(dataset_as_vec.begin(), dataset_as_vec.end());
|
||||
printf("%s: imatrix dataset='%s'\n", __func__, imatrix_datasets[0].c_str());
|
||||
}
|
||||
printf("%s: loaded %d importance matrix entries from %s computed on %d chunks\n", __func__, int(imatrix_data.size()), imatrix_file.c_str(), m_last_call);
|
||||
return m_last_call;
|
||||
}
|
||||
|
||||
static int load_imatrix(const std::string & imatrix_file, std::vector<std::string> & imatrix_datasets, std::unordered_map<std::string, std::vector<float>> & imatrix_data) {
|
||||
|
||||
struct ggml_context * ctx = nullptr;
|
||||
struct gguf_init_params meta_gguf_params = {
|
||||
/* .no_alloc = */ false, // the data is needed
|
||||
/* .ctx = */ &ctx,
|
||||
};
|
||||
struct gguf_context * ctx_gguf = gguf_init_from_file(imatrix_file.c_str(), meta_gguf_params);
|
||||
if (!ctx_gguf) {
|
||||
fprintf(stderr, "%s: imatrix file '%s' is using old format\n", __func__, imatrix_file.c_str());
|
||||
return load_legacy_imatrix(imatrix_file, imatrix_datasets, imatrix_data);
|
||||
}
|
||||
const int32_t n_entries = gguf_get_n_tensors(ctx_gguf);
|
||||
if (n_entries < 1) {
|
||||
fprintf(stderr, "%s: no data in file %s\n", __func__, imatrix_file.c_str());
|
||||
gguf_free(ctx_gguf);
|
||||
ggml_free(ctx);
|
||||
common_imatrix loaded;
|
||||
if (!common_imatrix_load(imatrix_file, loaded)) {
|
||||
fprintf(stderr, "%s: failed to load imatrix from '%s'\n", __func__, imatrix_file.c_str());
|
||||
exit(1);
|
||||
}
|
||||
|
||||
const int dataset_idx = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_DATASETS);
|
||||
const int chunk_count_idx = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_COUNT);
|
||||
const int chunk_size_idx = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_SIZE);
|
||||
if (dataset_idx < 0 || chunk_count_idx < 0 || chunk_size_idx < 0) {
|
||||
if (!loaded.is_legacy && !loaded.has_metadata) {
|
||||
fprintf(stderr, "%s: missing imatrix metadata in file %s\n", __func__, imatrix_file.c_str());
|
||||
gguf_free(ctx_gguf);
|
||||
ggml_free(ctx);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
const uint32_t chunk_size = gguf_get_val_u32(ctx_gguf, chunk_size_idx);
|
||||
|
||||
const std::string sums_suffix{ ".in_sum2" };
|
||||
const std::string counts_suffix{ ".counts" };
|
||||
|
||||
// Using an ordered map to get a deterministic iteration order.
|
||||
std::map<std::string, std::pair<struct ggml_tensor *, struct ggml_tensor *>> sums_counts_for;
|
||||
|
||||
for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
|
||||
std::string name = cur->name;
|
||||
|
||||
if (name.empty()) { continue; }
|
||||
|
||||
if (string_remove_suffix(name, sums_suffix)) {
|
||||
// in_sum2
|
||||
sums_counts_for[std::move(name)].first = cur;
|
||||
} else if (string_remove_suffix(name, counts_suffix)) {
|
||||
// counts
|
||||
sums_counts_for[std::move(name)].second = cur;
|
||||
} else {
|
||||
// ignore other tensors
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto & sc : sums_counts_for) {
|
||||
const std::string & name = sc.first;
|
||||
const struct ggml_tensor * sums = sc.second.first;
|
||||
const struct ggml_tensor * counts = sc.second.second;
|
||||
|
||||
if (!sums || !counts) {
|
||||
fprintf(stderr, "%s: mismatched sums and counts for %s\n", __func__, name.c_str());
|
||||
gguf_free(ctx_gguf);
|
||||
ggml_free(ctx);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
const int64_t ne0 = sums->ne[0];
|
||||
const int64_t ne1 = sums->ne[1];
|
||||
|
||||
for (const auto & [name, entry] : loaded.entries) {
|
||||
auto & e = imatrix_data[name];
|
||||
e.resize(ggml_nelements(sums));
|
||||
float max_count = 0.0f;
|
||||
for (int64_t j = 0; j < ne1; ++j) {
|
||||
const float count = ((const float *) counts->data)[j];
|
||||
if (count > 0.0f) {
|
||||
for (int64_t i = 0; i < ne0; ++i) {
|
||||
e[j*ne0 + i] = ((const float *) sums->data)[j*ne0 + i] / count;
|
||||
e.resize(entry.sums.size());
|
||||
|
||||
if (!loaded.is_legacy) {
|
||||
// GGUF format: normalize by per-expert counts
|
||||
const int64_t ncounts = entry.counts.size();
|
||||
const int64_t ne0 = (int64_t) entry.sums.size() / ncounts;
|
||||
|
||||
for (int64_t j = 0; j < ncounts; ++j) {
|
||||
const float count = (float) entry.counts[j];
|
||||
if (count > 0.0f) {
|
||||
for (int64_t i = 0; i < ne0; ++i) {
|
||||
e[j*ne0 + i] = entry.sums[j*ne0 + i] / count;
|
||||
}
|
||||
} else {
|
||||
for (int64_t i = 0; i < ne0; ++i) {
|
||||
e[j*ne0 + i] = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (getenv("LLAMA_TRACE")) {
|
||||
float max_count = 0.0f;
|
||||
for (int64_t j = 0; j < ncounts; ++j) {
|
||||
const float count = (float) entry.counts[j];
|
||||
if (count > max_count) {
|
||||
max_count = count;
|
||||
}
|
||||
}
|
||||
printf("%s: loaded data (size = %6d, n_tokens = %6d, n_chunks = %6d) for '%s'\n",
|
||||
__func__, int(e.size()), int(max_count), int(max_count / loaded.chunk_size), name.c_str());
|
||||
}
|
||||
} else {
|
||||
// Legacy format: sums contain (raw/count)*ncall, divide by ncall
|
||||
const int64_t ncall = entry.counts.empty() ? 0 : entry.counts[0];
|
||||
if (ncall > 0) {
|
||||
for (size_t i = 0; i < entry.sums.size(); ++i) {
|
||||
e[i] = entry.sums[i] / ncall;
|
||||
}
|
||||
} else {
|
||||
// Partial imatrix data, this tensor never got any input during calibration
|
||||
for (int64_t i = 0; i < ne0; ++i) {
|
||||
e[j*ne0 + i] = 1;
|
||||
for (size_t i = 0; i < entry.sums.size(); ++i) {
|
||||
e[i] = entry.sums[i];
|
||||
}
|
||||
}
|
||||
if (count > max_count) {
|
||||
max_count = count;
|
||||
|
||||
if (getenv("LLAMA_TRACE")) {
|
||||
printf("%s: loaded data (size = %6d, ncall = %6d) for '%s'\n",
|
||||
__func__, int(e.size()), int(ncall), name.c_str());
|
||||
}
|
||||
}
|
||||
if (getenv("LLAMA_TRACE")) {
|
||||
printf("%s: loaded data (size = %6d, n_tokens = %6d, n_chunks = %6d) for '%s'\n", __func__, int(e.size()), int(max_count), int(max_count / chunk_size), name.c_str());
|
||||
}
|
||||
|
||||
imatrix_datasets = std::move(loaded.datasets);
|
||||
|
||||
if (!imatrix_datasets.empty()) {
|
||||
printf("%s: imatrix datasets=['%s'", __func__, imatrix_datasets[0].c_str());
|
||||
for (size_t i = 1; i < imatrix_datasets.size(); ++i) {
|
||||
printf(", '%s'", imatrix_datasets[i].c_str());
|
||||
}
|
||||
printf("]\n");
|
||||
}
|
||||
|
||||
int m_last_chunk = gguf_get_val_u32(ctx_gguf, chunk_count_idx);
|
||||
printf("%s: loaded %d importance matrix entries from %s computed on %d chunks\n", __func__, int(imatrix_data.size()), imatrix_file.c_str(), loaded.chunk_count);
|
||||
|
||||
int64_t n_datasets = gguf_get_arr_n(ctx_gguf, dataset_idx);
|
||||
imatrix_datasets.reserve(n_datasets);
|
||||
for (int64_t i = 0; i < n_datasets; ++i) {
|
||||
imatrix_datasets.push_back(gguf_get_arr_str(ctx_gguf, dataset_idx, i));
|
||||
}
|
||||
printf("%s: imatrix datasets=['%s'", __func__, imatrix_datasets[0].c_str());
|
||||
for (size_t i = 1; i < imatrix_datasets.size(); ++i) {
|
||||
printf(", '%s'", imatrix_datasets[i].c_str());
|
||||
}
|
||||
printf("]\n");
|
||||
|
||||
printf("%s: loaded %d importance matrix entries from %s computed on %d chunks\n", __func__, int(imatrix_data.size()), imatrix_file.c_str(), m_last_chunk);
|
||||
|
||||
gguf_free(ctx_gguf);
|
||||
ggml_free(ctx);
|
||||
|
||||
return m_last_chunk;
|
||||
return loaded.chunk_count;
|
||||
}
|
||||
|
||||
static int prepare_imatrix(const std::string & imatrix_file,
|
||||
|
||||
@@ -2512,7 +2512,7 @@ private:
|
||||
llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), slot.id));
|
||||
|
||||
if (use_ckpt_dft) {
|
||||
slot.spec_ckpt.update_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
|
||||
slot.spec_ckpt.update_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
|
||||
}
|
||||
|
||||
slot.spec_prompt = slot.prompt.tokens.get_text_tokens();
|
||||
@@ -2551,7 +2551,7 @@ private:
|
||||
|
||||
if (ctx_dft) {
|
||||
if (use_ckpt_dft) {
|
||||
ckpt.load_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
|
||||
ckpt.load_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
|
||||
}
|
||||
|
||||
common_context_seq_rm(ctx_dft.get(), slot.id, ckpt.pos_max + 1, -1);
|
||||
@@ -2568,7 +2568,7 @@ private:
|
||||
if (use_ckpt_tgt) {
|
||||
//const int64_t t_start = ggml_time_us();
|
||||
|
||||
ckpt.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
|
||||
ckpt.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
|
||||
|
||||
//const int64_t t_total = ggml_time_us() - t_start;
|
||||
//printf("checkpoint total: %f ms\n", t_total / 1000.0);
|
||||
@@ -2580,7 +2580,7 @@ private:
|
||||
}
|
||||
|
||||
if (use_ckpt_dft) {
|
||||
ckpt.update_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
|
||||
ckpt.update_dft(ctx_dft.get(), slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3447,13 +3447,13 @@ private:
|
||||
SLT_DBG(slot, "restoring speculative checkpoint (pos_min = %d, pos_max = %d, size = %zu)\n", ckpt.pos_min, ckpt.pos_max, ckpt.size());
|
||||
|
||||
{
|
||||
ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
|
||||
ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
|
||||
|
||||
common_context_seq_rm(slot.ctx_tgt, slot.id, ckpt.pos_max + 1, -1);
|
||||
}
|
||||
|
||||
if (slot.ctx_dft) {
|
||||
ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
|
||||
ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
|
||||
|
||||
common_context_seq_rm(slot.ctx_dft, slot.id, ckpt.pos_max + 1, -1);
|
||||
}
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
engine-strict=true
|
||||
ignore-scripts=true
|
||||
|
||||
@@ -35,23 +35,27 @@
|
||||
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<Button
|
||||
{variant}
|
||||
{size}
|
||||
{disabled}
|
||||
onclick={(e: MouseEvent) => {
|
||||
if (stopPropagationOnClick) e.stopPropagation();
|
||||
<!-- prevent another nested button element -->
|
||||
{#snippet child({ props })}
|
||||
<Button
|
||||
{...props}
|
||||
{variant}
|
||||
{size}
|
||||
{disabled}
|
||||
onclick={(e: MouseEvent) => {
|
||||
if (stopPropagationOnClick) e.stopPropagation();
|
||||
|
||||
onclick?.(e);
|
||||
}}
|
||||
class="h-6 w-6 p-0 {className} flex hover:bg-transparent data-[state=open]:bg-transparent!"
|
||||
aria-label={ariaLabel || tooltip}
|
||||
>
|
||||
{#if icon}
|
||||
{@const IconComponent = icon}
|
||||
<IconComponent class={iconSize} />
|
||||
{/if}
|
||||
</Button>
|
||||
onclick?.(e);
|
||||
}}
|
||||
class="h-6 w-6 p-0 {className} flex hover:bg-transparent data-[state=open]:bg-transparent!"
|
||||
aria-label={ariaLabel || tooltip}
|
||||
>
|
||||
{#if icon}
|
||||
{@const IconComponent = icon}
|
||||
<IconComponent class={iconSize} />
|
||||
{/if}
|
||||
</Button>
|
||||
{/snippet}
|
||||
</Tooltip.Trigger>
|
||||
|
||||
<Tooltip.Content side={tooltipSide}>
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
<script lang="ts">
|
||||
import type { Snippet } from 'svelte';
|
||||
import type { HTMLButtonAttributes } from 'svelte/elements';
|
||||
|
||||
interface Props {
|
||||
interface Props extends HTMLButtonAttributes {
|
||||
children: Snippet;
|
||||
class?: string;
|
||||
icon?: Snippet;
|
||||
onclick?: () => void;
|
||||
}
|
||||
|
||||
let { children, class: className = '', icon, onclick }: Props = $props();
|
||||
let { children, class: className = '', icon, ...rest }: Props = $props();
|
||||
</script>
|
||||
|
||||
<button
|
||||
{...rest}
|
||||
class={[
|
||||
'inline-flex cursor-pointer items-center gap-1 rounded-sm bg-muted-foreground/15 px-1.5 py-0.75',
|
||||
className
|
||||
]}
|
||||
{onclick}
|
||||
>
|
||||
{#if icon}
|
||||
{@render icon()}
|
||||
|
||||
+3
-1
@@ -97,7 +97,9 @@
|
||||
{/snippet}
|
||||
|
||||
{#snippet removeButton()}
|
||||
<div class="absolute top-2 right-2 opacity-0 transition-opacity group-hover:opacity-100">
|
||||
<div
|
||||
class="absolute top-2 right-2 opacity-0 transition-opacity group-focus-within:opacity-100 group-hover:opacity-100"
|
||||
>
|
||||
<ActionIcon icon={X} tooltip="Remove" stopPropagationOnClick onclick={() => onRemove?.(id)} />
|
||||
</div>
|
||||
{/snippet}
|
||||
|
||||
+1
-1
@@ -51,7 +51,7 @@
|
||||
|
||||
{#if !readonly}
|
||||
<div
|
||||
class="absolute top-1 right-1 flex items-center justify-center opacity-0 transition-opacity group-hover:opacity-100"
|
||||
class="absolute top-1 right-1 flex items-center justify-center opacity-0 transition-opacity group-focus-within:opacity-100 group-hover:opacity-100"
|
||||
>
|
||||
<ActionIcon
|
||||
class="text-white"
|
||||
|
||||
+65
-86
@@ -6,6 +6,7 @@
|
||||
import type { ChatMessageAgenticTimings } from '$lib/types/chat';
|
||||
import { formatPerformanceTime } from '$lib/utils';
|
||||
import { MS_PER_SECOND, DEFAULT_PERFORMANCE_TIME } from '$lib/constants';
|
||||
import type { Component } from 'svelte';
|
||||
|
||||
interface Props {
|
||||
predictedTokens?: number;
|
||||
@@ -114,101 +115,79 @@
|
||||
let formattedAgenticTotalTime = $derived(formatPerformanceTime(agenticTotalTimeMs));
|
||||
</script>
|
||||
|
||||
{#snippet viewButton(opts: {
|
||||
view: ChatMessageStatsView;
|
||||
icon: Component;
|
||||
label: string;
|
||||
tooltipText: string;
|
||||
disabled?: boolean;
|
||||
})}
|
||||
{@const IconComponent = opts.icon}
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<!-- prevent another nested button element -->
|
||||
{#snippet child({ props })}
|
||||
<button
|
||||
{...props}
|
||||
type="button"
|
||||
class="inline-flex h-5 w-5 items-center justify-center rounded-sm transition-colors {activeView ===
|
||||
opts.view
|
||||
? 'bg-background text-foreground shadow-sm'
|
||||
: opts.disabled
|
||||
? 'cursor-not-allowed opacity-40'
|
||||
: 'hover:text-foreground'}"
|
||||
onclick={() => !opts.disabled && (activeView = opts.view)}
|
||||
disabled={opts.disabled}
|
||||
>
|
||||
<IconComponent class="h-3 w-3" />
|
||||
|
||||
<span class="sr-only">{opts.label}</span>
|
||||
</button>
|
||||
{/snippet}
|
||||
</Tooltip.Trigger>
|
||||
|
||||
<Tooltip.Content>
|
||||
<p>{opts.tooltipText}</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{/snippet}
|
||||
|
||||
<div class="inline-flex items-center text-xs text-muted-foreground">
|
||||
<div class="inline-flex items-center rounded-sm bg-muted-foreground/15 p-0.5">
|
||||
{#if hasPromptStats || isLive}
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<button
|
||||
type="button"
|
||||
class="inline-flex h-5 w-5 items-center justify-center rounded-sm transition-colors {activeView ===
|
||||
ChatMessageStatsView.READING
|
||||
? 'bg-background text-foreground shadow-sm'
|
||||
: 'hover:text-foreground'}"
|
||||
onclick={() => (activeView = ChatMessageStatsView.READING)}
|
||||
>
|
||||
<BookOpenText class="h-3 w-3" />
|
||||
|
||||
<span class="sr-only">Reading</span>
|
||||
</button>
|
||||
</Tooltip.Trigger>
|
||||
|
||||
<Tooltip.Content>
|
||||
<p>Reading (prompt processing)</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{@render viewButton({
|
||||
view: ChatMessageStatsView.READING,
|
||||
icon: BookOpenText,
|
||||
label: 'Reading',
|
||||
tooltipText: 'Reading (prompt processing)'
|
||||
})}
|
||||
{/if}
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<button
|
||||
type="button"
|
||||
class="inline-flex h-5 w-5 items-center justify-center rounded-sm transition-colors {activeView ===
|
||||
ChatMessageStatsView.GENERATION
|
||||
? 'bg-background text-foreground shadow-sm'
|
||||
: isGenerationDisabled
|
||||
? 'cursor-not-allowed opacity-40'
|
||||
: 'hover:text-foreground'}"
|
||||
onclick={() => !isGenerationDisabled && (activeView = ChatMessageStatsView.GENERATION)}
|
||||
disabled={isGenerationDisabled}
|
||||
>
|
||||
<Sparkles class="h-3 w-3" />
|
||||
|
||||
<span class="sr-only">Generation</span>
|
||||
</button>
|
||||
</Tooltip.Trigger>
|
||||
|
||||
<Tooltip.Content>
|
||||
<p>
|
||||
{isGenerationDisabled
|
||||
? 'Generation (waiting for tokens...)'
|
||||
: 'Generation (token output)'}
|
||||
</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{@render viewButton({
|
||||
view: ChatMessageStatsView.GENERATION,
|
||||
icon: Sparkles,
|
||||
label: 'Generation',
|
||||
tooltipText: isGenerationDisabled
|
||||
? 'Generation (waiting for tokens...)'
|
||||
: 'Generation (token output)',
|
||||
disabled: isGenerationDisabled
|
||||
})}
|
||||
|
||||
{#if hasAgenticStats}
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<button
|
||||
type="button"
|
||||
class="inline-flex h-5 w-5 items-center justify-center rounded-sm transition-colors {activeView ===
|
||||
ChatMessageStatsView.TOOLS
|
||||
? 'bg-background text-foreground shadow-sm'
|
||||
: 'hover:text-foreground'}"
|
||||
onclick={() => (activeView = ChatMessageStatsView.TOOLS)}
|
||||
>
|
||||
<Wrench class="h-3 w-3" />
|
||||
|
||||
<span class="sr-only">Tools</span>
|
||||
</button>
|
||||
</Tooltip.Trigger>
|
||||
|
||||
<Tooltip.Content>
|
||||
<p>Tool calls</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{@render viewButton({
|
||||
view: ChatMessageStatsView.TOOLS,
|
||||
icon: Wrench,
|
||||
label: 'Tools',
|
||||
tooltipText: 'Tool calls'
|
||||
})}
|
||||
|
||||
{#if !hideSummary}
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<button
|
||||
type="button"
|
||||
class="inline-flex h-5 w-5 items-center justify-center rounded-sm transition-colors {activeView ===
|
||||
ChatMessageStatsView.SUMMARY
|
||||
? 'bg-background text-foreground shadow-sm'
|
||||
: 'hover:text-foreground'}"
|
||||
onclick={() => (activeView = ChatMessageStatsView.SUMMARY)}
|
||||
>
|
||||
<Layers class="h-3 w-3" />
|
||||
|
||||
<span class="sr-only">Summary</span>
|
||||
</button>
|
||||
</Tooltip.Trigger>
|
||||
|
||||
<Tooltip.Content>
|
||||
<p>Agentic summary</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{@render viewButton({
|
||||
view: ChatMessageStatsView.SUMMARY,
|
||||
icon: Layers,
|
||||
label: 'Summary',
|
||||
tooltipText: 'Agentic summary'
|
||||
})}
|
||||
{/if}
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
+9
-6
@@ -21,13 +21,16 @@
|
||||
{#if tooltipLabel}
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<BadgeInfo class={className} onclick={handleClick}>
|
||||
{#snippet icon()}
|
||||
<IconComponent class="h-3 w-3" />
|
||||
{/snippet}
|
||||
<!-- prevent another nested button element -->
|
||||
{#snippet child({ props })}
|
||||
<BadgeInfo {...props} class={className} onclick={handleClick}>
|
||||
{#snippet icon()}
|
||||
<IconComponent class="h-3 w-3" />
|
||||
{/snippet}
|
||||
|
||||
{value}
|
||||
</BadgeInfo>
|
||||
{value}
|
||||
</BadgeInfo>
|
||||
{/snippet}
|
||||
</Tooltip.Trigger>
|
||||
<Tooltip.Content>
|
||||
<p>{tooltipLabel}</p>
|
||||
|
||||
@@ -41,16 +41,13 @@
|
||||
});
|
||||
</script>
|
||||
|
||||
<div
|
||||
class="pointer-events-{show
|
||||
? 'auto'
|
||||
: 'none'} relative z-50 mx-auto mb-4 flex max-w-[48rem] justify-center"
|
||||
>
|
||||
<div class="relative z-50 mx-auto mb-4 flex max-w-[48rem] justify-center">
|
||||
<Button
|
||||
onclick={scrollToBottom}
|
||||
variant="secondary"
|
||||
size="icon"
|
||||
class="pointer-events-all absolute h-10 w-10 rounded-full bg-background/80 shadow-lg backdrop-blur-sm transition-all duration-200 hover:bg-muted/80"
|
||||
disabled={!show}
|
||||
class="pointer-events-auto absolute h-10 w-10 rounded-full bg-background/80 shadow-lg backdrop-blur-sm transition-all duration-200 hover:bg-muted/80"
|
||||
style="bottom: {buttonBottom}; transform: translateY({show ? '0' : '2rem'}); opacity: {show
|
||||
? 1
|
||||
: 0};"
|
||||
|
||||
@@ -55,20 +55,20 @@
|
||||
}
|
||||
|
||||
$effect(() => {
|
||||
if (scrollContainer) {
|
||||
setTimeout(() => {
|
||||
updateScrollButtons();
|
||||
}, 0);
|
||||
}
|
||||
if (!scrollContainer) return;
|
||||
|
||||
const observer = new ResizeObserver(() => updateScrollButtons());
|
||||
observer.observe(scrollContainer);
|
||||
|
||||
return () => observer.disconnect();
|
||||
});
|
||||
</script>
|
||||
|
||||
<div class="relative {className}">
|
||||
<button
|
||||
class="absolute top-1/2 left-4 z-10 flex h-6 w-6 -translate-y-1/2 items-center justify-center rounded-full bg-background/25 shadow-md backdrop-blur-xs transition-opacity hover:bg-background/45 {canScrollLeft
|
||||
? 'opacity-100'
|
||||
: 'pointer-events-none opacity-0'}"
|
||||
class="absolute top-1/2 left-4 z-10 flex h-6 w-6 -translate-y-1/2 items-center justify-center rounded-full bg-background/25 shadow-md backdrop-blur-xs transition-opacity hover:bg-background/45 disabled:pointer-events-none disabled:opacity-0"
|
||||
onclick={scrollLeft}
|
||||
disabled={!canScrollLeft}
|
||||
aria-label="Scroll left"
|
||||
>
|
||||
<ChevronLeft class="h-4 w-4" />
|
||||
@@ -83,10 +83,9 @@
|
||||
</div>
|
||||
|
||||
<button
|
||||
class="absolute top-1/2 right-4 z-10 flex h-6 w-6 -translate-y-1/2 items-center justify-center rounded-full bg-background/25 shadow-md backdrop-blur-xs transition-opacity hover:bg-background/45 {canScrollRight
|
||||
? 'opacity-100'
|
||||
: 'pointer-events-none opacity-0'}"
|
||||
class="absolute top-1/2 right-4 z-10 flex h-6 w-6 -translate-y-1/2 items-center justify-center rounded-full bg-background/25 shadow-md backdrop-blur-xs transition-opacity hover:bg-background/45 disabled:pointer-events-none disabled:opacity-0"
|
||||
onclick={scrollRight}
|
||||
disabled={!canScrollRight}
|
||||
aria-label="Scroll right"
|
||||
>
|
||||
<ChevronRight class="h-4 w-4" />
|
||||
|
||||
@@ -27,8 +27,8 @@
|
||||
let shouldShow = $derived(model && (modelProp !== undefined || isModelMode));
|
||||
</script>
|
||||
|
||||
{#snippet badgeContent()}
|
||||
<BadgeInfo class={className} {onclick}>
|
||||
{#snippet badgeContent(triggerProps?: Record<string, unknown>)}
|
||||
<BadgeInfo {...triggerProps ?? {}} class={className} {onclick}>
|
||||
{#snippet icon()}
|
||||
<Package class="h-3 w-3" />
|
||||
{/snippet}
|
||||
@@ -47,7 +47,10 @@
|
||||
{#if showTooltip}
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
{@render badgeContent()}
|
||||
<!-- prevent another nested button element -->
|
||||
{#snippet child({ props })}
|
||||
{@render badgeContent(props)}
|
||||
{/snippet}
|
||||
</Tooltip.Trigger>
|
||||
|
||||
<Tooltip.Content>
|
||||
|
||||
@@ -116,52 +116,54 @@
|
||||
|
||||
{#if ms.isRouter}
|
||||
<DropdownMenu.Root bind:open={isOpen} onOpenChange={ms.handleOpenChange}>
|
||||
<DropdownMenu.Trigger
|
||||
class={[
|
||||
`inline-flex cursor-pointer items-center gap-1.5 rounded-sm bg-background px-1.5 py-1 text-xs shadow-sm transition hover:bg-muted-foreground/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-60 dark:bg-muted-foreground/15 dark:text-secondary-foreground`,
|
||||
!ms.isCurrentModelInCache
|
||||
? 'bg-red-400/10 !text-red-400 hover:bg-red-400/20 hover:text-red-400'
|
||||
: forceForegroundText
|
||||
? 'text-foreground'
|
||||
: ms.isHighlightedCurrentModelActive
|
||||
? 'text-foreground'
|
||||
: 'text-foreground',
|
||||
isOpen && 'text-foreground',
|
||||
'max-w-[min(calc(100vw-4rem) md:max-w-[min(calc(100cqw-9rem),25rem)]'
|
||||
]}
|
||||
disabled={disabled || ms.updating}
|
||||
>
|
||||
<Package class="h-3.5 w-3.5 shrink-0" />
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<!-- prevent another nested button element -->
|
||||
{#snippet child({ props })}
|
||||
<DropdownMenu.Trigger
|
||||
{...props}
|
||||
class={[
|
||||
`inline-grid cursor-pointer grid-cols-[1fr_auto_1fr] items-center gap-1.5 rounded-sm bg-background px-1.5 py-1 text-xs shadow-sm transition hover:bg-muted-foreground/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-60 dark:bg-muted-foreground/15 dark:text-secondary-foreground`,
|
||||
!ms.isCurrentModelInCache
|
||||
? 'bg-red-400/10 !text-red-400 hover:bg-red-400/20 hover:text-red-400'
|
||||
: forceForegroundText
|
||||
? 'text-foreground'
|
||||
: ms.isHighlightedCurrentModelActive
|
||||
? 'text-foreground'
|
||||
: 'text-foreground',
|
||||
isOpen && 'text-foreground',
|
||||
'max-w-[min(calc(100vw-4rem) md:max-w-[min(calc(100cqw-9rem),25rem)]'
|
||||
]}
|
||||
disabled={disabled || ms.updating}
|
||||
>
|
||||
<Package class="h-3.5 w-3.5 shrink-0" />
|
||||
|
||||
{#if selectedOption}
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<!-- prevent another nested button element -->
|
||||
{#snippet child({ props })}
|
||||
{#if selectedOption}
|
||||
<ModelId
|
||||
modelId={selectedOption.model}
|
||||
class="min-w-0 overflow-hidden"
|
||||
hideOrgName={false}
|
||||
hideQuantization
|
||||
{...props}
|
||||
/>
|
||||
{/snippet}
|
||||
</Tooltip.Trigger>
|
||||
{:else}
|
||||
<span class="min-w-0 font-medium">Select model</span>
|
||||
{/if}
|
||||
|
||||
<Tooltip.Content>
|
||||
<p class="font-mono">{selectedOption.model}</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{:else}
|
||||
<span class="min-w-0 font-medium">Select model</span>
|
||||
{/if}
|
||||
{#if ms.updating || ms.isLoadingModel}
|
||||
<Loader2 class="h-3 w-3.5 shrink-0 animate-spin" />
|
||||
{:else}
|
||||
<ChevronDown class="h-3 w-3.5 shrink-0" />
|
||||
{/if}
|
||||
</DropdownMenu.Trigger>
|
||||
{/snippet}
|
||||
</Tooltip.Trigger>
|
||||
|
||||
{#if ms.updating || ms.isLoadingModel}
|
||||
<Loader2 class="h-3 w-3.5 shrink-0 animate-spin" />
|
||||
{:else}
|
||||
<ChevronDown class="h-3 w-3.5 shrink-0" />
|
||||
{#if selectedOption}
|
||||
<Tooltip.Content>
|
||||
<p class="font-mono">{selectedOption.model}</p>
|
||||
</Tooltip.Content>
|
||||
{/if}
|
||||
</DropdownMenu.Trigger>
|
||||
</Tooltip.Root>
|
||||
|
||||
<DropdownMenu.Content
|
||||
align="end"
|
||||
@@ -234,49 +236,51 @@
|
||||
</DropdownMenu.Content>
|
||||
</DropdownMenu.Root>
|
||||
{:else}
|
||||
<button
|
||||
class={[
|
||||
`inline-flex cursor-pointer items-center gap-1.5 rounded-sm bg-background px-1.5 py-1 text-xs shadow-sm transition hover:bg-muted-foreground/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-60 dark:bg-muted-foreground/15 dark:text-secondary-foreground`,
|
||||
!ms.isCurrentModelInCache
|
||||
? 'bg-red-400/10 !text-red-400 hover:bg-red-400/20 hover:text-red-400'
|
||||
: forceForegroundText
|
||||
? 'text-foreground'
|
||||
: ms.isHighlightedCurrentModelActive
|
||||
? 'text-foreground'
|
||||
: 'text-foreground',
|
||||
isOpen && 'text-foreground'
|
||||
]}
|
||||
style="max-width: min(calc(100cqw - 6.5rem), 32rem)"
|
||||
onclick={() => ms.handleOpenChange(true)}
|
||||
disabled={disabled || ms.updating}
|
||||
>
|
||||
<Package class="h-3.5 w-3.5 shrink-0" />
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<!-- prevent another nested button element -->
|
||||
{#snippet child({ props })}
|
||||
<button
|
||||
{...props}
|
||||
class={[
|
||||
`inline-flex cursor-pointer items-center gap-1.5 rounded-sm bg-background px-1.5 py-1 text-xs shadow-sm transition hover:bg-muted-foreground/20 focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-60 dark:bg-muted-foreground/15 dark:text-secondary-foreground`,
|
||||
!ms.isCurrentModelInCache
|
||||
? 'bg-red-400/10 !text-red-400 hover:bg-red-400/20 hover:text-red-400'
|
||||
: forceForegroundText
|
||||
? 'text-foreground'
|
||||
: ms.isHighlightedCurrentModelActive
|
||||
? 'text-foreground'
|
||||
: 'text-foreground',
|
||||
isOpen && 'text-foreground'
|
||||
]}
|
||||
style="max-width: min(calc(100cqw - 6.5rem), 32rem)"
|
||||
onclick={() => ms.handleOpenChange(true)}
|
||||
disabled={disabled || ms.updating}
|
||||
>
|
||||
<Package class="h-3.5 w-3.5 shrink-0" />
|
||||
|
||||
{#if selectedOption}
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<!-- prevent another nested button element -->
|
||||
{#snippet child({ props })}
|
||||
{#if selectedOption}
|
||||
<ModelId
|
||||
modelId={selectedOption.model}
|
||||
class="min-w-0 overflow-hidden"
|
||||
hideOrgName={false}
|
||||
hideQuantization
|
||||
{...props}
|
||||
/>
|
||||
{/snippet}
|
||||
</Tooltip.Trigger>
|
||||
{/if}
|
||||
|
||||
<Tooltip.Content>
|
||||
<p class="font-mono">{selectedOption.model}</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{/if}
|
||||
{#if ms.updating}
|
||||
<Loader2 class="h-3 w-3.5 shrink-0 animate-spin" />
|
||||
{/if}
|
||||
</button>
|
||||
{/snippet}
|
||||
</Tooltip.Trigger>
|
||||
|
||||
{#if ms.updating}
|
||||
<Loader2 class="h-3 w-3.5 shrink-0 animate-spin" />
|
||||
{#if selectedOption}
|
||||
<Tooltip.Content>
|
||||
<p class="font-mono">{selectedOption.model}</p>
|
||||
</Tooltip.Content>
|
||||
{/if}
|
||||
</button>
|
||||
</Tooltip.Root>
|
||||
{/if}
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
@@ -34,24 +34,28 @@
|
||||
</script>
|
||||
|
||||
<DropdownMenu.Root bind:open>
|
||||
<DropdownMenu.Trigger
|
||||
class="flex h-6 w-6 cursor-pointer items-center justify-center rounded-md p-0 text-sm font-medium transition-colors hover:bg-accent hover:text-accent-foreground focus:bg-accent focus:text-accent-foreground focus:outline-none disabled:pointer-events-none disabled:opacity-50 data-[state=open]:bg-accent data-[state=open]:text-accent-foreground {triggerClass}"
|
||||
onclick={(e) => e.stopPropagation()}
|
||||
>
|
||||
{#if triggerTooltip}
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<!-- prevent another nested button element -->
|
||||
{#snippet child({ props })}
|
||||
<DropdownMenu.Trigger
|
||||
{...props}
|
||||
class="flex h-6 w-6 cursor-pointer items-center justify-center rounded-md p-0 text-sm font-medium transition-colors hover:bg-accent hover:text-accent-foreground focus:bg-accent focus:text-accent-foreground focus:outline-none disabled:pointer-events-none disabled:opacity-50 data-[state=open]:bg-accent data-[state=open]:text-accent-foreground {triggerClass}"
|
||||
onclick={(e) => e.stopPropagation()}
|
||||
>
|
||||
{@render iconComponent(triggerIcon, 'h-3 w-3')}
|
||||
<span class="sr-only">{triggerTooltip}</span>
|
||||
</Tooltip.Trigger>
|
||||
<Tooltip.Content>
|
||||
<p>{triggerTooltip}</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
{:else}
|
||||
{@render iconComponent(triggerIcon, 'h-3 w-3')}
|
||||
{#if triggerTooltip}
|
||||
<span class="sr-only">{triggerTooltip}</span>
|
||||
{/if}
|
||||
</DropdownMenu.Trigger>
|
||||
{/snippet}
|
||||
</Tooltip.Trigger>
|
||||
{#if triggerTooltip}
|
||||
<Tooltip.Content>
|
||||
<p>{triggerTooltip}</p>
|
||||
</Tooltip.Content>
|
||||
{/if}
|
||||
</DropdownMenu.Trigger>
|
||||
</Tooltip.Root>
|
||||
|
||||
<DropdownMenu.Content {align} class="z-[999999] w-48">
|
||||
{#each actions as action, index (action.label)}
|
||||
|
||||
+18
-7
@@ -105,6 +105,12 @@
|
||||
onclick={handleSelect}
|
||||
onmouseover={handleMouseOver}
|
||||
onmouseleave={handleMouseLeave}
|
||||
onfocusin={handleMouseOver}
|
||||
onfocusout={(e) => {
|
||||
if (!e.currentTarget.contains(e.relatedTarget as Node | null)) {
|
||||
handleMouseLeave();
|
||||
}
|
||||
}}
|
||||
>
|
||||
<div
|
||||
class="flex min-w-0 flex-1 items-center gap-2"
|
||||
@@ -113,12 +119,16 @@
|
||||
{#if depth > 0}
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger>
|
||||
<a
|
||||
href={RouterService.chat(conversation.forkedFromConversationId)}
|
||||
class="flex shrink-0 items-center text-muted-foreground transition-colors hover:text-foreground"
|
||||
>
|
||||
<GitBranch class="h-3.5 w-3.5" />
|
||||
</a>
|
||||
<!-- prevent another nested button element -->
|
||||
{#snippet child({ props })}
|
||||
<a
|
||||
{...props}
|
||||
href={RouterService.chat(conversation.forkedFromConversationId)}
|
||||
class="flex shrink-0 items-center text-muted-foreground transition-colors hover:text-foreground"
|
||||
>
|
||||
<GitBranch class="h-3.5 w-3.5" />
|
||||
</a>
|
||||
{/snippet}
|
||||
</Tooltip.Trigger>
|
||||
|
||||
<Tooltip.Content>
|
||||
@@ -195,7 +205,8 @@
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
&:is(:hover) :global([data-slot='dropdown-menu-trigger']) {
|
||||
&:is(:hover) :global([data-slot='dropdown-menu-trigger']),
|
||||
&:focus-within :global([data-slot='dropdown-menu-trigger']) {
|
||||
opacity: 1;
|
||||
}
|
||||
@media (max-width: 768px) {
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
<script module lang="ts">
|
||||
import { defineMeta } from '@storybook/addon-svelte-csf';
|
||||
import { Copy } from '@lucide/svelte';
|
||||
import ActionIcon from '$lib/components/app/actions/ActionIcon.svelte';
|
||||
import { expect } from 'storybook/test';
|
||||
|
||||
const { Story } = defineMeta({
|
||||
title: 'Components/ActionIcon/Accessibility',
|
||||
component: ActionIcon,
|
||||
parameters: {
|
||||
layout: 'centered'
|
||||
},
|
||||
tags: ['!dev']
|
||||
});
|
||||
</script>
|
||||
|
||||
<Story
|
||||
asChild
|
||||
name="SingleTabStop"
|
||||
play={async ({ canvas, userEvent }) => {
|
||||
const before = await canvas.findByRole('button', { name: 'before' });
|
||||
const target = await canvas.findByRole('button', { name: 'Copy' });
|
||||
|
||||
before.focus();
|
||||
await userEvent.tab();
|
||||
|
||||
await expect(target).toHaveFocus();
|
||||
}}
|
||||
>
|
||||
<div>
|
||||
<button type="button">before</button>
|
||||
<ActionIcon icon={Copy} tooltip="Copy" onclick={() => {}} />
|
||||
</div>
|
||||
</Story>
|
||||
@@ -0,0 +1,50 @@
|
||||
<script module lang="ts">
|
||||
import { defineMeta } from '@storybook/addon-svelte-csf';
|
||||
import ChatMessageStatistics from '$lib/components/app/chat/ChatMessages/ChatMessageStatistics/ChatMessageStatistics.svelte';
|
||||
import { expect } from 'storybook/test';
|
||||
|
||||
const { Story } = defineMeta({
|
||||
title: 'Components/ChatMessageStatistics/Accessibility',
|
||||
component: ChatMessageStatistics,
|
||||
parameters: {
|
||||
layout: 'centered'
|
||||
},
|
||||
tags: ['!dev']
|
||||
});
|
||||
</script>
|
||||
|
||||
<Story
|
||||
name="ViewButtonsSingleTabStop"
|
||||
args={{
|
||||
promptTokens: 100,
|
||||
promptMs: 500,
|
||||
predictedTokens: 200,
|
||||
predictedMs: 1000,
|
||||
agenticTimings: {
|
||||
turns: 1,
|
||||
toolCallsCount: 1,
|
||||
toolsMs: 500,
|
||||
llm: { predicted_n: 200, predicted_ms: 1000, prompt_n: 100, prompt_ms: 500 }
|
||||
},
|
||||
hideSummary: false,
|
||||
isLive: false
|
||||
}}
|
||||
play={async ({ canvas, userEvent }) => {
|
||||
const reading = await canvas.findByRole('button', { name: 'Reading' });
|
||||
const generation = await canvas.findByRole('button', { name: 'Generation' });
|
||||
const tools = await canvas.findByRole('button', { name: 'Tools' });
|
||||
const summary = await canvas.findByRole('button', { name: 'Summary' });
|
||||
|
||||
reading.focus();
|
||||
await expect(reading).toHaveFocus();
|
||||
|
||||
await userEvent.tab();
|
||||
await expect(generation).toHaveFocus();
|
||||
|
||||
await userEvent.tab();
|
||||
await expect(tools).toHaveFocus();
|
||||
|
||||
await userEvent.tab();
|
||||
await expect(summary).toHaveFocus();
|
||||
}}
|
||||
/>
|
||||
@@ -0,0 +1,69 @@
|
||||
<script module lang="ts">
|
||||
import { defineMeta } from '@storybook/addon-svelte-csf';
|
||||
import HorizontalScrollCarousel from '$lib/components/app/misc/HorizontalScrollCarousel.svelte';
|
||||
import { expect, waitFor } from 'storybook/test';
|
||||
|
||||
const { Story } = defineMeta({
|
||||
title: 'Components/HorizontalScrollCarousel/Accessibility',
|
||||
component: HorizontalScrollCarousel,
|
||||
parameters: {
|
||||
layout: 'centered'
|
||||
},
|
||||
tags: ['!dev']
|
||||
});
|
||||
</script>
|
||||
|
||||
<Story
|
||||
asChild
|
||||
name="ArrowsNotInTabOrderWhenNotScrollable"
|
||||
play={async ({ canvas, userEvent }) => {
|
||||
const before = await canvas.findByRole('button', { name: 'before' });
|
||||
const after = await canvas.findByRole('button', { name: 'after' });
|
||||
const leftArrow = await canvas.findByRole('button', { name: 'Scroll left' });
|
||||
|
||||
await waitFor(() => {
|
||||
expect(leftArrow).toBeDisabled();
|
||||
});
|
||||
|
||||
before.focus();
|
||||
await userEvent.tab();
|
||||
|
||||
await expect(after).toHaveFocus();
|
||||
}}
|
||||
>
|
||||
<div>
|
||||
<button type="button">before</button>
|
||||
<HorizontalScrollCarousel class="w-96">
|
||||
<div class="h-12 w-12 shrink-0 bg-muted"></div>
|
||||
<div class="h-12 w-12 shrink-0 bg-muted"></div>
|
||||
</HorizontalScrollCarousel>
|
||||
<button type="button">after</button>
|
||||
</div>
|
||||
</Story>
|
||||
|
||||
<Story
|
||||
asChild
|
||||
name="ArrowsInTabOrderWhenScrollable"
|
||||
play={async ({ canvas, userEvent }) => {
|
||||
const before = await canvas.findByRole('button', { name: 'before' });
|
||||
const rightArrow = await canvas.findByRole('button', { name: 'Scroll right' });
|
||||
|
||||
await waitFor(() => {
|
||||
expect(rightArrow).not.toBeDisabled();
|
||||
});
|
||||
|
||||
before.focus();
|
||||
await userEvent.tab();
|
||||
|
||||
await expect(rightArrow).toHaveFocus();
|
||||
}}
|
||||
>
|
||||
<div>
|
||||
<button type="button">before</button>
|
||||
<HorizontalScrollCarousel class="w-48">
|
||||
{#each [...Array(20).keys()] as i (i)}
|
||||
<div class="h-12 w-24 shrink-0 bg-muted">{i}</div>
|
||||
{/each}
|
||||
</HorizontalScrollCarousel>
|
||||
</div>
|
||||
</Story>
|
||||
@@ -0,0 +1,36 @@
|
||||
<script module lang="ts">
|
||||
import { defineMeta } from '@storybook/addon-svelte-csf';
|
||||
import SidebarNavigationConversationItem from '$lib/components/app/navigation/SidebarNavigation/SidebarNavigationConversationItem.svelte';
|
||||
import { expect } from 'storybook/test';
|
||||
|
||||
const mockForkedConversation: DatabaseConversation = {
|
||||
id: 'conv-2',
|
||||
name: 'Forked Conversation',
|
||||
lastModified: Date.now(),
|
||||
currNode: 'msg-2',
|
||||
forkedFromConversationId: 'conv-1'
|
||||
};
|
||||
|
||||
const { Story } = defineMeta({
|
||||
title: 'Components/SidebarNavigationConversationItem/Accessibility',
|
||||
component: SidebarNavigationConversationItem,
|
||||
parameters: {
|
||||
layout: 'centered'
|
||||
},
|
||||
tags: ['!dev']
|
||||
});
|
||||
</script>
|
||||
|
||||
<Story
|
||||
name="ForkIconSingleTabStop"
|
||||
args={{ conversation: mockForkedConversation, depth: 1 }}
|
||||
play={async ({ canvas, userEvent }) => {
|
||||
const row = await canvas.findByRole('button', { name: /Forked Conversation/ });
|
||||
const forkIcon = await canvas.findByRole('link');
|
||||
|
||||
row.focus();
|
||||
await userEvent.tab();
|
||||
|
||||
await expect(forkIcon).toHaveFocus();
|
||||
}}
|
||||
/>
|
||||
Reference in New Issue
Block a user