diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index e5c0767f0e..0ff4d45cf7 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -210,11 +210,9 @@ struct tensor_metadata { // dequantization // -// dequantizes [first_elem, first_elem + nelements) of the tensor into output[0, nelements) -// first_elem must be a multiple of the source block size, which holds for any whole-row range static void llama_tensor_dequantize_impl( ggml_tensor * tensor, std::vector> & output, std::vector & workers, - const size_t first_elem, const size_t nelements, const int nthread + const size_t nelements, const int nthread ) { if (output.size() < nelements) { output.resize(nelements); @@ -231,6 +229,19 @@ static void llama_tensor_dequantize_impl( throw std::runtime_error(format("cannot dequantize/convert tensor type %s", ggml_type_name(tensor->type))); } + if (nthread < 2) { + if (tensor->type == GGML_TYPE_F16) { + ggml_fp16_to_fp32_row((ggml_fp16_t *)tensor->data, f32_output, nelements); + } else if (tensor->type == GGML_TYPE_BF16) { + ggml_bf16_to_fp32_row((ggml_bf16_t *)tensor->data, f32_output, nelements); + } else if (ggml_is_quantized(tensor->type)) { + qtype->to_float(tensor->data, f32_output, nelements); + } else { + GGML_ABORT("fatal error"); // unreachable + } + return; + } + size_t block_size; if (tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) { @@ -241,24 +252,7 @@ static void llama_tensor_dequantize_impl( size_t block_size_bytes = ggml_type_size(tensor->type); - GGML_ASSERT(first_elem % block_size == 0); GGML_ASSERT(nelements % block_size == 0); - - uint8_t * f32_input = (uint8_t *) tensor->data + (first_elem / block_size) * block_size_bytes; - - if (nthread < 2) { - if (tensor->type == GGML_TYPE_F16) { - ggml_fp16_to_fp32_row((ggml_fp16_t *)f32_input, f32_output, nelements); - } else if (tensor->type == GGML_TYPE_BF16) { - ggml_bf16_to_fp32_row((ggml_bf16_t *)f32_input, f32_output, nelements); - } else if (ggml_is_quantized(tensor->type)) { - qtype->to_float(f32_input, f32_output, nelements); - } else { - GGML_ABORT("fatal error"); // unreachable - } - return; - } - size_t nblocks = nelements / block_size; size_t blocks_per_thread = nblocks / nthread; size_t spare_blocks = nblocks - (blocks_per_thread * nthread); // if blocks aren't divisible by thread count @@ -280,7 +274,7 @@ static void llama_tensor_dequantize_impl( qtype->to_float(inbuf, outbuf, nels); } }; - workers.emplace_back(compute, tensor->type, f32_input + in_buff_offs, f32_output + out_buff_offs, thr_elems); + workers.emplace_back(compute, tensor->type, (uint8_t *) tensor->data + in_buff_offs, f32_output + out_buff_offs, thr_elems); in_buff_offs += thr_block_bytes; out_buff_offs += thr_elems; } @@ -1220,6 +1214,8 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: new_size = tensor_size; LLAMA_LOG_INFO("size = %8.3f MiB\n", tensor_size/1024.0/1024.0); } else { + const int64_t nelements = ggml_nelements(tensor); + const float * imatrix = nullptr; if (imatrix_data) { auto it = imatrix_data->find(tm.remapped_imatrix_name); @@ -1251,19 +1247,22 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: throw std::runtime_error(format("Missing importance matrix for tensor %s in a very low-bit quantization", tensor->name)); } - if (ggml_is_quantized(tensor->type) && !params->allow_requantize) { + float * f32_data; + + if (tensor->type == GGML_TYPE_F32) { + f32_data = (float *) tensor->data; + } else if (ggml_is_quantized(tensor->type) && !params->allow_requantize) { throw std::runtime_error(format("requantizing from type %s is disabled", ggml_type_name(tensor->type))); + } else { + llama_tensor_dequantize_impl(tensor, f32_conv_buf, workers, nelements, nthread); + f32_data = (float *) f32_conv_buf.data(); } LLAMA_LOG_INFO("converting to %s .. ", ggml_type_name(new_type)); fflush(stdout); - // exact output size: ggml_row_size(new_type, ne0) per row, ne1 rows, ne2 slices - // this is what the loop writes and what new_size sums to - const size_t out_size = - ggml_row_size(new_type, tensor->ne[0]) * tensor->ne[1] * tensor->ne[2]; - if (work.size() < out_size) { - work.resize(out_size); + if (work.size() < (size_t)nelements * 4) { + work.resize(nelements * 4); // upper bound on size } new_data = work.data(); @@ -1277,38 +1276,14 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: const int64_t nchunk = (nelements_matrix + chunk_size - 1)/chunk_size; const int64_t nthread_use = nthread > 1 ? std::max((int64_t)1, std::min((int64_t)nthread, nchunk)) : 1; - // work in bands of whole rows to bound the f32 staging buffer: the whole tensor is hundreds of GiB for tables like per_layer_token_embd - // rows are independent and the imatrix is per column, so the band size cannot change any output byte - static const size_t max_band_bytes = 1024ull*1024*1024; // f32 staging cap per band - const int64_t nrows_per_chunk = chunk_size / n_per_row; - int64_t band_nrows = (int64_t) (max_band_bytes / (sizeof(float) * n_per_row)); - // keep enough rows per band to feed every worker thread, and at least one row - band_nrows = std::max(band_nrows, nrows_per_chunk * nthread_use); - band_nrows = std::max(band_nrows, (int64_t) 1); - band_nrows = std::min(band_nrows, nrows); - // quantize each expert separately since they have different importance matrices new_size = 0; for (int64_t i03 = 0; i03 < tensor->ne[2]; ++i03) { + const float * f32_data_03 = f32_data + i03 * nelements_matrix; + void * new_data_03 = (char *)new_data + ggml_row_size(new_type, n_per_row) * i03 * nrows; const float * imatrix_03 = imatrix ? imatrix + i03 * n_per_row : nullptr; - for (int64_t ir0 = 0; ir0 < nrows; ir0 += band_nrows) { - const int64_t band_rows = std::min(band_nrows, nrows - ir0); - const size_t first_elem = (size_t) (i03 * nelements_matrix + ir0 * n_per_row); - - const float * f32_band; - if (tensor->type == GGML_TYPE_F32) { - // already f32 on disk - no staging buffer, just point into the tensor - f32_band = (const float *) tensor->data + first_elem; - } else { - llama_tensor_dequantize_impl(tensor, f32_conv_buf, workers, first_elem, (size_t) (band_rows * n_per_row), nthread); - f32_band = (const float *) f32_conv_buf.data(); - } - - void * new_data_band = (char *) new_data + ggml_row_size(new_type, n_per_row) * (i03 * nrows + ir0); - - new_size += llama_tensor_quantize_impl(new_type, f32_band, new_data_band, chunk_size, band_rows, n_per_row, imatrix_03, workers, nthread_use); - } + new_size += llama_tensor_quantize_impl(new_type, f32_data_03, new_data_03, chunk_size, nrows, n_per_row, imatrix_03, workers, nthread_use); } LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB\n", tensor_size/1024.0/1024.0, new_size/1024.0/1024.0); }