@@ -2791,13 +2791,7 @@ struct llama_model_loader {
2791
2791
2792
2792
std::vector<no_init<uint8_t>> read_buf;
2793
2793
2794
- for (int i = 0; i < gguf_get_n_tensors(ctx_gguf); i++) {
2795
- struct ggml_tensor * cur = ggml_get_tensor(ctx, gguf_get_tensor_name(ctx_gguf, i));
2796
- if (!cur) {
2797
- // some tensors may be allocated in a different context
2798
- continue;
2799
- }
2800
-
2794
+ for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur != NULL; cur = ggml_get_next_tensor(ctx, cur)) {
2801
2795
if (progress_callback) {
2802
2796
if (!progress_callback((float) size_done / size_data, progress_callback_user_data)) {
2803
2797
return false;
@@ -3722,7 +3716,7 @@ static bool llm_load_tensors(
3722
3716
}
3723
3717
3724
3718
// create one context per buffer type
3725
- size_t ctx_size = ggml_tensor_overhead()*ml.n_tensors;
3719
+ size_t ctx_size = ggml_tensor_overhead()*( ml.n_tensors + 1); // +1 for models where tok_embd is duplicated as output
3726
3720
std::map<ggml_backend_buffer_type_t, ggml_context *> ctx_map;
3727
3721
for (auto & it : buft_layer_count) {
3728
3722
struct ggml_init_params params = {
@@ -3860,6 +3854,7 @@ static bool llm_load_tensors(
3860
3854
} else {
3861
3855
model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // needs to be on GPU
3862
3856
ml.n_created--; // artificial tensor
3857
+ ml.size_data += ggml_nbytes(model.output);
3863
3858
}
3864
3859
}
3865
3860
@@ -4396,6 +4391,7 @@ static bool llm_load_tensors(
4396
4391
model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
4397
4392
model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // same as tok_embd, duplicated to allow offloading
4398
4393
ml.n_created--; // artificial tensor
4394
+ ml.size_data += ggml_nbytes(model.output);
4399
4395
4400
4396
const int64_t n_ff = hparams.n_ff;
4401
4397
const int64_t n_embd_head_k = hparams.n_embd_head_k;
0 commit comments