Skip to content

Check for llama_get_logits_ith() errors #7448

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
Open

Check for llama_get_logits_ith() errors #7448

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
99291c0
Check for llama_get_logits_ith() errors
jart May 21, 2024
ac6bed1
Update llama.cpp
jart May 21, 2024
5224b65
Update llama.cpp
jart May 21, 2024
bc9a2e8
Update llama.cpp
jart May 21, 2024
cc363da
Update llama.cpp
jart May 21, 2024
6b17898
Update examples/perplexity/perplexity.cpp
jart May 21, 2024
aa3094c
Update common/sampling.cpp
jart May 21, 2024
a948952
Make sampling not throw exception
jart May 22, 2024
8be06dc
Update examples/server/server.cpp
jart May 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions common/sampling.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,9 @@ static llama_token llama_sampling_sample_impl(
llama_token id = 0;
// Get a pointer to the logits
float * logits = llama_get_logits_ith(ctx_main, idx);
if (!logits) {
throw std::runtime_error("llama_get_logits_ith failed");
}

if (temp < 0.0) {
// greedy sampling, with probs
Expand Down Expand Up @@ -284,6 +287,9 @@ static llama_token_data_array llama_sampling_prepare_impl(

// Get a pointer to the logits
float * logits = llama_get_logits_ith(ctx_main, idx);
if (!logits) {
throw std::runtime_error("llama_get_logits_ith failed");
}

if (ctx_sampling->grammar != NULL && !apply_grammar) {
GGML_ASSERT(original_logits != NULL);
Expand All @@ -298,6 +304,9 @@ static llama_token_data_array llama_sampling_prepare_impl(

if (ctx_cfg) {
float * logits_guidance = llama_get_logits_ith(ctx_cfg, idx);
if (!logits_guidance) {
throw std::runtime_error("llama_get_logits_ith failed");
}
llama_sample_apply_guidance(ctx_main, logits, logits_guidance, params.cfg_scale);
}

Expand Down
3 changes: 3 additions & 0 deletions examples/batched/batched.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,9 @@ int main(int argc, char ** argv) {

auto n_vocab = llama_n_vocab(model);
auto * logits = llama_get_logits_ith(ctx, i_batch[i]);
if (!logits) {
return 1;
}

std::vector<llama_token_data> candidates;
candidates.reserve(n_vocab);
Expand Down
6 changes: 6 additions & 0 deletions examples/gritlm/gritlm.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ static std::vector<std::vector<float>> encode(llama_context * ctx, const std::ve
// sum up all token embeddings
for (int32_t k = n_inst; k < n_toks; k++) {
float * emb = llama_get_embeddings_ith(ctx, k);
if (!emb) {
throw std::runtime_error("llama_get_embeddings_ith failed");
}
for (uint64_t j = 0; j < n_embd; j++) {
emb_unorm[j] += emb[j];
}
Expand Down Expand Up @@ -114,6 +117,9 @@ static std::string generate(llama_context * ctx, const std::string & prompt, boo

llama_decode(ctx, bat);
auto logits = llama_get_logits_ith(ctx, bat.n_tokens - 1);
if (!logits) {
throw std::runtime_error("llama_get_logits_ith failed");
}

auto candidates = std::vector<llama_token_data>(llama_n_vocab(mdl));
auto n_candidates = (int32_t)candidates.size();
Expand Down
3 changes: 3 additions & 0 deletions examples/llama.android/app/src/main/cpp/llama-android.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,9 @@ Java_com_example_llama_Llm_completion_1loop(

auto n_vocab = llama_n_vocab(model);
auto logits = llama_get_logits_ith(context, batch->n_tokens - 1);
if (!logits) {
throw std::runtime_error("llama_get_logits_ith failed");
}

std::vector<llama_token_data> candidates;
candidates.reserve(n_vocab);
Expand Down
3 changes: 3 additions & 0 deletions examples/passkey/passkey.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,9 @@ int main(int argc, char ** argv) {
{
auto n_vocab = llama_n_vocab(model);
auto * logits = llama_get_logits_ith(ctx, batch.n_tokens - 1);
if (!logits) {
return 1;
}

std::vector<llama_token_data> candidates;
candidates.reserve(n_vocab);
Expand Down
3 changes: 3 additions & 0 deletions examples/perplexity/perplexity.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -638,6 +638,9 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par

for (int seq = 0; seq < n_seq_batch; seq++) {
const float * all_logits = num_batches > 1 ? logits.data() : llama_get_logits_ith(ctx, seq*n_ctx + first);
if (!all_logits) {
return 1;
}

llama_token * tokens_data = tokens.data() + start + seq*n_ctx + first;
if (!params.logits_file.empty()) {
Expand Down
3 changes: 3 additions & 0 deletions examples/simple/simple.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,9 @@ int main(int argc, char ** argv) {
{
auto n_vocab = llama_n_vocab(model);
auto * logits = llama_get_logits_ith(ctx, batch.n_tokens - 1);
if (!logits) {
return 1;
}

std::vector<llama_token_data> candidates;
candidates.reserve(n_vocab);
Expand Down
119 changes: 58 additions & 61 deletions llama.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17301,42 +17301,39 @@ float * llama_get_logits(struct llama_context * ctx) {
return ctx->logits;
}

static float * llama_get_logits_ith_fail(int i, std::string reason) {
LLAMA_LOG_ERROR("%s: invalid logits id %d, reason: %s\n", __func__, i, reason.c_str());
#ifndef NDEBUG
GGML_ASSERT(false);
#endif
return nullptr;
}

float * llama_get_logits_ith(struct llama_context * ctx, int32_t i) {
int32_t j = -1;
llama_synchronize(ctx);

try {
if (ctx->logits == nullptr) {
throw std::runtime_error("no logits");
}

if (i < 0) {
j = ctx->n_outputs + i;
if (j < 0) {
throw std::runtime_error(format("negative index out of range [0, %d)", ctx->n_outputs));
}
} else if ((size_t) i >= ctx->output_ids.size()) {
throw std::runtime_error(format("out of range [0, %lu)", ctx->output_ids.size()));
} else {
j = ctx->output_ids[i];
}

if (ctx->logits == nullptr) {
// this can happen for embeddings models like bert
return llama_get_logits_ith_fail(i, "no logits");
}
if (i < 0) {
j = ctx->n_outputs + i;
if (j < 0) {
throw std::runtime_error(format("batch.logits[%d] != true", i));
}
if (j >= ctx->n_outputs) {
// This should not happen
throw std::runtime_error(format("corrupt output buffer (j=%d, n_outputs=%d)", j, ctx->n_outputs));
return llama_get_logits_ith_fail(i, format("negative index out of range [0, %d)", ctx->n_outputs));
}

return ctx->logits + j*ctx->model.hparams.n_vocab;
} catch (const std::exception & err) {
LLAMA_LOG_ERROR("%s: invalid logits id %d, reason: %s\n", __func__, i, err.what());
#ifndef NDEBUG
GGML_ASSERT(false);
#endif
return nullptr;
} else if ((size_t) i >= ctx->output_ids.size()) {
return llama_get_logits_ith_fail(i, format("out of range [0, %lu)", ctx->output_ids.size()));
} else {
j = ctx->output_ids[i];
}
if (j < 0) {
return llama_get_logits_ith_fail(i, format("batch.logits[%d] != true", i));
}
if (j >= ctx->n_outputs) {
// This should not happen
return llama_get_logits_ith_fail(i, format("corrupt output buffer (j=%d, n_outputs=%d)", j, ctx->n_outputs));
}
return ctx->logits + j*ctx->model.hparams.n_vocab;
}

float * llama_get_embeddings(struct llama_context * ctx) {
Expand All @@ -17345,43 +17342,43 @@ float * llama_get_embeddings(struct llama_context * ctx) {
return ctx->embd;
}

static float * llama_get_embeddings_ith_fail(int i, std::string reason) {
LLAMA_LOG_ERROR("%s: invalid embeddings id %d, reason: %s\n", __func__, i, reason.c_str());
#ifndef NDEBUG
GGML_ASSERT(false);
#endif
return nullptr;
}

float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i) {
int32_t j = -1;

llama_synchronize(ctx);

try {
if (ctx->embd == nullptr) {
throw std::runtime_error("no embeddings");
}

if (i < 0) {
j = ctx->n_outputs + i;
if (j < 0) {
throw std::runtime_error(format("negative index out of range [0, %d)", ctx->n_outputs));
}
} else if ((size_t) i >= ctx->output_ids.size()) {
throw std::runtime_error(format("out of range [0, %lu)", ctx->output_ids.size()));
} else {
j = ctx->output_ids[i];
}

if (ctx->embd == nullptr) {
return llama_get_embeddings_ith_fail(i, "no embeddings");
}
if (i < 0) {
j = ctx->n_outputs + i;
if (j < 0) {
throw std::runtime_error(format("batch.logits[%d] != true", i));
}
if (j >= ctx->n_outputs) {
// This should not happen
throw std::runtime_error(format("corrupt output buffer (j=%d, n_outputs=%d)", j, ctx->n_outputs));
return llama_get_embeddings_ith_fail(
i, format("negative index out of range [0, %d)", ctx->n_outputs));
}

return ctx->embd + j*ctx->model.hparams.n_embd;
} catch (const std::exception & err) {
LLAMA_LOG_ERROR("%s: invalid embeddings id %d, reason: %s\n", __func__, i, err.what());
#ifndef NDEBUG
GGML_ASSERT(false);
#endif
return nullptr;
} else if ((size_t) i >= ctx->output_ids.size()) {
return llama_get_embeddings_ith_fail(
i, format("out of range [0, %lu)", ctx->output_ids.size()));
} else {
j = ctx->output_ids[i];
}
if (j < 0) {
return llama_get_embeddings_ith_fail(
i, format("batch.logits[%d] != true", i));
}
if (j >= ctx->n_outputs) {
// This should not happen
return llama_get_embeddings_ith_fail(
i, format("corrupt output buffer (j=%d, n_outputs=%d)",
j, ctx->n_outputs));
}
return ctx->embd + j*ctx->model.hparams.n_embd;
}

float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id) {
Expand Down