diff --git a/main.cpp b/main.cpp index 01984e1..83816cb 100644 --- a/main.cpp +++ b/main.cpp @@ -1,454 +1,517 @@ -#include "llama.h" -#include "mtmd.h" -#include "mtmd-helper.h" #include "ggml.h" +#include "llama.h" +#include "mtmd-helper.h" +#include "mtmd.h" +#include +#include +#include #include #include -#include -#include +#include #include -#include +#include #include #include -#include -#include -#include -#include +#include static volatile bool g_interrupted = false; -static void sigint_handler(int signo) { - (void)signo; - g_interrupted = true; +static void sigint_handler(const int signo) { + (void)signo; + g_interrupted = true; } -static void batch_clear(struct llama_batch & batch) { - batch.n_tokens = 0; +static void batch_clear(llama_batch &batch) { batch.n_tokens = 0; } + +static void batch_add(llama_batch &batch, const llama_token id, + const llama_pos pos, + const std::vector &seq_ids, + const bool logits) { + batch.token[batch.n_tokens] = id; + batch.pos[batch.n_tokens] = pos; + batch.n_seq_id[batch.n_tokens] = static_cast(seq_ids.size()); + for (size_t i = 0; i < seq_ids.size(); ++i) { + batch.seq_id[batch.n_tokens][i] = seq_ids[i]; + } + batch.logits[batch.n_tokens] = logits ? 1 : 0; + batch.n_tokens++; } -static void batch_add(struct llama_batch & batch, llama_token id, llama_pos pos, const std::vector & seq_ids, bool logits) { - batch.token [batch.n_tokens] = id; - batch.pos [batch.n_tokens] = pos; - batch.n_seq_id[batch.n_tokens] = (int32_t)seq_ids.size(); - for (size_t i = 0; i < seq_ids.size(); ++i) { - batch.seq_id[batch.n_tokens][i] = seq_ids[i]; +static std::string token_to_piece(const llama_vocab *vocab, + const llama_token token) { + std::string piece; + piece.resize(256); + int32_t n = llama_token_to_piece( + vocab, token, &piece[0], static_cast(piece.size()), 0, false); + if (n < 0) { + piece.resize(-n); + n = llama_token_to_piece(vocab, token, &piece[0], + static_cast(piece.size()), 0, false); + } + piece.resize(std::max(0, n)); + return piece; +} + +static std::string json_escape(const std::string &s) { + std::string out; + out.reserve(s.size() + 2); + for (const char c : s) { + switch (c) { + case '"': + out += "\\\""; + break; + case '\\': + out += "\\\\"; + break; + case '\n': + out += "\\n"; + break; + case '\r': + out += "\\r"; + break; + case '\t': + out += "\\t"; + break; + default: + out += c; + break; } - batch.logits [batch.n_tokens] = logits ? 1 : 0; - batch.n_tokens++; -} - -static std::string token_to_piece(const struct llama_vocab * vocab, llama_token token) { - std::string piece; - piece.resize(256); - int32_t n = llama_token_to_piece(vocab, token, &piece[0], (int32_t)piece.size(), 0, false); - if (n < 0) { - piece.resize(-n); - n = llama_token_to_piece(vocab, token, &piece[0], (int32_t)piece.size(), 0, false); - } - piece.resize(std::max(0, n)); - return piece; -} - -static std::string json_escape(const std::string & s) { - std::string out; - out.reserve(s.size() + 2); - for (char c : s) { - switch (c) { - case '"': out += "\\\""; break; - case '\\': out += "\\\\"; break; - case '\n': out += "\\n"; break; - case '\r': out += "\\r"; break; - case '\t': out += "\\t"; break; - default: out += c; break; - } - } - return out; + } + return out; } static std::vector read_clipboard() { - const char * cmd = nullptr; - if (getenv("WAYLAND_DISPLAY")) { - cmd = "wl-paste 2>/dev/null"; - } else { - cmd = "xclip -selection clipboard -t image/png -o 2>/dev/null"; - } - std::vector data; - FILE * pipe = popen(cmd, "re"); - if (!pipe) return data; - unsigned char buf[65536]; - size_t n; - while ((n = fread(buf, 1, sizeof(buf), pipe)) > 0) { - data.insert(data.end(), buf, buf + n); - } - int st = pclose(pipe); - if (st != 0 || data.empty()) { - data.clear(); - // fallback for X11: try without explicit target - if (!getenv("WAYLAND_DISPLAY")) { - pipe = popen("xclip -selection clipboard -o 2>/dev/null", "re"); - if (pipe) { - while ((n = fread(buf, 1, sizeof(buf), pipe)) > 0) - data.insert(data.end(), buf, buf + n); - pclose(pipe); - } - } - } + const char *cmd; + if (getenv("WAYLAND_DISPLAY")) { + cmd = "wl-paste 2>/dev/null"; + } else { + cmd = "xclip -selection clipboard -t image/png -o 2>/dev/null"; + } + std::vector data; + FILE *pipe = popen(cmd, "re"); + if (!pipe) return data; + unsigned char buf[65536]; + size_t n; + while ((n = fread(buf, 1, sizeof(buf), pipe)) > 0) { + data.insert(data.end(), buf, buf + n); + } + int st = pclose(pipe); + if (st != 0 || data.empty()) { + data.clear(); + // fallback for X11: try without explicit target + if (!getenv("WAYLAND_DISPLAY")) { + pipe = popen("xclip -selection clipboard -o 2>/dev/null", "re"); + if (pipe) { + while ((n = fread(buf, 1, sizeof(buf), pipe)) > 0) + data.insert(data.end(), buf, buf + n); + pclose(pipe); + } + } + } + return data; } static std::string capture_screenshot() { - const char * tmpdir = getenv("TMPDIR"); - if (!tmpdir) tmpdir = "/tmp"; - std::string path = std::string(tmpdir) + "/ocr_screenshot.png"; + const char *tmpdir = getenv("TMPDIR"); + if (!tmpdir) + tmpdir = "/tmp"; + std::string path = std::string(tmpdir) + "/ocr_screenshot.png"; - fprintf(stderr, "Select a screen region to OCR...\n"); + fprintf(stderr, "Select a screen region to OCR...\n"); - auto check = [&]() -> bool { - struct stat st; - return stat(path.c_str(), &st) == 0 && st.st_size > 0; - }; + auto check = [&]() -> bool { + struct stat st {}; + return stat(path.c_str(), &st) == 0 && st.st_size > 0; + }; - // 1. gnome-screenshot (blocks until selection complete) - unlink(path.c_str()); - if (system(("gnome-screenshot --area -f " + path + " >/dev/null 2>&1").c_str()) == 0 && check()) - return path; + // 1. gnome-screenshot (blocks until selection complete) + unlink(path.c_str()); + if (system(("gnome-screenshot --area -f " + path + " >/dev/null 2>&1") + .c_str()) == 0 && + check()) + return path; - // 2. spectacle (KDE, blocks) - unlink(path.c_str()); - if (system(("spectacle --region -b -o " + path + " >/dev/null 2>&1").c_str()) == 0 && check()) - return path; + // 2. spectacle (KDE, blocks) + unlink(path.c_str()); + if (system( + ("spectacle --region -b -o " + path + " >/dev/null 2>&1").c_str()) == + 0 && + check()) + return path; - // 3. flameshot (writes raw PNG to stdout, blocks) - unlink(path.c_str()); - if (system(("flameshot gui -r > " + path + " 2>/dev/null").c_str()) == 0 && check()) - return path; + // 3. flameshot (writes raw PNG to stdout, blocks) + unlink(path.c_str()); + if (system(("flameshot gui -r > " + path + " 2>/dev/null").c_str()) == 0 && + check()) + return path; - // 4. maim (lightweight X11, blocks) - unlink(path.c_str()); - if (system(("maim -s " + path + " 2>/dev/null").c_str()) == 0 && check()) - return path; + // 4. maim (lightweight X11, blocks) + unlink(path.c_str()); + if (system(("maim -s " + path + " 2>/dev/null").c_str()) == 0 && check()) + return path; - // 5. slurp + grim (Sway/Wayland, blocks) - unlink(path.c_str()); - if (system(("slurp | grim -g - " + path + " 2>/dev/null").c_str()) == 0 && check()) - return path; + // 5. slurp + grim (Sway/Wayland, blocks) + unlink(path.c_str()); + if (system(("slurp | grim -g - " + path + " 2>/dev/null").c_str()) == 0 && + check()) + return path; - return ""; + return ""; } -static void print_usage(const char * prog) { - fprintf(stderr, - "Usage: %s -m --mmproj [options] []\n" - "\n" - "Options:\n" - " -m, --model model file path (GGUF)\n" - " --mmproj,--mm mmproj file path\n" - " -p OCR prompt (default: OCR)\n" - " -t number of threads (default: cpu cores)\n" - " --ngl GPU layers (-1 = all, default: -1)\n" - " -c context size (default: 8192)\n" - " --temp sampling temperature (0 = greedy, default: 0)\n" - " -s random seed\n" - " --json output in JSON format\n" - " --chat-template chat template (default: auto from model)\n" - " --screenshot interactively select screen region to OCR\n" - " -h show this help\n" - "\n" - "If is omitted, reads image from clipboard.\n" - "Requires wl-paste (Wayland) or xclip (X11) for clipboard support.\n" - "\n" - "Examples:\n" - " %s -m model.gguf --mmproj mmproj.gguf -p \"OCR\" image.png\n" - " %s -m model.gguf --mmproj mmproj.gguf # use clipboard image\n" - " %s -m model.gguf --mmproj mmproj.gguf --screenshot # select region\n", - prog, prog, prog, prog); +static void print_usage(const char *prog) { + fprintf( + stderr, + "Usage: %s -m --mmproj [options] []\n" + "\n" + "Options:\n" + " -m, --model model file path (GGUF)\n" + " --mmproj,--mm mmproj file path\n" + " -p OCR prompt (default: OCR)\n" + " -t number of threads (default: cpu cores)\n" + " --ngl GPU layers (-1 = all, default: -1)\n" + " -c context size (default: 8192)\n" + " --temp sampling temperature (0 = greedy, default: 0)\n" + " -s random seed\n" + " --json output in JSON format\n" + " --chat-template chat template (default: auto from model)\n" + " --screenshot interactively select screen region to OCR\n" + " -h show this help\n" + "\n" + "If is omitted, reads image from clipboard.\n" + "Requires wl-paste (Wayland) or xclip (X11) for clipboard support.\n" + "\n" + "Examples:\n" + " %s -m model.gguf --mmproj mmproj.gguf -p \"OCR\" image.png\n" + " %s -m model.gguf --mmproj mmproj.gguf # use clipboard image\n" + " %s -m model.gguf --mmproj mmproj.gguf --screenshot # select " + "region\n", + prog, prog, prog, prog); } -int main(int argc, char ** argv) { - std::string model_path; - std::string mmproj_path; - std::string image_path; - std::string prompt = "OCR"; - std::string chat_template; - bool screenshot_mode = false; - int n_threads = (int)std::thread::hardware_concurrency(); - int n_gpu_layers = -1; - int n_ctx = 8192; - float temp = 0.0f; - uint32_t seed = LLAMA_DEFAULT_SEED; - bool json_output = false; +int main(int argc, char **argv) { + std::string model_path; + std::string mmproj_path; + std::string image_path; + std::string prompt = "OCR"; + std::string chat_template; + bool screenshot_mode = false; + int n_threads = static_cast(std::thread::hardware_concurrency()); + int n_gpu_layers = -1; + int n_ctx = 8192; + float temp = 0.0f; + uint32_t seed = LLAMA_DEFAULT_SEED; + bool json_output = false; - // pre-process -mm to --mmproj so getopt handles it correctly - for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "-mm") == 0) { - argv[i] = (char *)"--mmproj"; - } + while (true) { + static option long_opts[] = { + {"temp", required_argument, nullptr, 0}, + {"json", no_argument, nullptr, 1}, + {"mmproj", required_argument, nullptr, 2}, + {"mm", required_argument, nullptr, 2}, + {"ngl", required_argument, nullptr, 3}, + {"model", required_argument, nullptr, 'm'}, + {"chat-template", required_argument, nullptr, 4}, + {"screenshot", no_argument, nullptr, 5}, + {"help", no_argument, nullptr, 'h'}, + {nullptr, 0, nullptr, 0}}; + int idx = 0; + int c = getopt_long(argc, argv, "m:p:t:c:s:h", long_opts, &idx); + if (c == -1) + break; + switch (c) { + case 0: + temp = std::stof(optarg); + break; + case 1: + json_output = true; + break; + case 2: + mmproj_path = optarg; + break; + case 3: + n_gpu_layers = std::stoi(optarg); + break; + case 4: + chat_template = optarg; + break; + case 5: + screenshot_mode = true; + break; + case 'm': + model_path = optarg; + break; + case 'p': + prompt = optarg; + break; + case 't': + n_threads = std::stoi(optarg); + break; + case 'c': + n_ctx = std::stoi(optarg); + break; + case 's': + seed = static_cast(std::stoul(optarg)); + break; + case 'h': + print_usage(argv[0]); + return 0; + default: + print_usage(argv[0]); + return 1; } + } - while (1) { - static struct option long_opts[] = { - {"temp", required_argument, nullptr, 0}, - {"json", no_argument, nullptr, 1}, - {"mmproj", required_argument, nullptr, 2}, - {"mm", required_argument, nullptr, 2}, - {"ngl", required_argument, nullptr, 3}, - {"model", required_argument, nullptr, 'm'}, - {"chat-template", required_argument, nullptr, 4}, - {"screenshot", no_argument, nullptr, 5}, - {"help", no_argument, nullptr, 'h'}, - {nullptr, 0, nullptr, 0} - }; - int idx = 0; - int c = getopt_long(argc, argv, "m:p:t:c:s:h", long_opts, &idx); - if (c == -1) break; - switch (c) { - case 0: temp = std::stof(optarg); break; - case 1: json_output = true; break; - case 2: mmproj_path = optarg; break; - case 3: n_gpu_layers = std::stoi(optarg); break; - case 4: chat_template = optarg; break; - case 5: screenshot_mode = true; break; - case 'm': model_path = optarg; break; - case 'p': prompt = optarg; break; - case 't': n_threads = std::stoi(optarg); break; - case 'c': n_ctx = std::stoi(optarg); break; - case 's': seed = (uint32_t)std::stoul(optarg); break; - case 'h': print_usage(argv[0]); return 0; - default: print_usage(argv[0]); return 1; - } + if (model_path.empty() || mmproj_path.empty()) { + fprintf(stderr, "ERROR: -m/--model and --mmproj/--mm are required\n"); + return 1; + } + + if (screenshot_mode) { + image_path = capture_screenshot(); + if (image_path.empty()) { + fprintf(stderr, + "ERROR: no screenshot tool found. Install gnome-screenshot, " + "flameshot, spectacle, maim, or slurp+grim.\n"); + return 1; } + } else if (optind < argc) { + image_path = argv[optind]; + } - if (model_path.empty() || mmproj_path.empty()) { - fprintf(stderr, "ERROR: -m/--model and --mmproj/--mm are required\n"); - return 1; + signal(SIGINT, sigint_handler); + + int64_t t_start_us = ggml_time_us(); + + llama_backend_init(); + + llama_model_params mparams = llama_model_default_params(); + mparams.n_gpu_layers = n_gpu_layers; + + llama_model *model = llama_model_load_from_file(model_path.c_str(), mparams); + if (!model) { + fprintf(stderr, "ERROR: failed to load model from %s\n", + model_path.c_str()); + return 1; + } + + llama_context_params cparams = llama_context_default_params(); + cparams.n_ctx = static_cast(n_ctx); + cparams.n_batch = 512; + cparams.n_ubatch = 512; + + llama_context *lctx = llama_init_from_model(model, cparams); + if (!lctx) { + fprintf(stderr, "ERROR: failed to create context\n"); + llama_model_free(model); + return 1; + } + + llama_set_n_threads(lctx, n_threads, n_threads); + + const llama_vocab *vocab = llama_model_get_vocab(model); + + mtmd_context_params mtmd_params = mtmd_context_params_default(); + mtmd_params.use_gpu = (n_gpu_layers != 0); + mtmd_params.n_threads = n_threads; + + mtmd_context *ctx_vision = + mtmd_init_from_file(mmproj_path.c_str(), model, mtmd_params); + if (!ctx_vision) { + fprintf(stderr, "ERROR: failed to load mmproj from %s\n", + mmproj_path.c_str()); + llama_free(lctx); + llama_model_free(model); + return 1; + } + + int64_t t_loaded_us = ggml_time_us(); + + mtmd_bitmap *bmp; + if (!image_path.empty()) { + bmp = mtmd_helper_bitmap_init_from_file(ctx_vision, image_path.c_str()); + if (!bmp) { + fprintf(stderr, "ERROR: failed to load image from %s\n", + image_path.c_str()); + if (screenshot_mode) + unlink(image_path.c_str()); + mtmd_free(ctx_vision); + llama_free(lctx); + llama_model_free(model); + return 1; } - - if (screenshot_mode) { - image_path = capture_screenshot(); - if (image_path.empty()) { - fprintf(stderr, "ERROR: no screenshot tool found. Install gnome-screenshot, flameshot, spectacle, maim, or slurp+grim.\n"); - return 1; - } - } else if (optind < argc) { - image_path = argv[optind]; + } else { + fprintf(stderr, "Reading image from clipboard...\n"); + auto clip = read_clipboard(); + if (clip.empty()) { + fprintf(stderr, + "ERROR: clipboard is empty or no image found.\n" + "Make sure wl-paste (Wayland) or xclip (X11) is installed.\n"); + mtmd_free(ctx_vision); + llama_free(lctx); + llama_model_free(model); + return 1; } - - signal(SIGINT, sigint_handler); - - int64_t t_start_us = ggml_time_us(); - - llama_backend_init(); - - llama_model_params mparams = llama_model_default_params(); - mparams.n_gpu_layers = n_gpu_layers; - - llama_model * model = llama_model_load_from_file(model_path.c_str(), mparams); - if (!model) { - fprintf(stderr, "ERROR: failed to load model from %s\n", model_path.c_str()); - return 1; + bmp = + mtmd_helper_bitmap_init_from_buf(ctx_vision, clip.data(), clip.size()); + if (!bmp) { + fprintf(stderr, "ERROR: failed to decode clipboard image.\n"); + mtmd_free(ctx_vision); + llama_free(lctx); + llama_model_free(model); + return 1; } + } - llama_context_params cparams = llama_context_default_params(); - cparams.n_ctx = (uint32_t)n_ctx; - cparams.n_batch = 512; - cparams.n_ubatch = 512; + // Build user message with image marker placeholder + std::string user_content = std::string(mtmd_default_marker()) + prompt; - llama_context * lctx = llama_init_from_model(model, cparams); - if (!lctx) { - fprintf(stderr, "ERROR: failed to create context\n"); - llama_model_free(model); - return 1; + // Apply chat template for any model + std::string full_prompt; + const char *tmpl = chat_template.empty() + ? llama_model_chat_template(model, nullptr) + : chat_template.c_str(); + if (tmpl) { + llama_chat_message msg[1] = {{"user", user_content.c_str()}}; + char buf[8192]; + int32_t n = llama_chat_apply_template(tmpl, msg, 1, true, buf, sizeof(buf)); + if (n > 0) { + full_prompt = std::string(buf, n); } + } - llama_set_n_threads(lctx, n_threads, n_threads); + // Fallback: use chatml format for models without recognizable template + if (full_prompt.empty()) { + full_prompt = "<|im_start|>user\n" + user_content + + "<|im_end|>\n<|im_start|>assistant\n"; + } - const struct llama_vocab * vocab = llama_model_get_vocab(model); + // Template output already includes role markers and special tokens (e.g. + // [gMASK]) So we don't add BOS separately — add_special=false + mtmd_input_chunks *chunks = mtmd_input_chunks_init(); - mtmd_context_params mtmd_params = mtmd_context_params_default(); - mtmd_params.use_gpu = (n_gpu_layers != 0); - mtmd_params.n_threads = n_threads; + mtmd_input_text in_text; + in_text.text = full_prompt.c_str(); + in_text.add_special = false; + in_text.parse_special = true; - mtmd_context * ctx_vision = mtmd_init_from_file(mmproj_path.c_str(), model, mtmd_params); - if (!ctx_vision) { - fprintf(stderr, "ERROR: failed to load mmproj from %s\n", mmproj_path.c_str()); - llama_free(lctx); - llama_model_free(model); - return 1; - } - - int64_t t_loaded_us = ggml_time_us(); - - mtmd_bitmap * bmp = nullptr; - if (!image_path.empty()) { - bmp = mtmd_helper_bitmap_init_from_file(ctx_vision, image_path.c_str()); - if (!bmp) { - fprintf(stderr, "ERROR: failed to load image from %s\n", image_path.c_str()); - if (screenshot_mode) unlink(image_path.c_str()); - mtmd_free(ctx_vision); - llama_free(lctx); - llama_model_free(model); - return 1; - } - } else { - fprintf(stderr, "Reading image from clipboard...\n"); - auto clip = read_clipboard(); - if (clip.empty()) { - fprintf(stderr, "ERROR: clipboard is empty or no image found.\n" - "Make sure wl-paste (Wayland) or xclip (X11) is installed.\n"); - mtmd_free(ctx_vision); - llama_free(lctx); - llama_model_free(model); - return 1; - } - bmp = mtmd_helper_bitmap_init_from_buf(ctx_vision, clip.data(), clip.size()); - if (!bmp) { - fprintf(stderr, "ERROR: failed to decode clipboard image.\n"); - mtmd_free(ctx_vision); - llama_free(lctx); - llama_model_free(model); - return 1; - } - } - - // Build user message with image marker placeholder - std::string user_content = std::string(mtmd_default_marker()) + prompt; - - // Apply chat template for any model - std::string full_prompt; - const char * tmpl = chat_template.empty() - ? llama_model_chat_template(model, nullptr) - : chat_template.c_str(); - if (tmpl) { - llama_chat_message msg[1] = {{"user", user_content.c_str()}}; - char buf[8192]; - int32_t n = llama_chat_apply_template(tmpl, msg, 1, true, buf, sizeof(buf)); - if (n > 0) { - full_prompt = std::string(buf, n); - } - } - - // Fallback: use chatml format for models without recognizable template - if (full_prompt.empty()) { - full_prompt = "<|im_start|>user\n" + user_content + "<|im_end|>\n<|im_start|>assistant\n"; - } - - // Template output already includes role markers and special tokens (e.g. [gMASK]) - // So we don't add BOS separately — add_special=false - mtmd_input_chunks * chunks = mtmd_input_chunks_init(); - - mtmd_input_text in_text; - in_text.text = full_prompt.c_str(); - in_text.add_special = false; - in_text.parse_special = true; - - const mtmd_bitmap * bitmaps[1] = { bmp }; - - int32_t res = mtmd_tokenize(ctx_vision, chunks, &in_text, bitmaps, 1); - if (res != 0) { - fprintf(stderr, "ERROR: mtmd_tokenize failed (%d)\n", res); - mtmd_input_chunks_free(chunks); - mtmd_bitmap_free(bmp); - if (screenshot_mode) unlink(image_path.c_str()); - mtmd_free(ctx_vision); - llama_free(lctx); - llama_model_free(model); - return 1; - } - - llama_pos n_past = 0; - res = mtmd_helper_eval_chunks(ctx_vision, lctx, chunks, n_past, 0, 512, true, &n_past); - if (res != 0) { - fprintf(stderr, "ERROR: mtmd_helper_eval_chunks failed (%d)\n", res); - mtmd_input_chunks_free(chunks); - mtmd_bitmap_free(bmp); - if (screenshot_mode) unlink(image_path.c_str()); - mtmd_free(ctx_vision); - llama_free(lctx); - llama_model_free(model); - return 1; - } + const mtmd_bitmap *bitmaps[1] = {bmp}; + int32_t res = mtmd_tokenize(ctx_vision, chunks, &in_text, bitmaps, 1); + if (res != 0) { + fprintf(stderr, "ERROR: mtmd_tokenize failed (%d)\n", res); mtmd_input_chunks_free(chunks); mtmd_bitmap_free(bmp); - if (screenshot_mode) unlink(image_path.c_str()); - - struct llama_sampler * smpl = nullptr; - if (temp <= 0.0f) { - struct llama_sampler_chain_params sparams = llama_sampler_chain_default_params(); - smpl = llama_sampler_chain_init(sparams); - llama_sampler_chain_add(smpl, llama_sampler_init_greedy()); - } else { - struct llama_sampler_chain_params sparams = llama_sampler_chain_default_params(); - smpl = llama_sampler_chain_init(sparams); - llama_sampler_chain_add(smpl, llama_sampler_init_top_k(40)); - llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.9f, 1)); - llama_sampler_chain_add(smpl, llama_sampler_init_temp(temp)); - llama_sampler_chain_add(smpl, llama_sampler_init_dist(seed)); - } - - struct llama_batch batch = llama_batch_init(1, 0, 1); - - int64_t t_infer_start_us = ggml_time_us(); - std::string ocr_text; - int n_generated = 0; - - for (int i = 0; i < 2048; i++) { - if (g_interrupted) break; - - llama_token token_id = llama_sampler_sample(smpl, lctx, -1); - llama_sampler_accept(smpl, token_id); - - if (llama_vocab_is_eog(vocab, token_id)) break; - - std::string piece = token_to_piece(vocab, token_id); - ocr_text += piece; - - if (!json_output) { - printf("%s", piece.c_str()); - fflush(stdout); - } - - batch_clear(batch); - batch_add(batch, token_id, n_past++, {0}, true); - - if (llama_decode(lctx, batch) != 0) { - fprintf(stderr, "\nERROR: llama_decode failed\n"); - break; - } - - n_generated++; - } - - int64_t t_end_us = ggml_time_us(); - - if (!json_output) { - printf("\n"); - } - - llama_batch_free(batch); - llama_sampler_free(smpl); + if (screenshot_mode) + unlink(image_path.c_str()); mtmd_free(ctx_vision); llama_free(lctx); llama_model_free(model); - llama_backend_free(); + return 1; + } - if (json_output) { - printf("{\n"); - printf(" \"text\": \"%s\",\n", json_escape(ocr_text).c_str()); - printf(" \"meta\": {\n"); - printf(" \"model\": \"%s\",\n", json_escape(model_path).c_str()); - printf(" \"mmproj\": \"%s\",\n", json_escape(mmproj_path).c_str()); - printf(" \"prompt\": \"%s\",\n", json_escape(prompt).c_str()); - printf(" \"n_prompt_tokens\": %" PRId64 ",\n", (int64_t)n_past); - printf(" \"n_generated\": %d,\n", n_generated); - printf(" \"timings_ms\": {\n"); - printf(" \"load\": %" PRId64 ",\n", (t_loaded_us - t_start_us) / 1000); - printf(" \"inference\": %" PRId64 "\n", (t_end_us - t_infer_start_us) / 1000); - printf(" }\n"); - printf(" }\n"); - printf("}\n"); + llama_pos n_past = 0; + res = mtmd_helper_eval_chunks(ctx_vision, lctx, chunks, n_past, 0, 512, true, + &n_past); + if (res != 0) { + fprintf(stderr, "ERROR: mtmd_helper_eval_chunks failed (%d)\n", res); + mtmd_input_chunks_free(chunks); + mtmd_bitmap_free(bmp); + if (screenshot_mode) + unlink(image_path.c_str()); + mtmd_free(ctx_vision); + llama_free(lctx); + llama_model_free(model); + return 1; + } + + mtmd_input_chunks_free(chunks); + mtmd_bitmap_free(bmp); + if (screenshot_mode) + unlink(image_path.c_str()); + + llama_sampler *smpl; + if (temp <= 0.0f) { + llama_sampler_chain_params sparams = llama_sampler_chain_default_params(); + smpl = llama_sampler_chain_init(sparams); + llama_sampler_chain_add(smpl, llama_sampler_init_greedy()); + } else { + llama_sampler_chain_params sparams = llama_sampler_chain_default_params(); + smpl = llama_sampler_chain_init(sparams); + llama_sampler_chain_add(smpl, llama_sampler_init_top_k(40)); + llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.9f, 1)); + llama_sampler_chain_add(smpl, llama_sampler_init_temp(temp)); + llama_sampler_chain_add(smpl, llama_sampler_init_dist(seed)); + } + + llama_batch batch = llama_batch_init(1, 0, 1); + + int64_t t_infer_start_us = ggml_time_us(); + std::string ocr_text; + int n_generated = 0; + + for (int i = 0; i < 2048; i++) { + if (g_interrupted) + break; + + llama_token token_id = llama_sampler_sample(smpl, lctx, -1); + llama_sampler_accept(smpl, token_id); + + if (llama_vocab_is_eog(vocab, token_id)) + break; + + std::string piece = token_to_piece(vocab, token_id); + ocr_text += piece; + + if (!json_output) { + printf("%s", piece.c_str()); + fflush(stdout); } - return g_interrupted ? 130 : 0; + batch_clear(batch); + batch_add(batch, token_id, n_past++, {0}, true); + + if (llama_decode(lctx, batch) != 0) { + fprintf(stderr, "\nERROR: llama_decode failed\n"); + break; + } + + n_generated++; + } + + int64_t t_end_us = ggml_time_us(); + + if (!json_output) { + printf("\n"); + } + + llama_batch_free(batch); + llama_sampler_free(smpl); + mtmd_free(ctx_vision); + llama_free(lctx); + llama_model_free(model); + llama_backend_free(); + + if (json_output) { + printf("{\n"); + printf(" \"text\": \"%s\",\n", json_escape(ocr_text).c_str()); + printf(" \"meta\": {\n"); + printf(" \"model\": \"%s\",\n", json_escape(model_path).c_str()); + printf(" \"mmproj\": \"%s\",\n", json_escape(mmproj_path).c_str()); + printf(" \"prompt\": \"%s\",\n", json_escape(prompt).c_str()); + printf(" \"n_prompt_tokens\": %" PRId64 ",\n", + static_cast(n_past)); + printf(" \"n_generated\": %d,\n", n_generated); + printf(" \"timings_ms\": {\n"); + printf(" \"load\": %" PRId64 ",\n", (t_loaded_us - t_start_us) / 1000); + printf(" \"inference\": %" PRId64 "\n", + (t_end_us - t_infer_start_us) / 1000); + printf(" }\n"); + printf(" }\n"); + printf("}\n"); + } + + return g_interrupted ? 130 : 0; }