wangjue/llm-ocr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: support reading image from clipboard when no image_path is given

Browse Source 
If <image_path> is omitted, reads image from clipboard using
wl-paste (Wayland) or xclip (X11), then passes the raw data to
mtmd_helper_bitmap_init_from_buf for decoding.
This commit is contained in:
wangjue 2026-05-29 14:59:22 +08:00
parent cdcd1f7202
commit 5180d7b508
1 changed files with 72 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
main.cpp
View File

@ -65,9 +65,40 @@ static std::string json_escape(const std::string & s) {
return out;
}
static std::vector<unsigned char> read_clipboard() {
const char * cmd = nullptr;
if (getenv("WAYLAND_DISPLAY")) {
cmd = "wl-paste 2>/dev/null";
} else {
cmd = "xclip -selection clipboard -t image/png -o 2>/dev/null";
}
std::vector<unsigned char> data;
FILE * pipe = popen(cmd, "re");
if (!pipe) return data;
unsigned char buf[65536];
size_t n;
while ((n = fread(buf, 1, sizeof(buf), pipe)) > 0) {
data.insert(data.end(), buf, buf + n);
}
int st = pclose(pipe);
if (st != 0 || data.empty()) {
data.clear();
// fallback for X11: try without explicit target
if (!getenv("WAYLAND_DISPLAY")) {
pipe = popen("xclip -selection clipboard -o 2>/dev/null", "re");
if (pipe) {
while ((n = fread(buf, 1, sizeof(buf), pipe)) > 0)
data.insert(data.end(), buf, buf + n);
pclose(pipe);
}
}
}
return data;
}
static void print_usage(const char * prog) {
fprintf(stderr,
"Usage: %s -m <model> --mmproj <mmproj> [options] <image_path>\n"
"Usage: %s -m <model> --mmproj <mmproj> [options] [<image_path>]\n"
"\n"
"Options:\n"
" -m, --model <path> model file path (GGUF)\n"
@ -81,9 +112,13 @@ static void print_usage(const char * prog) {
" --json output in JSON format\n"
" -h show this help\n"
"\n"
"Example:\n"
" %s -m model.gguf --mmproj mmproj.gguf -p \"OCR\" image.png\n",
prog, prog);
"If <image_path> is omitted, reads image from clipboard.\n"
"Requires wl-paste (Wayland) or xclip (X11) for clipboard support.\n"
"\n"
"Examples:\n"
" %s -m model.gguf --mmproj mmproj.gguf -p \"OCR\" image.png\n"
" %s -m model.gguf --mmproj mmproj.gguf # use clipboard image\n",
prog, prog, prog);
}
int main(int argc, char ** argv) {
@ -134,17 +169,15 @@ int main(int argc, char ** argv) {
}
}
if ((optind + 1) != argc) {
print_usage(argv[0]);
return 1;
}
image_path = argv[optind];
if (model_path.empty() || mmproj_path.empty()) {
fprintf(stderr, "ERROR: -m/--model and --mmproj/--mm are required\n");
return 1;
}
if (optind < argc) {
image_path = argv[optind];
}
signal(SIGINT, sigint_handler);
int64_t t_start_us = ggml_time_us();
@ -190,13 +223,35 @@ int main(int argc, char ** argv) {
int64_t t_loaded_us = ggml_time_us();
mtmd_bitmap * bmp = mtmd_helper_bitmap_init_from_file(ctx_vision, image_path.c_str());
if (!bmp) {
fprintf(stderr, "ERROR: failed to load image from %s\n", image_path.c_str());
mtmd_free(ctx_vision);
llama_free(lctx);
llama_model_free(model);
return 1;
mtmd_bitmap * bmp = nullptr;
if (!image_path.empty()) {
bmp = mtmd_helper_bitmap_init_from_file(ctx_vision, image_path.c_str());
if (!bmp) {
fprintf(stderr, "ERROR: failed to load image from %s\n", image_path.c_str());
mtmd_free(ctx_vision);
llama_free(lctx);
llama_model_free(model);
return 1;
}
} else {
fprintf(stderr, "Reading image from clipboard...\n");
auto clip = read_clipboard();
if (clip.empty()) {
fprintf(stderr, "ERROR: clipboard is empty or no image found.\n"
"Make sure wl-paste (Wayland) or xclip (X11) is installed.\n");
mtmd_free(ctx_vision);
llama_free(lctx);
llama_model_free(model);
return 1;
}
bmp = mtmd_helper_bitmap_init_from_buf(ctx_vision, clip.data(), clip.size());
if (!bmp) {
fprintf(stderr, "ERROR: failed to decode clipboard image.\n");
mtmd_free(ctx_vision);
llama_free(lctx);
llama_model_free(model);
return 1;
}
}
// Construct prompt using GLM-OCR chat template format