feat: support reading image from clipboard when no image_path is given
If <image_path> is omitted, reads image from clipboard using wl-paste (Wayland) or xclip (X11), then passes the raw data to mtmd_helper_bitmap_init_from_buf for decoding.
This commit is contained in:
parent
cdcd1f7202
commit
5180d7b508
89
main.cpp
89
main.cpp
@ -65,9 +65,40 @@ static std::string json_escape(const std::string & s) {
|
|||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static std::vector<unsigned char> read_clipboard() {
|
||||||
|
const char * cmd = nullptr;
|
||||||
|
if (getenv("WAYLAND_DISPLAY")) {
|
||||||
|
cmd = "wl-paste 2>/dev/null";
|
||||||
|
} else {
|
||||||
|
cmd = "xclip -selection clipboard -t image/png -o 2>/dev/null";
|
||||||
|
}
|
||||||
|
std::vector<unsigned char> data;
|
||||||
|
FILE * pipe = popen(cmd, "re");
|
||||||
|
if (!pipe) return data;
|
||||||
|
unsigned char buf[65536];
|
||||||
|
size_t n;
|
||||||
|
while ((n = fread(buf, 1, sizeof(buf), pipe)) > 0) {
|
||||||
|
data.insert(data.end(), buf, buf + n);
|
||||||
|
}
|
||||||
|
int st = pclose(pipe);
|
||||||
|
if (st != 0 || data.empty()) {
|
||||||
|
data.clear();
|
||||||
|
// fallback for X11: try without explicit target
|
||||||
|
if (!getenv("WAYLAND_DISPLAY")) {
|
||||||
|
pipe = popen("xclip -selection clipboard -o 2>/dev/null", "re");
|
||||||
|
if (pipe) {
|
||||||
|
while ((n = fread(buf, 1, sizeof(buf), pipe)) > 0)
|
||||||
|
data.insert(data.end(), buf, buf + n);
|
||||||
|
pclose(pipe);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
static void print_usage(const char * prog) {
|
static void print_usage(const char * prog) {
|
||||||
fprintf(stderr,
|
fprintf(stderr,
|
||||||
"Usage: %s -m <model> --mmproj <mmproj> [options] <image_path>\n"
|
"Usage: %s -m <model> --mmproj <mmproj> [options] [<image_path>]\n"
|
||||||
"\n"
|
"\n"
|
||||||
"Options:\n"
|
"Options:\n"
|
||||||
" -m, --model <path> model file path (GGUF)\n"
|
" -m, --model <path> model file path (GGUF)\n"
|
||||||
@ -81,9 +112,13 @@ static void print_usage(const char * prog) {
|
|||||||
" --json output in JSON format\n"
|
" --json output in JSON format\n"
|
||||||
" -h show this help\n"
|
" -h show this help\n"
|
||||||
"\n"
|
"\n"
|
||||||
"Example:\n"
|
"If <image_path> is omitted, reads image from clipboard.\n"
|
||||||
" %s -m model.gguf --mmproj mmproj.gguf -p \"OCR\" image.png\n",
|
"Requires wl-paste (Wayland) or xclip (X11) for clipboard support.\n"
|
||||||
prog, prog);
|
"\n"
|
||||||
|
"Examples:\n"
|
||||||
|
" %s -m model.gguf --mmproj mmproj.gguf -p \"OCR\" image.png\n"
|
||||||
|
" %s -m model.gguf --mmproj mmproj.gguf # use clipboard image\n",
|
||||||
|
prog, prog, prog);
|
||||||
}
|
}
|
||||||
|
|
||||||
int main(int argc, char ** argv) {
|
int main(int argc, char ** argv) {
|
||||||
@ -134,17 +169,15 @@ int main(int argc, char ** argv) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if ((optind + 1) != argc) {
|
|
||||||
print_usage(argv[0]);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
image_path = argv[optind];
|
|
||||||
|
|
||||||
if (model_path.empty() || mmproj_path.empty()) {
|
if (model_path.empty() || mmproj_path.empty()) {
|
||||||
fprintf(stderr, "ERROR: -m/--model and --mmproj/--mm are required\n");
|
fprintf(stderr, "ERROR: -m/--model and --mmproj/--mm are required\n");
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (optind < argc) {
|
||||||
|
image_path = argv[optind];
|
||||||
|
}
|
||||||
|
|
||||||
signal(SIGINT, sigint_handler);
|
signal(SIGINT, sigint_handler);
|
||||||
|
|
||||||
int64_t t_start_us = ggml_time_us();
|
int64_t t_start_us = ggml_time_us();
|
||||||
@ -190,13 +223,35 @@ int main(int argc, char ** argv) {
|
|||||||
|
|
||||||
int64_t t_loaded_us = ggml_time_us();
|
int64_t t_loaded_us = ggml_time_us();
|
||||||
|
|
||||||
mtmd_bitmap * bmp = mtmd_helper_bitmap_init_from_file(ctx_vision, image_path.c_str());
|
mtmd_bitmap * bmp = nullptr;
|
||||||
if (!bmp) {
|
if (!image_path.empty()) {
|
||||||
fprintf(stderr, "ERROR: failed to load image from %s\n", image_path.c_str());
|
bmp = mtmd_helper_bitmap_init_from_file(ctx_vision, image_path.c_str());
|
||||||
mtmd_free(ctx_vision);
|
if (!bmp) {
|
||||||
llama_free(lctx);
|
fprintf(stderr, "ERROR: failed to load image from %s\n", image_path.c_str());
|
||||||
llama_model_free(model);
|
mtmd_free(ctx_vision);
|
||||||
return 1;
|
llama_free(lctx);
|
||||||
|
llama_model_free(model);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fprintf(stderr, "Reading image from clipboard...\n");
|
||||||
|
auto clip = read_clipboard();
|
||||||
|
if (clip.empty()) {
|
||||||
|
fprintf(stderr, "ERROR: clipboard is empty or no image found.\n"
|
||||||
|
"Make sure wl-paste (Wayland) or xclip (X11) is installed.\n");
|
||||||
|
mtmd_free(ctx_vision);
|
||||||
|
llama_free(lctx);
|
||||||
|
llama_model_free(model);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
bmp = mtmd_helper_bitmap_init_from_buf(ctx_vision, clip.data(), clip.size());
|
||||||
|
if (!bmp) {
|
||||||
|
fprintf(stderr, "ERROR: failed to decode clipboard image.\n");
|
||||||
|
mtmd_free(ctx_vision);
|
||||||
|
llama_free(lctx);
|
||||||
|
llama_model_free(model);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Construct prompt using GLM-OCR chat template format
|
// Construct prompt using GLM-OCR chat template format
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user