Merge pull request #1441 from imgproxy/img-12-experiment-close-callback

img-12: load JPEG from source (bytes.Reader) instead of memory

vips/source.c

@@ -0,0 +1,76 @@
+#include "source.h"
+
+// define glib subtype for vips async source
+#define VIPS_TYPE_IMGPROXY_SOURCE (vips_imgproxy_source_get_type())
+G_DEFINE_FINAL_TYPE(VipsImgproxySource, vips_imgproxy_source, VIPS_TYPE_SOURCE)
+
+extern void closeImgproxyReader(uintptr_t handle);
+extern gint64 imgproxyReaderSeek(uintptr_t handle, gint64 offset, int whence);
+extern gint64 imgproxyReaderRead(uintptr_t handle, gpointer buffer, gint64 size);
+
+// dereferences source
+void
+unref_imgproxy_source(VipsImgproxySource *source)
+{
+  VIPS_UNREF(source);
+}
+
+// read function for vips imgproxy source
+static gint64
+vips_imgproxy_source_read(VipsSource *source, void *buffer, size_t length)
+{
+  VipsImgproxySource *self = (VipsImgproxySource *) source;
+
+  return imgproxyReaderRead(self->readerHandle, buffer, length);
+}
+
+// seek function for vips imgproxy source. whence can be SEEK_SET (0), SEEK_CUR (1), or SEEK_END (2).
+static gint64
+vips_imgproxy_source_seek(VipsSource *source, gint64 offset, int whence)
+{
+  VipsImgproxySource *self = (VipsImgproxySource *) source;
+
+  return imgproxyReaderSeek(self->readerHandle, offset, whence);
+}
+
+static void
+vips_imgproxy_source_dispose(GObject *gobject)
+{
+  VipsImgproxySource *source = (VipsImgproxySource *) gobject;
+
+  closeImgproxyReader(source->readerHandle);
+
+  G_OBJECT_CLASS(vips_imgproxy_source_parent_class)->dispose(gobject);
+}
+
+// attaches seek/read handlers to the imgproxy source class
+static void
+vips_imgproxy_source_class_init(VipsImgproxySourceClass *klass)
+{
+  GObjectClass *gobject_class = G_OBJECT_CLASS(klass);
+  VipsObjectClass *object_class = VIPS_OBJECT_CLASS(klass);
+  VipsSourceClass *source_class = VIPS_SOURCE_CLASS(klass);
+
+  object_class->nickname = "imgproxy_source";
+  object_class->description = "imgproxy input source";
+
+  gobject_class->dispose = vips_imgproxy_source_dispose;
+
+  source_class->read = vips_imgproxy_source_read;
+  source_class->seek = vips_imgproxy_source_seek;
+}
+
+// initializes the imgproxy source (nothing to do here yet)
+static void
+vips_imgproxy_source_init(VipsImgproxySource *source)
+{
+}
+
+// creates a new imgproxy source with the given reader handle
+VipsImgproxySource *
+vips_new_imgproxy_source(uintptr_t readerHandle)
+{
+  VipsImgproxySource *source = g_object_new(VIPS_TYPE_IMGPROXY_SOURCE, NULL);
+  source->readerHandle = readerHandle;
+  return source;
+}

vips/source.go

@@ -0,0 +1,70 @@
+package vips
+
+/*
+#cgo pkg-config: vips
+#cgo CFLAGS: -O3
+#cgo LDFLAGS: -lm
+#include "source.h"
+#include "vips.h"
+*/
+import "C"
+import (
+	"io"
+	"runtime/cgo"
+	"unsafe"
+)
+
+//export closeImgproxyReader
+func closeImgproxyReader(handle C.uintptr_t) {
+	h := cgo.Handle(handle)
+	h.Delete()
+}
+
+// calls seek() on the async reader via it's handle from the C side
+//
+//export imgproxyReaderSeek
+func imgproxyReaderSeek(handle C.uintptr_t, offset C.int64_t, whence int) C.int64_t {
+	h := cgo.Handle(handle)
+	reader, ok := h.Value().(io.ReadSeeker)
+	if !ok {
+		vipsError("imgproxyReaderSeek", "failed to cast handle to io.ReadSeeker")
+		return -1
+	}
+
+	pos, err := reader.Seek(int64(offset), whence)
+	if err != nil {
+		vipsError("imgproxyReaderSeek", "failed to seek: %v", err)
+		return -1
+	}
+
+	return C.int64_t(pos)
+}
+
+// calls read() on the async reader via it's handle from the C side
+//
+//export imgproxyReaderRead
+func imgproxyReaderRead(handle C.uintptr_t, pointer unsafe.Pointer, size C.int64_t) C.int64_t {
+	h := cgo.Handle(handle)
+	reader, ok := h.Value().(io.ReadSeeker)
+	if !ok {
+		vipsError("imgproxyReaderRead", "invalid reader handle")
+		return -1
+	}
+
+	buf := unsafe.Slice((*byte)(pointer), size)
+	n, err := reader.Read(buf)
+	if err == io.EOF {
+		return 0
+	} else if err != nil {
+		vipsError("imgproxyReaderRead", "error reading from imgproxy source: %v", err)
+		return -1
+	}
+
+	return C.int64_t(n)
+}
+
+// newVipsSource creates a new VipsAsyncSource from an io.ReadSeeker.
+func newVipsImgproxySource(r io.ReadSeeker) *C.VipsImgproxySource {
+	handler := cgo.NewHandle(r)
+	return C.vips_new_imgproxy_source(C.uintptr_t(handler))
+}

vips/source.h

@@ -0,0 +1,27 @@
+#include <stdlib.h>
+#include <stdint.h> // uintptr_t
+
+#include <vips/vips.h>
+#include <vips/connection.h>
+
+#ifndef VIPS_IMGPROXY_SOURCE_H
+#define VIPS_IMGPROXY_SOURCE_H
+
+// vips async source
+typedef struct _VipsImgproxySource {
+  VipsSource source;      // class designator
+  uintptr_t readerHandle; // async reader handler
+} VipsImgproxySource;
+
+// glib class for vips async source
+typedef struct _VipsImgproxySourceClass {
+  VipsSourceClass parent_class;
+} VipsImgproxySourceClass;
+
+// creates new vips async source from a reader handle
+VipsImgproxySource *vips_new_imgproxy_source(uintptr_t readerHandle);
+
+#endif
+
+// unreferences the source, which leads to reader close
+void unref_imgproxy_source(VipsImgproxySource *source);

vips/vips.c

@@ -55,37 +55,39 @@ vips_health()
   return res;
 }
 
+// loads jpeg from a source
 int
-vips_jpegload_go(void *buf, size_t len, int shrink, VipsImage **out)
+vips_jpegload_source_go(VipsImgproxySource *source, int shrink, VipsImage **out)
 {
   if (shrink > 1)
-    return vips_jpegload_buffer(buf, len, out, "access", VIPS_ACCESS_SEQUENTIAL, "shrink", shrink,
+    return vips_jpegload_source(VIPS_SOURCE(source), out, "access", VIPS_ACCESS_SEQUENTIAL, "shrink", shrink,
         NULL);
 
-  return vips_jpegload_buffer(buf, len, out, "access", VIPS_ACCESS_SEQUENTIAL, NULL);
+  return vips_jpegload_source(VIPS_SOURCE(source), out, "access", VIPS_ACCESS_SEQUENTIAL, NULL);
 }
 
+// loads xjl from source
 int
-vips_jxlload_go(void *buf, size_t len, int pages, VipsImage **out)
+vips_jxlload_source_go(VipsImgproxySource *source, int pages, VipsImage **out)
 {
-  return vips_jxlload_buffer(buf, len, out, "access", VIPS_ACCESS_SEQUENTIAL, "n", pages, NULL);
+  return vips_jxlload_source(VIPS_SOURCE(source), out, "access", VIPS_ACCESS_SEQUENTIAL, "n", pages, NULL);
 }
 
 int
-vips_pngload_go(void *buf, size_t len, VipsImage **out, int unlimited)
+vips_pngload_source_go(VipsImgproxySource *source, VipsImage **out, int unlimited)
 {
-  return vips_pngload_buffer(
-      buf, len, out,
+  return vips_pngload_source(
+      VIPS_SOURCE(source), out,
       "access", VIPS_ACCESS_SEQUENTIAL,
       "unlimited", unlimited,
       NULL);
 }
 
 int
-vips_webpload_go(void *buf, size_t len, double scale, int pages, VipsImage **out)
+vips_webpload_source_go(VipsImgproxySource *source, double scale, int pages, VipsImage **out)
 {
-  return vips_webpload_buffer(
-      buf, len, out,
+  return vips_webpload_source(
+      VIPS_SOURCE(source), out,
       "access", VIPS_ACCESS_SEQUENTIAL,
       "scale", scale,
       "n", pages,
@@ -93,13 +95,13 @@ vips_webpload_go(void *buf, size_t len, double scale, int pages, VipsImage **out
 }
 
 int
-vips_gifload_go(void *buf, size_t len, int pages, VipsImage **out)
+vips_gifload_source_go(VipsImgproxySource *source, int pages, VipsImage **out)
 {
-  return vips_gifload_buffer(buf, len, out, "access", VIPS_ACCESS_SEQUENTIAL, "n", pages, NULL);
+  return vips_gifload_source(VIPS_SOURCE(source), out, "access", VIPS_ACCESS_SEQUENTIAL, "n", pages, NULL);
 }
 
 int
-vips_svgload_go(void *buf, size_t len, double scale, VipsImage **out, int unlimited)
+vips_svgload_source_go(VipsImgproxySource *source, double scale, VipsImage **out, int unlimited)
 {
   // libvips limits the minimal scale to 0.001, so we have to scale down dpi
   // for lower scale values
@@ -109,8 +111,8 @@ vips_svgload_go(void *buf, size_t len, double scale, VipsImage **out, int unlimi
     scale = 0.001;
   }
 
-  return vips_svgload_buffer(
-      buf, len, out,
+  return vips_svgload_source(
+      VIPS_SOURCE(source), out,
       "access", VIPS_ACCESS_SEQUENTIAL,
       "scale", scale,
       "dpi", dpi,
@@ -119,19 +121,19 @@ vips_svgload_go(void *buf, size_t len, double scale, VipsImage **out, int unlimi
 }
 
 int
-vips_heifload_go(void *buf, size_t len, VipsImage **out, int thumbnail)
+vips_heifload_source_go(VipsImgproxySource *source, VipsImage **out, int thumbnail)
 {
-  return vips_heifload_buffer(
-      buf, len, out,
+  return vips_heifload_source(
+      VIPS_SOURCE(source), out,
       "access", VIPS_ACCESS_SEQUENTIAL,
       "thumbnail", thumbnail,
       NULL);
 }
 
 int
-vips_tiffload_go(void *buf, size_t len, VipsImage **out)
+vips_tiffload_source_go(VipsImgproxySource *source, VipsImage **out)
 {
-  return vips_tiffload_buffer(buf, len, out, "access", VIPS_ACCESS_SEQUENTIAL, NULL);
+  return vips_tiffload_source(VIPS_SOURCE(source), out, "access", VIPS_ACCESS_SEQUENTIAL, NULL);
 }
 
 int
@@ -1118,3 +1120,9 @@ vips_cleanup()
   vips_error_clear();
   vips_thread_shutdown();
 }
+
+void
+vips_error_go(const char *function, const char *message)
+{
+  vips_error(function, "%s", message);
+}

vips/vips.go

@@ -5,10 +5,13 @@ package vips
 #cgo CFLAGS: -O3
 #cgo LDFLAGS: -lm
 #include "vips.h"
+#include "source.h"
 */
 import "C"
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"math"
 	"net/http"
 	"os"
@@ -358,27 +361,29 @@ func (img *Image) Load(imgdata *imagedata.ImageData, shrink int, scale float64,
 
 	var tmp *C.VipsImage
 
-	data := unsafe.Pointer(&imgdata.Data[0])
-	dataSize := C.size_t(len(imgdata.Data))
 	err := C.int(0)
 
+	reader := bytes.NewReader(imgdata.Data)
+	source := newVipsImgproxySource(reader)
+	defer C.unref_imgproxy_source(source)
+
 	switch imgdata.Type {
 	case imagetype.JPEG:
-		err = C.vips_jpegload_go(data, dataSize, C.int(shrink), &tmp)
+		err = C.vips_jpegload_source_go(source, C.int(shrink), &tmp)
 	case imagetype.JXL:
-		err = C.vips_jxlload_go(data, dataSize, C.int(pages), &tmp)
+		err = C.vips_jxlload_source_go(source, C.int(pages), &tmp)
 	case imagetype.PNG:
-		err = C.vips_pngload_go(data, dataSize, &tmp, vipsConf.PngUnlimited)
+		err = C.vips_pngload_source_go(source, &tmp, vipsConf.PngUnlimited)
 	case imagetype.WEBP:
-		err = C.vips_webpload_go(data, dataSize, C.double(scale), C.int(pages), &tmp)
+		err = C.vips_webpload_source_go(source, C.double(scale), C.int(pages), &tmp)
 	case imagetype.GIF:
-		err = C.vips_gifload_go(data, dataSize, C.int(pages), &tmp)
+		err = C.vips_gifload_source_go(source, C.int(pages), &tmp)
 	case imagetype.SVG:
-		err = C.vips_svgload_go(data, dataSize, C.double(scale), &tmp, vipsConf.SvgUnlimited)
+		err = C.vips_svgload_source_go(source, C.double(scale), &tmp, vipsConf.SvgUnlimited)
 	case imagetype.HEIC, imagetype.AVIF:
-		err = C.vips_heifload_go(data, dataSize, &tmp, C.int(0))
+		err = C.vips_heifload_source_go(source, &tmp, C.int(0))
 	case imagetype.TIFF:
-		err = C.vips_tiffload_go(data, dataSize, &tmp)
+		err = C.vips_tiffload_source_go(source, &tmp)
 	default:
 		return newVipsError("Usupported image type to load")
 	}
@@ -406,10 +411,11 @@ func (img *Image) LoadThumbnail(imgdata *imagedata.ImageData) error {
 
 	var tmp *C.VipsImage
 
-	data := unsafe.Pointer(&imgdata.Data[0])
-	dataSize := C.size_t(len(imgdata.Data))
+	reader := bytes.NewReader(imgdata.Data)
+	source := newVipsImgproxySource(reader)
+	defer C.unref_imgproxy_source(source)
 
-	if err := C.vips_heifload_go(data, dataSize, &tmp, C.int(1)); err != 0 {
+	if err := C.vips_heifload_source_go(source, &tmp, C.int(1)); err != 0 {
 		return Error()
 	}
 
@@ -964,3 +970,15 @@ func (img *Image) StripAll() error {
 
 	return nil
 }
+
+func vipsError(fn string, msg string, args ...any) {
+	fnStr := C.CString(fn)
+	defer C.free(unsafe.Pointer(fnStr))
+
+	msg = fmt.Sprintf(msg, args...)
+
+	msgStr := C.CString(msg)
+	defer C.free(unsafe.Pointer(msgStr))
+
+	C.vips_error_go(fnStr, msgStr)
+}

vips/vips.h

@@ -1,8 +1,10 @@
 #include <stdlib.h>
+#include <stdint.h> // uintptr_t
 
 #include <vips/vips.h>
 #include <vips/vips7compat.h>
 #include <vips/vector.h>
+#include "source.h"
 
 typedef struct _RGB {
   double r;
@@ -21,14 +23,14 @@ int gif_resolution_limit();
 
 int vips_health();
 
-int vips_jpegload_go(void *buf, size_t len, int shrink, VipsImage **out);
-int vips_jxlload_go(void *buf, size_t len, int pages, VipsImage **out);
-int vips_pngload_go(void *buf, size_t len, VipsImage **out, int unlimited);
-int vips_webpload_go(void *buf, size_t len, double scale, int pages, VipsImage **out);
-int vips_gifload_go(void *buf, size_t len, int pages, VipsImage **out);
-int vips_svgload_go(void *buf, size_t len, double scale, VipsImage **out, int unlimited);
-int vips_heifload_go(void *buf, size_t len, VipsImage **out, int thumbnail);
-int vips_tiffload_go(void *buf, size_t len, VipsImage **out);
+int vips_jpegload_source_go(VipsImgproxySource *source, int shrink, VipsImage **out);
+int vips_jxlload_source_go(VipsImgproxySource *source, int pages, VipsImage **out);
+int vips_pngload_source_go(VipsImgproxySource *source, VipsImage **out, int unlimited);
+int vips_webpload_source_go(VipsImgproxySource *source, double scale, int pages, VipsImage **out);
+int vips_gifload_source_go(VipsImgproxySource *source, int pages, VipsImage **out);
+int vips_svgload_source_go(VipsImgproxySource *source, double scale, VipsImage **out, int unlimited);
+int vips_heifload_source_go(VipsImgproxySource *source, VipsImage **out, int thumbnail);
+int vips_tiffload_source_go(VipsImgproxySource *source, VipsImage **out);
 
 int vips_black_go(VipsImage **out, int width, int height, int bands);
 
@@ -99,3 +101,5 @@ int vips_avifsave_go(VipsImage *in, void **buf, size_t *len, int quality, int sp
 int vips_tiffsave_go(VipsImage *in, void **buf, size_t *len, int quality);
 
 void vips_cleanup();
+
+void vips_error_go(const char *function, const char *message);