libvncclient: add framebuffer viewport mode

CMakeLists.txt

@@ -704,6 +704,12 @@ if(WITH_LIBVNCSERVER)
   )
 endif(WITH_LIBVNCSERVER)
 
+if(WITH_LIBVNCCLIENT)
+  list(APPEND SIMPLETESTS
+    client_viewporttest
+  )
+endif(WITH_LIBVNCCLIENT)
+
 
 if(WITH_THREADS AND (CMAKE_USE_PTHREADS_INIT OR CMAKE_USE_WIN32_THREADS_INIT) AND WITH_LIBVNCSERVER AND WITH_LIBVNCCLIENT)
   set(SIMPLETESTS
@@ -759,6 +765,9 @@ endif(LIBVNCSERVER_WITH_WEBSOCKETS AND WITH_LIBVNCSERVER)
 if(WITH_LIBVNCSERVER)
   add_test(NAME cargs COMMAND test_cargstest)
 endif(WITH_LIBVNCSERVER)
+if(WITH_LIBVNCCLIENT)
+  add_test(NAME client_viewport COMMAND test_client_viewporttest)
+endif(WITH_LIBVNCCLIENT)
 if(UNIX)
   if(WITH_LIBVNCSERVER)
     add_test(NAME includetest_server COMMAND ${TESTS_DIR}/includetest.sh ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR} ${CMAKE_MAKE_PROGRAM} "rfb/rfb.h")

include/rfb/rfbclient.h

@@ -506,6 +506,22 @@ typedef struct _rfbClient {
         rfbBool isUpdateRectManagedByLib;
 
         GetX509CertFingerprintMismatchDecisionProc GetX509CertFingerprintMismatchDecision;
+
+	/**
+	 * Optional local framebuffer viewport.
+	 *
+	 * When enabled before rfbClientInitialise(), update requests still use
+	 * framebuffer coordinates from the remote server, but the default framebuffer
+	 * allocation and default drawing callbacks store only this sub-rectangle
+	 * locally. Encodings that require pixels outside the requested viewport are
+	 * not requested by the default encoding negotiation while this mode is
+	 * enabled.
+	 */
+	rfbBool useFrameBufferViewport;
+	int frameBufferViewportX;
+	int frameBufferViewportY;
+	int frameBufferViewportW;
+	int frameBufferViewportH;
 } rfbClient;
 
 /* cursor.c */
@@ -528,6 +544,40 @@ extern rfbBool rfbEnableClientLogging;
 typedef void (*rfbClientLogProc)(const char *format, ...);
 extern rfbClientLogProc rfbClientLog,rfbClientErr;
 extern rfbBool ConnectToRFBServer(rfbClient* client,const char *hostname, int port);
+/**
+ * Enables the default LibVNCClient framebuffer viewport mode.
+ *
+ * The supplied rectangle is expressed in remote framebuffer coordinates. This
+ * must be called before rfbClientInitialise() or before any application-provided
+ * framebuffer allocation. Once client->frameBuffer is allocated, changing or
+ * clearing the viewport is rejected/ignored because the framebuffer stride and
+ * size have already been chosen.
+ *
+ * When active, the default framebuffer allocator will allocate storage only for
+ * this viewport, and the default drawing callbacks will clip incoming
+ * rectangles to it before writing into client->frameBuffer. The update
+ * rectangle is set to the same viewport.
+ *
+ * This mode is intentionally conservative: the default SetEncodings request is
+ * limited to encodings whose default callbacks can safely draw into the reduced
+ * local framebuffer. If an application overrides drawing callbacks such as
+ * GotBitmap, GotFillRect or GotCopyRect, that application is responsible for
+ * applying the same remote-to-local viewport mapping.
+ *
+ * Returns FALSE if the client is NULL, the rectangle is invalid, the viewport is
+ * outside the known remote framebuffer size, or client->frameBuffer has already
+ * been allocated.
+ */
+extern rfbBool rfbClientSetFrameBufferViewport(rfbClient *client, int x, int y, int w, int h);
+/**
+ * Disables the default LibVNCClient framebuffer viewport mode.
+ *
+ * This must also be called before rfbClientInitialise() or before any
+ * framebuffer allocation. If client->frameBuffer is already allocated, the
+ * function leaves the existing viewport enabled to avoid making the allocated
+ * framebuffer inconsistent with the requested update rectangle.
+ */
+extern void rfbClientClearFrameBufferViewport(rfbClient *client);
 extern rfbBool ConnectToRFBRepeater(rfbClient* client,const char *repeaterHost, int repeaterPort, const char *destHost, int destPort);
 extern void SetClientAuthSchemes(rfbClient* client,const uint32_t *authSchemes, int size);
 extern rfbBool InitialiseRFBConnection(rfbClient* client);

src/libvncclient/rfbclient.c

@@ -1281,7 +1281,14 @@ SetFormatAndEncodings(rfbClient* client)
   se->pad = 0;
   se->nEncodings = 0;
 
-  if (client->appData.encodingsString) {
+  if (client->useFrameBufferViewport) {
+    rfbClientLog("Framebuffer viewport active: requesting viewport-safe encodings only\n");
+    encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingHextile);
+    encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingCoRRE);
+    encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingRRE);
+    encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingRaw);
+  }
+  else if (client->appData.encodingsString) {
     const char *encStr = client->appData.encodingsString;
     int encStrLen;
     do {
@@ -2021,6 +2028,57 @@ rfbClientProcessExtServerCutText(rfbClient* client, char *data, int len)
 }
 #endif
 
+rfbBool rfbClientSetFrameBufferViewport(rfbClient *client, int x, int y, int w, int h)
+{
+    rfbRectangle rect;
+
+    if (!client || x < 0 || y < 0 || w <= 0 || h <= 0)
+        return FALSE;
+
+    if (client->frameBuffer) {
+        rfbClientErr("Cannot change framebuffer viewport after framebuffer allocation\n");
+        return FALSE;
+    }
+
+    if ((client->width > 0 && (uint64_t)x + (uint64_t)w > (uint64_t)client->width) ||
+        (client->height > 0 && (uint64_t)y + (uint64_t)h > (uint64_t)client->height)) {
+        rfbClientErr("Framebuffer viewport %dx%d at (%d, %d) exceeds remote framebuffer %dx%d\n",
+                     w, h, x, y, client->width, client->height);
+        return FALSE;
+    }
+
+    client->useFrameBufferViewport = TRUE;
+    client->frameBufferViewportX = x;
+    client->frameBufferViewportY = y;
+    client->frameBufferViewportW = w;
+    client->frameBufferViewportH = h;
+
+    rect.x = x;
+    rect.y = y;
+    rect.w = w;
+    rect.h = h;
+    rfbClientSetUpdateRect(client, &rect);
+    return TRUE;
+}
+
+void rfbClientClearFrameBufferViewport(rfbClient *client)
+{
+    if (!client)
+        return;
+
+    if (client->frameBuffer) {
+        rfbClientErr("Cannot clear framebuffer viewport after framebuffer allocation\n");
+        return;
+    }
+
+    client->useFrameBufferViewport = FALSE;
+    client->frameBufferViewportX = 0;
+    client->frameBufferViewportY = 0;
+    client->frameBufferViewportW = 0;
+    client->frameBufferViewportH = 0;
+    rfbClientSetUpdateRect(client, NULL);
+}
+
 void rfbClientSetUpdateRect(rfbClient *client, rfbRectangle *rect) {
     if (rect) {
 	client->updateRect.x = rect->x;