diff --git a/examples/lvgldemo/Kconfig b/examples/lvgldemo/Kconfig index dfe26f1e046..30178bfc346 100644 --- a/examples/lvgldemo/Kconfig +++ b/examples/lvgldemo/Kconfig @@ -20,6 +20,14 @@ config EXAMPLES_LVGLDEMO_STACKSIZE int "lvgldemo stack size" default 16384 +config EXAMPLES_LVGLDEMO_UTOUCH_DEVPATH + string "Second pointer device path" + depends on INPUT_TOUCHSCREEN + ---help--- + Optional second pointer device, opened alongside the primary + touchscreen: both drive the same UI. A VNC server's remote + pointer device is the usual tenant. Leave empty for none. + config EXAMPLES_LVGLDEMO_INPUT_DEVPATH string "Touchscreen device path" default "/dev/input0" diff --git a/examples/lvgldemo/lvgldemo.c b/examples/lvgldemo/lvgldemo.c index b6ab1e5ab24..75b7875f6f2 100644 --- a/examples/lvgldemo/lvgldemo.c +++ b/examples/lvgldemo/lvgldemo.c @@ -115,6 +115,18 @@ int main(int argc, FAR char *argv[]) #ifdef CONFIG_INPUT_TOUCHSCREEN info.input_path = CONFIG_EXAMPLES_LVGLDEMO_INPUT_DEVPATH; +#ifdef CONFIG_EXAMPLES_LVGLDEMO_UTOUCH_DEVPATH + /* A second pointer device, a VNC server's remote pointer, say, + * alongside the physical touchscreen rather than instead of it. An + * empty string means none: a string option always exists, only its + * content says whether it was configured. + */ + + if (CONFIG_EXAMPLES_LVGLDEMO_UTOUCH_DEVPATH[0] != '\0') + { + info.utouch_path = CONFIG_EXAMPLES_LVGLDEMO_UTOUCH_DEVPATH; + } +#endif #endif lv_nuttx_init(&info, &result); diff --git a/include/netutils/fbvnc.h b/include/netutils/fbvnc.h new file mode 100644 index 00000000000..6623b1e1a4b --- /dev/null +++ b/include/netutils/fbvnc.h @@ -0,0 +1,178 @@ +/**************************************************************************** + * apps/include/netutils/fbvnc.h + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +#ifndef __APPS_INCLUDE_NETUTILS_FBVNC_H +#define __APPS_INCLUDE_NETUTILS_FBVNC_H + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include +#include + +/**************************************************************************** + * Public Types + ****************************************************************************/ + +/* Dirty rectangle descriptor */ + +struct fbvnc_rect_s +{ + uint16_t x; + uint16_t y; + uint16_t w; + uint16_t h; +}; + +/* Snapshot callback. + * + * Called by the server whenever a client asks for a framebuffer update. + * The caller owns the pixel data: this is what keeps the server free of + * a framebuffer of its own, so that it can stream the display's own + * memory without a copy. + * + * The callback fills in the rectangles that changed since the last call + * and returns the base of the framebuffer, or NULL if no snapshot could + * be taken. Returning zero rectangles is not an error; it means + * nothing changed. + * + * Note that this runs on the server thread. Do not block on a lock that + * the render thread may already hold. + */ + +typedef CODE FAR const uint8_t * + (*fbvnc_snapshot_t)(FAR struct fbvnc_rect_s *rects, + uint32_t maxrects, FAR uint32_t *nrects); + +/* Connection notification callback */ + +typedef CODE void (*fbvnc_event_t)(void); + +/* Remote input callbacks. Both run on the server thread: hand the event + * to the UI thread, do not call into the UI from here. + * + * Pointer: position in framebuffer coordinates plus the RFB button mask + * (bit 0 = left, 1 = middle, 2 = right, bits 3-6 = scroll wheel notches + * encoded as press+release pairs). + * + * Key: the X11 keysym as the client sent it. Printable characters + * arrive already shifted, Shift+a comes in as 'A', so only special + * keys (arrows, enter, backspace...) need translating. + */ + +typedef CODE void (*fbvnc_pointer_t)(uint16_t x, uint16_t y, + uint8_t buttons); +typedef CODE void (*fbvnc_key_t)(uint32_t keysym, bool pressed); + +/* Server configuration. Must remain valid for the lifetime of the + * server: the start function stores the pointer's contents, not a copy + * of the callbacks' arguments. + */ + +struct fbvnc_cfg_s +{ + fbvnc_snapshot_t snapshot; /* Mandatory */ + fbvnc_event_t on_connect; /* Optional, may be NULL */ + fbvnc_event_t on_disconnect; /* Optional, may be NULL */ + + /* Optional. Invoked when the client asks for a non-incremental + * update, i.e. it wants the whole screen again. Use it to force the + * next snapshot to report the full canvas as dirty. + */ + + fbvnc_event_t on_invalidate; + + /* Optional. Remote input; NULL means the events are discarded. */ + + fbvnc_pointer_t on_pointer; + fbvnc_key_t on_key; + + /* Geometry of the served framebuffer. Zero means the compile-time + * defaults; a daemon serving an arbitrary framebuffer fills these in + * from what the device reports. Only 16-bit RGB565 is served either + * way. + */ + + uint16_t width; + uint16_t height; + uint16_t stride; /* Bytes per row */ +}; + +/**************************************************************************** + * Public Function Prototypes + ****************************************************************************/ + +#ifdef __cplusplus +#define EXTERN extern "C" +extern "C" +{ +#else +#define EXTERN extern +#endif + +/**************************************************************************** + * Name: fbvnc_start + * + * Description: + * Start the VNC server thread. The server listens on + * CONFIG_NETUTILS_FBVNC_PORT and accepts one client at a time. + * + * Input Parameters: + * cfg - Server configuration. The snapshot callback is mandatory. + * + * Returned Value: + * Zero (OK) on success; a negated errno value on failure. + * + ****************************************************************************/ + +int fbvnc_start(FAR const struct fbvnc_cfg_s *cfg); + +/**************************************************************************** + * Name: fbvnc_stop + * + * Description: + * Stop the VNC server. Closes the client connection if there is one + * and terminates the server thread. + * + ****************************************************************************/ + +void fbvnc_stop(void); + +/**************************************************************************** + * Name: fbvnc_is_connected + * + * Description: + * Return true if a VNC client is currently connected. + * + ****************************************************************************/ + +bool fbvnc_is_connected(void); + +#undef EXTERN +#ifdef __cplusplus +} +#endif + +#endif /* __APPS_INCLUDE_NETUTILS_FBVNC_H */ diff --git a/netutils/fbvnc/CMakeLists.txt b/netutils/fbvnc/CMakeLists.txt new file mode 100644 index 00000000000..875e390d524 --- /dev/null +++ b/netutils/fbvnc/CMakeLists.txt @@ -0,0 +1,25 @@ +# ############################################################################## +# apps/netutils/fbvnc/CMakeLists.txt +# +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with this work for +# additional information regarding copyright ownership. The ASF licenses this +# file to you under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +# ############################################################################## + +if(CONFIG_NETUTILS_FBVNC) + target_sources(apps PRIVATE fbvnc.c) +endif() diff --git a/netutils/fbvnc/Kconfig b/netutils/fbvnc/Kconfig new file mode 100644 index 00000000000..9acd1eaffcd --- /dev/null +++ b/netutils/fbvnc/Kconfig @@ -0,0 +1,157 @@ +# +# For a description of the syntax of this configuration file, +# see the file kconfig-language.txt in the NuttX tools repository. +# + +menuconfig NETUTILS_FBVNC + bool "VNC server (RFB 3.7) streaming a caller-owned framebuffer" + default n + depends on NET_TCP + ---help--- + A VNC Remote Frame Buffer server that streams a framebuffer the + application owns, supplied through a snapshot callback. + + This is deliberately not the same thing as the VNC server in + drivers/video/vnc: that one allocates a framebuffer of its own + and registers it as another /dev/fbN, so it is a second, virtual + display. This one has no framebuffer at all and streams whatever + the caller points it at, which is what lets it mirror the display + the LCD is already showing without a duplicate copy. + +if NETUTILS_FBVNC + +config NETUTILS_FBVNC_PORT + int "TCP port" + default 5900 + ---help--- + The port clients connect to. Display N in a VNC client's + address is port 5900 + N. + +config NETUTILS_FBVNC_STACKSIZE + int "Server thread stack size" + default 8192 + +config NETUTILS_FBVNC_PRIORITY + int "Server thread priority" + default 100 + ---help--- + Keep this below the priority of the thread that renders the UI. + The remote display is worth less than the local one: if the two + compete, the local display should win. + + Note the inverted convention if you are porting a configuration + from Zephyr, where a smaller number means a higher priority. + +config NETUTILS_FBVNC_NAME + string "Desktop name advertised to clients" + default "NuttX" + +config NETUTILS_FBVNC_FB_WIDTH + int "Framebuffer width (pixels)" + default 1024 + +config NETUTILS_FBVNC_FB_HEIGHT + int "Framebuffer height (rows)" + default 600 + +config NETUTILS_FBVNC_FB_BYTESPP + int "Framebuffer bytes per pixel" + default 2 + range 2 2 + ---help--- + Only 2 (RGB565) is implemented so far. + + The wire format deliberately matches the framebuffer rather than + being promoted to 32bpp RGBA. Promoting doubles the bytes per + frame, and at this resolution that is the difference between + 1.2 MiB and 2.4 MiB per full redraw -- enough to exhaust the + Ethernet driver's buffer pool during sustained navigation. + +config NETUTILS_FBVNC_MAX_DIRTY + int "Maximum dirty rectangles per update" + default 32 + ---help--- + How many rectangles the snapshot callback may report. When the + application has more changed regions than this it should collapse + them into one full-screen rectangle rather than dropping any. + +config NETUTILS_FBVNC_SEND_CHUNK + int "Maximum bytes per send() call" + default 8192 + +config NETUTILS_FBVNC_ENCODING_HEXTILE + bool "Hextile encoding (RFB 7.7.4)" + default y + ---help--- + Tile-based encoding, 16x16 tiles, each described as a fill, as + runs of a foreground colour over a background, or as raw pixels. + + On a user interface most tiles are a single colour, and a solid + tile costs three bytes against 512 raw. That is the difference + between a first frame that arrives and one that does not: a + 1024x600 screen is 1.2 MiB raw, and this link cannot carry that + while the display is being redrawn. + + All of the codec is here -- no zlib, no jpeg. ZRLE and Tight + compress better but need both, and a per-session deflate + context besides. + + Falls back to Raw when the client does not ask for Hextile, or + when it has negotiated a pixel format other than the + framebuffer's own. + +config NETUTILS_FBVNC_MIN_UPDATE_MS + int "Minimum time between framebuffer updates (ms)" + default 100 + ---help--- + A floor on the time between two consecutive answers to a client + asking for a framebuffer update, no matter how fast it asks. + 100 ms is a ceiling of 10 updates per second. + + Clients ask again the instant they have finished parsing the + last answer, which while a list is being dragged means a full + screen is in flight permanently and the display never catches + up. Delaying the answer is not a protocol violation; the + client is already waiting for one. + + Set to 0 to answer as fast as the client asks. + +config NETUTILS_FBVNC_SEND_TIMEOUT + int "Send timeout (seconds)" + default 30 + ---help--- + Disconnect the client if a send blocks for longer than this. On + a wedged connection this is what gets the server back to + accepting new clients. + +endif # NETUTILS_FBVNC + +config NETUTILS_FBVNC_TRACE + bool "Log what every update cost" + default n + ---help--- + Report, for each update sent, how many rectangles it carried, + how many bytes they cover, and how long the snapshot and the + sending each took. + + This is how one tells a slow link from a slow application: a + large send time is the network, a large snapshot time is + whatever produced the pixels. It is a line of output per + update, which is far too much to leave on, so it is asked for + rather than assumed. + +config NETUTILS_FBVNC_ENCODING_TRLE + bool "TRLE encoding (RFB 7.7.5)" + default y + ---help--- + Send a tile of few colours as a palette and an index per pixel + rather than as a list of sub-rectangles. + + A widget toolkit draws flat panels and text, so most tiles have + one or two colours; two colours cost one bit per pixel here, + against a sub-rectangle each in Hextile. Offered to clients + that ask for it, and preferred over Hextile when both are + offered. + + Only the native pixel format is served this way. A client that + negotiates another gets Raw, converted. diff --git a/netutils/fbvnc/Make.defs b/netutils/fbvnc/Make.defs new file mode 100644 index 00000000000..5192e3ce628 --- /dev/null +++ b/netutils/fbvnc/Make.defs @@ -0,0 +1,25 @@ +############################################################################ +# apps/netutils/fbvnc/Make.defs +# +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +ifneq ($(CONFIG_NETUTILS_FBVNC),) +CONFIGURED_APPS += $(APPDIR)/netutils/fbvnc +endif diff --git a/netutils/fbvnc/Makefile b/netutils/fbvnc/Makefile new file mode 100644 index 00000000000..1d260b54808 --- /dev/null +++ b/netutils/fbvnc/Makefile @@ -0,0 +1,29 @@ +############################################################################ +# apps/netutils/fbvnc/Makefile +# +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +include $(APPDIR)/Make.defs + +# VNC server library + +CSRCS = fbvnc.c + +include $(APPDIR)/Application.mk diff --git a/netutils/fbvnc/fbvnc.c b/netutils/fbvnc/fbvnc.c new file mode 100644 index 00000000000..8f4aa5ed535 --- /dev/null +++ b/netutils/fbvnc/fbvnc.c @@ -0,0 +1,1870 @@ +/**************************************************************************** + * apps/netutils/fbvnc/fbvnc.c + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +#define VNC_PORT CONFIG_NETUTILS_FBVNC_PORT +#define VNC_STACKSIZE CONFIG_NETUTILS_FBVNC_STACKSIZE +#define VNC_PRIORITY CONFIG_NETUTILS_FBVNC_PRIORITY +#define VNC_NAME CONFIG_NETUTILS_FBVNC_NAME +#define VNC_WIDTH CONFIG_NETUTILS_FBVNC_FB_WIDTH +#define VNC_HEIGHT CONFIG_NETUTILS_FBVNC_FB_HEIGHT +#define VNC_BYTESPP CONFIG_NETUTILS_FBVNC_FB_BYTESPP +#define VNC_STRIDE (VNC_WIDTH * VNC_BYTESPP) +#define VNC_MAX_DIRTY CONFIG_NETUTILS_FBVNC_MAX_DIRTY +#define VNC_SEND_CHUNK CONFIG_NETUTILS_FBVNC_SEND_CHUNK +#define VNC_SEND_TIMEOUT CONFIG_NETUTILS_FBVNC_SEND_TIMEOUT +#define VNC_MIN_INTERVAL CONFIG_NETUTILS_FBVNC_MIN_UPDATE_MS + +/* How long to keep trying when the stack is out of buffers. Ten seconds + * in total: long enough to ride out a burst, short enough that a client + * that has really gone away is noticed. + */ + +#define VNC_SEND_RETRY_MS 10 + +/* Buffer exhaustion is invisible to poll(), so that one wait is on the + * clock, but a short one: the pool drains as the wire empties, not on + * any schedule of ours. + */ + +#define VNC_SEND_ENOMEM_US 1000 +/* Consecutive waits, of VNC_SEND_RETRY_MS each, before a send is called + * lost. Any progress at all resets it, so reaching this means the + * connection has stopped moving entirely, and a client waiting half a + * minute for a frame is worse off than one dropped and reconnected, which + * costs it a single screen. + */ + +#define VNC_SEND_RETRIES 300 + +/* The framebuffer's own layout. Everything the server sends is derived + * from this; see fbvnc_parsepixelfmt for what happens when a client + * asks for something else. + */ + +#define VNC_NATIVE_BPP 16 +#define VNC_NATIVE_RMAX 31 +#define VNC_NATIVE_GMAX 63 +#define VNC_NATIVE_BMAX 31 +#define VNC_NATIVE_RSHIFT 11 +#define VNC_NATIVE_GSHIFT 5 +#define VNC_NATIVE_BSHIFT 0 + +/* Hextile sub-encoding bits (RFB 7.7.4). rfb.h has the encoding number + * but not these. + */ + +#define RFB_HEXTILE_RAW 0x01 +#define RFB_HEXTILE_BG 0x02 +#define RFB_HEXTILE_FG 0x04 +#define RFB_HEXTILE_ANYSUBRECTS 0x08 +#define RFB_HEXTILE_SUBRECTSCOL 0x10 + +#define VNC_HEXTILE_TILE 16 + +/* TRLE uses the same tiling. Sixteen is also the largest palette that + * still packs into whole bits per index (four), which is where the gain + * over Hextile comes from on a flat interface. + */ + +#define VNC_TRLE_TILE 16 +#define VNC_TRLE_MAXPAL 16 +#define VNC_HEXTILE_MAX_SUBRECTS 128 + +/**************************************************************************** + * Private Types + ****************************************************************************/ + +/* The pixel format the client last asked for */ + +struct fbvnc_fmt_s +{ + uint8_t bpp; + uint8_t bytespp; + uint16_t rmax; + uint16_t gmax; + uint16_t bmax; + uint8_t rshift; + uint8_t gshift; + uint8_t bshift; + bool bigendian; + + /* True when the client's format is byte-for-byte the framebuffer's, so + * that a rectangle can go out straight from display memory. + */ + + bool native; +}; + +struct fbvnc_state_s +{ + volatile bool running; + volatile bool connected; + fbvnc_snapshot_t snapshot; + fbvnc_event_t on_connect; + fbvnc_event_t on_disconnect; + fbvnc_event_t on_invalidate; + fbvnc_pointer_t on_pointer; + fbvnc_key_t on_key; + pthread_t thread; + int listensock; +}; + +/**************************************************************************** + * Private Data + ****************************************************************************/ + +static struct fbvnc_state_s g_fbvnc; + +/* Served geometry: configuration defaults until fbvnc_start says + * otherwise. + */ + +static uint16_t g_w = VNC_WIDTH; +static uint16_t g_h = VNC_HEIGHT; +static uint16_t g_stride = VNC_STRIDE; + +/* The encoding picked for this session, decided once the client's + * SetEncodings message lands and then used for every rectangle. Raw is + * the fallback every RFB client is required to accept. + */ + +static int32_t g_encoding = RFB_ENCODING_RAW; + +/* When the last update finished, for the rate cap below */ + +static struct timespec g_lastupdate; + +/* Sending a frame takes as long as the frame is large, and while it is in + * flight the client's key and pointer messages sit unread in the socket. + * The reader and the sender are therefore separate: this thread only ever + * parses messages and dispatches input, and hands the sender one token per + * update the client asks for, so every request still gets exactly one + * answer. + */ + +static pthread_t g_sender; +static sem_t g_updatesem; + +/* Whether the client is owed a frame. It is a flag and not a count: a + * client that asks again while one is being sent is asking for what the + * screen looks like now, not for two frames, and answering every request + * separately means sending it a queue of pictures that were already out + * of date when they left. + */ + +static volatile bool g_updatereq; + +#ifdef CONFIG_NETUTILS_FBVNC_TRACE +/* Bytes handed to the network stack for the update being built */ + +static uint32_t g_wirebytes; +#endif +static volatile bool g_clientrun; +static struct fbvnc_fmt_s g_fmt; + +/* Scratch used only when the client's pixel format differs from the + * framebuffer's. In the native case nothing is copied at all. + */ + +static uint8_t g_fbvnc_cvt[VNC_SEND_CHUNK]; + +/**************************************************************************** + * Private Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: fbvnc_sendall + ****************************************************************************/ + +static int fbvnc_sendall(int sock, FAR const void *buf, size_t len) +{ + FAR const uint8_t *ptr = buf; +#ifdef CONFIG_NETUTILS_FBVNC_TRACE + size_t wire = len; +#endif + unsigned int retries = 0; + size_t chunk; + ssize_t nsent; + + while (len > 0) + { + chunk = len > VNC_SEND_CHUNK ? VNC_SEND_CHUNK : len; + + nsent = send(sock, ptr, chunk, 0); + if (nsent < 0) + { + if (errno == EINTR) + { + continue; + } + + /* Running out of buffers, or a full send queue, says to come + * back in a moment, not that the connection is finished. + * Treating it as fatal is what made a client drop and then + * reconnect, which costs a whole screen to start over with. + */ + + if (errno == ENOMEM || errno == EAGAIN || errno == EWOULDBLOCK) + { + struct pollfd pfd; + bool nobufs = errno == ENOMEM; + + if (++retries > VNC_SEND_RETRIES) + { + return -errno; + } + + /* Wait for the socket to take more, rather than for a fixed + * sleep to expire. Room appears with the next + * acknowledgement, which on a busy link is a fraction of a + * millisecond; sleeping ten of them per chunk spends most + * of a frame's time doing nothing, and it showed as a + * transfer running at a third of what the wire could carry. + */ + + pfd.fd = sock; + pfd.events = POLLOUT; + pfd.revents = 0; + + poll(&pfd, 1, VNC_SEND_RETRY_MS); + + if (nobufs) + { + usleep(VNC_SEND_ENOMEM_US); + } + + continue; + } + + return -errno; + } + else if (nsent == 0) + { + return -ECONNRESET; + } + + retries = 0; + ptr += nsent; + len -= nsent; + } + +#ifdef CONFIG_NETUTILS_FBVNC_TRACE + g_wirebytes += wire; +#endif + + return OK; +} + +/**************************************************************************** + * Name: fbvnc_recvall + ****************************************************************************/ + +static int fbvnc_recvall(int sock, FAR void *buf, size_t len) +{ + FAR uint8_t *ptr = buf; + ssize_t nrecvd; + + while (len > 0) + { + nrecvd = recv(sock, ptr, len, 0); + if (nrecvd < 0) + { + if (errno == EINTR) + { + continue; + } + + return -errno; + } + else if (nrecvd == 0) + { + return -ECONNRESET; + } + + ptr += nrecvd; + len -= nrecvd; + } + + return OK; +} + +/**************************************************************************** + * Name: fbvnc_isnative + * + * Description: + * Decide whether the client's format lets us send framebuffer memory + * verbatim. This is the whole point of advertising the native format: + * in the common case there is no conversion and no copy. + * + ****************************************************************************/ + +static bool fbvnc_isnative(FAR const struct fbvnc_fmt_s *fmt) +{ + return fmt->bpp == VNC_NATIVE_BPP && + fmt->rmax == VNC_NATIVE_RMAX && + fmt->gmax == VNC_NATIVE_GMAX && + fmt->bmax == VNC_NATIVE_BMAX && + fmt->rshift == VNC_NATIVE_RSHIFT && + fmt->gshift == VNC_NATIVE_GSHIFT && + fmt->bshift == VNC_NATIVE_BSHIFT && + !fmt->bigendian; +} + +/**************************************************************************** + * Name: fbvnc_setnative + ****************************************************************************/ + +static void fbvnc_setnative(FAR struct fbvnc_fmt_s *fmt) +{ + fmt->bpp = VNC_NATIVE_BPP; + fmt->bytespp = VNC_NATIVE_BPP / 8; + fmt->rmax = VNC_NATIVE_RMAX; + fmt->gmax = VNC_NATIVE_GMAX; + fmt->bmax = VNC_NATIVE_BMAX; + fmt->rshift = VNC_NATIVE_RSHIFT; + fmt->gshift = VNC_NATIVE_GSHIFT; + fmt->bshift = VNC_NATIVE_BSHIFT; + fmt->bigendian = false; + fmt->native = true; +} + +/**************************************************************************** + * Name: fbvnc_cvtrow + * + * Description: + * Repack one row of RGB565 into the layout the client asked for, + * whatever its width: a client that asks for eight bits per pixel and + * is answered in sixteen decodes noise, because in RFB the client + * chooses the format and the server obeys. + * + * Only reached when that layout is not the framebuffer's own. + * + ****************************************************************************/ + +static void fbvnc_cvtrow(FAR const uint8_t *src, FAR uint8_t *dst, + uint16_t npixels, + FAR const struct fbvnc_fmt_s *fmt) +{ + uint16_t pixel; + uint32_t value; + uint16_t i; + uint8_t n; + uint8_t r; + uint8_t g; + uint8_t b; + + for (i = 0; i < npixels; i++) + { + pixel = src[0] | (src[1] << 8); + src += 2; + + r = (pixel >> VNC_NATIVE_RSHIFT) & VNC_NATIVE_RMAX; + g = (pixel >> VNC_NATIVE_GSHIFT) & VNC_NATIVE_GMAX; + b = (pixel >> VNC_NATIVE_BSHIFT) & VNC_NATIVE_BMAX; + + value = ((uint32_t)(r * fmt->rmax / VNC_NATIVE_RMAX) << fmt->rshift) | + ((uint32_t)(g * fmt->gmax / VNC_NATIVE_GMAX) << fmt->gshift) | + ((uint32_t)(b * fmt->bmax / VNC_NATIVE_BMAX) << fmt->bshift); + + /* Byte order is the client's, and so is the width: one, two or + * four bytes, most significant first only if it asked for that. + */ + + for (n = 0; n < fmt->bytespp; n++) + { + *dst++ = fmt->bigendian ? + value >> ((fmt->bytespp - 1 - n) * 8) : + value >> (n * 8); + } + } +} + +/**************************************************************************** + * Name: fbvnc_sendrect + * + * Description: + * Send one rectangle using the Raw encoding. + * + ****************************************************************************/ + +static int fbvnc_sendrect(int sock, FAR const uint8_t *fb, + FAR const struct fbvnc_rect_s *rect, + FAR const struct fbvnc_fmt_s *fmt) +{ + struct rfb_rectangle_s hdr; + FAR const uint8_t *row; + uint16_t y; + int ret; + + rfb_putbe16(hdr.xpos, rect->x); + rfb_putbe16(hdr.ypos, rect->y); + rfb_putbe16(hdr.width, rect->w); + rfb_putbe16(hdr.height, rect->h); + rfb_putbe32(hdr.encoding, RFB_ENCODING_RAW); + + /* The data[] member is a placeholder for the pixels that follow, so + * only the header proper goes out here. + */ + + ret = fbvnc_sendall(sock, &hdr, SIZEOF_RFB_RECTANGE_S(0)); + if (ret < 0) + { + return ret; + } + + row = fb + rect->y * g_stride + rect->x * VNC_BYTESPP; + + if (fmt->native && rect->x == 0 && rect->w == g_w) + { + /* Full-width rectangle in the framebuffer's own format: the rows + * are contiguous, so the whole block leaves display memory in one + * call with nothing copied. + */ + + return fbvnc_sendall(sock, row, rect->h * g_stride); + } + + for (y = 0; y < rect->h; y++) + { + if (fmt->native) + { + ret = fbvnc_sendall(sock, row, rect->w * VNC_BYTESPP); + } + else + { + fbvnc_cvtrow(row, g_fbvnc_cvt, rect->w, fmt); + ret = fbvnc_sendall(sock, g_fbvnc_cvt, + rect->w * fmt->bytespp); + } + + if (ret < 0) + { + return ret; + } + + row += g_stride; + } + + return OK; +} + +#ifdef CONFIG_NETUTILS_FBVNC_ENCODING_HEXTILE + +/**************************************************************************** + * Name: fbvnc_tilepalette + * + * Description: + * Count the colours in a tile, giving up once there are more than two. + * Two is the interesting boundary: one colour is a fill, two can be + * described as a background plus runs of a foreground, and beyond that + * Raw is usually smaller than any description of the difference. + * + * Returned Value: + * The number of distinct colours, or 3 to mean "more than two". bg is + * set to the most frequent colour and fg to the other one. + * + ****************************************************************************/ + +static int fbvnc_tilepalette(FAR const uint8_t *fb, + uint16_t x, uint16_t y, + uint16_t w, uint16_t h, + FAR uint16_t *bg, FAR uint16_t *fg) +{ + FAR const uint16_t *row; + uint16_t colour[2]; + uint32_t count[2]; + uint16_t pixel; + uint16_t i; + uint16_t j; + int ncolours = 0; + + count[0] = 0; + count[1] = 0; + colour[0] = 0; + colour[1] = 0; + + for (i = 0; i < h; i++) + { + row = (FAR const uint16_t *)(fb + (y + i) * g_stride) + x; + + for (j = 0; j < w; j++) + { + pixel = row[j]; + + if (ncolours > 0 && pixel == colour[0]) + { + count[0]++; + } + else if (ncolours > 1 && pixel == colour[1]) + { + count[1]++; + } + else if (ncolours < 2) + { + colour[ncolours] = pixel; + count[ncolours] = 1; + ncolours++; + } + else + { + return 3; + } + } + } + + /* The background is the colour worth not describing */ + + if (ncolours == 2 && count[1] > count[0]) + { + *bg = colour[1]; + *fg = colour[0]; + } + else + { + *bg = colour[0]; + *fg = colour[1]; + } + + return ncolours; +} + +/**************************************************************************** + * Name: fbvnc_tilesubrects + * + * Description: + * Describe a two-colour tile as runs of the foreground colour over the + * background. Runs are found a row at a time and emitted as one-row + * subrectangles; merging vertically would cost maybe a fifth of the + * bytes but a good deal of the simplicity. + * + * Returned Value: + * The number of subrectangles written, or a negative value if there were + * more than would fit. + * + ****************************************************************************/ + +static int fbvnc_tilesubrects(FAR const uint8_t *fb, + uint16_t x, uint16_t y, + uint16_t w, uint16_t h, + uint16_t fg, FAR uint8_t *dest, + int maxsubrects) +{ + FAR const uint16_t *row; + uint16_t i; + uint16_t j; + uint16_t start; + int n = 0; + + for (i = 0; i < h; i++) + { + row = (FAR const uint16_t *)(fb + (y + i) * g_stride) + x; + + j = 0; + while (j < w) + { + if (row[j] != fg) + { + j++; + continue; + } + + start = j; + while (j < w && row[j] == fg) + { + j++; + } + + if (n >= maxsubrects) + { + return -1; + } + + /* x and y share a byte, as do width - 1 and height - 1 */ + + *dest++ = (start << 4) | i; + *dest++ = ((j - start - 1) << 4) | 0; + n++; + } + } + + return n; +} + +/**************************************************************************** + * Name: fbvnc_tilepalette_n + * + * Description: + * Collect the distinct colours of one tile. Returns how many there + * are, or zero if there are more than the caller can hold, which is + * the answer that says to send the tile as it is. + * + * The index each pixel resolved to is written out as it goes: finding + * it again while packing would mean searching the palette a second time + * for every pixel, and that search is what the encoding costs. + * + ****************************************************************************/ + +static int fbvnc_tilepalette_n(FAR const uint8_t *fb, + uint16_t x, uint16_t y, + uint16_t w, uint16_t h, + FAR uint16_t *palette, int max, + FAR uint8_t *index) +{ + FAR const uint16_t *row; + uint16_t pixel; + uint16_t i; + uint16_t j; + int n = 0; + int k; + + for (i = 0; i < h; i++) + { + row = (FAR const uint16_t *)(fb + (y + i) * g_stride) + x; + + for (j = 0; j < w; j++) + { + pixel = row[j]; + + for (k = 0; k < n; k++) + { + if (palette[k] == pixel) + { + break; + } + } + + if (k == n) + { + if (n == max) + { + return 0; + } + + palette[n++] = pixel; + } + + *index++ = k; + } + } + + return n; +} + +/**************************************************************************** + * Name: fbvnc_sendrect_trle + * + * Description: + * Send one rectangle using the TRLE encoding (RFB 7.7.5). + * + * Same sixteen pixel tiles as Hextile, but a tile of few colours is sent + * as a palette and an index per pixel rather than as a list of + * sub-rectangles. On an interface of flat panels and text, which is + * what a widget toolkit draws, most tiles have one or two colours, and + * two colours cost one bit per pixel instead of a rectangle each. + * + * The run-length subencodings are deliberately absent: they pay off on + * long horizontal runs, which the packed palette already covers at a + * fixed size, and leaving them out keeps this a single pass over each + * tile. + * + ****************************************************************************/ + +static int fbvnc_sendrect_trle(int sock, FAR const uint8_t *fb, + FAR const struct fbvnc_rect_s *r, + FAR const struct fbvnc_fmt_s *fmt) +{ + /* Static rather than automatic: larger than the server thread's stack, + * and there is only ever one client. + */ + + static uint8_t buf[VNC_SEND_CHUNK]; + + static uint8_t index[VNC_TRLE_TILE * VNC_TRLE_TILE]; + + uint16_t palette[VNC_TRLE_MAXPAL]; + struct rfb_rectangle_s hdr; + FAR const uint16_t *row; + size_t used = 0; + uint16_t tx; + uint16_t ty; + uint16_t i; + uint16_t j; + int npal; + int ret; + + UNUSED(fmt); + + rfb_putbe16(hdr.xpos, r->x); + rfb_putbe16(hdr.ypos, r->y); + rfb_putbe16(hdr.width, r->w); + rfb_putbe16(hdr.height, r->h); + rfb_putbe32(hdr.encoding, RFB_ENCODING_TRLE); + + ret = fbvnc_sendall(sock, &hdr, SIZEOF_RFB_RECTANGE_S(0)); + if (ret < 0) + { + return ret; + } + + for (ty = 0; ty < r->h; ty += VNC_TRLE_TILE) + { + uint16_t th = r->h - ty; + th = th > VNC_TRLE_TILE ? VNC_TRLE_TILE : th; + + for (tx = 0; tx < r->w; tx += VNC_TRLE_TILE) + { + uint16_t tw = r->w - tx; + FAR uint8_t *p; + size_t rawsize; + int bits; + + tw = tw > VNC_TRLE_TILE ? VNC_TRLE_TILE : tw; + + /* Flush when the largest possible tile would not fit */ + + if (used + 1 + tw * th * VNC_BYTESPP > sizeof(buf)) + { + ret = fbvnc_sendall(sock, buf, used); + if (ret < 0) + { + return ret; + } + + used = 0; + } + + p = &buf[used]; + rawsize = 1 + tw * th * VNC_BYTESPP; + npal = fbvnc_tilepalette_n(fb, r->x + tx, r->y + ty, + tw, th, palette, + VNC_TRLE_MAXPAL, index); + + /* One colour: the tile is that colour and nothing else */ + + if (npal == 1) + { + *p++ = 1; + *p++ = palette[0]; + *p++ = palette[0] >> 8; + used += p - &buf[used]; + continue; + } + + bits = npal == 2 ? 1 : (npal <= 4 ? 2 : 4); + + if (npal > 1 && + 1 + npal * VNC_BYTESPP + ((tw * bits + 7) / 8) * th < rawsize) + { + /* Packed palette: the colours, then an index per pixel, + * most significant bits first, each row padded to a byte. + */ + + *p++ = npal; + + for (i = 0; i < npal; i++) + { + *p++ = palette[i]; + *p++ = palette[i] >> 8; + } + + for (i = 0; i < th; i++) + { + uint8_t acc = 0; + int nbits = 0; + + for (j = 0; j < tw; j++) + { + acc = (acc << bits) | index[i * tw + j]; + nbits += bits; + + if (nbits == 8) + { + *p++ = acc; + acc = 0; + nbits = 0; + } + } + + if (nbits > 0) + { + *p++ = acc << (8 - nbits); + } + } + + used += p - &buf[used]; + continue; + } + + /* Too many colours to be worth describing: send the pixels */ + + *p++ = 0; + + for (i = 0; i < th; i++) + { + row = (FAR const uint16_t *) + (fb + (r->y + ty + i) * g_stride) + r->x + tx; + memcpy(p, row, tw * VNC_BYTESPP); + p += tw * VNC_BYTESPP; + } + + used += p - &buf[used]; + } + } + + if (used > 0) + { + return fbvnc_sendall(sock, buf, used); + } + + return OK; +} + +/**************************************************************************** + * Name: fbvnc_sendrect_hextile + * + * Description: + * Send one rectangle using the Hextile encoding (RFB 7.7.4). + * + * Only the native pixel format is handled. Anything else falls back to + * Raw, because a converted tile would have to be materialised first and + * the case does not arise with the clients this serves. + * + ****************************************************************************/ + +static int fbvnc_sendrect_hextile(int sock, FAR const uint8_t *fb, + FAR const struct fbvnc_rect_s *r, + FAR const struct fbvnc_fmt_s *fmt) +{ + /* Static rather than automatic: together these are larger than the + * server thread's stack, and there is only ever one client. + */ + + static uint8_t buf[VNC_SEND_CHUNK]; + static uint8_t subrects[2 * VNC_HEXTILE_MAX_SUBRECTS]; + + struct rfb_rectangle_s hdr; + size_t used = 0; + uint16_t lastbg = 0; + uint16_t lastfg = 0; + bool havebg = false; + bool havefg = false; + uint16_t tx; + uint16_t ty; + uint16_t i; + uint16_t bg; + uint16_t fg; + int ncolours; + int nsub; + int ret; + + rfb_putbe16(hdr.xpos, r->x); + rfb_putbe16(hdr.ypos, r->y); + rfb_putbe16(hdr.width, r->w); + rfb_putbe16(hdr.height, r->h); + rfb_putbe32(hdr.encoding, RFB_ENCODING_HEXTILE); + + ret = fbvnc_sendall(sock, &hdr, SIZEOF_RFB_RECTANGE_S(0)); + if (ret < 0) + { + return ret; + } + + for (ty = 0; ty < r->h; ty += VNC_HEXTILE_TILE) + { + uint16_t th = r->h - ty; + th = th > VNC_HEXTILE_TILE ? VNC_HEXTILE_TILE : th; + + for (tx = 0; tx < r->w; tx += VNC_HEXTILE_TILE) + { + uint16_t tw = r->w - tx; + size_t rawsize; + size_t subsize; + FAR uint8_t *p; + + tw = tw > VNC_HEXTILE_TILE ? VNC_HEXTILE_TILE : tw; + + /* Flush when the largest possible tile would not fit */ + + if (used + 5 + tw * th * VNC_BYTESPP > sizeof(buf)) + { + ret = fbvnc_sendall(sock, buf, used); + if (ret < 0) + { + return ret; + } + + used = 0; + } + + p = &buf[used]; + rawsize = 1 + tw * th * VNC_BYTESPP; + + ncolours = fbvnc_tilepalette(fb, r->x + tx, r->y + ty, + tw, th, &bg, &fg); + + if (ncolours <= 1) + { + *p++ = havebg && bg == lastbg ? 0 : RFB_HEXTILE_BG; + + if (!havebg || bg != lastbg) + { + *p++ = bg; + *p++ = bg >> 8; + lastbg = bg; + havebg = true; + } + + used += p - &buf[used]; + continue; + } + + if (ncolours == 2) + { + nsub = fbvnc_tilesubrects(fb, r->x + tx, r->y + ty, + tw, th, fg, subrects, + VNC_HEXTILE_MAX_SUBRECTS); + if (nsub > 0) + { + subsize = 1 + 1 + 2 * nsub + + ((havebg && bg == lastbg) ? 0 : VNC_BYTESPP) + + ((havefg && fg == lastfg) ? 0 : VNC_BYTESPP); + + if (subsize < rawsize) + { + FAR uint8_t *mask = p++; + + *mask = RFB_HEXTILE_ANYSUBRECTS; + + if (!havebg || bg != lastbg) + { + *mask |= RFB_HEXTILE_BG; + *p++ = bg; + *p++ = bg >> 8; + lastbg = bg; + havebg = true; + } + + if (!havefg || fg != lastfg) + { + *mask |= RFB_HEXTILE_FG; + *p++ = fg; + *p++ = fg >> 8; + lastfg = fg; + havefg = true; + } + + *p++ = nsub; + memcpy(p, subrects, 2 * nsub); + p += 2 * nsub; + + used += p - &buf[used]; + continue; + } + } + } + + /* Raw. A raw tile says nothing about the background or the + * foreground, so what the client remembers of them is unchanged. + */ + + *p++ = RFB_HEXTILE_RAW; + + for (i = 0; i < th; i++) + { + memcpy(p, fb + (r->y + ty + i) * g_stride + + (r->x + tx) * VNC_BYTESPP, tw * VNC_BYTESPP); + p += tw * VNC_BYTESPP; + } + + used += p - &buf[used]; + } + } + + if (used > 0) + { + return fbvnc_sendall(sock, buf, used); + } + + return OK; +} +#endif /* CONFIG_NETUTILS_FBVNC_ENCODING_HEXTILE */ + +/**************************************************************************** + * Name: fbvnc_sendupdate + ****************************************************************************/ + +static int fbvnc_sendupdate(int sock, + FAR const struct fbvnc_fmt_s *fmt) +{ + struct fbvnc_rect_s rects[VNC_MAX_DIRTY]; + struct rfb_framebufferupdate_s hdr; + FAR const uint8_t *fb; +#ifdef CONFIG_NETUTILS_FBVNC_TRACE + struct timespec t0; + struct timespec t1; + struct timespec t2; + uint32_t nbytes = 0; +#endif + uint32_t nrects = 0; + uint32_t i; + int ret; + +#ifdef CONFIG_NETUTILS_FBVNC_TRACE + clock_gettime(CLOCK_MONOTONIC, &t0); + g_wirebytes = 0; +#endif + + fb = g_fbvnc.snapshot(rects, VNC_MAX_DIRTY, &nrects); + +#ifdef CONFIG_NETUTILS_FBVNC_TRACE + clock_gettime(CLOCK_MONOTONIC, &t1); +#endif + if (fb == NULL) + { + syslog(LOG_WARNING, "fbvnc: snapshot failed\n"); + return -EIO; + } + + /* Nothing has changed. An empty update is a valid answer, but it is an + * answer the client will immediately ask again for, so the request is + * left standing instead: RFB lets the server reply when it has + * something, and every empty round trip costs a snapshot, which, + * where the dirty areas come from comparing frames, is the whole + * screen read twice. + */ + + if (nrects == 0) + { + return 0; + } + + memset(&hdr, 0, sizeof(hdr)); + hdr.msgtype = RFB_FBUPDATE_MSG; + rfb_putbe16(hdr.nrect, nrects); + + ret = fbvnc_sendall(sock, &hdr, SIZEOF_RFB_FRAMEBUFFERUPDATE_S(0)); + if (ret < 0) + { + return ret; + } + + for (i = 0; i < nrects; i++) + { +#ifdef CONFIG_NETUTILS_FBVNC_ENCODING_TRLE + if (g_encoding == RFB_ENCODING_TRLE && fmt->native) + { + ret = fbvnc_sendrect_trle(sock, fb, &rects[i], fmt); + } + else +#endif +#ifdef CONFIG_NETUTILS_FBVNC_ENCODING_HEXTILE + if (g_encoding == RFB_ENCODING_HEXTILE && fmt->native) + { + ret = fbvnc_sendrect_hextile(sock, fb, &rects[i], fmt); + } + else +#endif + { + ret = fbvnc_sendrect(sock, fb, &rects[i], fmt); + } + + if (ret < 0) + { + return ret; + } + +#ifdef CONFIG_NETUTILS_FBVNC_TRACE + nbytes += rects[i].w * rects[i].h * fmt->bytespp; +#endif + } + +#ifdef CONFIG_NETUTILS_FBVNC_TRACE + /* What an update costs is the whole question on a link this size, so + * say it rather than leave it to be guessed at. Splitting the snapshot + * from the sending is what says which side to look at. It is a line + * per update, so it is asked for rather than assumed. + */ + + clock_gettime(CLOCK_MONOTONIC, &t2); + + /* "queued" and not "sent": with write buffering the stack takes the + * bytes and returns, and they leave the wire afterwards. Reading this + * as delivery makes a frame look faster than the link can carry it. + */ + + syslog(LOG_INFO, "fbvnc: update: %" PRIu32 " rect(s), %" PRIu32 + " px of screen, %" PRIu32 " bytes on the wire, " + "snapshot %" PRIu32 " ms, queued %" PRIu32 " ms\n", + nrects, nbytes / VNC_BYTESPP, g_wirebytes, + (uint32_t)((t1.tv_sec - t0.tv_sec) * 1000 + + (t1.tv_nsec - t0.tv_nsec) / 1000000), + (uint32_t)((t2.tv_sec - t1.tv_sec) * 1000 + + (t2.tv_nsec - t1.tv_nsec) / 1000000)); +#endif + + return (int)nrects; +} + +/**************************************************************************** + * Name: fbvnc_handshake + * + * Description: + * RFB 3.7 handshake. 3.7 rather than 3.3 because it negotiates + * security with a list of types, which is what a password would need + * later; 3.8 only adds a SecurityResult message to the None path, + * which buys nothing here. + * + ****************************************************************************/ + +static int fbvnc_handshake(int sock, FAR struct fbvnc_fmt_s *fmt) +{ + struct rfb_serverinit_s sinit; + char version[sizeof(RFB_PROTOCOL_VERSION_3p7) - 1]; + uint8_t sectypes[2]; + uint8_t selected; + uint8_t shared; + size_t namelen; + int ret; + + ret = fbvnc_sendall(sock, RFB_PROTOCOL_VERSION_3p7, sizeof(version)); + if (ret < 0) + { + return ret; + } + + ret = fbvnc_recvall(sock, version, sizeof(version)); + if (ret < 0) + { + return ret; + } + + syslog(LOG_INFO, "fbvnc: client version %.11s\n", version); + + /* Offer exactly one security type. The count byte comes first, then + * the types themselves. + */ + + sectypes[0] = 1; + sectypes[1] = RFB_SECTYPE_NONE; + + ret = fbvnc_sendall(sock, sectypes, sizeof(sectypes)); + if (ret < 0) + { + return ret; + } + + ret = fbvnc_recvall(sock, &selected, sizeof(selected)); + if (ret < 0) + { + return ret; + } + + if (selected != RFB_SECTYPE_NONE) + { + syslog(LOG_WARNING, "fbvnc: client picked security type %u, " + "which was not offered\n", selected); + return -EPROTO; + } + + /* Under 3.7 the None type sends no SecurityResult, so ClientInit is + * next. Its shared flag is advisory and this server only ever has one + * client, so it is read and discarded. + */ + + ret = fbvnc_recvall(sock, &shared, sizeof(shared)); + if (ret < 0) + { + return ret; + } + + /* Advertise the framebuffer's own format rather than promoting to + * 32bpp RGBA. Promoting is convenient for canvas-based clients but + * doubles every byte on the wire, and at this resolution that is the + * difference between a 1.2 MiB and a 2.4 MiB full redraw. + */ + + namelen = strlen(VNC_NAME); + memset(&sinit, 0, sizeof(sinit)); + + rfb_putbe16(sinit.width, g_w); + rfb_putbe16(sinit.height, g_h); + + sinit.format.bpp = VNC_NATIVE_BPP; + sinit.format.depth = VNC_NATIVE_BPP; + sinit.format.truecolor = 1; + sinit.format.rshift = VNC_NATIVE_RSHIFT; + sinit.format.gshift = VNC_NATIVE_GSHIFT; + sinit.format.bshift = VNC_NATIVE_BSHIFT; + + rfb_putbe16(sinit.format.rmax, VNC_NATIVE_RMAX); + rfb_putbe16(sinit.format.gmax, VNC_NATIVE_GMAX); + rfb_putbe16(sinit.format.bmax, VNC_NATIVE_BMAX); + rfb_putbe32(sinit.namelen, namelen); + + ret = fbvnc_sendall(sock, &sinit, SIZEOF_RFB_SERVERINIT_S(0)); + if (ret < 0) + { + return ret; + } + + ret = fbvnc_sendall(sock, VNC_NAME, namelen); + if (ret < 0) + { + return ret; + } + + fbvnc_setnative(fmt); + + syslog(LOG_INFO, "fbvnc: handshake done, %dx%d %dbpp RGB565\n", + g_w, g_h, VNC_NATIVE_BPP); + + return OK; +} + +/**************************************************************************** + * Name: fbvnc_parsepixelfmt + * + * Description: + * Apply a SetPixelFormat message. Any 16-bit layout is honoured, which + * covers the RGB565/BGR565/RGB555 variants clients actually ask for. + * + * A request for a different depth is refused and the server keeps + * sending its native format. That is a deliberate deviation: honouring + * 32bpp would double the bytes per frame, and exhausting the network + * buffer pool is a worse failure than a client that has to accept 16bpp. + * + ****************************************************************************/ + +static void fbvnc_parsepixelfmt(FAR const uint8_t *buf, + FAR struct fbvnc_fmt_s *fmt) +{ + FAR const struct rfb_pixelfmt_s *pixelfmt; + + pixelfmt = (FAR const struct rfb_pixelfmt_s *)&buf[3]; + + /* Anything the conversion can produce is accepted. A colour map is + * not: it would mean sending the map and indices into it, so such a + * client keeps the native format and is told why. + */ + + if ((pixelfmt->bpp != 8 && pixelfmt->bpp != 16 && pixelfmt->bpp != 32) || + pixelfmt->truecolor == 0) + { + syslog(LOG_WARNING, "fbvnc: client asked for %ubpp%s, keeping " + "native %dbpp\n", pixelfmt->bpp, + pixelfmt->truecolor ? "" : " with a colour map", + VNC_NATIVE_BPP); + return; + } + + /* One converted row has to fit the buffer it is built in */ + + if ((uint32_t)g_w * (pixelfmt->bpp / 8) > sizeof(g_fbvnc_cvt)) + { + syslog(LOG_WARNING, "fbvnc: a %ubpp row of %u pixels does not fit " + "the %u byte conversion buffer, keeping native " + "%dbpp\n", pixelfmt->bpp, g_w, + (unsigned)sizeof(g_fbvnc_cvt), VNC_NATIVE_BPP); + return; + } + + fmt->bpp = pixelfmt->bpp; + fmt->bytespp = pixelfmt->bpp / 8; + fmt->bigendian = pixelfmt->bigendian != 0; + fmt->rmax = rfb_getbe16(pixelfmt->rmax); + fmt->gmax = rfb_getbe16(pixelfmt->gmax); + fmt->bmax = rfb_getbe16(pixelfmt->bmax); + fmt->rshift = pixelfmt->rshift; + fmt->gshift = pixelfmt->gshift; + fmt->bshift = pixelfmt->bshift; + fmt->native = fbvnc_isnative(fmt); + + syslog(LOG_INFO, "fbvnc: pixel format %ubpp rmax=%u gmax=%u bmax=%u " + "shifts=%u/%u/%u%s\n", + fmt->bpp, fmt->rmax, fmt->gmax, fmt->bmax, + fmt->rshift, fmt->gshift, fmt->bshift, + fmt->native ? " (native, zero copy)" : " (converted)"); +} + +/**************************************************************************** + * Name: fbvnc_sendthread + * + * Description: + * Waits for the reader to say that the client wants a frame, then spends + * however long the frame takes writing it, without holding up anything + * the client has to say in the meantime. + * + ****************************************************************************/ + +static FAR void *fbvnc_sendthread(FAR void *arg) +{ + int sock = (int)(intptr_t)arg; + struct timespec now; + int32_t elapsed; + int ret; + + while (g_clientrun) + { + if (sem_wait(&g_updatesem) < 0) + { + continue; + } + + if (!g_clientrun) + { + break; + } + + /* Cleared before the frame is built, so that a request arriving + * while it is being sent asks for the next one + */ + + g_updatereq = false; + + /* Hold off until the interval has passed since the last update went + * out. A client that asks again the instant it has finished + * parsing, which is what they do while a list is being dragged, + * otherwise keeps a full screen in flight permanently, and the + * display never catches up. Delaying the answer is not a protocol + * violation: the client is already waiting for one. + */ + + if (VNC_MIN_INTERVAL > 0) + { + clock_gettime(CLOCK_MONOTONIC, &now); + elapsed = (now.tv_sec - g_lastupdate.tv_sec) * 1000 + + (now.tv_nsec - g_lastupdate.tv_nsec) / 1000000; + + if (elapsed >= 0 && elapsed < VNC_MIN_INTERVAL) + { + usleep((VNC_MIN_INTERVAL - elapsed) * 1000); + } + } + + ret = fbvnc_sendupdate(sock, &g_fmt); + clock_gettime(CLOCK_MONOTONIC, &g_lastupdate); + + if (ret == 0) + { + /* Nothing to say yet: the client is still owed a frame, so + * look again after the interval rather than answering with an + * empty one. + */ + + g_updatereq = true; + sem_post(&g_updatesem); + continue; + } + + if (ret < 0) + { + syslog(LOG_ERR, "fbvnc: update failed: %d\n", ret); + g_clientrun = false; + + /* Wake the reader out of its recv so the client is dropped */ + + shutdown(sock, SHUT_RDWR); + break; + } + } + + return NULL; +} + +/**************************************************************************** + * Name: fbvnc_handleclient + ****************************************************************************/ + +static void fbvnc_handleclient(int sock) +{ + pthread_attr_t attr; + struct sched_param param; + uint8_t msgtype; + uint8_t buf[20]; + uint16_t nencodings; + uint16_t i; + bool hashextile; + bool hastrle; + int ret; + + ret = fbvnc_handshake(sock, &g_fmt); + if (ret < 0) + { + syslog(LOG_ERR, "fbvnc: handshake failed: %d\n", ret); + return; + } + + /* Older clients never send SetEncodings; they get Raw */ + + g_encoding = RFB_ENCODING_RAW; + clock_gettime(CLOCK_MONOTONIC, &g_lastupdate); + + g_fbvnc.connected = true; + if (g_fbvnc.on_connect != NULL) + { + g_fbvnc.on_connect(); + } + + /* The frames go out on a thread of their own, so that this one is always + * free to read what the client is saying + */ + + g_clientrun = true; + g_updatereq = false; + sem_init(&g_updatesem, 0, 0); + + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, VNC_STACKSIZE); + param.sched_priority = VNC_PRIORITY; + pthread_attr_setschedparam(&attr, ¶m); + + ret = pthread_create(&g_sender, &attr, fbvnc_sendthread, + (FAR void *)(intptr_t)sock); + pthread_attr_destroy(&attr); + + if (ret != 0) + { + syslog(LOG_ERR, "fbvnc: cannot start the sender: %d\n", ret); + sem_destroy(&g_updatesem); + return; + } + + pthread_setname_np(g_sender, "fbvncsend"); + + while (g_fbvnc.running && g_clientrun) + { + ret = fbvnc_recvall(sock, &msgtype, sizeof(msgtype)); + if (ret < 0) + { + syslog(LOG_INFO, "fbvnc: client disconnected\n"); + break; + } + + switch (msgtype) + { + case RFB_SETPIXELFMT_MSG: + + /* Three padding bytes then the 16-byte pixel format */ + + ret = fbvnc_recvall(sock, buf, 19); + if (ret < 0) + { + goto teardown; + } + + fbvnc_parsepixelfmt(buf, &g_fmt); + break; + + case RFB_SETENCODINGS_MSG: + ret = fbvnc_recvall(sock, buf, 3); + if (ret < 0) + { + goto teardown; + } + + nencodings = rfb_getbe16(&buf[1]); + hashextile = false; + hastrle = false; + + for (i = 0; i < nencodings; i++) + { + ret = fbvnc_recvall(sock, buf, 4); + if (ret < 0) + { + goto teardown; + } + + if ((int32_t)rfb_getbe32(buf) == RFB_ENCODING_HEXTILE) + { + hashextile = true; + } + else if ((int32_t)rfb_getbe32(buf) == RFB_ENCODING_TRLE) + { + hastrle = true; + } + } + + /* Fall back to Raw unless the client asked for something this + * server actually implements. Claiming an encoding it does + * not produce would corrupt the stream. + */ + + g_encoding = RFB_ENCODING_RAW; + +#ifdef CONFIG_NETUTILS_FBVNC_ENCODING_HEXTILE + if (hashextile) + { + g_encoding = RFB_ENCODING_HEXTILE; + } +#endif + + /* TRLE last, so that it wins where the client takes both: a + * tile of few colours costs bits per pixel there and a + * sub-rectangle each in Hextile. + */ + +#ifdef CONFIG_NETUTILS_FBVNC_ENCODING_TRLE + if (hastrle) + { + g_encoding = RFB_ENCODING_TRLE; + } +#endif + + syslog(LOG_INFO, "fbvnc: client offered %u encodings, " + "using %s\n", nencodings, + g_encoding == RFB_ENCODING_TRLE ? "TRLE" : + g_encoding == RFB_ENCODING_HEXTILE ? "Hextile" : "Raw"); + break; + + case RFB_FBUPDATEREQ_MSG: + ret = fbvnc_recvall(sock, buf, 9); + if (ret < 0) + { + goto teardown; + } + + /* buf[0] is the incremental flag. Zero means the client + * wants the whole screen, not just what changed. + */ + + if (buf[0] == 0 && g_fbvnc.on_invalidate != NULL) + { + g_fbvnc.on_invalidate(); + } + + /* Requests that arrive while a frame is on its way fold into + * the one after it + */ + + if (!g_updatereq) + { + g_updatereq = true; + sem_post(&g_updatesem); + } + + break; + + case RFB_KEYEVENT_MSG: + ret = fbvnc_recvall(sock, buf, 7); + if (ret < 0) + { + goto teardown; + } + + /* down-flag, 2 pad bytes, then the keysym */ + + if (g_fbvnc.on_key != NULL) + { + g_fbvnc.on_key(rfb_getbe32(&buf[3]), buf[0] != 0); + } + break; + + case RFB_POINTEREVENT_MSG: + ret = fbvnc_recvall(sock, buf, 5); + if (ret < 0) + { + goto teardown; + } + + /* button mask, then x and y */ + + if (g_fbvnc.on_pointer != NULL) + { + g_fbvnc.on_pointer(rfb_getbe16(&buf[1]), + rfb_getbe16(&buf[3]), buf[0]); + } + break; + + case RFB_CLIENTCUTTEXT_MSG: + ret = fbvnc_recvall(sock, buf, 7); + if (ret < 0) + { + goto teardown; + } + + /* Drop the text itself. It is read rather than ignored so + * that the stream stays in sync. + */ + + for (i = rfb_getbe32(&buf[3]); i > 0; i--) + { + ret = fbvnc_recvall(sock, buf, 1); + if (ret < 0) + { + goto teardown; + } + } + break; + + default: + syslog(LOG_WARNING, "fbvnc: unknown message type %u, " + "dropping client\n", msgtype); + goto teardown; + } + } + +teardown: + + /* The sender may be halfway through a frame: drop the connection under + * it so its write fails, then wait for it before the socket is closed. + */ + + g_clientrun = false; + shutdown(sock, SHUT_RDWR); + sem_post(&g_updatesem); + pthread_join(g_sender, NULL); + sem_destroy(&g_updatesem); +} + +/**************************************************************************** + * Name: fbvnc_thread + ****************************************************************************/ + +static FAR void *fbvnc_thread(FAR void *arg) +{ + struct timeval tv; + int listensock = g_fbvnc.listensock; + int clientsock; + + while (g_fbvnc.running) + { + struct pollfd pfd; + + /* Wait for a client with a timeout rather than in accept(): a + * thread blocked in accept() cannot be woken, not by closing the + * socket under it, and not by a receive timeout, so a server + * that is asked to stop would hold its port until someone + * happened to connect. + */ + + pfd.fd = listensock; + pfd.events = POLLIN; + pfd.revents = 0; + + if (poll(&pfd, 1, 500) <= 0 || (pfd.revents & POLLIN) == 0) + { + continue; + } + + clientsock = accept(listensock, NULL, NULL); + if (clientsock < 0) + { + /* A connection that died while it sat in the backlog comes out + * of accept() as an error. That is that connection's problem, + * not the listener's: a server that exits here goes silent + * the first time a client gives up waiting, which is exactly + * how it was found. + * + * Which errno that is depends on how far the connection got + * before it died, ECONNABORTED, ENOTCONN and ETIMEDOUT have + * all been seen, so listing the survivable ones is a list + * that is always missing its next entry. Only the ones that + * say the listening socket itself is finished end the loop. + */ + + if (errno == EBADF || errno == EINVAL || errno == ENOTSOCK) + { + syslog(LOG_ERR, "fbvnc: accept failed: %d\n", errno); + break; + } + + syslog(LOG_WARNING, "fbvnc: dropped a connection that died " + "waiting: %d\n", errno); + continue; + } + + syslog(LOG_INFO, "fbvnc: client connected\n"); + + /* A stuck send must not park the server forever: the timeout is + * what gets it back to accepting clients when a connection wedges. + */ + + tv.tv_sec = VNC_SEND_TIMEOUT; + tv.tv_usec = 0; + setsockopt(clientsock, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)); + + fbvnc_handleclient(clientsock); + + close(clientsock); + g_fbvnc.connected = false; + + if (g_fbvnc.on_disconnect != NULL) + { + g_fbvnc.on_disconnect(); + } + } + + close(listensock); + g_fbvnc.listensock = -1; + g_fbvnc.running = false; + return NULL; +} + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: fbvnc_start + ****************************************************************************/ + +int fbvnc_start(FAR const struct fbvnc_cfg_s *cfg) +{ + struct sockaddr_in addr; + pthread_attr_t attr; + struct sched_param param; + int optval; + int ret; + + if (cfg == NULL || cfg->snapshot == NULL) + { + return -EINVAL; + } + + if (g_fbvnc.running) + { + return -EALREADY; + } + + g_fbvnc.snapshot = cfg->snapshot; + g_fbvnc.on_connect = cfg->on_connect; + g_fbvnc.on_disconnect = cfg->on_disconnect; + g_fbvnc.on_invalidate = cfg->on_invalidate; + g_fbvnc.on_pointer = cfg->on_pointer; + g_fbvnc.on_key = cfg->on_key; + + g_w = cfg->width != 0 ? cfg->width : VNC_WIDTH; + g_h = cfg->height != 0 ? cfg->height : VNC_HEIGHT; + g_stride = cfg->stride != 0 ? cfg->stride : g_w * VNC_BYTESPP; + g_fbvnc.connected = false; + + /* Listen before reporting success: a port already in use has to reach + * whoever asked for the server, not a thread that exits on its own. + */ + + g_fbvnc.listensock = socket(AF_INET, SOCK_STREAM, 0); + if (g_fbvnc.listensock < 0) + { + return -errno; + } + + optval = 1; + setsockopt(g_fbvnc.listensock, SOL_SOCKET, SO_REUSEADDR, &optval, + sizeof(optval)); + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = HTONS(VNC_PORT); + addr.sin_addr.s_addr = HTONL(INADDR_ANY); + + if (bind(g_fbvnc.listensock, (FAR struct sockaddr *)&addr, + sizeof(addr)) < 0 || + listen(g_fbvnc.listensock, 1) < 0) + { + ret = -errno; + close(g_fbvnc.listensock); + g_fbvnc.listensock = -1; + return ret; + } + + syslog(LOG_INFO, "fbvnc: listening on port %d\n", VNC_PORT); + g_fbvnc.running = true; + + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, VNC_STACKSIZE); + + param.sched_priority = VNC_PRIORITY; + pthread_attr_setschedparam(&attr, ¶m); + + ret = pthread_create(&g_fbvnc.thread, &attr, fbvnc_thread, NULL); + pthread_attr_destroy(&attr); + + if (ret != 0) + { + g_fbvnc.running = false; + close(g_fbvnc.listensock); + g_fbvnc.listensock = -1; + return -ret; + } + + pthread_setname_np(g_fbvnc.thread, "fbvnc"); + return OK; +} + +/**************************************************************************** + * Name: fbvnc_stop + ****************************************************************************/ + +void fbvnc_stop(void) +{ + if (g_fbvnc.running) + { + g_fbvnc.running = false; + + /* The accept() timeout bounds how long this takes; the thread + * closes the socket on its way out, so the port is free by the + * time this returns. + */ + + pthread_join(g_fbvnc.thread, NULL); + } +} + +/**************************************************************************** + * Name: fbvnc_is_connected + ****************************************************************************/ + +bool fbvnc_is_connected(void) +{ + return g_fbvnc.connected; +} diff --git a/system/fbvnc/CMakeLists.txt b/system/fbvnc/CMakeLists.txt new file mode 100644 index 00000000000..5e140729962 --- /dev/null +++ b/system/fbvnc/CMakeLists.txt @@ -0,0 +1,25 @@ +# ############################################################################## +# apps/system/fbvnc/CMakeLists.txt +# +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with this work for +# additional information regarding copyright ownership. The ASF licenses this +# file to you under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +# ############################################################################## + +if(CONFIG_SYSTEM_FBVNC) + target_sources(apps PRIVATE fbvnc_main.c) +endif() diff --git a/system/fbvnc/Kconfig b/system/fbvnc/Kconfig new file mode 100644 index 00000000000..846de83a40d --- /dev/null +++ b/system/fbvnc/Kconfig @@ -0,0 +1,34 @@ +# +# For a description of the syntax of this configuration file, +# see the file kconfig-language.txt in the NuttX tools repository. +# + +config SYSTEM_FBVNC + tristate "VNC framebuffer mirror daemon" + default n + depends on NETUTILS_FBVNC && VIDEO_FB + select FB_UPDATE + ---help--- + Serves any framebuffer device over VNC with no application + cooperation: what changed comes from the kernel's dirty-area + reporting, and remote input enters through the uinput devices. + + Selects FB_UPDATE, which is what carries an application's + report of what it changed into the kernel. Without it the + daemon still works, sending a whole frame per request. + +if SYSTEM_FBVNC + +config SYSTEM_FBVNC_PROGNAME + string "Program name" + default "fbvnc" + +config SYSTEM_FBVNC_PRIORITY + int "Task priority" + default 100 + +config SYSTEM_FBVNC_STACKSIZE + int "Stack size" + default 4096 + +endif diff --git a/system/fbvnc/Make.defs b/system/fbvnc/Make.defs new file mode 100644 index 00000000000..f3fc85c0664 --- /dev/null +++ b/system/fbvnc/Make.defs @@ -0,0 +1,25 @@ +############################################################################ +# apps/system/fbvnc/Make.defs +# +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +ifneq ($(CONFIG_SYSTEM_FBVNC),) +CONFIGURED_APPS += $(APPDIR)/system/fbvnc +endif diff --git a/system/fbvnc/Makefile b/system/fbvnc/Makefile new file mode 100644 index 00000000000..4469f42f3a1 --- /dev/null +++ b/system/fbvnc/Makefile @@ -0,0 +1,32 @@ +############################################################################ +# apps/system/fbvnc/Makefile +# +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +include $(APPDIR)/Make.defs + +PROGNAME = $(CONFIG_SYSTEM_FBVNC_PROGNAME) +PRIORITY = $(CONFIG_SYSTEM_FBVNC_PRIORITY) +STACKSIZE = $(CONFIG_SYSTEM_FBVNC_STACKSIZE) +MODULE = $(CONFIG_SYSTEM_FBVNC) + +MAINSRC = fbvnc_main.c + +include $(APPDIR)/Application.mk diff --git a/system/fbvnc/fbvnc_main.c b/system/fbvnc/fbvnc_main.c new file mode 100644 index 00000000000..860039edfc7 --- /dev/null +++ b/system/fbvnc/fbvnc_main.c @@ -0,0 +1,897 @@ +/**************************************************************************** + * apps/system/fbvnc/fbvnc_main.c + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/* A VNC server for any framebuffer, no application cooperation required. + * + * fbvnc start [/dev/fb0] [--diff | --full] + * fbvnc stop + * fbvnc status + * + * It is a service rather than part of an application: it serves whatever + * is on the framebuffer, and the application that draws there neither + * knows nor needs to. Starting it before or after that application makes + * no difference, and it survives the application being restarted. + * + * The framebuffer is mapped and served as-is; what changed comes from the + * kernel's dirty-area reporting (FBIOC_WATCHAREA), which every application + * that issues FBIO_UPDATE feeds, LVGL's fbdev driver among them. An + * An application that writes to the framebuffer without saying so, a game + * rendering straight into the mapping, say, is served with --diff, which + * keeps a copy of the last frame sent and compares against it, or with + * --full, a whole frame per update request. + * + * Remote input goes to the uinput devices when they exist: the pointer as + * touch samples on /dev/utouch, keys as keyboard events on /dev/ukeyboard. + * + * TODO: known limitation, with LVGL as the application. + * + * A key of LVGL's on-screen keyboard, clicked once, is typed twice, and + * holding the click types the key over and over. Everything up to the + * device is known good: one click writes exactly one TOUCH_DOWN and one + * TOUCH_UP here, and the samples arrive at /dev/utouch. + * + * The suspect is lv_nuttx_touchscreen.c, which remembers the state of + * the last sample but not its position, while LVGL clears the sample + * structure before every read. A pointer that reports only when + * something changes, this one, or any touch controller with an + * interrupt, therefore reads as pressed at the origin between its + * samples, which is a press leaving the widget and coming back. + * Repeating the position while the button is held was tried and did not + * change the doubling, so this is not established, only where to look + * next: count what the driver delivers per click against what + * lv_buttonmatrix acts on. + * + * Typing into LVGL widgets from a remote keyboard does not work at all, + * for a plainer reason: LVGL's NuttX port has a touchscreen driver and + * no keyboard one, so nothing reads /dev/ukeyboard into an indev. An + * application that reads the keyboard itself, lvglterm does, is + * unaffected, and works. + * Whatever reads those, an LVGL touchscreen driver, an lvglterm, + * receives the remote user exactly as it would a local one. + */ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +#define FBVNC_RFB_BUTTON1 (1 << 0) + +/* Rows compared as a unit in --diff. Larger bands mean fewer memcmp calls + * and coarser rectangles; 16 is the height of a Hextile tile, so a band + * that changed costs whole tiles either way. + */ + +#define FBVNC_BAND 16 + +/* The only pixel format served; the framebuffer is refused at startup + * if it is anything else. + */ + +#define FBVNC_BYTESPP 2 + +/* Keys held down at once, tracked so that a client that vanishes mid-game + * does not leave the application with Ctrl still down. Enough for the + * modifiers plus a couple of ordinary keys. + */ + +#define FBVNC_MAXHELD 8 + +/**************************************************************************** + * Private Types + ****************************************************************************/ + +struct fbvnc_key_s +{ + uint32_t code; + uint32_t type; +}; + +/**************************************************************************** + * Private Data + ****************************************************************************/ + +static int g_fbfd = -1; +static int g_touchfd = -1; +static int g_kbdfd = -1; +static FAR const uint8_t *g_fb; +static uint16_t g_width; +static uint16_t g_height; +static bool g_fullonly; +static bool g_diff; +static bool g_watching; +static FAR uint8_t *g_shadow; +static uint32_t g_stride; +static uint32_t g_fblen; +static volatile bool g_invalid = true; +static uint8_t g_buttons; + +/* Keys currently down, with the event type that releases each: a special + * key is released with KEYBOARD_SPECREL, and releasing it as an ordinary + * key would report a character instead. + */ + +static struct fbvnc_key_s g_held[FBVNC_MAXHELD]; +static uint8_t g_nheld; +static pid_t g_daemon = -1; + +/**************************************************************************** + * Private Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: fbvnc_snapshot + * + * Description: + * Runs on the server thread. Drains the kernel's dirty queue; a full + * screen when asked to start over or when running --full; in --diff, + * the bands that differ from the last frame sent. + * + ****************************************************************************/ + +static FAR const uint8_t * +fbvnc_snapshot(FAR struct fbvnc_rect_s *rects, uint32_t maxrects, + FAR uint32_t *nrects) +{ + struct fb_area_s area; + uint32_t total = 0; + uint32_t n = 0; + uint32_t idx; + + if (g_invalid || g_fullonly) + { + /* Drop anything stale first */ + + while (ioctl(g_fbfd, FBIOC_GETDIRTY, (unsigned long)&area) == OK); + + rects[0].x = 0; + rects[0].y = 0; + rects[0].w = g_width; + rects[0].h = g_height; + *nrects = 1; + g_invalid = false; + + if (g_shadow != NULL) + { + memcpy(g_shadow, g_fb, g_fblen); + } + + return g_fb; + } + + if (g_diff && !g_watching) + { + uint32_t bandlen = g_stride * FBVNC_BAND; + uint32_t y; + + for (y = 0; y < g_height && n < maxrects; y += FBVNC_BAND) + { + uint32_t rows = MIN(FBVNC_BAND, g_height - y); + uint32_t off = y * g_stride; + uint32_t len = rows == FBVNC_BAND ? bandlen : rows * g_stride; + + if (memcmp(g_shadow + off, g_fb + off, len) == 0) + { + continue; + } + + memcpy(g_shadow + off, g_fb + off, len); + + /* Bands that changed together are one rectangle */ + + if (n > 0 && rects[n - 1].y + rects[n - 1].h == y) + { + rects[n - 1].h += rows; + continue; + } + + rects[n].x = 0; + rects[n].y = y; + rects[n].w = g_width; + rects[n].h = rows; + n++; + } + + *nrects = n; + return g_fb; + } + + while (ioctl(g_fbfd, FBIOC_GETDIRTY, (unsigned long)&area) == OK) + { + uint32_t best = maxrects; + int32_t bestcost = 0; + uint32_t i; + + /* Areas reported for one redraw overlap: a widget, then the panel + * it sits on, then the screen behind that. Sending each of them + * sends the same pixels several times over. A tab change was + * measured at fifteen rectangles covering four and a half + * screens, and a burst that large empties the network buffers + * and stalls the send for seconds. + * + * So areas are joined, but only where joining is not itself + * expensive: the rectangle around two of them can be far larger + * than the two, and answering a change in two corners by sending + * everything between them is how a small edit becomes a whole + * screen. The cost of a join is what the surrounding rectangle + * covers beyond the pair; a join that costs nothing is taken, and + * the cheapest one wins. + */ + + for (i = 0; i < n; i++) + { + uint16_t x1 = MIN(rects[i].x, area.x); + uint16_t y1 = MIN(rects[i].y, area.y); + uint16_t x2 = MAX(rects[i].x + rects[i].w, area.x + area.w); + uint16_t y2 = MAX(rects[i].y + rects[i].h, area.y + area.h); + int32_t cost = (int32_t)((uint32_t)(x2 - x1) * (y2 - y1)) - + (int32_t)((uint32_t)rects[i].w * rects[i].h) - + (int32_t)((uint32_t)area.w * area.h); + + if (best == maxrects || cost < bestcost) + { + best = i; + bestcost = cost; + } + } + + /* Somewhere to put it: a join worth making, or room for one more. + * With neither, the cheapest join is taken anyway, a rectangle + * too many is worse than a rectangle too large. + */ + + if (best < n && (bestcost <= 0 || n == maxrects)) + { + uint16_t x2 = MAX(rects[best].x + rects[best].w, area.x + area.w); + uint16_t y2 = MAX(rects[best].y + rects[best].h, area.y + area.h); + + rects[best].x = MIN(rects[best].x, area.x); + rects[best].y = MIN(rects[best].y, area.y); + rects[best].w = x2 - rects[best].x; + rects[best].h = y2 - rects[best].y; + } + else + { + rects[n].x = area.x; + rects[n].y = area.y; + rects[n].w = area.w; + rects[n].h = area.h; + n++; + } + } + + /* What is going to be sent, which is not what arrived: the areas + * overlap, and counting them as they came calls a fraction of the + * screen a screenful and sends everything for no reason. + */ + + for (idx = 0; idx < n; idx++) + { + total += (uint32_t)rects[idx].w * rects[idx].h; + } + + /* What the application reported is what it redrew, which is not the + * same as what changed: a toolkit that animates one element inside a + * panel invalidates the panel, and half a screen arrives as dirty for + * the sake of a moving handle. Where there is a shadow to compare + * against, each reported area is narrowed to the bands inside it that + * actually differ, bounded by the area, not by the screen, so it + * costs a fraction of comparing everything. + */ + + if (g_shadow != NULL) + { + uint32_t out = 0; + + for (idx = 0; idx < n; idx++) + { + uint32_t rowlen = (uint32_t)rects[idx].w * FBVNC_BYTESPP; + int32_t first = -1; + int32_t last = -1; + uint16_t y; + + for (y = 0; y < rects[idx].h; y++) + { + uint32_t off = (uint32_t)(rects[idx].y + y) * g_stride + + (uint32_t)rects[idx].x * FBVNC_BYTESPP; + + if (memcmp(g_shadow + off, g_fb + off, rowlen) != 0) + { + if (first < 0) + { + first = y; + } + + last = y; + memcpy(g_shadow + off, g_fb + off, rowlen); + } + } + + if (first < 0) + { + continue; /* reported, but nothing in it changed */ + } + + rects[out] = rects[idx]; + rects[out].y += first; + rects[out].h = last - first + 1; + out++; + } + + n = out; + total = 0; + + for (idx = 0; idx < n; idx++) + { + total += (uint32_t)rects[idx].w * rects[idx].h; + } + } + + /* Past a whole screen's worth there is nothing to be gained by being + * careful about which parts changed + */ + + if (total >= (uint32_t)g_width * g_height) + { + rects[0].x = 0; + rects[0].y = 0; + rects[0].w = g_width; + rects[0].h = g_height; + n = 1; + } + + *nrects = n; + return g_fb; +} + +/**************************************************************************** + * Name: fbvnc_invalidate + ****************************************************************************/ + +static void fbvnc_invalidate(void) +{ + g_invalid = true; +} + +/**************************************************************************** + * Name: fbvnc_track_key + * + * Description: + * Remember what is down, so that it can be let go of if the client + * disappears while holding it. + * + ****************************************************************************/ + +static void fbvnc_track_key(uint32_t code, uint32_t type, bool pressed) +{ + uint8_t i; + + for (i = 0; i < g_nheld; i++) + { + if (g_held[i].code == code) + { + if (!pressed) + { + g_held[i] = g_held[--g_nheld]; + } + + return; + } + } + + if (pressed && g_nheld < FBVNC_MAXHELD) + { + g_held[g_nheld].code = code; + g_held[g_nheld].type = type == KEYBOARD_SPECPRESS ? + KEYBOARD_SPECREL : KEYBOARD_RELEASE; + g_nheld++; + } +} + +/**************************************************************************** + * Name: fbvnc_release_input + * + * Description: + * A client that vanishes mid-press must not leave the UI with a finger + * or a key stuck down. + * + ****************************************************************************/ + +static void fbvnc_release_input(void) +{ + if ((g_buttons & FBVNC_RFB_BUTTON1) != 0 && g_touchfd >= 0) + { + struct touch_sample_s sample; + + memset(&sample, 0, sizeof(sample)); + sample.npoints = 1; + sample.point[0].flags = TOUCH_UP | TOUCH_ID_VALID; + write(g_touchfd, &sample, sizeof(sample)); + } + + while (g_nheld > 0 && g_kbdfd >= 0) + { + struct keyboard_event_s ev; + + g_nheld--; + ev.code = g_held[g_nheld].code; + ev.type = g_held[g_nheld].type; + write(g_kbdfd, &ev, sizeof(ev)); + } + + g_buttons = 0; + g_nheld = 0; +} + +/**************************************************************************** + * Name: fbvnc_on_disconnect + ****************************************************************************/ + +static void fbvnc_on_disconnect(void) +{ + fbvnc_release_input(); +} + +/**************************************************************************** + * Name: fbvnc_pointer + * + * Description: + * Button edges and drags become touch samples; hover is nothing, which + * is the contract a touch consumer expects. + * + ****************************************************************************/ + +static void fbvnc_pointer(uint16_t x, uint16_t y, uint8_t buttons) +{ + struct touch_sample_s sample; + uint8_t pressed = buttons & FBVNC_RFB_BUTTON1; + uint8_t waspressed = g_buttons & FBVNC_RFB_BUTTON1; + + g_buttons = buttons; + + if (g_touchfd < 0 || (!pressed && !waspressed)) + { + return; + } + + memset(&sample, 0, sizeof(sample)); + sample.npoints = 1; + sample.point[0].x = x; + sample.point[0].y = y; + sample.point[0].flags = TOUCH_ID_VALID | TOUCH_POS_VALID; + + if (pressed && !waspressed) + { + sample.point[0].flags |= TOUCH_DOWN; + } + else if (pressed) + { + sample.point[0].flags |= TOUCH_MOVE; + } + else + { + sample.point[0].flags |= TOUCH_UP; + } + + write(g_touchfd, &sample, sizeof(sample)); +} + +/**************************************************************************** + * Name: fbvnc_key + * + * Description: + * Printables pass through, RFB sends them already shifted. Enter is + * a line feed, the arrows and their kin go as SPEC events with kbd_codec + * keycodes: all four event types, so nothing is dropped silently. + * + ****************************************************************************/ + +static void fbvnc_key(uint32_t keysym, bool pressed) +{ + struct keyboard_event_s ev; + uint32_t type = pressed ? KEYBOARD_PRESS : KEYBOARD_RELEASE; + uint32_t code; + + if (g_kbdfd < 0) + { + return; + } + + if (keysym >= 0x20 && keysym <= 0x7e) + { + code = keysym; + } + else + { + switch (keysym) + { + case XK_Return: + case XK_KP_Enter: + code = '\n'; + break; + + case XK_BackSpace: + code = '\b'; + break; + + case XK_Tab: + code = '\t'; + break; + + case XK_Escape: + code = 0x1b; + break; + + case XK_Up: + code = KEYCODE_UP; + type = pressed ? KEYBOARD_SPECPRESS : KEYBOARD_SPECREL; + break; + + case XK_Down: + code = KEYCODE_DOWN; + type = pressed ? KEYBOARD_SPECPRESS : KEYBOARD_SPECREL; + break; + + case XK_Left: + code = KEYCODE_LEFT; + type = pressed ? KEYBOARD_SPECPRESS : KEYBOARD_SPECREL; + break; + + case XK_Right: + code = KEYCODE_RIGHT; + type = pressed ? KEYBOARD_SPECPRESS : KEYBOARD_SPECREL; + break; + + case XK_Delete: + code = KEYCODE_FWDDEL; + type = pressed ? KEYBOARD_SPECPRESS : KEYBOARD_SPECREL; + break; + + case XK_Insert: + code = KEYCODE_INSERT; + type = pressed ? KEYBOARD_SPECPRESS : KEYBOARD_SPECREL; + break; + + case XK_Page_Up: + code = KEYCODE_PAGEUP; + type = pressed ? KEYBOARD_SPECPRESS : KEYBOARD_SPECREL; + break; + + case XK_Page_Down: + code = KEYCODE_PAGEDOWN; + type = pressed ? KEYBOARD_SPECPRESS : KEYBOARD_SPECREL; + break; + + /* The modifiers are keys in their own right to whatever is + * reading: a game binds fire to Ctrl and strafe to Alt, and + * dropping them as decoration leaves it unplayable. + */ + + case XK_Control_L: + case XK_Control_R: + code = KEYCODE_LCTRL; + type = pressed ? KEYBOARD_SPECPRESS : KEYBOARD_SPECREL; + break; + + case XK_Shift_L: + case XK_Shift_R: + code = KEYCODE_LSHIFT; + type = pressed ? KEYBOARD_SPECPRESS : KEYBOARD_SPECREL; + break; + + case XK_Alt_L: + case XK_Alt_R: + code = KEYCODE_LALT; + type = pressed ? KEYBOARD_SPECPRESS : KEYBOARD_SPECREL; + break; + + default: + if (keysym >= XK_F1 && keysym <= XK_F12) + { + code = KEYCODE_F1 + (keysym - XK_F1); + type = pressed ? KEYBOARD_SPECPRESS : KEYBOARD_SPECREL; + break; + } + + return; + } + } + + fbvnc_track_key(code, type, pressed); + + ev.code = code; + ev.type = type; + write(g_kbdfd, &ev, sizeof(ev)); +} + +static int fbvnc_daemon(int argc, FAR char *argv[]) +{ + struct fbvnc_cfg_s cfg; + struct fb_videoinfo_s vinfo; + struct fb_planeinfo_s pinfo; + FAR const char *fbdev = "/dev/fb0"; + int ret; + int i; + + for (i = 1; i < argc; i++) + { + if (argv[i][0] != '-') + { + /* The framebuffer to serve, said plainly rather than behind a + * flag: which screen this is about is the one thing the + * command is always about. + */ + + fbdev = argv[i]; + } + else if (strcmp(argv[i], "--full") == 0) + { + g_fullonly = true; + } + else if (strcmp(argv[i], "--diff") == 0) + { + g_diff = true; + } + else + { + fprintf(stderr, "fbvnc: unknown option %s\n", argv[i]); + g_daemon = -1; + return EXIT_FAILURE; + } + } + + g_fbfd = open(fbdev, O_RDWR); + if (g_fbfd < 0) + { + perror("fbvnc: cannot open framebuffer"); + g_daemon = -1; + return EXIT_FAILURE; + } + + if (ioctl(g_fbfd, FBIOGET_VIDEOINFO, (unsigned long)&vinfo) < 0 || + ioctl(g_fbfd, FBIOGET_PLANEINFO, (unsigned long)&pinfo) < 0) + { + perror("fbvnc: cannot query framebuffer"); + g_daemon = -1; + return EXIT_FAILURE; + } + + if (pinfo.bpp != 16) + { + fprintf(stderr, "fbvnc: %u bpp framebuffer; only 16 is served\n", + pinfo.bpp); + g_daemon = -1; + return EXIT_FAILURE; + } + + g_fb = mmap(NULL, pinfo.fblen, PROT_READ, MAP_SHARED | MAP_FILE, + g_fbfd, 0); + if (g_fb == MAP_FAILED) + { + perror("fbvnc: mmap"); + g_daemon = -1; + return EXIT_FAILURE; + } + + g_width = vinfo.xres; + g_height = vinfo.yres; + g_stride = pinfo.stride; + g_fblen = pinfo.fblen; + + if (g_diff) + { + g_shadow = malloc(g_fblen); + if (g_shadow == NULL) + { + fprintf(stderr, "fbvnc: no memory for a %lu byte shadow " + "frame\n", (unsigned long)g_fblen); + return EXIT_FAILURE; + } + } + + if (!g_fullonly && ioctl(g_fbfd, FBIOC_WATCHAREA, 1) == OK) + { + g_watching = true; + } + else if (!g_fullonly && !g_diff) + { + printf("fbvnc: no dirty reporting; falling back to full frames\n"); + g_fullonly = true; + } + + /* Never blocking: the thread that writes these is the one reading the + * client, and a device whose buffer is full would stop it reading, + * which is the whole connection stalled by a fast drag. A sample + * dropped costs nothing: the next one carries where the pointer is + * now, which is what the application wants anyway. + */ + + g_touchfd = open("/dev/utouch", O_WRONLY | O_NONBLOCK); + g_kbdfd = open("/dev/ukeyboard", O_WRONLY | O_NONBLOCK); + + memset(&cfg, 0, sizeof(cfg)); + cfg.snapshot = fbvnc_snapshot; + cfg.on_connect = fbvnc_invalidate; + cfg.on_invalidate = fbvnc_invalidate; + cfg.on_disconnect = fbvnc_on_disconnect; + cfg.on_pointer = fbvnc_pointer; + cfg.on_key = fbvnc_key; + cfg.width = vinfo.xres; + cfg.height = vinfo.yres; + cfg.stride = pinfo.stride; + + ret = fbvnc_start(&cfg); + if (ret < 0) + { + fprintf(stderr, "fbvnc: fbvnc_start failed: %d\n", ret); + g_daemon = -1; + return EXIT_FAILURE; + } + + printf("fbvnc: serving %s (%ux%u) on port %d%s\n", fbdev, + vinfo.xres, vinfo.yres, CONFIG_NETUTILS_FBVNC_PORT, + g_fullonly ? " (full frames)" : g_diff ? " (compared frames)" : ""); + + /* Nothing left to do but stay alive: the server has a thread of its + * own, and it belongs to this task. + */ + + while (g_daemon >= 0) + { + sleep(1); + } + + fbvnc_stop(); + munmap((FAR void *)g_fb, g_fblen); + close(g_fbfd); + + if (g_touchfd >= 0) + { + close(g_touchfd); + } + + if (g_kbdfd >= 0) + { + close(g_kbdfd); + } + + free(g_shadow); + g_shadow = NULL; + printf("fbvnc: stopped\n"); + return EXIT_SUCCESS; +} + +/**************************************************************************** + * Name: fbvnc_usage + ****************************************************************************/ + +static int fbvnc_usage(FAR const char *progname) +{ + fprintf(stderr, + "Usage: %s start [] [--diff | --full]\n" + " %s stop\n" + " %s status\n" + "\n" + " what to serve, /dev/fb0 by default\n" + " --diff work out what changed by comparing frames, for\n" + " an application that redraws without saying so\n" + " --full a whole frame per update request\n", + progname, progname, progname); + return EXIT_FAILURE; +} + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +int main(int argc, FAR char *argv[]) +{ + if (argc < 2) + { + return fbvnc_usage(argv[0]); + } + + if (strcmp(argv[1], "start") == 0) + { + if (g_daemon >= 0) + { + fprintf(stderr, "fbvnc: already running as task %d\n", + g_daemon); + return EXIT_FAILURE; + } + + /* The daemon owns the server thread, so it has to outlive this + * command rather than run inside it. + */ + + g_daemon = task_create("fbvnc", + CONFIG_SYSTEM_FBVNC_PRIORITY, + CONFIG_SYSTEM_FBVNC_STACKSIZE, + fbvnc_daemon, + argc > 2 ? &argv[1] : NULL); + if (g_daemon < 0) + { + fprintf(stderr, "fbvnc: cannot start: %d\n", errno); + g_daemon = -1; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; + } + + if (strcmp(argv[1], "stop") == 0) + { + if (g_daemon < 0) + { + fprintf(stderr, "fbvnc: not running\n"); + return EXIT_FAILURE; + } + + g_daemon = -1; + return EXIT_SUCCESS; + } + + if (strcmp(argv[1], "status") == 0) + { + if (g_daemon < 0) + { + printf("fbvnc: not running\n"); + } + else + { + printf("fbvnc: serving %ux%u on port %d, %s\n", + g_width, g_height, CONFIG_NETUTILS_FBVNC_PORT, + fbvnc_is_connected() ? "client connected" : + "waiting for a client"); + } + + return EXIT_SUCCESS; + } + + return fbvnc_usage(argv[0]); +}